2023-09-23T19:19:54.125 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T19:19:54.125 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812 --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --owner scheduled_teuthology@teuthology --verbose --name teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812/orig.config.yaml 2023-09-23T19:19:54.180 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T19:19:54.274 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7401812' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4997 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi135.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi144.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T19:19:54.274 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T19:19:54.275 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T19:19:54.275 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T19:19:54.277 INFO:teuthology.task.internal:Checking packages... 2023-09-23T19:19:54.298 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T19:19:54.299 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T19:19:54.299 INFO:teuthology.packaging:ref: None 2023-09-23T19:19:54.299 INFO:teuthology.packaging:tag: None 2023-09-23T19:19:54.299 INFO:teuthology.packaging:branch: reef 2023-09-23T19:19:54.299 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:19:54.299 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T19:19:54.553 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460.gbb0daeff 2023-09-23T19:19:54.553 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T19:19:54.560 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T19:19:54.560 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T19:19:54.644 INFO:teuthology.task.internal:Saving configuration 2023-09-23T19:19:54.666 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T19:19:54.672 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T19:19:54.699 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi135.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 19:14:57.983037', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T19:19:54.719 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi144.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 19:14:57.984771', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T19:19:54.719 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T19:19:54.723 INFO:teuthology.task.internal:roles: ubuntu@smithi135.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T19:19:54.723 INFO:teuthology.task.internal:roles: ubuntu@smithi144.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T19:19:54.723 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T19:19:54.791 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f9c3393eaf0>, signals=[15]) 2023-09-23T19:19:54.791 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T19:19:54.796 INFO:teuthology.task.internal:Opening connections... 2023-09-23T19:19:54.797 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi135.front.sepia.ceph.com 2023-09-23T19:19:54.798 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T19:19:54.865 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi144.front.sepia.ceph.com 2023-09-23T19:19:54.865 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T19:19:54.931 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T19:19:54.937 DEBUG:teuthology.orchestra.run.smithi135:> uname -m 2023-09-23T19:19:54.957 INFO:teuthology.orchestra.run.smithi135.stdout:x86_64 2023-09-23T19:19:54.957 DEBUG:teuthology.orchestra.run.smithi135:> cat /etc/os-release 2023-09-23T19:19:55.016 INFO:teuthology.orchestra.run.smithi135.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T19:19:55.016 INFO:teuthology.orchestra.run.smithi135.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T19:19:55.016 INFO:teuthology.orchestra.run.smithi135.stdout:ID="rhel" 2023-09-23T19:19:55.016 INFO:teuthology.orchestra.run.smithi135.stdout:ID_LIKE="fedora" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:VERSION_ID="8.6" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:PLATFORM_ID="platform:el8" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:ANSI_COLOR="0;31" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T19:19:55.017 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T19:19:55.018 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T19:19:55.018 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T19:19:55.018 INFO:teuthology.lock.ops:Updating smithi135.front.sepia.ceph.com on lock server 2023-09-23T19:19:55.043 DEBUG:teuthology.orchestra.run.smithi144:> uname -m 2023-09-23T19:19:55.064 INFO:teuthology.orchestra.run.smithi144.stdout:x86_64 2023-09-23T19:19:55.064 DEBUG:teuthology.orchestra.run.smithi144:> cat /etc/os-release 2023-09-23T19:19:55.122 INFO:teuthology.orchestra.run.smithi144.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T19:19:55.122 INFO:teuthology.orchestra.run.smithi144.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T19:19:55.122 INFO:teuthology.orchestra.run.smithi144.stdout:ID="rhel" 2023-09-23T19:19:55.122 INFO:teuthology.orchestra.run.smithi144.stdout:ID_LIKE="fedora" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:VERSION_ID="8.6" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:PLATFORM_ID="platform:el8" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:ANSI_COLOR="0;31" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T19:19:55.123 INFO:teuthology.orchestra.run.smithi144.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T19:19:55.124 INFO:teuthology.orchestra.run.smithi144.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T19:19:55.124 INFO:teuthology.lock.ops:Updating smithi144.front.sepia.ceph.com on lock server 2023-09-23T19:19:55.143 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T19:19:55.151 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T19:19:55.156 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T19:19:55.156 DEBUG:teuthology.orchestra.run.smithi135:> test '!' -e /home/ubuntu/cephtest 2023-09-23T19:19:55.159 DEBUG:teuthology.orchestra.run.smithi144:> test '!' -e /home/ubuntu/cephtest 2023-09-23T19:19:55.180 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T19:19:55.188 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T19:19:55.188 DEBUG:teuthology.orchestra.run.smithi135:> test -z $(ls -A /var/lib/ceph) 2023-09-23T19:19:55.218 DEBUG:teuthology.orchestra.run.smithi144:> test -z $(ls -A /var/lib/ceph) 2023-09-23T19:19:55.248 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T19:19:55.305 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T19:19:55.319 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T19:19:55.319 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T19:19:55.319 DEBUG:teuthology.orchestra.run.smithi135:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T19:19:55.320 DEBUG:teuthology.orchestra.run.smithi144:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T19:19:55.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:19:55.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:19:55.338 DEBUG:teuthology.orchestra.run.smithi144:> uname -r 2023-09-23T19:19:55.339 DEBUG:teuthology.orchestra.run.smithi135:> uname -r 2023-09-23T19:19:55.397 INFO:teuthology.orchestra.run.smithi144.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T19:19:55.397 INFO:teuthology.task.kernel:Running kernel on smithi144: 4.18.0-372.9.1.el8.x86_64 2023-09-23T19:19:55.397 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum install -y kernel 2023-09-23T19:19:55.398 INFO:teuthology.orchestra.run.smithi135.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T19:19:55.399 INFO:teuthology.task.kernel:Running kernel on smithi135: 4.18.0-372.9.1.el8.x86_64 2023-09-23T19:19:55.399 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum install -y kernel 2023-09-23T19:19:56.580 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:19:56.580 INFO:teuthology.orchestra.run.smithi135.stdout:Unable to read consumer identity 2023-09-23T19:19:56.614 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:19:56.614 INFO:teuthology.orchestra.run.smithi144.stdout:Unable to read consumer identity 2023-09-23T19:19:58.135 INFO:teuthology.orchestra.run.smithi135.stdout:Copr repo for python3-asyncssh owned by ceph 12 kB/s | 3.5 kB 00:00 2023-09-23T19:19:58.155 INFO:teuthology.orchestra.run.smithi144.stdout:Copr repo for python3-asyncssh owned by ceph 20 kB/s | 3.5 kB 00:00 2023-09-23T19:19:58.930 INFO:teuthology.orchestra.run.smithi144.stdout:Extra Packages for Enterprise Linux 21 MB/s | 16 MB 00:00 2023-09-23T19:19:58.951 INFO:teuthology.orchestra.run.smithi135.stdout:Extra Packages for Enterprise Linux 20 MB/s | 16 MB 00:00 2023-09-23T19:20:02.430 INFO:teuthology.orchestra.run.smithi144.stdout:lab-extras 413 kB/s | 24 kB 00:00 2023-09-23T19:20:02.493 INFO:teuthology.orchestra.run.smithi135.stdout:lab-extras 299 kB/s | 24 kB 00:00 2023-09-23T19:20:03.150 INFO:teuthology.orchestra.run.smithi144.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T19:20:03.196 INFO:teuthology.orchestra.run.smithi135.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T19:20:03.201 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:20:03.202 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:20:03.202 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:20:03.248 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:20:03.249 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:20:03.250 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:20:03.322 DEBUG:teuthology.orchestra.run.smithi144:> echo no | sudo yum reinstall kernel || true 2023-09-23T19:20:03.409 DEBUG:teuthology.orchestra.run.smithi135:> echo no | sudo yum reinstall kernel || true 2023-09-23T19:20:03.655 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:20:03.655 INFO:teuthology.orchestra.run.smithi144.stdout:Unable to read consumer identity 2023-09-23T19:20:03.744 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:20:03.745 INFO:teuthology.orchestra.run.smithi135.stdout:Unable to read consumer identity 2023-09-23T19:20:03.918 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 07:20:02 PM UTC. 2023-09-23T19:20:03.962 INFO:teuthology.orchestra.run.smithi144.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T19:20:04.003 INFO:teuthology.orchestra.run.smithi144.stderr:Error: No packages marked for reinstall. 2023-09-23T19:20:04.008 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 07:20:02 PM UTC. 2023-09-23T19:20:04.051 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum reinstall -y kernel || true 2023-09-23T19:20:04.052 INFO:teuthology.orchestra.run.smithi135.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T19:20:04.089 INFO:teuthology.orchestra.run.smithi135.stderr:Error: No packages marked for reinstall. 2023-09-23T19:20:04.136 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum reinstall -y kernel || true 2023-09-23T19:20:04.382 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:20:04.382 INFO:teuthology.orchestra.run.smithi144.stdout:Unable to read consumer identity 2023-09-23T19:20:04.465 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:20:04.466 INFO:teuthology.orchestra.run.smithi135.stdout:Unable to read consumer identity 2023-09-23T19:20:04.638 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 07:20:02 PM UTC. 2023-09-23T19:20:04.678 INFO:teuthology.orchestra.run.smithi144.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T19:20:04.700 INFO:teuthology.orchestra.run.smithi144.stderr:Error: No packages marked for reinstall. 2023-09-23T19:20:04.721 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 07:20:02 PM UTC. 2023-09-23T19:20:04.746 DEBUG:teuthology.orchestra.run.smithi144:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T19:20:04.761 INFO:teuthology.orchestra.run.smithi135.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T19:20:04.806 INFO:teuthology.orchestra.run.smithi135.stderr:Error: No packages marked for reinstall. 2023-09-23T19:20:04.826 INFO:teuthology.orchestra.run.smithi144.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T19:20:04.826 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T19:20:04.827 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T19:20:04.827 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T19:20:04.827 DEBUG:teuthology.orchestra.run.smithi144:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T19:20:04.855 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T19:20:04.860 INFO:teuthology.orchestra.run.smithi144.stdout:ttyS1 2023-09-23T19:20:04.873 DEBUG:teuthology.parallel:result is None 2023-09-23T19:20:04.936 INFO:teuthology.orchestra.run.smithi135.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T19:20:04.936 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T19:20:04.936 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T19:20:04.936 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T19:20:04.936 DEBUG:teuthology.orchestra.run.smithi135:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T19:20:04.972 INFO:teuthology.orchestra.run.smithi135.stdout:ttyS1 2023-09-23T19:20:04.992 DEBUG:teuthology.parallel:result is None 2023-09-23T19:20:04.992 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T19:20:04.999 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T19:20:05.000 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T19:20:05.034 DEBUG:teuthology.orchestra.run.smithi144:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T19:20:05.050 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T19:20:05.057 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T19:20:05.066 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T19:20:05.066 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T19:20:05.091 DEBUG:teuthology.orchestra.run.smithi144:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T19:20:05.112 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T19:20:05.118 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T19:20:05.119 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T19:20:05.148 DEBUG:teuthology.orchestra.run.smithi144:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T19:20:05.174 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T19:20:05.184 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T19:20:05.189 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T19:20:05.199 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T19:20:05.200 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T19:20:05.207 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T19:20:05.207 DEBUG:teuthology.orchestra.run.smithi135:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T19:20:05.242 DEBUG:teuthology.orchestra.run.smithi144:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T19:20:05.269 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T19:20:05.277 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T19:20:05.277 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T19:20:05.310 DEBUG:teuthology.orchestra.run.smithi144:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T19:20:05.326 DEBUG:teuthology.orchestra.run.smithi135:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T19:20:05.407 DEBUG:teuthology.orchestra.run.smithi135:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T19:20:05.457 DEBUG:teuthology.orchestra.run.smithi135:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T19:20:05.540 DEBUG:teuthology.orchestra.run.smithi135:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T19:20:05.581 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:20:05.582 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T19:20:05.653 DEBUG:teuthology.orchestra.run.smithi144:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T19:20:05.698 DEBUG:teuthology.orchestra.run.smithi144:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T19:20:05.744 DEBUG:teuthology.orchestra.run.smithi144:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T19:20:05.828 DEBUG:teuthology.orchestra.run.smithi144:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T19:20:05.870 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:20:05.871 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T19:20:05.938 DEBUG:teuthology.orchestra.run.smithi135:> sudo service rsyslog restart 2023-09-23T19:20:05.941 DEBUG:teuthology.orchestra.run.smithi144:> sudo service rsyslog restart 2023-09-23T19:20:05.998 INFO:teuthology.orchestra.run.smithi135.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T19:20:06.038 INFO:teuthology.orchestra.run.smithi144.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T19:20:06.393 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T19:20:06.402 INFO:teuthology.task.internal:Starting timer... 2023-09-23T19:20:06.402 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T19:20:06.448 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T19:20:06.456 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T19:20:06.456 DEBUG:teuthology.orchestra.run.smithi135:> sudo service auditd rotate 2023-09-23T19:20:06.515 INFO:teuthology.orchestra.run.smithi135.stdout:Rotating logs: 2023-09-23T19:20:06.517 DEBUG:teuthology.orchestra.run.smithi144:> sudo service auditd rotate 2023-09-23T19:20:06.580 INFO:teuthology.orchestra.run.smithi144.stdout:Rotating logs: 2023-09-23T19:20:06.581 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T19:20:06.582 DEBUG:teuthology.orchestra.run.smithi135:> /usr/sbin/getenforce 2023-09-23T19:20:06.604 INFO:teuthology.orchestra.run.smithi135.stdout:Permissive 2023-09-23T19:20:06.605 DEBUG:teuthology.orchestra.run.smithi144:> /usr/sbin/getenforce 2023-09-23T19:20:06.628 INFO:teuthology.orchestra.run.smithi144.stdout:Permissive 2023-09-23T19:20:06.628 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi135.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi144.front.sepia.ceph.com': 'permissive'} 2023-09-23T19:20:06.628 DEBUG:teuthology.orchestra.run.smithi135:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T19:20:06.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:20:06.674 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T19:20:06.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:20:06.701 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T19:20:06.701 DEBUG:teuthology.orchestra.run.smithi135:> sudo /usr/sbin/setenforce permissive 2023-09-23T19:20:06.741 DEBUG:teuthology.orchestra.run.smithi144:> sudo /usr/sbin/setenforce permissive 2023-09-23T19:20:06.769 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T19:20:06.779 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T19:20:06.783 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T19:20:06.945 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T19:20:06.985 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T19:20:06.986 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi135.front.sepia.ceph.com,smithi144.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T19:28:41.570 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi135.front.sepia.ceph.com'), Remote(name='ubuntu@smithi144.front.sepia.ceph.com')] 2023-09-23T19:28:41.571 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-09-23T19:28:41.571 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T19:28:41.640 DEBUG:teuthology.orchestra.run.smithi135:> true 2023-09-23T19:28:41.707 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-09-23T19:28:41.708 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2023-09-23T19:28:41.708 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T19:28:41.775 DEBUG:teuthology.orchestra.run.smithi144:> true 2023-09-23T19:28:41.844 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi144.front.sepia.ceph.com' 2023-09-23T19:28:41.844 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T19:28:41.856 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T19:28:41.856 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T19:28:41.857 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T19:28:41.859 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T19:28:41.859 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T19:28:41.888 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T19:28:41.902 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T19:28:41.916 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T19:28:41.931 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T19:28:41.972 INFO:teuthology.orchestra.run.smithi135.stderr:sudo: ntpd: command not found 2023-09-23T19:28:41.984 INFO:teuthology.orchestra.run.smithi135.stdout:506 Cannot talk to daemon 2023-09-23T19:28:41.999 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T19:28:42.006 INFO:teuthology.orchestra.run.smithi144.stderr:sudo: ntpd: command not found 2023-09-23T19:28:42.013 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T19:28:42.018 INFO:teuthology.orchestra.run.smithi144.stdout:506 Cannot talk to daemon 2023-09-23T19:28:42.033 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T19:28:42.046 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T19:28:42.080 INFO:teuthology.orchestra.run.smithi135.stderr:bash: ntpq: command not found 2023-09-23T19:28:42.083 INFO:teuthology.orchestra.run.smithi135.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T19:28:42.084 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================== 2023-09-23T19:28:42.084 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T19:28:42.084 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T19:28:42.084 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T19:28:42.084 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T19:28:42.114 INFO:teuthology.orchestra.run.smithi144.stderr:bash: ntpq: command not found 2023-09-23T19:28:42.118 INFO:teuthology.orchestra.run.smithi144.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T19:28:42.118 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================== 2023-09-23T19:28:42.118 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T19:28:42.118 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T19:28:42.118 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T19:28:42.118 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T19:28:42.119 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T19:28:42.129 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T19:28:42.130 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi135.front.sepia.ceph.com 2023-09-23T19:28:42.130 DEBUG:teuthology.orchestra.run.smithi135:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T19:28:42.131 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi144.front.sepia.ceph.com 2023-09-23T19:28:42.131 DEBUG:teuthology.orchestra.run.smithi144:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T19:28:42.504 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:28:42.531 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:28:43.346 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:26 ago on Sat 23 Sep 2023 07:27:16 PM UTC. 2023-09-23T19:28:43.362 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:26 ago on Sat 23 Sep 2023 07:27:16 PM UTC. 2023-09-23T19:28:45.059 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:28:45.060 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:28:45.060 INFO:teuthology.orchestra.run.smithi144.stdout: Package Architecture Version Repository Size 2023-09-23T19:28:45.061 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:28:45.061 INFO:teuthology.orchestra.run.smithi144.stdout:Resetting modules: 2023-09-23T19:28:45.061 INFO:teuthology.orchestra.run.smithi144.stdout: container-tools 2023-09-23T19:28:45.061 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:28:45.061 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:28:45.061 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:28:45.061 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:28:45.094 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:28:45.095 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:28:45.095 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2023-09-23T19:28:45.095 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:28:45.095 INFO:teuthology.orchestra.run.smithi135.stdout:Resetting modules: 2023-09-23T19:28:45.095 INFO:teuthology.orchestra.run.smithi135.stdout: container-tools 2023-09-23T19:28:45.096 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:28:45.096 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:28:45.096 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:28:45.096 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:28:46.040 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:29 ago on Sat 23 Sep 2023 07:27:16 PM UTC. 2023-09-23T19:28:46.065 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:29 ago on Sat 23 Sep 2023 07:27:16 PM UTC. 2023-09-23T19:28:48.801 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:28:48.817 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:28:49.240 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:28:49.274 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:28:50.052 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:33 ago on Sat 23 Sep 2023 07:27:16 PM UTC. 2023-09-23T19:28:50.066 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:33 ago on Sat 23 Sep 2023 07:27:16 PM UTC. 2023-09-23T19:28:51.743 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:28:51.783 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:28:51.834 INFO:teuthology.orchestra.run.smithi144.stdout:========================================================================================================================== 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout:========================================================================================================================== 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout:Installing group/module packages: 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:28:51.835 INFO:teuthology.orchestra.run.smithi144.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout:Downgrading: 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T19:28:51.836 INFO:teuthology.orchestra.run.smithi144.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout:Installing module profiles: 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout: container-tools/common 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout:Enabling module streams: 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout: container-tools 3.0 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout:========================================================================================================================== 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout:Install 7 Packages 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout:Downgrade 12 Packages 2023-09-23T19:28:51.837 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:28:51.838 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 51 M 2023-09-23T19:28:51.838 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:28:51.880 INFO:teuthology.orchestra.run.smithi135.stdout:========================================================================================================================== 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout:========================================================================================================================== 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout:Installing group/module packages: 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T19:28:51.881 INFO:teuthology.orchestra.run.smithi135.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout:Downgrading: 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T19:28:51.882 INFO:teuthology.orchestra.run.smithi135.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout:Installing module profiles: 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout: container-tools/common 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout:Enabling module streams: 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout: container-tools 3.0 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout:========================================================================================================================== 2023-09-23T19:28:51.883 INFO:teuthology.orchestra.run.smithi135.stdout:Install 7 Packages 2023-09-23T19:28:51.884 INFO:teuthology.orchestra.run.smithi135.stdout:Downgrade 12 Packages 2023-09-23T19:28:51.884 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:28:51.884 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 51 M 2023-09-23T19:28:51.884 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:28:51.932 INFO:teuthology.orchestra.run.smithi144.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 565 kB/s | 53 kB 00:00 2023-09-23T19:28:51.957 INFO:teuthology.orchestra.run.smithi144.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 431 kB/s | 51 kB 00:00 2023-09-23T19:28:51.968 INFO:teuthology.orchestra.run.smithi135.stdout:(1/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 617 kB/s | 51 kB 00:00 2023-09-23T19:28:51.983 INFO:teuthology.orchestra.run.smithi144.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 630 kB/s | 91 kB 00:00 2023-09-23T19:28:51.985 INFO:teuthology.orchestra.run.smithi135.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 528 kB/s | 53 kB 00:00 2023-09-23T19:28:52.010 INFO:teuthology.orchestra.run.smithi135.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 723 kB/s | 91 kB 00:00 2023-09-23T19:28:52.016 INFO:teuthology.orchestra.run.smithi144.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 6.0 MB/s | 511 kB 00:00 2023-09-23T19:28:52.035 INFO:teuthology.orchestra.run.smithi135.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.3 MB/s | 69 kB 00:00 2023-09-23T19:28:52.069 INFO:teuthology.orchestra.run.smithi135.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 4.9 MB/s | 511 kB 00:00 2023-09-23T19:28:52.071 INFO:teuthology.orchestra.run.smithi144.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 608 kB/s | 69 kB 00:00 2023-09-23T19:28:52.094 INFO:teuthology.orchestra.run.smithi135.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.2 MB/s | 72 kB 00:00 2023-09-23T19:28:52.100 INFO:teuthology.orchestra.run.smithi144.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 861 kB/s | 72 kB 00:00 2023-09-23T19:28:52.176 INFO:teuthology.orchestra.run.smithi144.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 15 MB/s | 2.9 MB 00:00 2023-09-23T19:28:52.192 INFO:teuthology.orchestra.run.smithi135.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 16 MB/s | 2.9 MB 00:00 2023-09-23T19:28:52.200 INFO:teuthology.orchestra.run.smithi144.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 558 kB/s | 56 kB 00:00 2023-09-23T19:28:52.220 INFO:teuthology.orchestra.run.smithi135.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 446 kB/s | 56 kB 00:00 2023-09-23T19:28:52.226 INFO:teuthology.orchestra.run.smithi144.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 1.0 MB/s | 51 kB 00:00 2023-09-23T19:28:52.245 INFO:teuthology.orchestra.run.smithi135.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 966 kB/s | 51 kB 00:00 2023-09-23T19:28:52.293 INFO:teuthology.orchestra.run.smithi144.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 4.7 MB/s | 322 kB 00:00 2023-09-23T19:28:52.296 INFO:teuthology.orchestra.run.smithi135.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-09-23T19:28:52.344 INFO:teuthology.orchestra.run.smithi144.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 818 kB/s | 41 kB 00:00 2023-09-23T19:28:52.354 INFO:teuthology.orchestra.run.smithi135.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 699 kB/s | 41 kB 00:00 2023-09-23T19:28:52.461 INFO:teuthology.orchestra.run.smithi144.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 28 MB/s | 11 MB 00:00 2023-09-23T19:28:52.505 INFO:teuthology.orchestra.run.smithi135.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 25 MB/s | 11 MB 00:00 2023-09-23T19:28:52.553 INFO:teuthology.orchestra.run.smithi144.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 11 MB/s | 2.2 MB 00:00 2023-09-23T19:28:52.614 INFO:teuthology.orchestra.run.smithi135.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 8.5 MB/s | 2.2 MB 00:00 2023-09-23T19:28:52.719 INFO:teuthology.orchestra.run.smithi144.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 25 MB/s | 6.4 MB 00:00 2023-09-23T19:28:52.764 INFO:teuthology.orchestra.run.smithi135.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 25 MB/s | 6.4 MB 00:00 2023-09-23T19:28:52.823 INFO:teuthology.orchestra.run.smithi135.stdout:(15/19): cockpit-podman-29-2.module+el8.6.0+148 18 MB/s | 1.1 MB 00:00 2023-09-23T19:28:52.828 INFO:teuthology.orchestra.run.smithi144.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 25 MB/s | 6.8 MB 00:00 2023-09-23T19:28:52.870 INFO:teuthology.orchestra.run.smithi144.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 7.1 MB/s | 1.1 MB 00:00 2023-09-23T19:28:52.923 INFO:teuthology.orchestra.run.smithi135.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 22 MB/s | 6.8 MB 00:00 2023-09-23T19:28:53.079 INFO:teuthology.orchestra.run.smithi144.stdout:(17/19): containernetworking-plugins-0.9.1-1.mo 22 MB/s | 19 MB 00:00 2023-09-23T19:28:53.104 INFO:teuthology.orchestra.run.smithi144.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 216 kB/s | 51 kB 00:00 2023-09-23T19:28:53.124 INFO:teuthology.orchestra.run.smithi135.stdout:(17/19): containernetworking-plugins-0.9.1-1.mo 21 MB/s | 19 MB 00:00 2023-09-23T19:28:53.129 INFO:teuthology.orchestra.run.smithi144.stdout:(19/19): crun-0.18-3.module+el8.6.0+14874+64436 615 kB/s | 185 kB 00:00 2023-09-23T19:28:53.130 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:28:53.130 INFO:teuthology.orchestra.run.smithi144.stdout:Total 39 MB/s | 51 MB 00:01 2023-09-23T19:28:53.149 INFO:teuthology.orchestra.run.smithi135.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 224 kB/s | 51 kB 00:00 2023-09-23T19:28:53.174 INFO:teuthology.orchestra.run.smithi135.stdout:(19/19): crun-0.18-3.module+el8.6.0+14874+64436 527 kB/s | 185 kB 00:00 2023-09-23T19:28:53.175 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:28:53.175 INFO:teuthology.orchestra.run.smithi135.stdout:Total 39 MB/s | 51 MB 00:01 2023-09-23T19:28:53.605 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:28:53.647 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:28:53.647 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:28:53.655 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:28:53.697 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:28:53.697 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:28:54.088 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:28:54.092 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:28:54.138 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:28:54.142 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:28:55.273 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:28:55.317 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:28:55.324 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T19:28:55.357 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T19:28:55.515 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T19:28:55.522 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T19:28:55.524 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T19:28:55.532 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T19:28:55.548 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T19:28:55.549 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T19:28:55.747 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T19:28:55.773 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T19:29:12.736 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T19:29:12.869 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T19:29:14.503 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T19:29:14.637 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T19:29:14.758 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T19:29:14.871 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T19:29:14.981 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T19:29:15.099 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T19:29:15.184 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T19:29:15.269 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T19:29:15.387 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T19:29:15.387 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T19:29:15.388 INFO:teuthology.orchestra.run.smithi135.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T19:29:15.388 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:29:15.453 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T19:29:15.453 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T19:29:15.453 INFO:teuthology.orchestra.run.smithi144.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T19:29:15.453 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:29:16.245 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T19:29:16.255 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T19:29:16.486 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T19:29:16.503 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T19:29:17.935 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T19:29:17.951 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T19:29:18.246 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T19:29:18.307 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T19:29:18.644 INFO:teuthology.orchestra.run.smithi144.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T19:29:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T19:29:18.918 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T19:29:18.934 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T19:29:19.636 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T19:29:19.669 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T19:29:20.373 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T19:29:20.411 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T19:29:20.626 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T19:29:20.685 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T19:29:20.834 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T19:29:20.876 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T19:29:21.061 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T19:29:21.086 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T19:29:21.192 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T19:29:21.192 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T19:29:21.208 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T19:29:21.208 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T19:29:21.517 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T19:29:21.520 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T19:29:21.674 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T19:29:21.674 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T19:29:21.685 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T19:29:21.685 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T19:29:21.845 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T19:29:21.853 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T19:29:21.986 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T19:29:22.019 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T19:29:22.162 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T19:29:22.162 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T19:29:22.294 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T19:29:22.312 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T19:29:22.485 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T19:29:22.516 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T19:29:22.611 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T19:29:22.653 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T19:29:22.744 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T19:29:22.778 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T19:29:22.929 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T19:29:22.929 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T19:29:22.962 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T19:29:22.962 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T19:29:23.110 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T19:29:23.127 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T19:29:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T19:29:23.644 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T19:29:24.034 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T19:29:24.034 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T19:29:24.035 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T19:29:24.036 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T19:29:24.037 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T19:29:24.037 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T19:29:24.037 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T19:29:24.037 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T19:29:24.037 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T19:29:24.037 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T19:29:24.037 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T19:29:24.037 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T19:29:24.069 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T19:29:24.070 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T19:29:24.071 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T19:29:24.072 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T19:29:24.072 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T19:29:24.072 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T19:29:25.408 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T19:29:25.408 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:08 ago on Sat 23 Sep 2023 07:27:16 PM UTC. 2023-09-23T19:29:25.540 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T19:29:25.540 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:09 ago on Sat 23 Sep 2023 07:27:16 PM UTC. 2023-09-23T19:29:27.990 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout:Downgraded: 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.189 INFO:teuthology.orchestra.run.smithi135.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.190 INFO:teuthology.orchestra.run.smithi135.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T19:29:28.191 INFO:teuthology.orchestra.run.smithi135.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T19:29:28.191 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:29:28.191 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:29:28.227 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:29:28.419 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:29:28.419 INFO:teuthology.orchestra.run.smithi144.stdout:Downgraded: 2023-09-23T19:29:28.419 INFO:teuthology.orchestra.run.smithi144.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.419 INFO:teuthology.orchestra.run.smithi144.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:29:28.420 INFO:teuthology.orchestra.run.smithi144.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.421 INFO:teuthology.orchestra.run.smithi144.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T19:29:28.421 INFO:teuthology.orchestra.run.smithi144.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.421 INFO:teuthology.orchestra.run.smithi144.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.421 INFO:teuthology.orchestra.run.smithi144.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T19:29:28.421 INFO:teuthology.orchestra.run.smithi144.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T19:29:28.421 INFO:teuthology.orchestra.run.smithi144.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T19:29:28.421 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:29:28.421 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:29:28.440 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:28.606 DEBUG:teuthology.parallel:result is None 2023-09-23T19:29:28.606 INFO:teuthology.run_tasks:Running task install... 2023-09-23T19:29:28.614 DEBUG:teuthology.task.install:project ceph 2023-09-23T19:29:28.614 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T19:29:28.614 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T19:29:28.614 INFO:teuthology.task.install:Using flavor: default 2023-09-23T19:29:28.620 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T19:29:28.620 INFO:teuthology.task.install:extra packages: [] 2023-09-23T19:29:28.620 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T19:29:28.620 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T19:29:28.620 INFO:teuthology.packaging:ref: None 2023-09-23T19:29:28.621 INFO:teuthology.packaging:tag: None 2023-09-23T19:29:28.621 INFO:teuthology.packaging:branch: quincy 2023-09-23T19:29:28.621 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:29:28.621 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T19:29:28.623 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T19:29:28.623 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T19:29:28.623 INFO:teuthology.packaging:ref: None 2023-09-23T19:29:28.623 INFO:teuthology.packaging:tag: None 2023-09-23T19:29:28.623 INFO:teuthology.packaging:branch: quincy 2023-09-23T19:29:28.623 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:29:28.623 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T19:29:28.857 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T19:29:28.858 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T19:29:28.958 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T19:29:28.958 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T19:29:28.978 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T19:29:28.979 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:29:28.979 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T19:29:29.006 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T19:29:29.006 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T19:29:29.006 INFO:teuthology.packaging:ref: None 2023-09-23T19:29:29.006 INFO:teuthology.packaging:tag: None 2023-09-23T19:29:29.006 INFO:teuthology.packaging:branch: quincy 2023-09-23T19:29:29.007 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:29:29.007 DEBUG:teuthology.orchestra.run.smithi144:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T19:29:29.073 DEBUG:teuthology.orchestra.run.smithi144:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T19:29:29.146 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T19:29:29.146 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:29:29.146 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T19:29:29.149 DEBUG:teuthology.orchestra.run.smithi144:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T19:29:29.176 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T19:29:29.176 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T19:29:29.176 INFO:teuthology.packaging:ref: None 2023-09-23T19:29:29.177 INFO:teuthology.packaging:tag: None 2023-09-23T19:29:29.177 INFO:teuthology.packaging:branch: quincy 2023-09-23T19:29:29.177 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:29:29.177 DEBUG:teuthology.orchestra.run.smithi135:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T19:29:29.214 INFO:teuthology.orchestra.run.smithi144.stdout:check_obsoletes = 1 2023-09-23T19:29:29.216 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum clean all 2023-09-23T19:29:29.245 DEBUG:teuthology.orchestra.run.smithi135:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T19:29:29.325 DEBUG:teuthology.orchestra.run.smithi135:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T19:29:29.391 INFO:teuthology.orchestra.run.smithi135.stdout:check_obsoletes = 1 2023-09-23T19:29:29.393 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum clean all 2023-09-23T19:29:29.586 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:29:29.762 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:29:30.580 INFO:teuthology.orchestra.run.smithi144.stdout:44 files removed 2023-09-23T19:29:30.633 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-radosgw 2023-09-23T19:29:30.686 INFO:teuthology.orchestra.run.smithi135.stdout:44 files removed 2023-09-23T19:29:30.738 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-radosgw 2023-09-23T19:29:30.964 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:29:31.069 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:29:32.356 INFO:teuthology.orchestra.run.smithi144.stdout:ceph packages for x86_64 277 kB/s | 78 kB 00:00 2023-09-23T19:29:32.513 INFO:teuthology.orchestra.run.smithi135.stdout:ceph packages for x86_64 215 kB/s | 78 kB 00:00 2023-09-23T19:29:32.657 INFO:teuthology.orchestra.run.smithi144.stdout:ceph noarch packages 34 kB/s | 9.1 kB 00:00 2023-09-23T19:29:32.822 INFO:teuthology.orchestra.run.smithi135.stdout:ceph noarch packages 33 kB/s | 9.1 kB 00:00 2023-09-23T19:29:32.949 INFO:teuthology.orchestra.run.smithi144.stdout:ceph source packages 5.4 kB/s | 1.5 kB 00:00 2023-09-23T19:29:33.107 INFO:teuthology.orchestra.run.smithi135.stdout:ceph source packages 5.5 kB/s | 1.5 kB 00:00 2023-09-23T19:29:33.109 INFO:teuthology.orchestra.run.smithi144.stdout:Copr repo for python3-asyncssh owned by ceph 25 kB/s | 3.5 kB 00:00 2023-09-23T19:29:33.258 INFO:teuthology.orchestra.run.smithi135.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2023-09-23T19:29:35.259 INFO:teuthology.orchestra.run.smithi144.stdout:Extra Packages for Enterprise Linux 7.4 MB/s | 16 MB 00:02 2023-09-23T19:29:35.386 INFO:teuthology.orchestra.run.smithi135.stdout:Extra Packages for Enterprise Linux 7.5 MB/s | 16 MB 00:02 2023-09-23T19:29:38.767 INFO:teuthology.orchestra.run.smithi144.stdout:lab-extras 390 kB/s | 24 kB 00:00 2023-09-23T19:29:38.900 INFO:teuthology.orchestra.run.smithi135.stdout:lab-extras 292 kB/s | 24 kB 00:00 2023-09-23T19:29:39.669 INFO:teuthology.orchestra.run.smithi144.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 64 MB/s | 53 MB 00:00 2023-09-23T19:29:39.836 INFO:teuthology.orchestra.run.smithi135.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 61 MB/s | 53 MB 00:00 2023-09-23T19:29:47.962 INFO:teuthology.orchestra.run.smithi144.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 58 MB/s | 47 MB 00:00 2023-09-23T19:29:48.013 INFO:teuthology.orchestra.run.smithi135.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 54 MB/s | 47 MB 00:00 2023-09-23T19:29:59.050 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:29:59.053 INFO:teuthology.orchestra.run.smithi135.stdout:==================================================================================================== 2023-09-23T19:29:59.053 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:29:59.053 INFO:teuthology.orchestra.run.smithi135.stdout:==================================================================================================== 2023-09-23T19:29:59.053 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:29:59.053 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T19:29:59.053 INFO:teuthology.orchestra.run.smithi135.stdout:Upgrading: 2023-09-23T19:29:59.053 INFO:teuthology.orchestra.run.smithi135.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T19:29:59.053 INFO:teuthology.orchestra.run.smithi135.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T19:29:59.054 INFO:teuthology.orchestra.run.smithi135.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T19:29:59.055 INFO:teuthology.orchestra.run.smithi135.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T19:29:59.056 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:29:59.056 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:29:59.056 INFO:teuthology.orchestra.run.smithi135.stdout:==================================================================================================== 2023-09-23T19:29:59.056 INFO:teuthology.orchestra.run.smithi135.stdout:Install 21 Packages 2023-09-23T19:29:59.056 INFO:teuthology.orchestra.run.smithi135.stdout:Upgrade 2 Packages 2023-09-23T19:29:59.056 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:29:59.056 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 63 M 2023-09-23T19:29:59.057 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:29:59.152 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:29:59.154 INFO:teuthology.orchestra.run.smithi144.stdout:==================================================================================================== 2023-09-23T19:29:59.154 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:29:59.154 INFO:teuthology.orchestra.run.smithi144.stdout:==================================================================================================== 2023-09-23T19:29:59.154 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout:Upgrading: 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T19:29:59.155 INFO:teuthology.orchestra.run.smithi144.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T19:29:59.156 INFO:teuthology.orchestra.run.smithi144.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T19:29:59.157 INFO:teuthology.orchestra.run.smithi144.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T19:29:59.157 INFO:teuthology.orchestra.run.smithi144.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T19:29:59.157 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:29:59.157 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:29:59.157 INFO:teuthology.orchestra.run.smithi144.stdout:==================================================================================================== 2023-09-23T19:29:59.157 INFO:teuthology.orchestra.run.smithi144.stdout:Install 21 Packages 2023-09-23T19:29:59.157 INFO:teuthology.orchestra.run.smithi144.stdout:Upgrade 2 Packages 2023-09-23T19:29:59.157 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:29:59.158 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 63 M 2023-09-23T19:29:59.158 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:30:01.029 INFO:teuthology.orchestra.run.smithi144.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 3.0 MB/s | 5.2 MB 00:01 2023-09-23T19:30:01.162 INFO:teuthology.orchestra.run.smithi144.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 182 kB/s | 24 kB 00:00 2023-09-23T19:30:01.339 INFO:teuthology.orchestra.run.smithi135.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 2.8 MB/s | 5.2 MB 00:01 2023-09-23T19:30:01.464 INFO:teuthology.orchestra.run.smithi135.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 194 kB/s | 24 kB 00:00 2023-09-23T19:30:01.496 INFO:teuthology.orchestra.run.smithi144.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 2.3 MB/s | 796 kB 00:00 2023-09-23T19:30:01.713 INFO:teuthology.orchestra.run.smithi144.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 2.3 MB/s | 506 kB 00:00 2023-09-23T19:30:01.873 INFO:teuthology.orchestra.run.smithi135.stdout:(3/23): ceph-common-17.2.6-1123.g61431f6b.el8.x 9.7 MB/s | 23 MB 00:02 2023-09-23T19:30:01.915 INFO:teuthology.orchestra.run.smithi135.stdout:(4/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 1.7 MB/s | 796 kB 00:00 2023-09-23T19:30:02.032 INFO:teuthology.orchestra.run.smithi135.stdout:(5/23): libradosstriper1-17.2.6-1123.g61431f6b. 3.1 MB/s | 506 kB 00:00 2023-09-23T19:30:02.116 INFO:teuthology.orchestra.run.smithi135.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 536 kB/s | 45 kB 00:00 2023-09-23T19:30:02.199 INFO:teuthology.orchestra.run.smithi135.stdout:(7/23): python3-ceph-common-17.2.6-1123.g61431f 1.3 MB/s | 115 kB 00:00 2023-09-23T19:30:02.275 INFO:teuthology.orchestra.run.smithi135.stdout:(8/23): python3-cephfs-17.2.6-1123.g61431f6b.el 2.5 MB/s | 190 kB 00:00 2023-09-23T19:30:02.383 INFO:teuthology.orchestra.run.smithi135.stdout:(9/23): python3-rados-17.2.6-1123.g61431f6b.el8 3.3 MB/s | 371 kB 00:00 2023-09-23T19:30:02.467 INFO:teuthology.orchestra.run.smithi135.stdout:(10/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 4.2 MB/s | 358 kB 00:00 2023-09-23T19:30:02.551 INFO:teuthology.orchestra.run.smithi135.stdout:(11/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 1.2 MB/s | 106 kB 00:00 2023-09-23T19:30:02.730 INFO:teuthology.orchestra.run.smithi144.stdout:(5/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8. 4.9 MB/s | 17 MB 00:03 2023-09-23T19:30:02.785 INFO:teuthology.orchestra.run.smithi135.stdout:(12/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.3 MB/s | 306 kB 00:00 2023-09-23T19:30:02.838 INFO:teuthology.orchestra.run.smithi144.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 412 kB/s | 45 kB 00:00 2023-09-23T19:30:02.852 INFO:teuthology.orchestra.run.smithi135.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 884 kB/s | 59 kB 00:00 2023-09-23T19:30:02.902 INFO:teuthology.orchestra.run.smithi135.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.5 MB/s | 75 kB 00:00 2023-09-23T19:30:02.930 INFO:teuthology.orchestra.run.smithi144.stdout:(7/23): python3-ceph-common-17.2.6-1123.g61431f 1.2 MB/s | 115 kB 00:00 2023-09-23T19:30:03.006 INFO:teuthology.orchestra.run.smithi144.stdout:(8/23): python3-cephfs-17.2.6-1123.g61431f6b.el 2.5 MB/s | 190 kB 00:00 2023-09-23T19:30:03.094 INFO:teuthology.orchestra.run.smithi135.stdout:(15/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8 4.6 MB/s | 17 MB 00:03 2023-09-23T19:30:03.162 INFO:teuthology.orchestra.run.smithi144.stdout:(9/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 3.8 MB/s | 5.5 MB 00:01 2023-09-23T19:30:03.178 INFO:teuthology.orchestra.run.smithi135.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 564 kB/s | 47 kB 00:00 2023-09-23T19:30:03.197 INFO:teuthology.orchestra.run.smithi144.stdout:(10/23): python3-rados-17.2.6-1123.g61431f6b.el 1.9 MB/s | 371 kB 00:00 2023-09-23T19:30:03.220 INFO:teuthology.orchestra.run.smithi135.stdout:(17/23): thrift-0.13.0-2.el8.x86_64.rpm 5.5 MB/s | 1.7 MB 00:00 2023-09-23T19:30:03.256 INFO:teuthology.orchestra.run.smithi144.stdout:(11/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 1.8 MB/s | 106 kB 00:00 2023-09-23T19:30:03.282 INFO:teuthology.orchestra.run.smithi144.stdout:(12/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 2.9 MB/s | 358 kB 00:00 2023-09-23T19:30:03.295 INFO:teuthology.orchestra.run.smithi135.stdout:(18/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_ 4.0 MB/s | 5.5 MB 00:01 2023-09-23T19:30:03.329 INFO:teuthology.orchestra.run.smithi135.stdout:(19/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.7 MB/s | 259 kB 00:00 2023-09-23T19:30:03.363 INFO:teuthology.orchestra.run.smithi135.stdout:(20/23): python3-prettytable-0.7.2-14.el8.noarc 309 kB/s | 44 kB 00:00 2023-09-23T19:30:03.396 INFO:teuthology.orchestra.run.smithi135.stdout:(21/23): librdkafka-0.11.4-3.el8.x86_64.rpm 3.6 MB/s | 354 kB 00:00 2023-09-23T19:30:03.541 INFO:teuthology.orchestra.run.smithi144.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 227 kB/s | 59 kB 00:00 2023-09-23T19:30:03.641 INFO:teuthology.orchestra.run.smithi144.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 752 kB/s | 75 kB 00:00 2023-09-23T19:30:03.666 INFO:teuthology.orchestra.run.smithi144.stdout:(15/23): gperftools-libs-2.7-9.el8.x86_64.rpm 748 kB/s | 306 kB 00:00 2023-09-23T19:30:03.688 INFO:teuthology.orchestra.run.smithi135.stdout:(22/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 11 MB/s | 3.6 MB 00:00 2023-09-23T19:30:03.733 INFO:teuthology.orchestra.run.smithi144.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 703 kB/s | 47 kB 00:00 2023-09-23T19:30:03.775 INFO:teuthology.orchestra.run.smithi144.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.0 MB/s | 259 kB 00:00 2023-09-23T19:30:03.788 INFO:teuthology.orchestra.run.smithi135.stdout:(23/23): librados2-17.2.6-1123.g61431f6b.el8.x8 7.6 MB/s | 3.5 MB 00:00 2023-09-23T19:30:03.791 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:30:03.791 INFO:teuthology.orchestra.run.smithi135.stdout:Total 13 MB/s | 63 MB 00:04 2023-09-23T19:30:03.817 INFO:teuthology.orchestra.run.smithi144.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 1.0 MB/s | 44 kB 00:00 2023-09-23T19:30:03.831 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:30:03.868 INFO:teuthology.orchestra.run.smithi144.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 6.9 MB/s | 354 kB 00:00 2023-09-23T19:30:03.874 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:30:03.875 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:30:03.977 INFO:teuthology.orchestra.run.smithi144.stdout:(20/23): thrift-0.13.0-2.el8.x86_64.rpm 5.2 MB/s | 1.7 MB 00:00 2023-09-23T19:30:04.217 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:30:04.221 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:30:04.360 INFO:teuthology.orchestra.run.smithi144.stdout:(21/23): ceph-common-17.2.6-1123.g61431f6b.el8. 4.7 MB/s | 23 MB 00:05 2023-09-23T19:30:04.552 INFO:teuthology.orchestra.run.smithi144.stdout:(22/23): librados2-17.2.6-1123.g61431f6b.el8.x8 5.1 MB/s | 3.5 MB 00:00 2023-09-23T19:30:04.986 INFO:teuthology.orchestra.run.smithi144.stdout:(23/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 3.6 MB/s | 3.6 MB 00:01 2023-09-23T19:30:04.990 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:30:04.990 INFO:teuthology.orchestra.run.smithi144.stdout:Total 11 MB/s | 63 MB 00:05 2023-09-23T19:30:05.036 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:30:05.081 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:30:05.081 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:30:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:30:05.433 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:30:05.437 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:30:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T19:30:05.886 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T19:30:06.070 INFO:teuthology.orchestra.run.smithi135.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T19:30:06.165 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T19:30:06.359 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T19:30:06.487 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:30:06.561 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T19:30:06.661 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T19:30:06.754 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T19:30:07.040 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T19:30:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T19:30:07.244 INFO:teuthology.orchestra.run.smithi144.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T19:30:07.342 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T19:30:07.395 INFO:teuthology.orchestra.run.smithi135.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T19:30:07.495 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T19:30:07.517 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T19:30:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T19:30:07.748 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T19:30:07.759 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T19:30:07.935 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T19:30:08.073 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T19:30:08.319 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T19:30:08.510 INFO:teuthology.orchestra.run.smithi144.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T19:30:08.630 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T19:30:08.753 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T19:30:08.768 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T19:30:08.817 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T19:30:08.945 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T19:30:09.011 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T19:30:09.130 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T19:30:09.152 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T19:30:09.548 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T19:30:09.813 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T19:30:09.990 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T19:30:10.186 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T19:30:10.258 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T19:30:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T19:30:10.380 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T19:30:10.493 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T19:30:10.555 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T19:30:10.621 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T19:30:10.718 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T19:30:10.846 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T19:30:10.871 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T19:30:11.045 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T19:30:11.086 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T19:30:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T19:30:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T19:30:11.501 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T19:30:11.627 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T19:30:11.824 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T19:30:12.059 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T19:30:12.268 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T19:30:14.740 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T19:30:15.486 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T19:30:16.021 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T19:30:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T19:30:16.234 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T19:30:16.374 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T19:30:16.460 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T19:30:16.610 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T19:30:16.805 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T19:30:17.050 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T19:30:17.135 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T19:30:17.300 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T19:30:35.381 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T19:30:35.381 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /sys 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /proc 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /dev 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /run 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /mnt 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /var/tmp 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /home 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /tmp 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /dev 2023-09-23T19:30:35.382 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:30:36.056 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T19:30:36.056 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /sys 2023-09-23T19:30:36.056 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /proc 2023-09-23T19:30:36.057 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /dev 2023-09-23T19:30:36.057 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /run 2023-09-23T19:30:36.057 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /mnt 2023-09-23T19:30:36.057 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /var/tmp 2023-09-23T19:30:36.057 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /home 2023-09-23T19:30:36.057 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /tmp 2023-09-23T19:30:36.057 INFO:teuthology.orchestra.run.smithi144.stdout:skipping the directory /dev 2023-09-23T19:30:36.057 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:30:37.062 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T19:30:37.142 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T19:30:37.143 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T19:30:37.376 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T19:30:37.376 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T19:30:37.746 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T19:30:37.827 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T19:30:37.827 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T19:30:37.981 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T19:30:37.981 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T19:30:37.981 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T19:30:37.982 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T19:30:37.983 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T19:30:37.984 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T19:30:38.024 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T19:30:38.024 INFO:teuthology.orchestra.run.smithi144.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T19:30:38.647 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T19:30:38.647 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T19:30:38.647 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T19:30:38.647 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T19:30:38.647 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T19:30:38.648 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T19:30:38.648 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T19:30:38.648 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T19:30:38.648 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T19:30:38.649 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T19:30:38.649 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T19:30:38.650 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T19:30:38.651 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T19:30:38.651 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T19:30:38.651 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T19:30:38.651 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T19:30:39.360 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T19:30:39.361 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:52 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:30:40.025 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T19:30:40.025 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:00:53 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:30:41.838 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:30:42.024 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:30:42.024 INFO:teuthology.orchestra.run.smithi135.stdout:Upgraded: 2023-09-23T19:30:42.024 INFO:teuthology.orchestra.run.smithi135.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.024 INFO:teuthology.orchestra.run.smithi135.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.024 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.025 INFO:teuthology.orchestra.run.smithi135.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:30:42.026 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:30:42.195 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-test 2023-09-23T19:30:42.464 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:30:42.542 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:30:42.648 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:30:42.648 INFO:teuthology.orchestra.run.smithi144.stdout:Upgraded: 2023-09-23T19:30:42.648 INFO:teuthology.orchestra.run.smithi144.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.648 INFO:teuthology.orchestra.run.smithi144.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.648 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T19:30:42.649 INFO:teuthology.orchestra.run.smithi144.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T19:30:42.650 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:30:42.651 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:30:42.795 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-test 2023-09-23T19:30:43.142 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:30:43.363 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:56 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:30:43.955 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:00:56 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:30:44.726 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout:============================================================================================ 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout:============================================================================================ 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T19:30:44.727 INFO:teuthology.orchestra.run.smithi135.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T19:30:44.728 INFO:teuthology.orchestra.run.smithi135.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T19:30:44.728 INFO:teuthology.orchestra.run.smithi135.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T19:30:44.728 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:30:44.728 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:30:44.728 INFO:teuthology.orchestra.run.smithi135.stdout:============================================================================================ 2023-09-23T19:30:44.728 INFO:teuthology.orchestra.run.smithi135.stdout:Install 6 Packages 2023-09-23T19:30:44.728 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:30:44.729 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 53 M 2023-09-23T19:30:44.729 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 233 M 2023-09-23T19:30:44.729 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:30:44.772 INFO:teuthology.orchestra.run.smithi135.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.6 MB/s | 70 kB 00:00 2023-09-23T19:30:44.864 INFO:teuthology.orchestra.run.smithi135.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 3.8 MB/s | 323 kB 00:00 2023-09-23T19:30:44.915 INFO:teuthology.orchestra.run.smithi135.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 3.7 MB/s | 187 kB 00:00 2023-09-23T19:30:44.965 INFO:teuthology.orchestra.run.smithi135.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.0 MB/s | 202 kB 00:00 2023-09-23T19:30:45.090 INFO:teuthology.orchestra.run.smithi135.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 483 kB/s | 174 kB 00:00 2023-09-23T19:30:45.312 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:30:45.313 INFO:teuthology.orchestra.run.smithi144.stdout:============================================================================================ 2023-09-23T19:30:45.313 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:30:45.313 INFO:teuthology.orchestra.run.smithi144.stdout:============================================================================================ 2023-09-23T19:30:45.314 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:30:45.314 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:30:45.316 INFO:teuthology.orchestra.run.smithi144.stdout:============================================================================================ 2023-09-23T19:30:45.317 INFO:teuthology.orchestra.run.smithi144.stdout:Install 6 Packages 2023-09-23T19:30:45.317 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:30:45.317 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 53 M 2023-09-23T19:30:45.317 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 233 M 2023-09-23T19:30:45.317 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:30:45.369 INFO:teuthology.orchestra.run.smithi144.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.3 MB/s | 70 kB 00:00 2023-09-23T19:30:45.444 INFO:teuthology.orchestra.run.smithi144.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.8 MB/s | 323 kB 00:00 2023-09-23T19:30:45.486 INFO:teuthology.orchestra.run.smithi144.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.5 MB/s | 187 kB 00:00 2023-09-23T19:30:45.528 INFO:teuthology.orchestra.run.smithi144.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-09-23T19:30:45.820 INFO:teuthology.orchestra.run.smithi144.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 346 kB/s | 174 kB 00:00 2023-09-23T19:30:47.282 INFO:teuthology.orchestra.run.smithi135.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 20 MB/s | 52 MB 00:02 2023-09-23T19:30:47.284 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:30:47.284 INFO:teuthology.orchestra.run.smithi135.stdout:Total 21 MB/s | 53 MB 00:02 2023-09-23T19:30:47.322 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:30:47.342 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:30:47.343 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:30:47.617 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:30:47.620 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:30:48.071 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:30:48.214 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T19:30:48.310 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T19:30:48.517 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T19:30:48.698 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T19:30:48.872 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T19:30:49.047 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T19:30:53.311 INFO:teuthology.orchestra.run.smithi144.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 6.5 MB/s | 52 MB 00:07 2023-09-23T19:30:53.313 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:30:53.313 INFO:teuthology.orchestra.run.smithi144.stdout:Total 6.6 MB/s | 53 MB 00:07 2023-09-23T19:30:53.349 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:30:53.370 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:30:53.370 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:30:53.643 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:30:53.647 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:30:53.955 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T19:30:54.108 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:30:54.227 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T19:30:54.235 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T19:30:54.313 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T19:30:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T19:30:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T19:30:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T19:30:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T19:30:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T19:30:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T19:30:54.547 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T19:30:54.711 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T19:30:54.909 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T19:30:55.035 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T19:30:55.537 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T19:30:55.538 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:08 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:30:58.285 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout: jq-1.6-3.el8.x86_64 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T19:30:58.466 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:30:58.467 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:30:58.621 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph 2023-09-23T19:30:58.955 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:30:59.785 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:12 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:30:59.969 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T19:31:00.232 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T19:31:00.443 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T19:31:00.443 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T19:31:00.443 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T19:31:00.443 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T19:31:00.443 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T19:31:00.443 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T19:31:01.152 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================= 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================= 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T19:31:01.156 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T19:31:01.157 INFO:teuthology.orchestra.run.smithi135.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T19:31:01.158 INFO:teuthology.orchestra.run.smithi135.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================= 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout:Install 32 Packages 2023-09-23T19:31:01.159 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:01.160 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 28 M 2023-09-23T19:31:01.160 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 110 M 2023-09-23T19:31:01.160 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:31:01.532 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T19:31:01.532 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:14 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:01.564 INFO:teuthology.orchestra.run.smithi135.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 31 kB/s | 6.4 kB 00:00 2023-09-23T19:31:01.847 INFO:teuthology.orchestra.run.smithi135.stdout:(2/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 4.7 MB/s | 2.3 MB 00:00 2023-09-23T19:31:01.903 INFO:teuthology.orchestra.run.smithi135.stdout:(3/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 2.9 MB/s | 1.6 MB 00:00 2023-09-23T19:31:02.015 INFO:teuthology.orchestra.run.smithi135.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1123.g6143 2.1 MB/s | 238 kB 00:00 2023-09-23T19:31:02.115 INFO:teuthology.orchestra.run.smithi135.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 436 kB/s | 44 kB 00:00 2023-09-23T19:31:02.199 INFO:teuthology.orchestra.run.smithi135.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 2.2 MB/s | 185 kB 00:00 2023-09-23T19:31:02.249 INFO:teuthology.orchestra.run.smithi135.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 3.4 MB/s | 173 kB 00:00 2023-09-23T19:31:02.324 INFO:teuthology.orchestra.run.smithi135.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 5.0 MB/s | 384 kB 00:00 2023-09-23T19:31:02.366 INFO:teuthology.orchestra.run.smithi135.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 266 kB/s | 11 kB 00:00 2023-09-23T19:31:02.409 INFO:teuthology.orchestra.run.smithi135.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 419 kB/s | 18 kB 00:00 2023-09-23T19:31:02.459 INFO:teuthology.orchestra.run.smithi135.stdout:(11/32): python3-logutils-0.3.5-11.el8.noarch.r 976 kB/s | 49 kB 00:00 2023-09-23T19:31:02.509 INFO:teuthology.orchestra.run.smithi135.stdout:(12/32): python3-more-itertools-7.2.0-3.el8.noa 1.1 MB/s | 59 kB 00:00 2023-09-23T19:31:02.560 INFO:teuthology.orchestra.run.smithi135.stdout:(13/32): python3-pecan-1.3.2-9.el8.noarch.rpm 5.5 MB/s | 283 kB 00:00 2023-09-23T19:31:02.785 INFO:teuthology.orchestra.run.smithi135.stdout:(14/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86 18 MB/s | 17 MB 00:00 2023-09-23T19:31:02.810 INFO:teuthology.orchestra.run.smithi135.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 62 kB/s | 16 kB 00:00 2023-09-23T19:31:02.837 INFO:teuthology.orchestra.run.smithi135.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 372 kB/s | 19 kB 00:00 2023-09-23T19:31:02.903 INFO:teuthology.orchestra.run.smithi135.stdout:(17/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86 2.9 MB/s | 3.9 MB 00:01 2023-09-23T19:31:02.920 INFO:teuthology.orchestra.run.smithi135.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 219 kB/s | 24 kB 00:00 2023-09-23T19:31:02.937 INFO:teuthology.orchestra.run.smithi135.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 290 kB/s | 29 kB 00:00 2023-09-23T19:31:02.987 INFO:teuthology.orchestra.run.smithi135.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 5.0 MB/s | 255 kB 00:00 2023-09-23T19:31:03.006 INFO:teuthology.orchestra.run.smithi135.stdout:(21/32): python3-trustme-0.6.0-4.el8.noarch.rpm 272 kB/s | 27 kB 00:00 2023-09-23T19:31:03.030 INFO:teuthology.orchestra.run.smithi135.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 2.0 MB/s | 85 kB 00:00 2023-09-23T19:31:03.080 INFO:teuthology.orchestra.run.smithi135.stdout:(23/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.7 MB/s | 251 kB 00:00 2023-09-23T19:31:03.105 INFO:teuthology.orchestra.run.smithi135.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 228 kB/s | 23 kB 00:00 2023-09-23T19:31:03.173 INFO:teuthology.orchestra.run.smithi135.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 766 kB/s | 109 kB 00:00 2023-09-23T19:31:03.198 INFO:teuthology.orchestra.run.smithi135.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 2.5 MB/s | 238 kB 00:00 2023-09-23T19:31:03.223 INFO:teuthology.orchestra.run.smithi135.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.1 MB/s | 54 kB 00:00 2023-09-23T19:31:03.249 INFO:teuthology.orchestra.run.smithi135.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 773 kB/s | 39 kB 00:00 2023-09-23T19:31:03.265 INFO:teuthology.orchestra.run.smithi135.stdout:(29/32): python3-mako-1.0.6-13.el8.noarch.rpm 3.7 MB/s | 157 kB 00:00 2023-09-23T19:31:03.308 INFO:teuthology.orchestra.run.smithi135.stdout:(30/32): python3-cryptography-3.2.1-5.el8.x86_6 2.4 MB/s | 559 kB 00:00 2023-09-23T19:31:03.324 INFO:teuthology.orchestra.run.smithi135.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.7 MB/s | 103 kB 00:00 2023-09-23T19:31:03.358 INFO:teuthology.orchestra.run.smithi135.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.1 MB/s | 457 kB 00:00 2023-09-23T19:31:03.360 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:31:03.360 INFO:teuthology.orchestra.run.smithi135.stdout:Total 13 MB/s | 28 MB 00:02 2023-09-23T19:31:03.452 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:31:03.489 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:31:03.489 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:31:03.693 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:31:03.697 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:31:03.858 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:31:04.042 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:04.042 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:31:04.042 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:04.042 INFO:teuthology.orchestra.run.smithi144.stdout: jq-1.6-3.el8.x86_64 2023-09-23T19:31:04.042 INFO:teuthology.orchestra.run.smithi144.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:04.042 INFO:teuthology.orchestra.run.smithi144.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T19:31:04.043 INFO:teuthology.orchestra.run.smithi144.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T19:31:04.043 INFO:teuthology.orchestra.run.smithi144.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T19:31:04.043 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:04.043 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:31:04.187 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph 2023-09-23T19:31:04.522 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:04.798 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:31:05.083 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T19:31:05.323 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:18 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T19:31:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T19:31:05.795 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T19:31:06.005 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T19:31:06.214 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T19:31:06.421 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T19:31:06.707 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:31:06.710 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================= 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================= 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T19:31:06.711 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T19:31:06.712 INFO:teuthology.orchestra.run.smithi144.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T19:31:06.713 INFO:teuthology.orchestra.run.smithi144.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T19:31:06.714 INFO:teuthology.orchestra.run.smithi144.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T19:31:06.714 INFO:teuthology.orchestra.run.smithi144.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T19:31:06.714 INFO:teuthology.orchestra.run.smithi144.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T19:31:06.714 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:06.714 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:31:06.714 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================= 2023-09-23T19:31:06.714 INFO:teuthology.orchestra.run.smithi144.stdout:Install 32 Packages 2023-09-23T19:31:06.714 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:06.715 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 28 M 2023-09-23T19:31:06.715 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 110 M 2023-09-23T19:31:06.715 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:31:06.814 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T19:31:06.962 INFO:teuthology.orchestra.run.smithi144.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 57 kB/s | 6.4 kB 00:00 2023-09-23T19:31:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T19:31:07.244 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T19:31:07.304 INFO:teuthology.orchestra.run.smithi144.stdout:(2/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 12 MB/s | 3.9 MB 00:00 2023-09-23T19:31:07.444 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T19:31:07.529 INFO:teuthology.orchestra.run.smithi144.stdout:(3/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 2.3 MB/s | 1.6 MB 00:00 2023-09-23T19:31:07.579 INFO:teuthology.orchestra.run.smithi144.stdout:(4/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 3.2 MB/s | 2.3 MB 00:00 2023-09-23T19:31:07.630 INFO:teuthology.orchestra.run.smithi144.stdout:(5/32): ceph-mgr-modules-core-17.2.6-1123.g6143 2.3 MB/s | 238 kB 00:00 2023-09-23T19:31:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T19:31:07.896 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T19:31:08.068 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T19:31:08.138 INFO:teuthology.orchestra.run.smithi144.stdout:(6/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86_ 20 MB/s | 17 MB 00:00 2023-09-23T19:31:08.265 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T19:31:08.422 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T19:31:08.614 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T19:31:08.745 INFO:teuthology.orchestra.run.smithi144.stdout:(7/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 37 kB/s | 44 kB 00:01 2023-09-23T19:31:08.750 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T19:31:08.990 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T19:31:09.139 INFO:teuthology.orchestra.run.smithi144.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 123 kB/s | 185 kB 00:01 2023-09-23T19:31:09.191 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T19:31:09.389 INFO:teuthology.orchestra.run.smithi144.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 44 kB/s | 11 kB 00:00 2023-09-23T19:31:09.408 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T19:31:09.590 INFO:teuthology.orchestra.run.smithi144.stdout:(10/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 119 kB/s | 173 kB 00:01 2023-09-23T19:31:09.647 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T19:31:09.665 INFO:teuthology.orchestra.run.smithi144.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 64 kB/s | 18 kB 00:00 2023-09-23T19:31:09.692 INFO:teuthology.orchestra.run.smithi144.stdout:(12/32): python3-cherrypy-18.4.0-1.el8.noarch.r 407 kB/s | 384 kB 00:00 2023-09-23T19:31:09.874 INFO:teuthology.orchestra.run.smithi144.stdout:(13/32): python3-logutils-0.3.5-11.el8.noarch.r 172 kB/s | 49 kB 00:00 2023-09-23T19:31:09.932 INFO:teuthology.orchestra.run.smithi144.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 220 kB/s | 59 kB 00:00 2023-09-23T19:31:09.933 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T19:31:10.041 INFO:teuthology.orchestra.run.smithi144.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 807 kB/s | 283 kB 00:00 2023-09-23T19:31:10.124 INFO:teuthology.orchestra.run.smithi144.stdout:(16/32): python3-portend-2.6-1.el8.noarch.rpm 62 kB/s | 16 kB 00:00 2023-09-23T19:31:10.175 INFO:teuthology.orchestra.run.smithi144.stdout:(17/32): python3-simplegeneric-0.8.1-17.el8.noa 77 kB/s | 19 kB 00:00 2023-09-23T19:31:10.292 INFO:teuthology.orchestra.run.smithi144.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 95 kB/s | 24 kB 00:00 2023-09-23T19:31:10.392 INFO:teuthology.orchestra.run.smithi144.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 109 kB/s | 29 kB 00:00 2023-09-23T19:31:10.434 INFO:teuthology.orchestra.run.smithi144.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 106 kB/s | 27 kB 00:00 2023-09-23T19:31:10.634 INFO:teuthology.orchestra.run.smithi144.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 746 kB/s | 255 kB 00:00 2023-09-23T19:31:10.718 INFO:teuthology.orchestra.run.smithi144.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 301 kB/s | 85 kB 00:00 2023-09-23T19:31:10.776 INFO:teuthology.orchestra.run.smithi144.stdout:(23/32): python3-pycparser-2.14-14.el8.noarch.r 1.8 MB/s | 109 kB 00:00 2023-09-23T19:31:10.823 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T19:31:10.835 INFO:teuthology.orchestra.run.smithi144.stdout:(24/32): python3-cryptography-3.2.1-5.el8.x86_6 9.3 MB/s | 559 kB 00:00 2023-09-23T19:31:10.861 INFO:teuthology.orchestra.run.smithi144.stdout:(25/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 536 kB/s | 251 kB 00:00 2023-09-23T19:31:10.877 INFO:teuthology.orchestra.run.smithi144.stdout:(26/32): python3-zc-lockfile-2.0-2.el8.noarch.r 94 kB/s | 23 kB 00:00 2023-09-23T19:31:10.911 INFO:teuthology.orchestra.run.smithi144.stdout:(27/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 3.1 MB/s | 238 kB 00:00 2023-09-23T19:31:10.936 INFO:teuthology.orchestra.run.smithi144.stdout:(28/32): python3-pytz-2017.2-9.el8.noarch.rpm 718 kB/s | 54 kB 00:00 2023-09-23T19:31:10.953 INFO:teuthology.orchestra.run.smithi144.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 554 kB/s | 39 kB 00:00 2023-09-23T19:31:11.050 INFO:teuthology.orchestra.run.smithi144.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.2 MB/s | 157 kB 00:00 2023-09-23T19:31:11.168 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T19:31:11.239 INFO:teuthology.orchestra.run.smithi144.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 362 kB/s | 103 kB 00:00 2023-09-23T19:31:11.389 INFO:teuthology.orchestra.run.smithi144.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 1.0 MB/s | 457 kB 00:00 2023-09-23T19:31:11.393 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:31:11.393 INFO:teuthology.orchestra.run.smithi144.stdout:Total 6.0 MB/s | 28 MB 00:04 2023-09-23T19:31:11.486 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:31:11.491 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T19:31:11.523 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:31:11.523 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:31:11.783 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:31:11.788 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:31:11.821 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T19:31:11.974 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T19:31:13.098 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:31:13.304 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T19:31:13.446 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T19:31:13.599 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T19:31:13.657 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T19:31:13.819 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T19:31:14.048 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T19:31:14.049 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T19:31:14.185 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T19:31:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T19:31:14.400 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T19:31:14.468 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T19:31:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T19:31:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T19:31:14.675 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T19:31:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T19:31:15.027 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T19:31:15.076 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T19:31:15.077 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T19:31:15.077 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T19:31:15.077 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T19:31:15.077 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T19:31:15.077 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T19:31:15.077 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T19:31:15.077 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T19:31:15.078 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T19:31:15.079 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T19:31:15.080 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T19:31:15.080 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T19:31:15.080 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T19:31:15.326 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T19:31:15.464 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T19:31:15.639 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T19:31:15.848 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T19:31:16.056 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T19:31:16.255 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T19:31:16.427 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T19:31:16.565 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T19:31:16.565 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:29 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:16.566 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T19:31:16.734 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T19:31:16.854 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T19:31:17.060 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T19:31:17.277 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T19:31:17.445 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T19:31:17.717 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T19:31:17.952 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T19:31:18.181 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T19:31:18.475 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T19:31:18.785 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T19:31:18.980 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:31:19.136 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T19:31:19.166 INFO:teuthology.orchestra.run.smithi135.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T19:31:19.167 INFO:teuthology.orchestra.run.smithi135.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T19:31:19.168 INFO:teuthology.orchestra.run.smithi135.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T19:31:19.169 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:19.169 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:31:19.297 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T19:31:19.315 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-base 2023-09-23T19:31:19.659 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:20.521 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:33 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:20.770 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T19:31:20.947 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T19:31:21.338 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T19:31:21.489 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T19:31:21.705 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T19:31:21.785 INFO:teuthology.orchestra.run.smithi135.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:31:21.856 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T19:31:21.873 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T19:31:21.895 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:31:21.896 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:31:21.896 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:31:22.020 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T19:31:22.410 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T19:31:22.410 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T19:31:22.410 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T19:31:22.410 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T19:31:22.410 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T19:31:22.410 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T19:31:22.410 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T19:31:22.411 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T19:31:22.412 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T19:31:22.413 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T19:31:22.413 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T19:31:22.413 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T19:31:22.413 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T19:31:22.413 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T19:31:22.666 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install cephadm 2023-09-23T19:31:23.014 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:23.900 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T19:31:23.900 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:36 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:24.558 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:37 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:25.989 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:31:25.989 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2023-09-23T19:31:25.990 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:25.991 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 80 k 2023-09-23T19:31:25.991 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 363 k 2023-09-23T19:31:25.991 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:31:26.360 INFO:teuthology.orchestra.run.smithi135.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 217 kB/s | 80 kB 00:00 2023-09-23T19:31:26.361 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:31:26.361 INFO:teuthology.orchestra.run.smithi135.stdout:Total 216 kB/s | 80 kB 00:00 2023-09-23T19:31:26.361 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:31:26.369 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:31:26.369 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:31:26.371 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:31:26.377 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:31:26.417 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:31:26.490 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:31:26.674 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:26.674 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:31:26.674 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:26.674 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:26.674 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T19:31:26.675 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T19:31:26.676 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T19:31:26.677 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T19:31:26.677 INFO:teuthology.orchestra.run.smithi144.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T19:31:26.677 INFO:teuthology.orchestra.run.smithi144.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T19:31:26.677 INFO:teuthology.orchestra.run.smithi144.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T19:31:26.677 INFO:teuthology.orchestra.run.smithi144.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T19:31:26.677 INFO:teuthology.orchestra.run.smithi144.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T19:31:26.677 INFO:teuthology.orchestra.run.smithi144.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T19:31:26.677 INFO:teuthology.orchestra.run.smithi144.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T19:31:26.678 INFO:teuthology.orchestra.run.smithi144.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T19:31:26.678 INFO:teuthology.orchestra.run.smithi144.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T19:31:26.678 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:26.678 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:31:26.829 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-base 2023-09-23T19:31:27.184 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:28.090 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:41 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:28.120 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T19:31:28.310 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T19:31:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T19:31:29.342 INFO:teuthology.orchestra.run.smithi144.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:31:29.455 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:31:29.456 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:31:29.456 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:31:29.571 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install cephadm 2023-09-23T19:31:29.708 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T19:31:29.708 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:42 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:29.914 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:30.770 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:43 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:32.133 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:31:32.134 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:31:32.134 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:31:32.134 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout:Install 1 Package 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 80 k 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 363 k 2023-09-23T19:31:32.135 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:31:32.303 INFO:teuthology.orchestra.run.smithi144.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 477 kB/s | 80 kB 00:00 2023-09-23T19:31:32.303 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:31:32.304 INFO:teuthology.orchestra.run.smithi144.stdout:Total 473 kB/s | 80 kB 00:00 2023-09-23T19:31:32.304 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:31:32.312 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:31:32.312 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:31:32.314 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:31:32.317 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:31:32.366 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:31:33.580 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:31:33.762 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:33.762 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:31:33.762 INFO:teuthology.orchestra.run.smithi135.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:31:33.762 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:33.762 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:31:33.896 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T19:31:34.253 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:34.656 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T19:31:34.828 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T19:31:34.996 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T19:31:35.239 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:48 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:36.079 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T19:31:36.079 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:49 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:36.677 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repo Size 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:31:36.678 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2023-09-23T19:31:36.679 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:36.679 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 161 k 2023-09-23T19:31:36.679 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 490 k 2023-09-23T19:31:36.679 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:31:37.111 INFO:teuthology.orchestra.run.smithi135.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 373 kB/s | 161 kB 00:00 2023-09-23T19:31:37.112 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:31:37.112 INFO:teuthology.orchestra.run.smithi135.stdout:Total 372 kB/s | 161 kB 00:00 2023-09-23T19:31:37.112 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:31:37.123 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:31:37.123 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:31:37.176 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:31:37.179 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:31:37.292 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:31:37.507 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T19:31:37.842 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T19:31:38.845 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T19:31:38.845 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:51 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:39.339 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:31:39.520 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:39.520 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:31:39.520 INFO:teuthology.orchestra.run.smithi144.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:31:39.521 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:39.521 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:31:39.646 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T19:31:40.003 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:40.903 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:53 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:41.794 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:31:41.976 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:41.976 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:31:41.976 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:41.976 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:41.976 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:31:42.111 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr 2023-09-23T19:31:42.262 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:31:42.262 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:31:42.262 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repo Size 2023-09-23T19:31:42.263 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:31:42.263 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:31:42.263 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T19:31:42.263 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:42.263 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:31:42.263 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:31:42.263 INFO:teuthology.orchestra.run.smithi144.stdout:Install 1 Package 2023-09-23T19:31:42.263 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:42.264 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 161 k 2023-09-23T19:31:42.264 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 490 k 2023-09-23T19:31:42.264 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:31:42.456 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:42.625 INFO:teuthology.orchestra.run.smithi144.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 446 kB/s | 161 kB 00:00 2023-09-23T19:31:42.626 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:31:42.626 INFO:teuthology.orchestra.run.smithi144.stdout:Total 444 kB/s | 161 kB 00:00 2023-09-23T19:31:42.626 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:31:42.637 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:31:42.637 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:31:42.688 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:31:42.691 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:31:42.800 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:31:42.986 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T19:31:43.334 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T19:31:43.355 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:56 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:44.389 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T19:31:44.390 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:01:57 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:44.655 INFO:teuthology.orchestra.run.smithi135.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:31:44.762 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:31:44.763 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:31:44.763 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:31:45.242 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T19:31:45.589 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:47.118 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:00 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:47.804 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:31:47.987 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:47.987 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:31:47.987 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:31:47.987 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:47.987 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:31:48.108 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-mgr 2023-09-23T19:31:48.455 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:48.552 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:31:48.553 INFO:teuthology.orchestra.run.smithi135.stdout:=================================================================================================== 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout:=================================================================================================== 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T19:31:48.554 INFO:teuthology.orchestra.run.smithi135.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T19:31:48.555 INFO:teuthology.orchestra.run.smithi135.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T19:31:48.555 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:48.555 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:31:48.555 INFO:teuthology.orchestra.run.smithi135.stdout:=================================================================================================== 2023-09-23T19:31:48.555 INFO:teuthology.orchestra.run.smithi135.stdout:Install 6 Packages 2023-09-23T19:31:48.555 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:48.556 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 2.0 M 2023-09-23T19:31:48.556 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 8.0 M 2023-09-23T19:31:48.556 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:31:48.948 INFO:teuthology.orchestra.run.smithi135.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 103 kB/s | 23 kB 00:00 2023-09-23T19:31:48.964 INFO:teuthology.orchestra.run.smithi135.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 60 kB/s | 14 kB 00:00 2023-09-23T19:31:49.515 INFO:teuthology.orchestra.run.smithi135.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 2.1 MB/s | 1.7 MB 00:00 2023-09-23T19:31:49.790 INFO:teuthology.orchestra.run.smithi135.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 155 kB/s | 43 kB 00:00 2023-09-23T19:31:49.940 INFO:teuthology.orchestra.run.smithi135.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 34 kB/s | 34 kB 00:00 2023-09-23T19:31:49.970 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:02 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:50.599 INFO:teuthology.orchestra.run.smithi135.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 120 kB/s | 196 kB 00:01 2023-09-23T19:31:50.603 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:31:50.603 INFO:teuthology.orchestra.run.smithi135.stdout:Total 999 kB/s | 2.0 MB 00:02 2023-09-23T19:31:50.616 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:31:50.627 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:31:50.627 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:31:50.688 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:31:50.692 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:31:51.018 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:31:51.231 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T19:31:51.244 INFO:teuthology.orchestra.run.smithi144.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:31:51.351 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:31:51.353 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:31:51.353 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:31:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T19:31:51.453 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T19:31:51.633 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T19:31:51.736 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T19:31:51.797 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:31:52.091 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T19:31:52.308 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T19:31:52.540 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T19:31:52.540 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T19:31:52.541 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T19:31:52.541 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T19:31:52.541 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T19:31:52.541 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T19:31:52.885 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:05 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:53.565 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T19:31:53.565 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:06 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:54.263 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:31:54.264 INFO:teuthology.orchestra.run.smithi144.stdout:=================================================================================================== 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout:=================================================================================================== 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:54.265 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi144.stdout:=================================================================================================== 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi144.stdout:Install 6 Packages 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 2.0 M 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 8.0 M 2023-09-23T19:31:54.266 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:31:54.573 INFO:teuthology.orchestra.run.smithi144.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1123.g6143 99 kB/s | 14 kB 00:00 2023-09-23T19:31:54.697 INFO:teuthology.orchestra.run.smithi144.stdout:(2/6): ceph-grafana-dashboards-17.2.6-1123.g614 85 kB/s | 23 kB 00:00 2023-09-23T19:31:54.790 INFO:teuthology.orchestra.run.smithi144.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 155 kB/s | 34 kB 00:00 2023-09-23T19:31:54.890 INFO:teuthology.orchestra.run.smithi144.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 428 kB/s | 43 kB 00:00 2023-09-23T19:31:54.949 INFO:teuthology.orchestra.run.smithi144.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 780 kB/s | 196 kB 00:00 2023-09-23T19:31:54.991 INFO:teuthology.orchestra.run.smithi144.stdout:(6/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 3.0 MB/s | 1.7 MB 00:00 2023-09-23T19:31:54.994 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:31:54.994 INFO:teuthology.orchestra.run.smithi144.stdout:Total 2.7 MB/s | 2.0 MB 00:00 2023-09-23T19:31:55.005 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:31:55.014 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:31:55.014 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:31:55.071 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:31:55.075 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:31:55.381 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:31:55.534 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T19:31:56.000 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T19:31:56.187 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T19:31:56.290 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T19:31:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T19:31:56.778 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T19:31:57.013 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T19:31:57.014 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T19:31:57.014 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T19:31:57.014 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T19:31:57.014 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T19:31:57.014 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T19:31:58.047 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T19:31:58.047 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:11 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:31:59.696 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:31:59.879 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:59.879 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:31:59.879 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:31:59.880 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:31:59.880 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:31:59.880 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T19:31:59.880 INFO:teuthology.orchestra.run.smithi135.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T19:31:59.880 INFO:teuthology.orchestra.run.smithi135.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T19:31:59.880 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:31:59.880 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:32:00.016 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T19:32:00.362 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:32:01.945 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:14 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:03.322 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:32:03.323 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================================= 2023-09-23T19:32:03.323 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:32:03.324 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================================= 2023-09-23T19:32:03.324 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:32:03.324 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T19:32:03.324 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:32:03.324 INFO:teuthology.orchestra.run.smithi135.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T19:32:03.324 INFO:teuthology.orchestra.run.smithi135.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T19:32:03.325 INFO:teuthology.orchestra.run.smithi135.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T19:32:03.325 INFO:teuthology.orchestra.run.smithi135.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T19:32:03.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T19:32:03.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T19:32:03.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T19:32:03.325 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:03.326 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:32:03.326 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================================= 2023-09-23T19:32:03.326 INFO:teuthology.orchestra.run.smithi135.stdout:Install 8 Packages 2023-09-23T19:32:03.326 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:03.327 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 35 M 2023-09-23T19:32:03.327 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 210 M 2023-09-23T19:32:03.327 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:32:03.458 INFO:teuthology.orchestra.run.smithi135.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.3 MB/s | 171 kB 00:00 2023-09-23T19:32:03.658 INFO:teuthology.orchestra.run.smithi135.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 1.9 MB/s | 644 kB 00:00 2023-09-23T19:32:03.725 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:03.909 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:32:03.969 INFO:teuthology.orchestra.run.smithi135.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 27 MB/s | 14 MB 00:00 2023-09-23T19:32:04.044 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T19:32:04.151 INFO:teuthology.orchestra.run.smithi135.stdout:(4/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 7.6 MB/s | 3.7 MB 00:00 2023-09-23T19:32:04.234 INFO:teuthology.orchestra.run.smithi135.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 844 kB/s | 225 kB 00:00 2023-09-23T19:32:04.326 INFO:teuthology.orchestra.run.smithi135.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 28 MB/s | 4.9 MB 00:00 2023-09-23T19:32:04.395 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:32:04.543 INFO:teuthology.orchestra.run.smithi135.stdout:(7/8): ceph-mgr-diskprediction-local-17.2.6-112 6.1 MB/s | 7.4 MB 00:01 2023-09-23T19:32:04.676 INFO:teuthology.orchestra.run.smithi135.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 11 MB/s | 4.7 MB 00:00 2023-09-23T19:32:04.677 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:32:04.678 INFO:teuthology.orchestra.run.smithi135.stdout:Total 26 MB/s | 35 MB 00:01 2023-09-23T19:32:04.914 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:32:04.931 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:32:04.932 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:32:05.224 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:32:05.227 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:32:05.775 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:32:05.897 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:18 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:05.902 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T19:32:06.055 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T19:32:06.194 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T19:32:06.791 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T19:32:06.952 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T19:32:07.366 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================================= 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================================= 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T19:32:07.368 INFO:teuthology.orchestra.run.smithi144.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T19:32:07.369 INFO:teuthology.orchestra.run.smithi144.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T19:32:07.369 INFO:teuthology.orchestra.run.smithi144.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T19:32:07.369 INFO:teuthology.orchestra.run.smithi144.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T19:32:07.369 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:07.369 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:32:07.369 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================================= 2023-09-23T19:32:07.369 INFO:teuthology.orchestra.run.smithi144.stdout:Install 8 Packages 2023-09-23T19:32:07.369 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:07.370 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 35 M 2023-09-23T19:32:07.370 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 210 M 2023-09-23T19:32:07.370 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:32:07.488 INFO:teuthology.orchestra.run.smithi144.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 5.3 MB/s | 644 kB 00:00 2023-09-23T19:32:07.529 INFO:teuthology.orchestra.run.smithi144.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.0 MB/s | 171 kB 00:00 2023-09-23T19:32:07.550 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T19:32:07.730 INFO:teuthology.orchestra.run.smithi144.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 19 MB/s | 3.7 MB 00:00 2023-09-23T19:32:07.760 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T19:32:07.922 INFO:teuthology.orchestra.run.smithi144.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 32 MB/s | 14 MB 00:00 2023-09-23T19:32:07.947 INFO:teuthology.orchestra.run.smithi144.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.0 MB/s | 225 kB 00:00 2023-09-23T19:32:08.139 INFO:teuthology.orchestra.run.smithi144.stdout:(6/8): openblas-0.3.15-3.el8.x86_64.rpm 25 MB/s | 4.7 MB 00:00 2023-09-23T19:32:08.239 INFO:teuthology.orchestra.run.smithi144.stdout:(7/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 15 MB/s | 4.9 MB 00:00 2023-09-23T19:32:08.449 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T19:32:08.489 INFO:teuthology.orchestra.run.smithi144.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 6.6 MB/s | 7.4 MB 00:01 2023-09-23T19:32:08.491 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:32:08.491 INFO:teuthology.orchestra.run.smithi144.stdout:Total 32 MB/s | 35 MB 00:01 2023-09-23T19:32:08.726 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:32:08.744 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:32:08.744 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:32:08.788 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T19:32:08.969 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T19:32:09.038 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:32:09.041 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:32:09.623 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:32:09.747 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T19:32:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T19:32:10.047 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T19:32:10.646 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T19:32:10.721 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T19:32:10.797 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T19:32:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T19:32:11.588 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T19:32:11.974 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T19:32:12.151 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T19:32:12.261 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T19:32:12.405 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T19:32:12.406 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T19:32:12.406 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T19:32:12.406 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T19:32:12.406 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T19:32:12.406 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T19:32:12.406 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T19:32:12.406 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T19:32:12.533 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T19:32:12.749 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T19:32:13.479 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T19:32:13.479 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:26 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:14.492 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T19:32:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T19:32:15.913 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T19:32:16.175 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T19:32:16.175 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T19:32:16.175 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T19:32:16.175 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T19:32:16.176 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T19:32:16.176 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T19:32:16.176 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T19:32:16.176 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T19:32:17.245 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T19:32:17.245 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:30 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:19.353 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:32:19.534 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:19.534 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:32:19.534 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:19.534 INFO:teuthology.orchestra.run.smithi135.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T19:32:19.534 INFO:teuthology.orchestra.run.smithi135.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T19:32:19.534 INFO:teuthology.orchestra.run.smithi135.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T19:32:19.534 INFO:teuthology.orchestra.run.smithi135.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T19:32:19.535 INFO:teuthology.orchestra.run.smithi135.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T19:32:19.535 INFO:teuthology.orchestra.run.smithi135.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T19:32:19.535 INFO:teuthology.orchestra.run.smithi135.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T19:32:19.535 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:19.535 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:32:19.728 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr-rook 2023-09-23T19:32:20.072 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:32:21.494 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:34 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:22.681 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:32:22.864 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:22.864 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:32:22.864 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:22.864 INFO:teuthology.orchestra.run.smithi144.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T19:32:22.865 INFO:teuthology.orchestra.run.smithi144.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T19:32:22.865 INFO:teuthology.orchestra.run.smithi144.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T19:32:22.865 INFO:teuthology.orchestra.run.smithi144.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T19:32:22.865 INFO:teuthology.orchestra.run.smithi144.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T19:32:22.865 INFO:teuthology.orchestra.run.smithi144.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T19:32:22.865 INFO:teuthology.orchestra.run.smithi144.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T19:32:22.865 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:22.865 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:32:22.874 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:32:22.876 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================== 2023-09-23T19:32:22.876 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:32:22.876 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================== 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T19:32:22.877 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T19:32:22.878 INFO:teuthology.orchestra.run.smithi135.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T19:32:22.878 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T19:32:22.878 INFO:teuthology.orchestra.run.smithi135.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T19:32:22.878 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:22.878 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:32:22.878 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================================== 2023-09-23T19:32:22.878 INFO:teuthology.orchestra.run.smithi135.stdout:Install 13 Packages 2023-09-23T19:32:22.878 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:22.879 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 1.9 M 2023-09-23T19:32:22.879 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 25 M 2023-09-23T19:32:22.879 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:32:23.015 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-mgr-rook 2023-09-23T19:32:23.173 INFO:teuthology.orchestra.run.smithi135.stdout:(1/13): python3-cachetools-3.1.1-4.el8.noarch.r 229 kB/s | 33 kB 00:00 2023-09-23T19:32:23.198 INFO:teuthology.orchestra.run.smithi135.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 96 kB/s | 16 kB 00:00 2023-09-23T19:32:23.240 INFO:teuthology.orchestra.run.smithi135.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.2 MB/s | 81 kB 00:00 2023-09-23T19:32:23.282 INFO:teuthology.orchestra.run.smithi135.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 1.5 MB/s | 64 kB 00:00 2023-09-23T19:32:23.299 INFO:teuthology.orchestra.run.smithi135.stdout:(5/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 175 kB/s | 47 kB 00:00 2023-09-23T19:32:23.342 INFO:teuthology.orchestra.run.smithi135.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 1.4 MB/s | 61 kB 00:00 2023-09-23T19:32:23.360 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:32:23.417 INFO:teuthology.orchestra.run.smithi135.stdout:(7/13): python3-kubernetes-11.0.0-6.el8.noarch. 5.3 MB/s | 1.2 MB 00:00 2023-09-23T19:32:23.442 INFO:teuthology.orchestra.run.smithi135.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 300 kB/s | 43 kB 00:00 2023-09-23T19:32:23.467 INFO:teuthology.orchestra.run.smithi135.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.1 MB/s | 155 kB 00:00 2023-09-23T19:32:23.534 INFO:teuthology.orchestra.run.smithi135.stdout:(10/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.9 MB/s | 126 kB 00:00 2023-09-23T19:32:23.559 INFO:teuthology.orchestra.run.smithi135.stdout:(11/13): python3-pyasn1-modules-0.3.7-6.el8.noa 774 kB/s | 110 kB 00:00 2023-09-23T19:32:23.576 INFO:teuthology.orchestra.run.smithi135.stdout:(12/13): python3-jsonpatch-1.21-2.el8.noarch.rp 206 kB/s | 27 kB 00:00 2023-09-23T19:32:23.601 INFO:teuthology.orchestra.run.smithi135.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 298 kB/s | 20 kB 00:00 2023-09-23T19:32:23.603 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:32:23.603 INFO:teuthology.orchestra.run.smithi135.stdout:Total 2.7 MB/s | 1.9 MB 00:00 2023-09-23T19:32:23.662 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:32:23.677 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:32:23.677 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:32:23.746 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:32:23.750 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:32:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:32:24.399 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:37 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:24.424 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T19:32:24.660 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T19:32:24.814 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T19:32:25.015 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T19:32:25.210 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T19:32:25.470 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T19:32:25.651 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T19:32:25.759 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:32:25.761 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================== 2023-09-23T19:32:25.761 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:32:25.761 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================== 2023-09-23T19:32:25.761 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:32:25.761 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T19:32:25.761 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:32:25.761 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T19:32:25.761 INFO:teuthology.orchestra.run.smithi144.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T19:32:25.762 INFO:teuthology.orchestra.run.smithi144.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T19:32:25.763 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:25.763 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:32:25.763 INFO:teuthology.orchestra.run.smithi144.stdout:======================================================================================================== 2023-09-23T19:32:25.763 INFO:teuthology.orchestra.run.smithi144.stdout:Install 13 Packages 2023-09-23T19:32:25.763 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:25.763 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 1.9 M 2023-09-23T19:32:25.764 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 25 M 2023-09-23T19:32:25.764 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:32:25.873 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T19:32:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T19:32:26.237 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T19:32:26.379 INFO:teuthology.orchestra.run.smithi144.stdout:(1/13): python3-cachetools-3.1.1-4.el8.noarch.r 73 kB/s | 33 kB 00:00 2023-09-23T19:32:26.404 INFO:teuthology.orchestra.run.smithi144.stdout:(2/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 99 kB/s | 47 kB 00:00 2023-09-23T19:32:26.421 INFO:teuthology.orchestra.run.smithi144.stdout:(3/13): python3-certifi-2018.10.15-7.el8.noarch 33 kB/s | 16 kB 00:00 2023-09-23T19:32:26.563 INFO:teuthology.orchestra.run.smithi144.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 442 kB/s | 81 kB 00:00 2023-09-23T19:32:26.672 INFO:teuthology.orchestra.run.smithi144.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 564 kB/s | 61 kB 00:00 2023-09-23T19:32:26.822 INFO:teuthology.orchestra.run.smithi144.stdout:(6/13): python3-requests-oauthlib-1.0.0-1.el8.n 285 kB/s | 43 kB 00:00 2023-09-23T19:32:26.939 INFO:teuthology.orchestra.run.smithi144.stdout:(7/13): python3-rsa-4.9-2.el8.noarch.rpm 124 kB/s | 64 kB 00:00 2023-09-23T19:32:26.972 INFO:teuthology.orchestra.run.smithi144.stdout:(8/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.0 MB/s | 155 kB 00:00 2023-09-23T19:32:27.114 INFO:teuthology.orchestra.run.smithi144.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 195 kB/s | 27 kB 00:00 2023-09-23T19:32:27.131 INFO:teuthology.orchestra.run.smithi144.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 572 kB/s | 110 kB 00:00 2023-09-23T19:32:27.281 INFO:teuthology.orchestra.run.smithi144.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 758 kB/s | 126 kB 00:00 2023-09-23T19:32:27.290 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T19:32:27.350 INFO:teuthology.orchestra.run.smithi144.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 91 kB/s | 20 kB 00:00 2023-09-23T19:32:27.507 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T19:32:27.691 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T19:32:27.915 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T19:32:27.915 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T19:32:27.915 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T19:32:27.915 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T19:32:27.915 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T19:32:27.915 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T19:32:27.915 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T19:32:27.915 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T19:32:27.916 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T19:32:27.916 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T19:32:27.916 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T19:32:27.916 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T19:32:27.916 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T19:32:28.074 INFO:teuthology.orchestra.run.smithi144.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 715 kB/s | 1.2 MB 00:01 2023-09-23T19:32:28.077 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:32:28.077 INFO:teuthology.orchestra.run.smithi144.stdout:Total 854 kB/s | 1.9 MB 00:02 2023-09-23T19:32:28.142 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:32:28.158 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:32:28.158 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:32:28.230 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:32:28.234 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:32:28.727 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:32:28.977 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T19:32:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T19:32:29.071 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:42 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:29.196 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T19:32:29.383 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T19:32:29.534 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T19:32:29.720 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T19:32:29.947 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T19:32:30.146 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T19:32:30.342 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T19:32:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T19:32:30.682 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T19:32:31.782 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T19:32:31.906 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:32:32.002 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T19:32:32.090 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:32.090 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:32:32.090 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:32.090 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T19:32:32.090 INFO:teuthology.orchestra.run.smithi135.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T19:32:32.091 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:32.092 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:32:32.160 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T19:32:32.383 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T19:32:32.383 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T19:32:32.383 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T19:32:32.383 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T19:32:32.383 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T19:32:32.383 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T19:32:32.384 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T19:32:32.384 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T19:32:32.384 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T19:32:32.384 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T19:32:32.384 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T19:32:32.384 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T19:32:32.384 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T19:32:32.432 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T19:32:32.785 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:32:33.508 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T19:32:33.508 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:46 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:34.089 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:47 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:35.444 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:32:35.445 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================================== 2023-09-23T19:32:35.445 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:32:35.445 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================================== 2023-09-23T19:32:35.445 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:32:35.445 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T19:32:35.445 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:32:35.445 INFO:teuthology.orchestra.run.smithi135.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T19:32:35.446 INFO:teuthology.orchestra.run.smithi135.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T19:32:35.446 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T19:32:35.446 INFO:teuthology.orchestra.run.smithi135.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T19:32:35.446 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:35.446 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:32:35.446 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================================== 2023-09-23T19:32:35.446 INFO:teuthology.orchestra.run.smithi135.stdout:Install 5 Packages 2023-09-23T19:32:35.446 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:35.447 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 6.0 M 2023-09-23T19:32:35.447 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 26 M 2023-09-23T19:32:35.447 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:32:35.800 INFO:teuthology.orchestra.run.smithi135.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 633 kB/s | 121 kB 00:00 2023-09-23T19:32:36.117 INFO:teuthology.orchestra.run.smithi135.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 1.7 MB/s | 538 kB 00:00 2023-09-23T19:32:36.267 INFO:teuthology.orchestra.run.smithi135.stdout:(3/5): python3-natsort-7.1.1-2.el8.noarch.rpm 91 kB/s | 60 kB 00:00 2023-09-23T19:32:36.375 INFO:teuthology.orchestra.run.smithi135.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 19 MB/s | 4.8 MB 00:00 2023-09-23T19:32:36.575 INFO:teuthology.orchestra.run.smithi135.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 492 kB/s | 476 kB 00:00 2023-09-23T19:32:36.576 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:32:36.576 INFO:teuthology.orchestra.run.smithi135.stdout:Total 5.3 MB/s | 6.0 MB 00:01 2023-09-23T19:32:36.630 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:32:36.642 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:32:36.642 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:32:36.720 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:32:36.724 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:32:37.574 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:32:37.697 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T19:32:37.759 INFO:teuthology.orchestra.run.smithi144.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T19:32:37.760 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T19:32:37.760 INFO:teuthology.orchestra.run.smithi144.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T19:32:37.760 INFO:teuthology.orchestra.run.smithi144.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T19:32:37.760 INFO:teuthology.orchestra.run.smithi144.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T19:32:37.760 INFO:teuthology.orchestra.run.smithi144.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T19:32:37.760 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:37.760 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:32:37.897 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T19:32:38.058 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T19:32:38.247 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:32:38.274 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T19:32:38.593 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T19:32:38.874 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T19:32:39.064 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T19:32:39.287 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T19:32:39.287 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T19:32:39.287 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T19:32:39.287 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T19:32:39.287 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T19:32:39.826 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:52 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:40.321 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T19:32:40.321 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:53 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:41.195 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:32:41.195 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================================== 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================================== 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T19:32:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:41.197 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:32:41.197 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================================== 2023-09-23T19:32:41.197 INFO:teuthology.orchestra.run.smithi144.stdout:Install 5 Packages 2023-09-23T19:32:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:41.197 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 6.0 M 2023-09-23T19:32:41.198 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 26 M 2023-09-23T19:32:41.198 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:32:41.548 INFO:teuthology.orchestra.run.smithi144.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 639 kB/s | 121 kB 00:00 2023-09-23T19:32:41.573 INFO:teuthology.orchestra.run.smithi144.stdout:(2/5): python3-natsort-7.1.1-2.el8.noarch.rpm 280 kB/s | 60 kB 00:00 2023-09-23T19:32:41.665 INFO:teuthology.orchestra.run.smithi144.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.5 MB/s | 476 kB 00:00 2023-09-23T19:32:41.707 INFO:teuthology.orchestra.run.smithi144.stdout:(4/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.3 MB/s | 538 kB 00:00 2023-09-23T19:32:41.799 INFO:teuthology.orchestra.run.smithi144.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 21 MB/s | 4.8 MB 00:00 2023-09-23T19:32:41.800 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:32:41.801 INFO:teuthology.orchestra.run.smithi144.stdout:Total 9.9 MB/s | 6.0 MB 00:00 2023-09-23T19:32:41.853 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:32:41.864 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:32:41.865 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:32:41.943 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:32:41.947 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:32:42.889 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:32:44.461 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T19:32:44.676 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T19:32:45.023 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T19:32:45.260 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T19:32:45.364 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:32:45.449 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T19:32:45.548 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:45.548 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:32:45.548 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:45.548 INFO:teuthology.orchestra.run.smithi135.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T19:32:45.548 INFO:teuthology.orchestra.run.smithi135.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T19:32:45.548 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T19:32:45.549 INFO:teuthology.orchestra.run.smithi135.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T19:32:45.549 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:45.549 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:32:45.680 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T19:32:45.680 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T19:32:45.680 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T19:32:45.680 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T19:32:45.681 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T19:32:45.694 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-fuse 2023-09-23T19:32:46.045 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:32:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T19:32:46.718 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:02:59 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:47.341 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:00 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:48.720 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:32:48.721 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:32:48.721 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-09-23T19:32:48.721 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:32:48.721 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:32:48.721 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-fuse 2023-09-23T19:32:48.721 INFO:teuthology.orchestra.run.smithi135.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T19:32:48.721 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-09-23T19:32:48.721 INFO:teuthology.orchestra.run.smithi135.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T19:32:48.722 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:48.722 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:32:48.722 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:32:48.722 INFO:teuthology.orchestra.run.smithi135.stdout:Install 2 Packages 2023-09-23T19:32:48.722 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:48.722 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 957 k 2023-09-23T19:32:48.723 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 3.0 M 2023-09-23T19:32:48.723 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:32:49.062 INFO:teuthology.orchestra.run.smithi135.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 244 kB/s | 83 kB 00:00 2023-09-23T19:32:49.112 INFO:teuthology.orchestra.run.smithi135.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 2.2 MB/s | 874 kB 00:00 2023-09-23T19:32:49.112 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:32:49.113 INFO:teuthology.orchestra.run.smithi135.stdout:Total 2.4 MB/s | 957 kB 00:00 2023-09-23T19:32:49.124 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:32:49.145 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:32:49.145 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:32:49.219 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:32:49.223 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:32:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:32:49.646 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T19:32:49.828 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T19:32:50.142 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T19:32:50.142 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T19:32:51.236 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:32:51.240 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T19:32:51.241 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:04 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:51.421 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:51.421 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:32:51.422 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T19:32:51.422 INFO:teuthology.orchestra.run.smithi144.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T19:32:51.422 INFO:teuthology.orchestra.run.smithi144.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T19:32:51.422 INFO:teuthology.orchestra.run.smithi144.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T19:32:51.422 INFO:teuthology.orchestra.run.smithi144.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T19:32:51.422 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:51.422 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:32:51.559 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install ceph-fuse 2023-09-23T19:32:51.911 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:32:53.196 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:06 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:54.557 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:32:54.557 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:32:54.557 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repository Size 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-fuse 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout:Install 2 Packages 2023-09-23T19:32:54.558 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:32:54.559 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 957 k 2023-09-23T19:32:54.559 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 3.0 M 2023-09-23T19:32:54.559 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:32:54.694 INFO:teuthology.orchestra.run.smithi144.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 614 kB/s | 83 kB 00:00 2023-09-23T19:32:54.811 INFO:teuthology.orchestra.run.smithi144.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 3.4 MB/s | 874 kB 00:00 2023-09-23T19:32:54.812 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:32:54.812 INFO:teuthology.orchestra.run.smithi144.stdout:Total 3.7 MB/s | 957 kB 00:00 2023-09-23T19:32:54.822 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:32:54.845 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:32:54.845 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:32:54.919 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:32:54.922 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:32:55.314 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:32:55.589 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T19:32:55.738 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T19:32:56.120 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T19:32:56.121 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T19:32:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T19:32:57.157 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:10 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:32:58.121 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:32:58.304 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:58.304 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:32:58.305 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T19:32:58.305 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:32:58.305 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:32:58.451 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install librados-devel 2023-09-23T19:32:58.809 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:00.084 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:13 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:00.446 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:33:00.627 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:00.627 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:33:00.627 INFO:teuthology.orchestra.run.smithi144.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T19:33:00.627 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:00.627 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:00.769 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install librados-devel 2023-09-23T19:33:01.124 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:01.444 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:01.444 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:01.444 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repo Size 2023-09-23T19:33:01.444 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:01.444 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:33:01.445 INFO:teuthology.orchestra.run.smithi135.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T19:33:01.445 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:01.445 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:33:01.445 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:01.445 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2023-09-23T19:33:01.445 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:01.446 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 130 k 2023-09-23T19:33:01.446 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 464 k 2023-09-23T19:33:01.446 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:33:01.725 INFO:teuthology.orchestra.run.smithi135.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 464 kB/s | 130 kB 00:00 2023-09-23T19:33:01.726 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:33:01.726 INFO:teuthology.orchestra.run.smithi135.stdout:Total 462 kB/s | 130 kB 00:00 2023-09-23T19:33:01.727 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:33:01.732 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:33:01.732 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:33:01.786 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:33:01.789 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:33:01.905 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:33:01.996 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:14 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:02.108 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:03.354 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:03.354 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:16 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:03.431 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:03.432 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:03.432 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repo Size 2023-09-23T19:33:03.432 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:03.432 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:33:03.432 INFO:teuthology.orchestra.run.smithi144.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T19:33:03.432 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:03.432 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:33:03.433 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:03.433 INFO:teuthology.orchestra.run.smithi144.stdout:Install 1 Package 2023-09-23T19:33:03.433 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:03.433 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 130 k 2023-09-23T19:33:03.433 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 464 k 2023-09-23T19:33:03.434 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:33:03.632 INFO:teuthology.orchestra.run.smithi144.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 653 kB/s | 130 kB 00:00 2023-09-23T19:33:03.633 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:33:03.633 INFO:teuthology.orchestra.run.smithi144.stdout:Total 649 kB/s | 130 kB 00:00 2023-09-23T19:33:03.633 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:33:03.639 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:33:03.639 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:33:03.693 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:33:03.697 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:33:03.823 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:33:04.010 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:04.247 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:05.295 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:05.296 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:18 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:06.569 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:33:06.752 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:06.752 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:33:06.752 INFO:teuthology.orchestra.run.smithi135.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:33:06.752 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:06.753 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:06.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install libcephfs2 2023-09-23T19:33:07.247 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:08.814 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:21 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:10.107 INFO:teuthology.orchestra.run.smithi135.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:10.216 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:10.217 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:33:10.217 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:10.322 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install libcephfs-devel 2023-09-23T19:33:10.676 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:11.135 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:33:11.315 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:11.549 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:33:11.549 INFO:teuthology.orchestra.run.smithi144.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:33:11.549 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:11.549 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:11.550 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install libcephfs2 2023-09-23T19:33:11.905 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:12.287 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:25 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:13.407 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:26 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:13.645 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:13.645 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:13.645 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repo Size 2023-09-23T19:33:13.646 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:13.646 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:33:13.646 INFO:teuthology.orchestra.run.smithi135.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T19:33:13.646 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:13.646 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:33:13.646 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:13.646 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2023-09-23T19:33:13.646 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:13.647 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 25 k 2023-09-23T19:33:13.647 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 109 k 2023-09-23T19:33:13.647 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:33:13.787 INFO:teuthology.orchestra.run.smithi135.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 177 kB/s | 25 kB 00:00 2023-09-23T19:33:13.788 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:33:13.788 INFO:teuthology.orchestra.run.smithi135.stdout:Total 175 kB/s | 25 kB 00:00 2023-09-23T19:33:13.788 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:33:13.792 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:33:13.792 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:33:13.794 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:33:13.797 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:33:13.845 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:33:14.014 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:14.258 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:14.649 INFO:teuthology.orchestra.run.smithi144.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:14.756 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:14.758 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:33:14.758 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:14.890 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install libcephfs-devel 2023-09-23T19:33:15.244 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:15.246 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:15.246 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:28 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:16.382 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:29 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:17.817 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:17.818 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:17.818 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repo Size 2023-09-23T19:33:17.818 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:17.818 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:33:17.818 INFO:teuthology.orchestra.run.smithi144.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T19:33:17.818 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:17.819 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:33:17.819 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:17.819 INFO:teuthology.orchestra.run.smithi144.stdout:Install 1 Package 2023-09-23T19:33:17.819 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:17.819 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 25 k 2023-09-23T19:33:17.819 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 109 k 2023-09-23T19:33:17.819 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:33:18.047 INFO:teuthology.orchestra.run.smithi144.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 109 kB/s | 25 kB 00:00 2023-09-23T19:33:18.048 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:33:18.048 INFO:teuthology.orchestra.run.smithi144.stdout:Total 108 kB/s | 25 kB 00:00 2023-09-23T19:33:18.048 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:33:18.052 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:33:18.052 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:33:18.054 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:33:18.058 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:33:18.109 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:33:18.257 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:18.460 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:19.441 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:19.441 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:32 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:20.202 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:33:20.384 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:20.385 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:33:20.385 INFO:teuthology.orchestra.run.smithi135.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:33:20.385 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:20.385 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:20.550 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install librados2 2023-09-23T19:33:20.902 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:22.450 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:35 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:23.405 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:33:23.583 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:23.583 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:33:23.583 INFO:teuthology.orchestra.run.smithi144.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:33:23.583 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:23.583 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:23.707 INFO:teuthology.orchestra.run.smithi135.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:23.710 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install librados2 2023-09-23T19:33:23.816 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:23.817 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:33:23.817 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:23.941 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install librbd1 2023-09-23T19:33:24.063 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:24.293 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:25.571 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:38 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:25.774 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:38 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:26.845 INFO:teuthology.orchestra.run.smithi144.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:26.959 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:26.960 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:33:26.960 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:27.028 INFO:teuthology.orchestra.run.smithi135.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:27.102 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install librbd1 2023-09-23T19:33:27.136 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:27.138 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:33:27.138 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:27.253 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install python3-rados 2023-09-23T19:33:27.455 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:27.606 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:29.064 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:42 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:29.222 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:42 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:30.323 INFO:teuthology.orchestra.run.smithi144.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:30.431 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:30.432 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:33:30.432 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:30.482 INFO:teuthology.orchestra.run.smithi135.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:30.562 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install python3-rados 2023-09-23T19:33:30.591 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:30.592 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:33:30.592 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:30.707 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install python3-rgw 2023-09-23T19:33:30.913 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:31.065 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:32.511 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:45 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:32.580 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:45 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:33.761 INFO:teuthology.orchestra.run.smithi144.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:33.839 INFO:teuthology.orchestra.run.smithi135.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:33.871 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:33.872 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:33:33.873 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:33.947 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:33.948 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:33:33.949 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:33.987 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install python3-rgw 2023-09-23T19:33:34.073 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install python3-cephfs 2023-09-23T19:33:34.338 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:34.426 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:35.307 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:48 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:35.389 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:48 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:36.563 INFO:teuthology.orchestra.run.smithi144.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:36.637 INFO:teuthology.orchestra.run.smithi135.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:36.671 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:36.673 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:33:36.673 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:36.748 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:36.749 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:33:36.749 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:36.778 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install python3-cephfs 2023-09-23T19:33:36.860 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install python3-rbd 2023-09-23T19:33:37.130 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:37.211 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:38.583 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:51 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:38.599 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:51 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:39.909 INFO:teuthology.orchestra.run.smithi135.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:39.936 INFO:teuthology.orchestra.run.smithi144.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:40.018 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:40.020 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2023-09-23T19:33:40.020 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:40.044 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:40.045 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:33:40.045 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:40.135 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install rbd-fuse 2023-09-23T19:33:40.161 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install python3-rbd 2023-09-23T19:33:40.489 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:40.513 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:42.080 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:55 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:42.094 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:55 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:43.353 INFO:teuthology.orchestra.run.smithi144.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T19:33:43.435 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:43.436 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:43.436 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2023-09-23T19:33:43.436 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:43.437 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:33:43.437 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T19:33:43.437 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:43.437 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:33:43.437 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:43.437 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2023-09-23T19:33:43.437 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:43.438 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 90 k 2023-09-23T19:33:43.438 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 264 k 2023-09-23T19:33:43.438 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:33:43.462 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:43.463 INFO:teuthology.orchestra.run.smithi144.stdout:Nothing to do. 2023-09-23T19:33:43.463 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:43.577 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install rbd-fuse 2023-09-23T19:33:43.605 INFO:teuthology.orchestra.run.smithi135.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 537 kB/s | 90 kB 00:00 2023-09-23T19:33:43.606 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:33:43.606 INFO:teuthology.orchestra.run.smithi135.stdout:Total 534 kB/s | 90 kB 00:00 2023-09-23T19:33:43.607 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:33:43.616 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:33:43.616 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:33:43.669 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:33:43.673 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:33:43.782 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:33:43.930 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:43.968 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:44.204 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:45.232 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:03:58 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:45.427 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:03:58 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:46.788 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:46.788 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:46.788 INFO:teuthology.orchestra.run.smithi144.stdout: Package Architecture Version Repository Size 2023-09-23T19:33:46.789 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:46.789 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:33:46.789 INFO:teuthology.orchestra.run.smithi144.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T19:33:46.789 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:46.789 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:33:46.789 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:46.789 INFO:teuthology.orchestra.run.smithi144.stdout:Install 1 Package 2023-09-23T19:33:46.789 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:46.790 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 90 k 2023-09-23T19:33:46.790 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 264 k 2023-09-23T19:33:46.790 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:33:47.090 INFO:teuthology.orchestra.run.smithi144.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 301 kB/s | 90 kB 00:00 2023-09-23T19:33:47.090 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:33:47.091 INFO:teuthology.orchestra.run.smithi144.stdout:Total 299 kB/s | 90 kB 00:00 2023-09-23T19:33:47.091 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:33:47.101 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:33:47.101 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:33:47.156 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:33:47.159 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:33:47.269 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:33:47.402 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:47.639 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:48.696 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:48.696 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:04:01 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:48.774 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:33:48.954 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:48.954 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:33:48.954 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:33:48.954 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:48.954 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:33:49.088 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install rbd-mirror 2023-09-23T19:33:49.441 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:50.357 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:04:03 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:51.718 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:33:51.719 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:51.719 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repo Size 2023-09-23T19:33:51.719 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:51.719 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:33:51.719 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T19:33:51.719 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:51.719 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:33:51.719 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:33:51.720 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2023-09-23T19:33:51.720 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:33:51.720 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 3.5 M 2023-09-23T19:33:51.720 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 16 M 2023-09-23T19:33:51.720 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:33:52.274 INFO:teuthology.orchestra.run.smithi135.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 6.2 MB/s | 3.5 MB 00:00 2023-09-23T19:33:52.274 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:33:52.275 INFO:teuthology.orchestra.run.smithi135.stdout:Total 6.2 MB/s | 3.5 MB 00:00 2023-09-23T19:33:52.275 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:33:52.286 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:33:52.287 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:33:52.354 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:33:52.357 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:33:52.583 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:33:52.764 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:52.764 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:33:52.764 INFO:teuthology.orchestra.run.smithi144.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:33:52.764 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:52.764 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:33:52.806 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:33:52.899 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install rbd-mirror 2023-09-23T19:33:52.995 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:53.253 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:33:53.369 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:54.481 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:04:07 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:54.707 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:04:07 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:33:56.073 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:33:56.073 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:56.073 INFO:teuthology.orchestra.run.smithi144.stdout: Package Arch Version Repo Size 2023-09-23T19:33:56.073 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:56.073 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:33:56.073 INFO:teuthology.orchestra.run.smithi144.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T19:33:56.074 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:56.074 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:33:56.074 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:33:56.074 INFO:teuthology.orchestra.run.smithi144.stdout:Install 1 Package 2023-09-23T19:33:56.074 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:33:56.074 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 3.5 M 2023-09-23T19:33:56.075 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 16 M 2023-09-23T19:33:56.075 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:33:56.444 INFO:teuthology.orchestra.run.smithi144.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 9.4 MB/s | 3.5 MB 00:00 2023-09-23T19:33:56.444 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:33:56.445 INFO:teuthology.orchestra.run.smithi144.stdout:Total 9.3 MB/s | 3.5 MB 00:00 2023-09-23T19:33:56.445 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:33:56.457 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:33:56.457 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:33:56.524 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:33:56.528 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:33:56.966 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:33:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:57.529 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:58.542 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:33:58.542 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:04:11 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:34:00.629 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:34:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:34:00.812 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:34:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:34:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:34:00.812 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:34:00.945 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install rbd-nbd 2023-09-23T19:34:01.301 INFO:teuthology.orchestra.run.smithi135.stdout:Updating Subscription Management repositories. 2023-09-23T19:34:02.788 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:04:15 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:34:03.220 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:34:03.403 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:34:03.403 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:34:03.403 INFO:teuthology.orchestra.run.smithi144.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:34:03.403 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:34:03.403 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:34:03.535 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum -y install rbd-nbd 2023-09-23T19:34:03.891 INFO:teuthology.orchestra.run.smithi144.stdout:Updating Subscription Management repositories. 2023-09-23T19:34:04.150 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-09-23T19:34:04.151 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:34:04.151 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2023-09-23T19:34:04.151 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:34:04.151 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-09-23T19:34:04.151 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T19:34:04.152 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:34:04.152 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-09-23T19:34:04.152 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-09-23T19:34:04.152 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2023-09-23T19:34:04.152 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:34:04.152 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 173 k 2023-09-23T19:34:04.153 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 552 k 2023-09-23T19:34:04.153 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-09-23T19:34:04.343 INFO:teuthology.orchestra.run.smithi135.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 908 kB/s | 173 kB 00:00 2023-09-23T19:34:04.343 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:34:04.344 INFO:teuthology.orchestra.run.smithi135.stdout:Total 903 kB/s | 173 kB 00:00 2023-09-23T19:34:04.344 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-09-23T19:34:04.354 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-09-23T19:34:04.355 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-09-23T19:34:04.409 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-09-23T19:34:04.413 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-09-23T19:34:04.530 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-09-23T19:34:04.682 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:34:04.905 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:34:05.498 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:04:18 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:34:06.020 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:34:06.020 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:04:19 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:34:06.876 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2023-09-23T19:34:06.877 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:34:06.877 INFO:teuthology.orchestra.run.smithi144.stdout: Package Architecture Version Repository Size 2023-09-23T19:34:06.877 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:34:06.877 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2023-09-23T19:34:06.877 INFO:teuthology.orchestra.run.smithi144.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T19:34:06.877 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:34:06.877 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2023-09-23T19:34:06.878 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2023-09-23T19:34:06.878 INFO:teuthology.orchestra.run.smithi144.stdout:Install 1 Package 2023-09-23T19:34:06.878 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:34:06.878 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 173 k 2023-09-23T19:34:06.878 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 552 k 2023-09-23T19:34:06.879 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2023-09-23T19:34:07.066 INFO:teuthology.orchestra.run.smithi144.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 922 kB/s | 173 kB 00:00 2023-09-23T19:34:07.066 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:34:07.067 INFO:teuthology.orchestra.run.smithi144.stdout:Total 916 kB/s | 173 kB 00:00 2023-09-23T19:34:07.067 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2023-09-23T19:34:07.077 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2023-09-23T19:34:07.077 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2023-09-23T19:34:07.131 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2023-09-23T19:34:07.135 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2023-09-23T19:34:07.250 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2023-09-23T19:34:07.424 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:34:07.653 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:34:08.710 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T19:34:08.710 INFO:teuthology.orchestra.run.smithi144.stdout:Last metadata expiration check: 0:04:21 ago on Sat 23 Sep 2023 07:29:47 PM UTC. 2023-09-23T19:34:11.546 INFO:teuthology.orchestra.run.smithi135.stdout:Installed products updated. 2023-09-23T19:34:11.727 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:34:11.727 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-09-23T19:34:11.727 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:34:11.728 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:34:11.728 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-09-23T19:34:11.884 DEBUG:teuthology.parallel:result is None 2023-09-23T19:34:16.146 INFO:teuthology.orchestra.run.smithi144.stdout:Installed products updated. 2023-09-23T19:34:16.327 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:34:16.328 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2023-09-23T19:34:16.328 INFO:teuthology.orchestra.run.smithi144.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T19:34:16.328 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:34:16.328 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2023-09-23T19:34:16.455 DEBUG:teuthology.parallel:result is None 2023-09-23T19:34:16.455 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T19:34:16.455 INFO:teuthology.packaging:ref: None 2023-09-23T19:34:16.455 INFO:teuthology.packaging:tag: None 2023-09-23T19:34:16.455 INFO:teuthology.packaging:branch: quincy 2023-09-23T19:34:16.455 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:34:16.456 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T19:34:16.674 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T19:34:16.701 INFO:teuthology.orchestra.run.smithi135.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T19:34:16.702 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T19:34:16.702 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T19:34:16.702 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T19:34:16.703 INFO:teuthology.packaging:ref: None 2023-09-23T19:34:16.703 INFO:teuthology.packaging:tag: None 2023-09-23T19:34:16.703 INFO:teuthology.packaging:branch: quincy 2023-09-23T19:34:16.703 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:34:16.703 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T19:34:17.001 DEBUG:teuthology.orchestra.run.smithi144:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T19:34:17.026 INFO:teuthology.orchestra.run.smithi144.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T19:34:17.026 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T19:34:17.026 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T19:34:17.027 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T19:34:17.027 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:34:17.027 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T19:34:17.061 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:34:17.061 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T19:34:17.097 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T19:34:17.097 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:34:17.098 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T19:34:17.131 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T19:34:17.200 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:34:17.200 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T19:34:17.230 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T19:34:17.301 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T19:34:17.301 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:34:17.302 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T19:34:17.329 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T19:34:17.396 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:34:17.397 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T19:34:17.428 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T19:34:17.498 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T19:34:17.498 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:34:17.498 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T19:34:17.525 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T19:34:17.592 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:34:17.592 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T19:34:17.621 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T19:34:17.690 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T19:34:17.754 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T19:34:17.755 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-23T19:34:17.755 INFO:tasks.cephadm:Cluster fsid is 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:34:17.755 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T19:34:17.755 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.135', 'mon.c': '[v2:172.21.15.135:3301,v1:172.21.15.135:6790]', 'mon.b': '172.21.15.144'} 2023-09-23T19:34:17.755 INFO:tasks.cephadm:First mon is mon.a on smithi135 2023-09-23T19:34:17.756 INFO:tasks.cephadm:First mgr is y 2023-09-23T19:34:17.756 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T19:34:17.756 DEBUG:teuthology.orchestra.run.smithi135:> sudo hostname $(hostname -s) 2023-09-23T19:34:17.783 DEBUG:teuthology.orchestra.run.smithi144:> sudo hostname $(hostname -s) 2023-09-23T19:34:17.813 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-23T19:34:17.813 DEBUG:teuthology.orchestra.run.smithi135:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T19:34:17.954 INFO:teuthology.orchestra.run.smithi135.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 19:34 /home/ubuntu/cephtest/cephadm 2023-09-23T19:34:17.954 DEBUG:teuthology.orchestra.run.smithi144:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T19:34:18.090 INFO:teuthology.orchestra.run.smithi144.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 19:34 /home/ubuntu/cephtest/cephadm 2023-09-23T19:34:18.091 DEBUG:teuthology.orchestra.run.smithi135:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T19:34:18.109 DEBUG:teuthology.orchestra.run.smithi144:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T19:34:18.138 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-23T19:34:18.138 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T19:34:18.151 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T19:34:19.297 INFO:teuthology.orchestra.run.smithi135.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T19:34:19.329 INFO:teuthology.orchestra.run.smithi144.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T19:34:58.738 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:34:58.738 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T19:34:58.739 INFO:teuthology.orchestra.run.smithi135.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T19:34:58.739 INFO:teuthology.orchestra.run.smithi135.stdout: "repo_digests": [ 2023-09-23T19:34:58.739 INFO:teuthology.orchestra.run.smithi135.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T19:34:58.739 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:34:58.739 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:34:59.115 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:34:59.115 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T19:34:59.115 INFO:teuthology.orchestra.run.smithi144.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T19:34:59.115 INFO:teuthology.orchestra.run.smithi144.stdout: "repo_digests": [ 2023-09-23T19:34:59.115 INFO:teuthology.orchestra.run.smithi144.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T19:34:59.116 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:34:59.116 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:34:59.142 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /etc/ceph 2023-09-23T19:34:59.178 DEBUG:teuthology.orchestra.run.smithi144:> sudo mkdir -p /etc/ceph 2023-09-23T19:34:59.212 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 777 /etc/ceph 2023-09-23T19:34:59.247 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod 777 /etc/ceph 2023-09-23T19:34:59.280 INFO:tasks.cephadm:Writing seed config... 2023-09-23T19:34:59.281 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T19:34:59.281 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T19:34:59.281 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T19:34:59.281 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T19:34:59.282 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T19:34:59.282 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-09-23T19:34:59.282 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T19:34:59.282 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T19:34:59.282 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T19:34:59.282 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T19:34:59.282 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T19:34:59.283 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:34:59.283 DEBUG:teuthology.orchestra.run.smithi135:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T19:34:59.307 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2f95c858-5a48-11ee-8db4-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T19:34:59.307 DEBUG:teuthology.orchestra.run.smithi135:mon.a> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T19:34:59.349 DEBUG:teuthology.orchestra.run.smithi135:mgr.y> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T19:34:59.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:34:59.379 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T19:34:59.379 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.135 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T19:34:59.407 INFO:journalctl@ceph.mgr.y.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:34:59.582 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-09-23T19:34:59.583 INFO:teuthology.orchestra.run.smithi135.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '2f95c858-5a48-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.135', '--skip-admin-label'] 2023-09-23T19:34:59.648 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stdout 3.0.1 2023-09-23T19:34:59.648 INFO:teuthology.orchestra.run.smithi135.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-23T19:34:59.648 INFO:teuthology.orchestra.run.smithi135.stdout:Verifying podman|docker is present... 2023-09-23T19:34:59.715 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stdout 3.0.1 2023-09-23T19:34:59.715 INFO:teuthology.orchestra.run.smithi135.stdout:Verifying lvm2 is present... 2023-09-23T19:34:59.715 INFO:teuthology.orchestra.run.smithi135.stdout:Verifying time synchronization is in place... 2023-09-23T19:34:59.722 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T19:34:59.722 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T19:34:59.728 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T19:34:59.728 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout inactive 2023-09-23T19:34:59.734 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout enabled 2023-09-23T19:34:59.740 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout active 2023-09-23T19:34:59.740 INFO:teuthology.orchestra.run.smithi135.stdout:Unit chronyd.service is enabled and running 2023-09-23T19:34:59.741 INFO:teuthology.orchestra.run.smithi135.stdout:Repeating the final host check... 2023-09-23T19:34:59.799 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stdout 3.0.1 2023-09-23T19:34:59.799 INFO:teuthology.orchestra.run.smithi135.stdout:podman (/bin/podman) version 3.0.1 is present 2023-09-23T19:34:59.799 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl is present 2023-09-23T19:34:59.799 INFO:teuthology.orchestra.run.smithi135.stdout:lvcreate is present 2023-09-23T19:34:59.805 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T19:34:59.805 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T19:34:59.811 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T19:34:59.811 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout inactive 2023-09-23T19:34:59.816 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout enabled 2023-09-23T19:34:59.822 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout active 2023-09-23T19:34:59.823 INFO:teuthology.orchestra.run.smithi135.stdout:Unit chronyd.service is enabled and running 2023-09-23T19:34:59.823 INFO:teuthology.orchestra.run.smithi135.stdout:Host looks OK 2023-09-23T19:34:59.823 INFO:teuthology.orchestra.run.smithi135.stdout:Cluster fsid: 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:34:59.823 INFO:teuthology.orchestra.run.smithi135.stdout:Acquiring lock 139741480196584 on /run/cephadm/2f95c858-5a48-11ee-8db4-212e2dc638e7.lock 2023-09-23T19:34:59.823 INFO:teuthology.orchestra.run.smithi135.stdout:Lock 139741480196584 acquired on /run/cephadm/2f95c858-5a48-11ee-8db4-212e2dc638e7.lock 2023-09-23T19:34:59.824 INFO:teuthology.orchestra.run.smithi135.stdout:Verifying IP 172.21.15.135 port 3300 ... 2023-09-23T19:34:59.824 INFO:teuthology.orchestra.run.smithi135.stdout:Verifying IP 172.21.15.135 port 6789 ... 2023-09-23T19:34:59.824 INFO:teuthology.orchestra.run.smithi135.stdout:Base mon IP(s) is [172.21.15.135:3300, 172.21.15.135:6789], mon addrv is [v2:172.21.15.135:3300,v1:172.21.15.135:6789] 2023-09-23T19:34:59.827 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-09-23T19:34:59.827 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.135 metric 100 2023-09-23T19:34:59.830 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T19:34:59.830 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-09-23T19:34:59.830 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-09-23T19:34:59.832 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T19:34:59.832 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-23T19:34:59.832 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T19:34:59.833 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-09-23T19:34:59.833 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout inet6 fe80::5f35:1bf3:232a:13a3/64 scope link noprefixroute 2023-09-23T19:34:59.833 INFO:teuthology.orchestra.run.smithi135.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T19:34:59.834 INFO:teuthology.orchestra.run.smithi135.stdout:Mon IP `172.21.15.135` is in CIDR network `172.21.0.0/20` 2023-09-23T19:34:59.834 INFO:teuthology.orchestra.run.smithi135.stdout:Mon IP `172.21.15.135` is in CIDR network `172.21.0.0/20` 2023-09-23T19:34:59.834 INFO:teuthology.orchestra.run.smithi135.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T19:34:59.834 INFO:teuthology.orchestra.run.smithi135.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T19:34:59.834 INFO:teuthology.orchestra.run.smithi135.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T19:35:00.279 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stdout 0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T19:35:00.279 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T19:35:00.280 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stderr Getting image source signatures 2023-09-23T19:35:00.280 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stderr Copying blob sha256:af7d028f70e220a466755c6119effe51eebe359d9c00eb6b4883db3fabc6becd 2023-09-23T19:35:00.280 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T19:35:00.280 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stderr Copying config sha256:0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T19:35:00.280 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-23T19:35:00.280 INFO:teuthology.orchestra.run.smithi135.stdout:/bin/podman: stderr Storing signatures 2023-09-23T19:35:01.416 INFO:teuthology.orchestra.run.smithi135.stdout:ceph: stdout ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T19:35:01.417 INFO:teuthology.orchestra.run.smithi135.stdout:Ceph version: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T19:35:01.417 INFO:teuthology.orchestra.run.smithi135.stdout:Extracting ceph user uid/gid from container image... 2023-09-23T19:35:02.419 INFO:teuthology.orchestra.run.smithi135.stdout:stat: stdout 167 167 2023-09-23T19:35:02.419 INFO:teuthology.orchestra.run.smithi135.stdout:Creating initial keys... 2023-09-23T19:35:03.498 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph-authtool: stdout AQDmPQ9lil6MLxAAIq/e715sdPcjFMHvzplzPw== 2023-09-23T19:35:04.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph-authtool: stdout AQDnPQ9lWgjNMhAAL+ATN0wDWinWzKsEtY4Wpw== 2023-09-23T19:35:05.525 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph-authtool: stdout AQDoPQ9lfJ3JNBAA496PYVl/0nCJWKTbgCFjww== 2023-09-23T19:35:05.526 INFO:teuthology.orchestra.run.smithi135.stdout:Creating initial monmap... 2023-09-23T19:35:06.596 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T19:35:06.596 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-23T19:35:06.596 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:06.596 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T19:35:06.597 INFO:teuthology.orchestra.run.smithi135.stdout:monmaptool for a [v2:172.21.15.135:3300,v1:172.21.15.135:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T19:35:06.597 INFO:teuthology.orchestra.run.smithi135.stdout:setting min_mon_release = octopus 2023-09-23T19:35:06.597 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/monmaptool: set fsid to 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:06.597 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T19:35:06.597 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:35:06.597 INFO:teuthology.orchestra.run.smithi135.stdout:Creating mon... 2023-09-23T19:35:07.676 INFO:teuthology.orchestra.run.smithi135.stdout:create mon.a on 2023-09-23T19:35:07.773 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T19:35:07.861 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T19:35:07.952 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7.target. 2023-09-23T19:35:07.952 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7.target. 2023-09-23T19:35:08.072 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T19:35:08.072 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T19:35:08.159 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7.target.wants/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@.service. 2023-09-23T19:35:08.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:08 smithi135 systemd[1]: Starting Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:35:08.767 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T19:35:08.767 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout disabled 2023-09-23T19:35:08.776 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T19:35:08.776 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout inactive 2023-09-23T19:35:08.776 INFO:teuthology.orchestra.run.smithi135.stdout:firewalld.service is not enabled 2023-09-23T19:35:08.776 INFO:teuthology.orchestra.run.smithi135.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T19:35:08.777 INFO:teuthology.orchestra.run.smithi135.stdout:Waiting for mon to start... 2023-09-23T19:35:08.777 INFO:teuthology.orchestra.run.smithi135.stdout:Waiting for mon... 2023-09-23T19:35:08.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:08 smithi135 bash[99236]: 865d3c889f0fd8e9465dcc80fe46e11b25565153d71bcdd1b5e2ed1fd190689e 2023-09-23T19:35:08.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:08 smithi135 systemd[1]: Started Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:35:08.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:08 smithi135 ceph-mon[99301]: mkfs 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:08.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:08 smithi135 ceph-mon[99301]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T19:35:10.069 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout cluster: 2023-09-23T19:35:10.069 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout id: 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout services: 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.530991s) 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout data: 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-23T19:35:10.070 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-23T19:35:10.071 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-23T19:35:10.071 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout pgs: 2023-09-23T19:35:10.071 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:10.071 INFO:teuthology.orchestra.run.smithi135.stdout:mon is available 2023-09-23T19:35:10.071 INFO:teuthology.orchestra.run.smithi135.stdout:Assimilating anything we can from ceph.conf... 2023-09-23T19:35:10.074 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:09 smithi135 ceph-mon[99301]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T19:35:10.074 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:09 smithi135 ceph-mon[99301]: monmap e1: 1 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} removed_ranks: {} 2023-09-23T19:35:10.074 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:09 smithi135 ceph-mon[99301]: fsmap 2023-09-23T19:35:10.074 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:09 smithi135 ceph-mon[99301]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T19:35:10.074 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:09 smithi135 ceph-mon[99301]: mgrmap e1: no daemons active 2023-09-23T19:35:10.075 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:09 smithi135 ceph-mon[99301]: from='client.? 172.21.15.135:0/1600458185' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T19:35:10.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:10 smithi135 ceph-mon[99301]: from='client.? 172.21.15.135:0/2869417718' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T19:35:10.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:10 smithi135 ceph-mon[99301]: from='client.? 172.21.15.135:0/2869417718' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T19:35:11.286 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:11.286 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T19:35:11.286 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout fsid = 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:11.286 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.135:3300,v1:172.21.15.135:6789] 2023-09-23T19:35:11.286 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T19:35:11.286 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T19:35:11.286 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T19:35:11.287 INFO:teuthology.orchestra.run.smithi135.stdout:Generating new minimal ceph.conf... 2023-09-23T19:35:12.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:11 smithi135 ceph-mon[99301]: from='client.? 172.21.15.135:0/4042563163' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:35:12.401 INFO:teuthology.orchestra.run.smithi135.stdout:Restarting the monitor... 2023-09-23T19:35:12.676 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:12 smithi135 systemd[1]: Stopping Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:35:12.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:12 smithi135 conmon[99277]: 2023-09-23T19:35:12.675+0000 7fc753c01700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:35:12.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:12 smithi135 conmon[99277]: 2023-09-23T19:35:12.675+0000 7fc753c01700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T19:35:12.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:12 smithi135 bash[99857]: 865d3c889f0fd8e9465dcc80fe46e11b25565153d71bcdd1b5e2ed1fd190689e 2023-09-23T19:35:13.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:13 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T19:35:13.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:13 smithi135 systemd[1]: Stopped Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:35:13.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:13 smithi135 systemd[1]: Starting Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:35:14.020 INFO:teuthology.orchestra.run.smithi135.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-23T19:35:14.255 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:35:14.255 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T19:35:14.255 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: pidfile_write: ignore empty --pid-file 2023-09-23T19:35:14.255 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: load: jerasure load: lrc 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: RocksDB version: 6.15.5 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Compile date Sep 22 2023 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: DB SUMMARY 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: DB Session ID: BHQJOHLIM1F77N6AZHOX 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: CURRENT file: CURRENT 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83617 ; 2023-09-23T19:35:14.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.error_if_exists: 0 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.create_if_missing: 0 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.paranoid_checks: 1 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.env: 0x556fcfaa8a80 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.fs: Posix File System 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.info_log: 0x556fd19d25a0 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.statistics: (nil) 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.use_fsync: 0 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_log_file_size: 0 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:35:14.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.allow_fallocate: 1 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.use_direct_reads: 0 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.db_log_dir: 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T19:35:14.258 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.write_buffer_manager: 0x556fd2744900 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T19:35:14.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.unordered_write: 0 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:35:14.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.row_cache: None 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.wal_filter: None 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.preserve_deletes: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.two_write_queues: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.atomic_flush: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.log_readahead_size: 0 2023-09-23T19:35:14.261 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_background_jobs: 2 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_background_compactions: -1 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_subcompactions: 1 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:35:14.262 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_open_files: -1 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:35:14.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_background_flushes: -1 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Compression algorithms supported: 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: kZSTD supported: 0 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: kXpressCompression supported: 0 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: kZlibCompression supported: 1 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: kSnappyCompression supported: 1 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: kLZ4Compression supported: 1 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: kBZip2Compression supported: 0 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-23T19:35:14.264 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.merge_operator: 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_filter: None 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556fd1a24fb8) 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout: cache_index_and_filter_blocks: 1 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:35:14.265 INFO:journalctl@ceph.mon.a.smithi135.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: pin_top_level_index_and_filter: 1 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: index_type: 0 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: data_block_index_type: 0 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: index_shortening: 1 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: hash_index_allow_collision: 1 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: checksum: 1 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: no_block_cache: 0 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_cache: 0x556fd1a57090 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_cache_name: BinnedLRUCache 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_cache_options: 2023-09-23T19:35:14.266 INFO:journalctl@ceph.mon.a.smithi135.stdout: capacity : 536870912 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: num_shard_bits : 4 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: strict_capacity_limit : 0 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: high_pri_pool_ratio: 0.000 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_cache_compressed: (nil) 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: persistent_cache: (nil) 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_size: 4096 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_size_deviation: 10 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_restart_interval: 16 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: index_block_restart_interval: 1 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: metadata_block_size: 4096 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: partition_filters: 0 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: use_delta_encoding: 1 2023-09-23T19:35:14.267 INFO:journalctl@ceph.mon.a.smithi135.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout: whole_key_filtering: 1 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout: verify_compression: 0 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout: read_amp_bytes_per_bit: 0 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout: format_version: 4 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout: enable_index_compression: 1 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_align: 0 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compression: NoCompression 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:35:14.268 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.num_levels: 7 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:35:14.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:35:14.270 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:35:14.270 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:35:14.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T19:35:14.272 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:35:14.273 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.table_properties_collectors: 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.inplace_update_support: 0 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.bloom_locality: 0 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.max_successive_merges: 0 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:35:14.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.ttl: 2592000 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.enable_blob_files: false 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.min_blob_size: 0 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:35:14.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497714010110, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497714010791, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80164, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78432, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72640, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695497714, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "11f3b44b-1225-4e1c-aa8f-687af70847a9", "db_session_id": "BHQJOHLIM1F77N6AZHOX"}} 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497714013808, "job": 1, "event": "recovery_finished"} 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:35:14.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x556fd19eb180 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: DB pointer 0x556fd1ac0000 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** DB Stats ** 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:35:14.277 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 171.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 171.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 171.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 171.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:35:14.278 INFO:journalctl@ceph.mon.a.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative compaction: 0.00 GB write, 13.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval compaction: 0.00 GB write, 13.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:35:14.279 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 171.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 171.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 171.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:35:14.280 INFO:journalctl@ceph.mon.a.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative compaction: 0.00 GB write, 13.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: starting mon.a rank 0 at public addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] at bind addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:14.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 bash[100186]: 1e28ea7cdc97a851a11b1a682dfa04021558ba887460e5e4d8f5aa740a5445e5 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a@-1(???) e1 preinit fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a@-1(???).mds e1 new map 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a@-1(???).mds e1 print_map 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout: e1 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout: legacy client fscid: -1 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout: No filesystems configured 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:35:14.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:35:14.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:35:14.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T19:35:14.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 systemd[1]: Started Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:35:14.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T19:35:14.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: monmap e1: 1 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} removed_ranks: {} 2023-09-23T19:35:14.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: fsmap 2023-09-23T19:35:14.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T19:35:14.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:14 smithi135 ceph-mon[100251]: mgrmap e1: no daemons active 2023-09-23T19:35:15.285 INFO:teuthology.orchestra.run.smithi135.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-23T19:35:15.285 INFO:teuthology.orchestra.run.smithi135.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T19:35:15.286 INFO:teuthology.orchestra.run.smithi135.stdout:Creating mgr... 2023-09-23T19:35:15.286 INFO:teuthology.orchestra.run.smithi135.stdout:Verifying port 9283 ... 2023-09-23T19:35:15.407 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T19:35:15.407 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T19:35:15.497 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7.target.wants/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@.service. 2023-09-23T19:35:15.570 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:15 smithi135 systemd[1]: Starting Ceph mgr.y for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:35:15.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:15 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2661678869' entity='client.admin' 2023-09-23T19:35:16.158 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T19:35:16.158 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout disabled 2023-09-23T19:35:16.165 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T19:35:16.165 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout inactive 2023-09-23T19:35:16.165 INFO:teuthology.orchestra.run.smithi135.stdout:firewalld.service is not enabled 2023-09-23T19:35:16.165 INFO:teuthology.orchestra.run.smithi135.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T19:35:16.175 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T19:35:16.175 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout disabled 2023-09-23T19:35:16.183 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T19:35:16.183 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout inactive 2023-09-23T19:35:16.183 INFO:teuthology.orchestra.run.smithi135.stdout:firewalld.service is not enabled 2023-09-23T19:35:16.183 INFO:teuthology.orchestra.run.smithi135.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T19:35:16.183 INFO:teuthology.orchestra.run.smithi135.stdout:Waiting for mgr to start... 2023-09-23T19:35:16.183 INFO:teuthology.orchestra.run.smithi135.stdout:Waiting for mgr... 2023-09-23T19:35:16.393 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:16 smithi135 bash[100642]: 7c00c1a623308619c8d0c5941281f0a796425f78245d05c4ea698ccc322e5e31 2023-09-23T19:35:16.393 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:16 smithi135 systemd[1]: Started Ceph mgr.y for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:35:16.393 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:16 smithi135 conmon[100683]: 2023-09-23T19:35:16.261+0000 7f4a22a5c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:35:16.393 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:16 smithi135 conmon[100683]: 2023-09-23T19:35:16.341+0000 7f4a22a5c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:35:16.706 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:16 smithi135 conmon[100683]: 2023-09-23T19:35:16.436+0000 7f4a22a5c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:35:16.706 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:16 smithi135 conmon[100683]: 2023-09-23T19:35:16.530+0000 7f4a22a5c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:35:16.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:16 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1543296838' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T19:35:16.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:16 smithi135 conmon[100683]: 2023-09-23T19:35:16.703+0000 7f4a22a5c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:35:17.270 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:17 smithi135 conmon[100683]: 2023-09-23T19:35:17.051+0000 7f4a22a5c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:35:17.270 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:17 smithi135 conmon[100683]: 2023-09-23T19:35:17.141+0000 7f4a22a5c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:35:17.492 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:17.494 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout { 2023-09-23T19:35:17.494 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "fsid": "2f95c858-5a48-11ee-8db4-212e2dc638e7", 2023-09-23T19:35:17.494 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 0 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T19:35:17.495 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T19:35:17.496 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T19:35:17.497 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T19:35:17.497 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T19:35:17.500 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T19:35:17.501 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T19:35:08.773577+0000", 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T19:35:17.502 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout } 2023-09-23T19:35:17.503 INFO:teuthology.orchestra.run.smithi135.stdout:mgr not available, waiting (1/15)... 2023-09-23T19:35:17.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:17 smithi135 conmon[100683]: 2023-09-23T19:35:17.270+0000 7f4a22a5c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:35:17.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:17 smithi135 conmon[100683]: 2023-09-23T19:35:17.361+0000 7f4a22a5c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:35:18.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:17 smithi135 conmon[100683]: 2023-09-23T19:35:17.778+0000 7f4a22a5c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:35:18.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:18 smithi135 conmon[100683]: 2023-09-23T19:35:18.340+0000 7f4a22a5c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:35:18.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:18 smithi135 conmon[100683]: 2023-09-23T19:35:18.506+0000 7f4a22a5c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:35:19.502 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:19 smithi135 conmon[100683]: 2023-09-23T19:35:19.251+0000 7f4a22a5c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:35:19.502 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:19 smithi135 conmon[100683]: 2023-09-23T19:35:19.337+0000 7f4a22a5c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:35:19.764 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:19 smithi135 conmon[100683]: 2023-09-23T19:35:19.501+0000 7f4a22a5c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:35:20.140 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:19 smithi135 conmon[100683]: 2023-09-23T19:35:19.791+0000 7f4a22a5c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:35:20.140 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:19 smithi135 conmon[100683]: 2023-09-23T19:35:19.878+0000 7f4a22a5c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:35:20.453 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:20 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2098796179' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T19:35:20.453 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:20 smithi135 conmon[100683]: 2023-09-23T19:35:20.177+0000 7f4a22a5c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:35:20.718 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout { 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "fsid": "2f95c858-5a48-11ee-8db4-212e2dc638e7", 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T19:35:20.719 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 0 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T19:35:20.720 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:20.721 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T19:35:20.721 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:20.721 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T19:35:20.721 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T19:35:20.721 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T19:35:20.721 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T19:35:20.721 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T19:35:20.721 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T19:35:20.724 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:20.724 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T19:35:20.724 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T19:35:20.724 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T19:35:20.725 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T19:35:20.726 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:20.727 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T19:35:20.727 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:20.727 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T19:35:08.773577+0000", 2023-09-23T19:35:20.727 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T19:35:20.727 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:20.727 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T19:35:20.727 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout } 2023-09-23T19:35:20.727 INFO:teuthology.orchestra.run.smithi135.stdout:mgr not available, waiting (2/15)... 2023-09-23T19:35:20.727 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:20 smithi135 conmon[100683]: 2023-09-23T19:35:20.452+0000 7f4a22a5c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:35:21.460 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:21 smithi135 conmon[100683]: 2023-09-23T19:35:21.151+0000 7f4a22a5c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:35:21.460 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:21 smithi135 conmon[100683]: 2023-09-23T19:35:21.277+0000 7f4a22a5c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:35:21.460 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:21 smithi135 conmon[100683]: 2023-09-23T19:35:21.459+0000 7f4a22a5c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:35:21.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:21 smithi135 conmon[100683]: 2023-09-23T19:35:21.569+0000 7f4a22a5c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:35:22.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:21 smithi135 conmon[100683]: 2023-09-23T19:35:21.775+0000 7f4a22a5c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:35:22.906 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: Activating manager daemon y 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: mgrmap e2: y(active, starting, since 0.0029506s) 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: Manager daemon y is now available 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:35:22.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:35:22.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' 2023-09-23T19:35:22.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:22 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' 2023-09-23T19:35:22.908 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:22 smithi135 conmon[100683]: 2023-09-23T19:35:22.609+0000 7f4a22a5c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:35:23.910 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout { 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "fsid": "2f95c858-5a48-11ee-8db4-212e2dc638e7", 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 0 2023-09-23T19:35:23.911 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T19:35:23.912 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:23.913 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T19:35:23.913 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T19:35:23.913 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T19:35:23.913 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T19:35:23.913 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T19:35:23.913 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T19:35:23.916 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:23.916 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T19:35:23.916 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T19:35:23.916 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T19:35:23.916 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T19:35:23.916 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:23.917 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:23.918 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T19:35:08.773577+0000", 2023-09-23T19:35:23.919 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T19:35:23.919 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:23.919 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T19:35:23.919 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout } 2023-09-23T19:35:23.919 INFO:teuthology.orchestra.run.smithi135.stdout:mgr not available, waiting (3/15)... 2023-09-23T19:35:23.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:23 smithi135 ceph-mon[100251]: from='mgr.14100 172.21.15.135:0/3125428954' entity='mgr.y' 2023-09-23T19:35:23.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:23 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1223794377' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T19:35:23.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:23 smithi135 ceph-mon[100251]: mgrmap e3: y(active, since 1.00578s) 2023-09-23T19:35:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:26 smithi135 ceph-mon[100251]: mgrmap e4: y(active, since 2s) 2023-09-23T19:35:27.235 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:27.235 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout { 2023-09-23T19:35:27.235 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "fsid": "2f95c858-5a48-11ee-8db4-212e2dc638e7", 2023-09-23T19:35:27.235 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T19:35:27.235 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 0 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:27.236 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T19:35:27.237 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T19:35:27.238 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T19:35:27.238 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T19:35:27.241 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:27.241 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T19:35:27.241 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T19:35:27.241 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T19:35:27.241 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T19:35:27.241 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T19:35:27.241 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T19:35:27.241 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T19:35:27.242 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ], 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T19:35:08.773577+0000", 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout }, 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T19:35:27.243 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout } 2023-09-23T19:35:27.244 INFO:teuthology.orchestra.run.smithi135.stdout:mgr is available 2023-09-23T19:35:27.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:27 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3312609157' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T19:35:28.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:28 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3903729290' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T19:35:28.483 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:28.483 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T19:35:28.483 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout fsid = 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:28.483 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T19:35:28.483 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T19:35:28.483 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T19:35:28.483 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T19:35:28.484 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:28.484 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T19:35:28.484 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T19:35:28.484 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 2023-09-23T19:35:28.484 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T19:35:28.484 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T19:35:28.484 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T19:35:28.484 INFO:teuthology.orchestra.run.smithi135.stdout:Enabling cephadm module... 2023-09-23T19:35:30.339 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:30 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2705966990' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T19:35:30.339 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:30 smithi135 conmon[100683]: ignoring --setuser ceph since I am not root 2023-09-23T19:35:30.339 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:30 smithi135 conmon[100683]: ignoring --setgroup ceph since I am not root 2023-09-23T19:35:30.340 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:30 smithi135 conmon[100683]: 2023-09-23T19:35:30.188+0000 7f287a0e81c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:35:30.340 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:30 smithi135 conmon[100683]: 2023-09-23T19:35:30.267+0000 7f287a0e81c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:35:30.612 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:30 smithi135 conmon[100683]: 2023-09-23T19:35:30.357+0000 7f287a0e81c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:35:30.612 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:30 smithi135 conmon[100683]: 2023-09-23T19:35:30.448+0000 7f287a0e81c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:35:30.612 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:30 smithi135 conmon[100683]: 2023-09-23T19:35:30.612+0000 7f287a0e81c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:35:31.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:31 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2705966990' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T19:35:31.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:31 smithi135 ceph-mon[100251]: mgrmap e5: y(active, since 7s) 2023-09-23T19:35:31.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:30 smithi135 conmon[100683]: 2023-09-23T19:35:30.930+0000 7f287a0e81c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:35:31.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:31 smithi135 conmon[100683]: 2023-09-23T19:35:31.029+0000 7f287a0e81c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:35:31.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:31 smithi135 conmon[100683]: 2023-09-23T19:35:31.169+0000 7f287a0e81c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:35:31.688 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:31 smithi135 conmon[100683]: 2023-09-23T19:35:31.266+0000 7f287a0e81c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:35:31.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:31 smithi135 conmon[100683]: 2023-09-23T19:35:31.702+0000 7f287a0e81c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:35:32.090 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout { 2023-09-23T19:35:32.090 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-23T19:35:32.090 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T19:35:32.090 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T19:35:32.090 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T19:35:32.091 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout } 2023-09-23T19:35:32.091 INFO:teuthology.orchestra.run.smithi135.stdout:Waiting for the mgr to restart... 2023-09-23T19:35:32.091 INFO:teuthology.orchestra.run.smithi135.stdout:Waiting for mgr epoch 5... 2023-09-23T19:35:32.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:32 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2278124487' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T19:35:32.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:32 smithi135 conmon[100683]: 2023-09-23T19:35:32.274+0000 7f287a0e81c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:35:32.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:32 smithi135 conmon[100683]: 2023-09-23T19:35:32.451+0000 7f287a0e81c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:35:33.467 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:33 smithi135 conmon[100683]: 2023-09-23T19:35:33.213+0000 7f287a0e81c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:35:33.468 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:33 smithi135 conmon[100683]: 2023-09-23T19:35:33.302+0000 7f287a0e81c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:35:33.468 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:33 smithi135 conmon[100683]: 2023-09-23T19:35:33.467+0000 7f287a0e81c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:35:34.125 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:33 smithi135 conmon[100683]: 2023-09-23T19:35:33.736+0000 7f287a0e81c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:35:34.125 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:33 smithi135 conmon[100683]: 2023-09-23T19:35:33.821+0000 7f287a0e81c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:35:34.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:34 smithi135 conmon[100683]: 2023-09-23T19:35:34.124+0000 7f287a0e81c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:35:34.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:34 smithi135 conmon[100683]: 2023-09-23T19:35:34.375+0000 7f287a0e81c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:35:35.380 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:35 smithi135 conmon[100683]: 2023-09-23T19:35:35.066+0000 7f287a0e81c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:35:35.380 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:35 smithi135 conmon[100683]: 2023-09-23T19:35:35.195+0000 7f287a0e81c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:35:35.700 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:35 smithi135 conmon[100683]: 2023-09-23T19:35:35.379+0000 7f287a0e81c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:35:35.700 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:35 smithi135 conmon[100683]: 2023-09-23T19:35:35.490+0000 7f287a0e81c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:35:36.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:35 smithi135 conmon[100683]: 2023-09-23T19:35:35.699+0000 7f287a0e81c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:35:36.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:36 smithi135 conmon[100683]: 2023-09-23T19:35:36.537+0000 7f287a0e81c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:35:36.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: Active manager daemon y restarted 2023-09-23T19:35:36.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: Activating manager daemon y 2023-09-23T19:35:36.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: osdmap e2: 0 total, 0 up, 0 in 2023-09-23T19:35:36.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: mgrmap e6: y(active, starting, since 0.0054022s) 2023-09-23T19:35:36.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:35:36.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:35:36.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:35:36.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:35:36.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:35:36.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: Manager daemon y is now available 2023-09-23T19:35:36.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:36.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:36 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:37 smithi135 ceph-mon[100251]: Found migration_current of "None". Setting to last migration. 2023-09-23T19:35:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:37 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:37 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:35:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:37 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:35:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:37 smithi135 ceph-mon[100251]: mgrmap e7: y(active, since 1.00476s) 2023-09-23T19:35:38.181 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout { 2023-09-23T19:35:38.181 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-23T19:35:38.181 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T19:35:38.181 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout } 2023-09-23T19:35:38.181 INFO:teuthology.orchestra.run.smithi135.stdout:mgr epoch 5 is available 2023-09-23T19:35:38.181 INFO:teuthology.orchestra.run.smithi135.stdout:Setting orchestrator backend to cephadm... 2023-09-23T19:35:38.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:38 smithi135 ceph-mon[100251]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T19:35:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:38 smithi135 ceph-mon[100251]: [23/Sep/2023:19:35:37] ENGINE Bus STARTING 2023-09-23T19:35:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:38 smithi135 ceph-mon[100251]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T19:35:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:38 smithi135 ceph-mon[100251]: [23/Sep/2023:19:35:37] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:35:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:38 smithi135 ceph-mon[100251]: [23/Sep/2023:19:35:37] ENGINE Bus STARTED 2023-09-23T19:35:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:38 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:38 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:39.750 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:39 smithi135 ceph-mon[100251]: mgrmap e8: y(active, since 2s) 2023-09-23T19:35:39.750 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:39 smithi135 ceph-mon[100251]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:35:39.750 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:39 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:39.750 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:39 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:40.797 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-23T19:35:40.798 INFO:teuthology.orchestra.run.smithi135.stdout:Generating ssh key... 2023-09-23T19:35:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:41 smithi135 ceph-mon[100251]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:35:41.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: Generating public/private rsa key pair. 2023-09-23T19:35:41.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: Your identification has been saved in /tmp/tmpzq0ui6cg/key. 2023-09-23T19:35:41.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: Your public key has been saved in /tmp/tmpzq0ui6cg/key.pub. 2023-09-23T19:35:41.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: The key fingerprint is: 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: SHA256:wdLTnqYp5PhSGDuGXfBAE/McxlUfZLK8PfVVbYN62Eg ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: The key's randomart image is: 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: +---[RSA 3072]----+ 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | .=oo...o.+ . o| 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | o*.+ o E o .+| 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | ++ = = * ..o| 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | . o. + B + ..| 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | o =. S = + .| 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | . *+. + . | 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | ..oo o | 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | .. . | 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: | .. | 2023-09-23T19:35:41.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:41 smithi135 conmon[100683]: +----[SHA256]-----+ 2023-09-23T19:35:42.890 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:42 smithi135 ceph-mon[100251]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:35:42.890 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:42 smithi135 ceph-mon[100251]: Generating ssh key... 2023-09-23T19:35:42.890 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:42 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:42.890 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:42 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:43.489 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7C7LhMJGL+S/iQP/Q0tJgOLfTgJV24JHegh8F6uDJNG9UXtLSeFIQeiL8IdGLxH/YdZOj4TJnI6jrFBbam/UMKWNYD91SDev29Vm0Kx3KDpurqiuJA+q9BycsJKDVpp3uakYYy4BU3LbNTbqTA8mDQ+Kuw00QxNfIh5aFiwGTOHrbTGYWVPU03GIjeBuZZst9tAXix6JOGwu25bOCDlcldcmxUPYCKuU3BOtY0Yi4icjUpOCv0x/z2Xt7UTKwn4h1ZBdL1KwmF/ZDbFATCHRzFTnKBEE5tllJ7Dld0CmyFkdeJkO+yk8K9EYxxooX3XqAp6ivoX+RwhqLRmKk0+L01wgBDRV8UvnJE87HlLRvtp/738s0ofGHhR2By1Lj1msVuonkgSmrTIHiO9Il+TbsGOuH0/I49311kczfOFf5rWHF3b47muJtTmOq2tOHRD1gk+uhw1aHym42FZ7D7PSvoswBozFYoHSgrS4GF/rbeyrUvUrE5c7l4woi+FJnRM= ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:35:43.489 INFO:teuthology.orchestra.run.smithi135.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T19:35:43.489 INFO:teuthology.orchestra.run.smithi135.stdout:Adding key to root@localhost authorized_keys... 2023-09-23T19:35:43.490 INFO:teuthology.orchestra.run.smithi135.stdout:Adding host smithi135... 2023-09-23T19:35:43.757 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:43 smithi135 ceph-mon[100251]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:35:45.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:45 smithi135 ceph-mon[100251]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi135", "addr": "172.21.15.135", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:35:45.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:45 smithi135 ceph-mon[100251]: Deploying cephadm binary to smithi135 2023-09-23T19:35:46.588 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout Added host 'smithi135' with addr '172.21.15.135' 2023-09-23T19:35:46.588 INFO:teuthology.orchestra.run.smithi135.stdout:Deploying unmanaged mon service... 2023-09-23T19:35:47.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:46 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:47.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:46 smithi135 ceph-mon[100251]: Added host smithi135 2023-09-23T19:35:47.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:46 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:35:48.448 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-23T19:35:48.448 INFO:teuthology.orchestra.run.smithi135.stdout:Deploying unmanaged mgr service... 2023-09-23T19:35:48.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:48 smithi135 ceph-mon[100251]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:35:48.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:48 smithi135 ceph-mon[100251]: Saving service mon spec with placement count:5 2023-09-23T19:35:48.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:48 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:49.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:49 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:49.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:49 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:49.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:49 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:50.079 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-23T19:35:50.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:50 smithi135 ceph-mon[100251]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:35:50.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:50 smithi135 ceph-mon[100251]: Saving service mgr spec with placement count:2 2023-09-23T19:35:51.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:51 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1804098391' entity='client.admin' 2023-09-23T19:35:51.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:51 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:51.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:51 smithi135 ceph-mon[100251]: from='mgr.14120 172.21.15.135:0/253246005' entity='mgr.y' 2023-09-23T19:35:52.795 INFO:teuthology.orchestra.run.smithi135.stdout:Enabling the dashboard module... 2023-09-23T19:35:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:53 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1045753524' entity='client.admin' 2023-09-23T19:35:54.421 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:54 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2179188800' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T19:35:54.421 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:54 smithi135 conmon[100683]: ignoring --setuser ceph since I am not root 2023-09-23T19:35:54.421 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:54 smithi135 conmon[100683]: ignoring --setgroup ceph since I am not root 2023-09-23T19:35:54.421 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:54 smithi135 conmon[100683]: 2023-09-23T19:35:54.340+0000 7f7993e721c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:35:54.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:54 smithi135 conmon[100683]: 2023-09-23T19:35:54.420+0000 7f7993e721c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:35:54.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:54 smithi135 conmon[100683]: 2023-09-23T19:35:54.510+0000 7f7993e721c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:35:54.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:54 smithi135 conmon[100683]: 2023-09-23T19:35:54.602+0000 7f7993e721c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:35:55.068 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:54 smithi135 conmon[100683]: 2023-09-23T19:35:54.768+0000 7f7993e721c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:35:55.320 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:55 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2179188800' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T19:35:55.320 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:55 smithi135 ceph-mon[100251]: mgrmap e9: y(active, since 17s) 2023-09-23T19:35:55.320 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:55 smithi135 conmon[100683]: 2023-09-23T19:35:55.093+0000 7f7993e721c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:35:55.320 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:55 smithi135 conmon[100683]: 2023-09-23T19:35:55.184+0000 7f7993e721c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:35:55.618 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:55 smithi135 conmon[100683]: 2023-09-23T19:35:55.319+0000 7f7993e721c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:35:55.618 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:55 smithi135 conmon[100683]: 2023-09-23T19:35:55.418+0000 7f7993e721c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:35:55.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:55 smithi135 conmon[100683]: 2023-09-23T19:35:55.856+0000 7f7993e721c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:35:56.178 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout { 2023-09-23T19:35:56.179 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-23T19:35:56.179 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T19:35:56.179 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T19:35:56.179 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T19:35:56.179 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout } 2023-09-23T19:35:56.179 INFO:teuthology.orchestra.run.smithi135.stdout:Waiting for the mgr to restart... 2023-09-23T19:35:56.179 INFO:teuthology.orchestra.run.smithi135.stdout:Waiting for mgr epoch 9... 2023-09-23T19:35:56.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:35:56 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3412187348' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T19:35:56.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:56 smithi135 conmon[100683]: 2023-09-23T19:35:56.441+0000 7f7993e721c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:35:56.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:56 smithi135 conmon[100683]: 2023-09-23T19:35:56.616+0000 7f7993e721c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:35:57.641 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:57 smithi135 conmon[100683]: 2023-09-23T19:35:57.381+0000 7f7993e721c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:35:57.641 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:57 smithi135 conmon[100683]: 2023-09-23T19:35:57.470+0000 7f7993e721c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:35:57.913 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:57 smithi135 conmon[100683]: 2023-09-23T19:35:57.641+0000 7f7993e721c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:35:58.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:57 smithi135 conmon[100683]: 2023-09-23T19:35:57.913+0000 7f7993e721c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:35:58.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:58 smithi135 conmon[100683]: 2023-09-23T19:35:57.999+0000 7f7993e721c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:35:58.548 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:58 smithi135 conmon[100683]: 2023-09-23T19:35:58.290+0000 7f7993e721c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:35:58.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:58 smithi135 conmon[100683]: 2023-09-23T19:35:58.547+0000 7f7993e721c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:35:59.566 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:59 smithi135 conmon[100683]: 2023-09-23T19:35:59.246+0000 7f7993e721c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:35:59.566 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:59 smithi135 conmon[100683]: 2023-09-23T19:35:59.376+0000 7f7993e721c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:35:59.893 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:59 smithi135 conmon[100683]: 2023-09-23T19:35:59.565+0000 7f7993e721c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:35:59.893 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:59 smithi135 conmon[100683]: 2023-09-23T19:35:59.679+0000 7f7993e721c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:36:00.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:35:59 smithi135 conmon[100683]: 2023-09-23T19:35:59.892+0000 7f7993e721c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:36:01.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:36:00 smithi135 conmon[100683]: 2023-09-23T19:36:00.770+0000 7f7993e721c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:36:01.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: Active manager daemon y restarted 2023-09-23T19:36:01.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: Activating manager daemon y 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: osdmap e3: 0 total, 0 up, 0 in 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: mgrmap e10: y(active, starting, since 0.00341521s) 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: Manager daemon y is now available 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:36:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:36:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:01 smithi135 ceph-mon[100251]: mgrmap e11: y(active, since 1.00605s) 2023-09-23T19:36:02.413 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout { 2023-09-23T19:36:02.413 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-23T19:36:02.413 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T19:36:02.413 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout } 2023-09-23T19:36:02.413 INFO:teuthology.orchestra.run.smithi135.stdout:mgr epoch 9 is available 2023-09-23T19:36:02.413 INFO:teuthology.orchestra.run.smithi135.stdout:Generating a dashboard self-signed certificate... 2023-09-23T19:36:02.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:02 smithi135 ceph-mon[100251]: [23/Sep/2023:19:36:01] ENGINE Bus STARTING 2023-09-23T19:36:02.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:02 smithi135 ceph-mon[100251]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T19:36:02.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:02 smithi135 ceph-mon[100251]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T19:36:02.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:02 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:02.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:02 smithi135 ceph-mon[100251]: [23/Sep/2023:19:36:01] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:36:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:02 smithi135 ceph-mon[100251]: [23/Sep/2023:19:36:01] ENGINE Bus STARTED 2023-09-23T19:36:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:02 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:02 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:04.298 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-23T19:36:04.299 INFO:teuthology.orchestra.run.smithi135.stdout:Creating initial admin user... 2023-09-23T19:36:04.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:04 smithi135 ceph-mon[100251]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:04.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:04.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:04.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:04 smithi135 ceph-mon[100251]: mgrmap e12: y(active, since 2s) 2023-09-23T19:36:05.542 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:05 smithi135 ceph-mon[100251]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:05.978 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$13g1a/amhV4GRK/HEn.DK.W9OehB7fh1lGo309J1oRGH0O7.llCfa", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695497765, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T19:36:05.979 INFO:teuthology.orchestra.run.smithi135.stdout:Fetching dashboard port number... 2023-09-23T19:36:06.665 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:06 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:07.143 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stdout 8443 2023-09-23T19:36:07.153 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T19:36:07.153 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout disabled 2023-09-23T19:36:07.161 INFO:teuthology.orchestra.run.smithi135.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T19:36:07.161 INFO:teuthology.orchestra.run.smithi135.stdout:systemctl: stdout inactive 2023-09-23T19:36:07.161 INFO:teuthology.orchestra.run.smithi135.stdout:firewalld.service is not enabled 2023-09-23T19:36:07.161 INFO:teuthology.orchestra.run.smithi135.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T19:36:07.162 INFO:teuthology.orchestra.run.smithi135.stdout:Ceph Dashboard is now available at: 2023-09-23T19:36:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: URL: https://smithi135:8443/ 2023-09-23T19:36:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: User: admin 2023-09-23T19:36:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: Password: hxxzqkj0q6 2023-09-23T19:36:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:07.163 INFO:teuthology.orchestra.run.smithi135.stdout:Saving cluster configuration to /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config directory 2023-09-23T19:36:07.164 INFO:teuthology.orchestra.run.smithi135.stdout:Enabling autotune for osd_memory_target 2023-09-23T19:36:07.423 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:07 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1448511120' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T19:36:08.674 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:08 smithi135 ceph-mon[100251]: mgrmap e13: y(active, since 6s) 2023-09-23T19:36:09.851 INFO:teuthology.orchestra.run.smithi135.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-23T19:36:09.851 INFO:teuthology.orchestra.run.smithi135.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T19:36:09.851 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:09.851 INFO:teuthology.orchestra.run.smithi135.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T19:36:09.851 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout:Or, if you are only running a single cluster on this host: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: ceph telemetry on 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout:For more information see: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:09.852 INFO:teuthology.orchestra.run.smithi135.stdout:Bootstrap complete. 2023-09-23T19:36:09.890 INFO:tasks.cephadm:Fetching config... 2023-09-23T19:36:09.890 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:36:09.891 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T19:36:09.907 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T19:36:09.907 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:36:09.907 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T19:36:09.964 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T19:36:09.964 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:36:09.964 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T19:36:10.031 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T19:36:10.031 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:36:10.032 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T19:36:10.089 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T19:36:10.089 DEBUG:teuthology.orchestra.run.smithi135:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7C7LhMJGL+S/iQP/Q0tJgOLfTgJV24JHegh8F6uDJNG9UXtLSeFIQeiL8IdGLxH/YdZOj4TJnI6jrFBbam/UMKWNYD91SDev29Vm0Kx3KDpurqiuJA+q9BycsJKDVpp3uakYYy4BU3LbNTbqTA8mDQ+Kuw00QxNfIh5aFiwGTOHrbTGYWVPU03GIjeBuZZst9tAXix6JOGwu25bOCDlcldcmxUPYCKuU3BOtY0Yi4icjUpOCv0x/z2Xt7UTKwn4h1ZBdL1KwmF/ZDbFATCHRzFTnKBEE5tllJ7Dld0CmyFkdeJkO+yk8K9EYxxooX3XqAp6ivoX+RwhqLRmKk0+L01wgBDRV8UvnJE87HlLRvtp/738s0ofGHhR2By1Lj1msVuonkgSmrTIHiO9Il+TbsGOuH0/I49311kczfOFf5rWHF3b47muJtTmOq2tOHRD1gk+uhw1aHym42FZ7D7PSvoswBozFYoHSgrS4GF/rbeyrUvUrE5c7l4woi+FJnRM= ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T19:36:10.169 INFO:teuthology.orchestra.run.smithi135.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7C7LhMJGL+S/iQP/Q0tJgOLfTgJV24JHegh8F6uDJNG9UXtLSeFIQeiL8IdGLxH/YdZOj4TJnI6jrFBbam/UMKWNYD91SDev29Vm0Kx3KDpurqiuJA+q9BycsJKDVpp3uakYYy4BU3LbNTbqTA8mDQ+Kuw00QxNfIh5aFiwGTOHrbTGYWVPU03GIjeBuZZst9tAXix6JOGwu25bOCDlcldcmxUPYCKuU3BOtY0Yi4icjUpOCv0x/z2Xt7UTKwn4h1ZBdL1KwmF/ZDbFATCHRzFTnKBEE5tllJ7Dld0CmyFkdeJkO+yk8K9EYxxooX3XqAp6ivoX+RwhqLRmKk0+L01wgBDRV8UvnJE87HlLRvtp/738s0ofGHhR2By1Lj1msVuonkgSmrTIHiO9Il+TbsGOuH0/I49311kczfOFf5rWHF3b47muJtTmOq2tOHRD1gk+uhw1aHym42FZ7D7PSvoswBozFYoHSgrS4GF/rbeyrUvUrE5c7l4woi+FJnRM= ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:36:10.181 DEBUG:teuthology.orchestra.run.smithi144:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7C7LhMJGL+S/iQP/Q0tJgOLfTgJV24JHegh8F6uDJNG9UXtLSeFIQeiL8IdGLxH/YdZOj4TJnI6jrFBbam/UMKWNYD91SDev29Vm0Kx3KDpurqiuJA+q9BycsJKDVpp3uakYYy4BU3LbNTbqTA8mDQ+Kuw00QxNfIh5aFiwGTOHrbTGYWVPU03GIjeBuZZst9tAXix6JOGwu25bOCDlcldcmxUPYCKuU3BOtY0Yi4icjUpOCv0x/z2Xt7UTKwn4h1ZBdL1KwmF/ZDbFATCHRzFTnKBEE5tllJ7Dld0CmyFkdeJkO+yk8K9EYxxooX3XqAp6ivoX+RwhqLRmKk0+L01wgBDRV8UvnJE87HlLRvtp/738s0ofGHhR2By1Lj1msVuonkgSmrTIHiO9Il+TbsGOuH0/I49311kczfOFf5rWHF3b47muJtTmOq2tOHRD1gk+uhw1aHym42FZ7D7PSvoswBozFYoHSgrS4GF/rbeyrUvUrE5c7l4woi+FJnRM= ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T19:36:10.226 INFO:teuthology.orchestra.run.smithi144.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7C7LhMJGL+S/iQP/Q0tJgOLfTgJV24JHegh8F6uDJNG9UXtLSeFIQeiL8IdGLxH/YdZOj4TJnI6jrFBbam/UMKWNYD91SDev29Vm0Kx3KDpurqiuJA+q9BycsJKDVpp3uakYYy4BU3LbNTbqTA8mDQ+Kuw00QxNfIh5aFiwGTOHrbTGYWVPU03GIjeBuZZst9tAXix6JOGwu25bOCDlcldcmxUPYCKuU3BOtY0Yi4icjUpOCv0x/z2Xt7UTKwn4h1ZBdL1KwmF/ZDbFATCHRzFTnKBEE5tllJ7Dld0CmyFkdeJkO+yk8K9EYxxooX3XqAp6ivoX+RwhqLRmKk0+L01wgBDRV8UvnJE87HlLRvtp/738s0ofGHhR2By1Lj1msVuonkgSmrTIHiO9Il+TbsGOuH0/I49311kczfOFf5rWHF3b47muJtTmOq2tOHRD1gk+uhw1aHym42FZ7D7PSvoswBozFYoHSgrS4GF/rbeyrUvUrE5c7l4woi+FJnRM= ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:36:10.239 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T19:36:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:10 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4205213333' entity='client.admin' 2023-09-23T19:36:10.576 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:13.284 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:13.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:13.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi135", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:36:13.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:13.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:13.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:13.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2022442432' entity='client.admin' 2023-09-23T19:36:13.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:13.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:13.287 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T19:36:13.287 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T19:36:14.117 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:16.524 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi144 2023-09-23T19:36:16.525 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:36:16.525 DEBUG:teuthology.orchestra.run.smithi144:> dd of=/etc/ceph/ceph.conf 2023-09-23T19:36:16.545 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:36:16.545 DEBUG:teuthology.orchestra.run.smithi144:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:36:16.605 INFO:tasks.cephadm:Adding host smithi144 to orchestrator... 2023-09-23T19:36:16.605 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi144 2023-09-23T19:36:16.946 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:17.148 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:17.148 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:17.148 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:17.148 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:17.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:17.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:36:17.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:17.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:17.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:17.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:16 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:18.200 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:17 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:36:18.200 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:17 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:36:20.378 INFO:teuthology.orchestra.run.smithi135.stdout:Added host 'smithi144' with addr '172.21.15.144' 2023-09-23T19:36:20.402 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:20 smithi135 ceph-mon[100251]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi144", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:20.402 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:20 smithi135 ceph-mon[100251]: Deploying cephadm binary to smithi144 2023-09-23T19:36:20.998 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T19:36:21.330 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:21.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:21.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:21 smithi135 ceph-mon[100251]: Added host smithi144 2023-09-23T19:36:21.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:21.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:21.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:21 smithi135 ceph-mon[100251]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:21.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:23.007 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:36:23.007 INFO:teuthology.orchestra.run.smithi135.stdout:[{"addr": "172.21.15.135", "hostname": "smithi135", "labels": [], "status": ""}, {"addr": "172.21.15.144", "hostname": "smithi144", "labels": [], "status": ""}] 2023-09-23T19:36:23.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:22 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:23.630 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T19:36:23.630 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T19:36:23.964 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:24.219 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:23 smithi135 ceph-mon[100251]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:24.219 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:23 smithi135 ceph-mon[100251]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:36:26.346 INFO:teuthology.orchestra.run.smithi135.stderr:adjusted tunables profile to default 2023-09-23T19:36:26.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:26.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:26.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:26.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:26.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:26.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi144", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:36:26.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:26.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:26.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:26 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2719823303' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T19:36:26.971 INFO:tasks.cephadm:Adding mon.a on smithi135 2023-09-23T19:36:26.971 INFO:tasks.cephadm:Adding mon.c on smithi135 2023-09-23T19:36:26.972 INFO:tasks.cephadm:Adding mon.b on smithi144 2023-09-23T19:36:26.972 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi135:172.21.15.135=a;smithi135:[v2:172.21.15.135:3301,v1:172.21.15.135:6790]=c;smithi144:172.21.15.144=b' 2023-09-23T19:36:27.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:36:27.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:27.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:36:27.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:36:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2719823303' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T19:36:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:27 smithi135 ceph-mon[100251]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:27.346 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:28 smithi135 ceph-mon[100251]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:28.901 INFO:teuthology.orchestra.run.smithi144.stdout:Scheduled mon update... 2023-09-23T19:36:29.550 DEBUG:teuthology.orchestra.run.smithi135:mon.c> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T19:36:29.552 DEBUG:teuthology.orchestra.run.smithi144:mon.b> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T19:36:29.555 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T19:36:29.556 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T19:36:29.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:36:29.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:36:30.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:30.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi135:172.21.15.135=a;smithi135:[v2:172.21.15.135:3301,v1:172.21.15.135:6790]=c;smithi144:172.21.15.144=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: Saving service mon spec with placement smithi135:172.21.15.135=a;smithi135:[v2:172.21.15.135:3301,v1:172.21.15.135:6790]=c;smithi144:172.21.15.144=b;count:3 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:29 smithi135 ceph-mon[100251]: Deploying daemon mon.b on smithi144 2023-09-23T19:36:30.565 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:32.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:31 smithi135 ceph-mon[100251]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:32.473 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 systemd[1]: Starting Ceph mon.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:36:32.946 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:36:32.946 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","modified":"2023-09-23T19:35:05.884970Z","created":"2023-09-23T19:35:05.884970Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:3300","nonce":0},{"type":"v1","addr":"172.21.15.135:6789","nonce":0}]},"addr":"172.21.15.135:6789/0","public_addr":"172.21.15.135:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T19:36:32.948 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2023-09-23T19:36:33.233 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: pidfile_write: ignore empty --pid-file 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: load: jerasure load: lrc 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: RocksDB version: 6.15.5 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Compile date Sep 22 2023 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: DB SUMMARY 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: DB Session ID: QHS2LRJO9KFLMTDM2YSE 2023-09-23T19:36:33.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: CURRENT file: CURRENT 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.error_if_exists: 0 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.create_if_missing: 0 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.paranoid_checks: 1 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:36:33.235 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.env: 0x563413cd5a80 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.fs: Posix File System 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.info_log: 0x56341495e5a0 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.statistics: (nil) 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.use_fsync: 0 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_log_file_size: 0 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:36:33.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:36:33.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.allow_fallocate: 1 2023-09-23T19:36:33.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:36:33.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:36:33.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.use_direct_reads: 0 2023-09-23T19:36:33.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:36:33.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:36:33.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.db_log_dir: 2023-09-23T19:36:33.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-09-23T19:36:33.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:36:33.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:36:33.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:36:33.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:36:33.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:36:33.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:36:33.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:36:33.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.write_buffer_manager: 0x5634156d0900 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:36:33.239 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.unordered_write: 0 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.row_cache: None 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.wal_filter: None 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:36:33.240 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.preserve_deletes: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.two_write_queues: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.atomic_flush: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.log_readahead_size: 0 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:36:33.241 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_background_jobs: 2 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_background_compactions: -1 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_subcompactions: 1 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:36:33.242 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_open_files: -1 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:36:33.243 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_background_flushes: -1 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Compression algorithms supported: 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: kZSTD supported: 0 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: kXpressCompression supported: 0 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: kZlibCompression supported: 1 2023-09-23T19:36:33.244 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: kSnappyCompression supported: 1 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: kLZ4Compression supported: 1 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: kBZip2Compression supported: 0 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.merge_operator: 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_filter: None 2023-09-23T19:36:33.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5634149b0fb8) 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout: cache_index_and_filter_blocks: 1 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout: pin_top_level_index_and_filter: 1 2023-09-23T19:36:33.246 INFO:journalctl@ceph.mon.b.smithi144.stdout: index_type: 0 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: data_block_index_type: 0 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: index_shortening: 1 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: hash_index_allow_collision: 1 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: checksum: 1 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: no_block_cache: 0 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_cache: 0x5634149e3090 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_cache_name: BinnedLRUCache 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_cache_options: 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: capacity : 536870912 2023-09-23T19:36:33.247 INFO:journalctl@ceph.mon.b.smithi144.stdout: num_shard_bits : 4 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: strict_capacity_limit : 0 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: high_pri_pool_ratio: 0.000 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_cache_compressed: (nil) 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: persistent_cache: (nil) 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_size: 4096 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_size_deviation: 10 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_restart_interval: 16 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: index_block_restart_interval: 1 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: metadata_block_size: 4096 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: partition_filters: 0 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: use_delta_encoding: 1 2023-09-23T19:36:33.248 INFO:journalctl@ceph.mon.b.smithi144.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout: whole_key_filtering: 1 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout: verify_compression: 0 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout: read_amp_bytes_per_bit: 0 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout: format_version: 4 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout: enable_index_compression: 1 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_align: 0 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compression: NoCompression 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:36:33.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.num_levels: 7 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:36:33.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:36:33.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:36:33.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:36:33.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.table_properties_collectors: 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.inplace_update_support: 0 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.bloom_locality: 0 2023-09-23T19:36:33.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.max_successive_merges: 0 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.ttl: 2592000 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.enable_blob_files: false 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.min_blob_size: 0 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:36:33.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497792934633, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497792934918, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695497792, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "e93b43bd-2451-4a01-afe5-c78e66c9762b", "db_session_id": "QHS2LRJO9KFLMTDM2YSE"}} 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497792935246, "job": 1, "event": "recovery_finished"} 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:36:33.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x563414977180 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: DB pointer 0x563414a4c000 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** DB Stats ** 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:36:33.257 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** Compaction Stats [default] ** 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** Compaction Stats [default] ** 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:33.258 INFO:journalctl@ceph.mon.b.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** Compaction Stats [default] ** 2023-09-23T19:36:33.259 INFO:journalctl@ceph.mon.b.smithi144.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** Compaction Stats [default] ** 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T19:36:33.260 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: using public_addr v2:172.21.15.144:0/0 -> [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: starting mon.b rank -1 at public addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] at bind addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:36:33.261 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(???) e0 preinit fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 bash[104909]: 40abc2a02dd4d4e43a1fd1407dd7d20476c09eaa27f89e70f31461c176e994d8 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 systemd[1]: Started Ceph mon.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).mds e1 new map 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).mds e1 print_map 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout: e1 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout: legacy client fscid: -1 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:36:33.262 INFO:journalctl@ceph.mon.b.smithi144.stdout: No filesystems configured 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:36:33.263 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2719823303' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi135:172.21.15.135=a;smithi135:[v2:172.21.15.135:3301,v1:172.21.15.135:6790]=c;smithi144:172.21.15.144=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:33.264 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: Saving service mon spec with placement smithi135:172.21.15.135=a;smithi135:[v2:172.21.15.135:3301,v1:172.21.15.135:6790]=c;smithi144:172.21.15.144=b;count:3 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: Deploying daemon mon.b on smithi144 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T19:36:33.265 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:32 smithi144 ceph-mon[104994]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-09-23T19:36:34.692 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T19:36:34.736 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T19:36:35.094 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: mon.a calling monitor election 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: mon.b calling monitor election 2023-09-23T19:36:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: monmap e2: 2 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0]} removed_ranks: {} 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: fsmap 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: mgrmap e13: y(active, since 37s) 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: overall HEALTH_OK 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:36:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: mon.a calling monitor election 2023-09-23T19:36:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: mon.b calling monitor election 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: monmap e2: 2 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0]} removed_ranks: {} 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: fsmap 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: mgrmap e13: y(active, since 37s) 2023-09-23T19:36:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: overall HEALTH_OK 2023-09-23T19:36:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:39.335 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:39 smithi135 ceph-mon[100251]: Deploying daemon mon.c on smithi135 2023-09-23T19:36:39.335 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:39 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:39 smithi144 ceph-mon[104994]: Deploying daemon mon.c on smithi135 2023-09-23T19:36:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:39 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:39.750 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:36:39.750 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":2,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","modified":"2023-09-23T19:36:32.963137Z","created":"2023-09-23T19:35:05.884970Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:3300","nonce":0},{"type":"v1","addr":"172.21.15.135:6789","nonce":0}]},"addr":"172.21.15.135:6789/0","public_addr":"172.21.15.135:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T19:36:39.752 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 2 2023-09-23T19:36:40.078 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:36:39 smithi135 conmon[100683]: 2023-09-23T19:36:39.961+0000 7f794f11f700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T19:36:40.340 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:40 smithi144 ceph-mon[104994]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:40.340 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:40 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/2476900316' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:36:40.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:40 smithi135 ceph-mon[100251]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:40.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:40 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/2476900316' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:36:40.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:40 smithi135 systemd[1]: Starting Ceph mon.c for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:36:41.358 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T19:36:41.359 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T19:36:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:36:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T19:36:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: pidfile_write: ignore empty --pid-file 2023-09-23T19:36:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: load: jerasure load: lrc 2023-09-23T19:36:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: RocksDB version: 6.15.5 2023-09-23T19:36:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T19:36:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Compile date Sep 22 2023 2023-09-23T19:36:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: DB SUMMARY 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: DB Session ID: X8376WMVWRD9C318SH52 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: CURRENT file: CURRENT 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.error_if_exists: 0 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.create_if_missing: 0 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.paranoid_checks: 1 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.env: 0x563870314a80 2023-09-23T19:36:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.fs: Posix File System 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.info_log: 0x5638715b05a0 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.statistics: (nil) 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.use_fsync: 0 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_log_file_size: 0 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.allow_fallocate: 1 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:36:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.use_direct_reads: 0 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.db_log_dir: 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:36:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.write_buffer_manager: 0x563872322930 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:36:41.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.unordered_write: 0 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.row_cache: None 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.wal_filter: None 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.preserve_deletes: 0 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.two_write_queues: 0 2023-09-23T19:36:41.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.atomic_flush: 0 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.log_readahead_size: 0 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:36:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_background_jobs: 2 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_background_compactions: -1 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_subcompactions: 1 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:36:41.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_open_files: -1 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_background_flushes: -1 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Compression algorithms supported: 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: kZSTD supported: 0 2023-09-23T19:36:41.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: kXpressCompression supported: 0 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: kZlibCompression supported: 1 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: kSnappyCompression supported: 1 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: kLZ4Compression supported: 1 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: kBZip2Compression supported: 0 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.merge_operator: 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_filter: None 2023-09-23T19:36:41.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563871602fb8) 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout: cache_index_and_filter_blocks: 1 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout: pin_top_level_index_and_filter: 1 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_type: 0 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout: data_block_index_type: 0 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_shortening: 1 2023-09-23T19:36:41.482 INFO:journalctl@ceph.mon.c.smithi135.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: hash_index_allow_collision: 1 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: checksum: 1 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: no_block_cache: 0 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache: 0x563871635090 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_name: BinnedLRUCache 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_options: 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: capacity : 536870912 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: num_shard_bits : 4 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: strict_capacity_limit : 0 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: high_pri_pool_ratio: 0.000 2023-09-23T19:36:41.483 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_compressed: (nil) 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: persistent_cache: (nil) 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_size: 4096 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_size_deviation: 10 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_restart_interval: 16 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_block_restart_interval: 1 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: metadata_block_size: 4096 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: partition_filters: 0 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: use_delta_encoding: 1 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: whole_key_filtering: 1 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: verify_compression: 0 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: read_amp_bytes_per_bit: 0 2023-09-23T19:36:41.484 INFO:journalctl@ceph.mon.c.smithi135.stdout: format_version: 4 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout: enable_index_compression: 1 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_align: 0 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compression: NoCompression 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.num_levels: 7 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:36:41.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:36:41.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:36:41.487 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T19:36:41.488 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:36:41.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.table_properties_collectors: 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.inplace_update_support: 0 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.bloom_locality: 0 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.max_successive_merges: 0 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:36:41.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.ttl: 2592000 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.enable_blob_files: false 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.min_blob_size: 0 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T19:36:41.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497801103280, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497801103634, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695497801, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5908b8e5-a4bc-450b-a85e-3b3781f3364d", "db_session_id": "X8376WMVWRD9C318SH52"}} 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695497801104059, "job": 1, "event": "recovery_finished"} 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5638715c9180 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: DB pointer 0x56387169e000 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T19:36:41.492 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** DB Stats ** 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:36:41.493 INFO:journalctl@ceph.mon.c.smithi135.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:36:41.495 INFO:journalctl@ceph.mon.c.smithi135.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:36:41.496 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative compaction: 0.00 GB write, 0.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: using public_addrv [v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0] 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: starting mon.c rank -1 at public addrs [v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0] at bind addrs [v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(???) e0 preinit fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 bash[118650]: 3ddce49c9396205eb730b554c987a1933596df3521539a0090a6d864bf609009 2023-09-23T19:36:41.497 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).mds e1 new map 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout: e1 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout: legacy client fscid: -1 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout: No filesystems configured 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T19:36:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:41.499 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2719823303' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi135:172.21.15.135=a;smithi135:[v2:172.21.15.135:3301,v1:172.21.15.135:6790]=c;smithi144:172.21.15.144=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: Saving service mon spec with placement smithi135:172.21.15.135=a;smithi135:[v2:172.21.15.135:3301,v1:172.21.15.135:6790]=c;smithi144:172.21.15.144=b;count:3 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:41.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: Deploying daemon mon.b on smithi144 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.a calling monitor election 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.b calling monitor election 2023-09-23T19:36:41.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: monmap e2: 2 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0]} removed_ranks: {} 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: fsmap 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mgrmap e13: y(active, since 37s) 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: overall HEALTH_OK 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:41.502 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: Deploying daemon mon.c on smithi135 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/2476900316' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 ceph-mon[118715]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-23T19:36:41.503 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:41 smithi135 systemd[1]: Started Ceph mon.c for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:36:41.716 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:36:46.577 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-09-23T19:36:46.577 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":3,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","modified":"2023-09-23T19:36:41.126061Z","created":"2023-09-23T19:35:05.884970Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:3300","nonce":0},{"type":"v1","addr":"172.21.15.135:6789","nonce":0}]},"addr":"172.21.15.135:6789/0","public_addr":"172.21.15.135:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:3301","nonce":0},{"type":"v1","addr":"172.21.15.135:6790","nonce":0}]},"addr":"172.21.15.135:6790/0","public_addr":"172.21.15.135:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-09-23T19:36:46.579 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 3 2023-09-23T19:36:46.599 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: mon.a calling monitor election 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: mon.b calling monitor election 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: mon.c calling monitor election 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: fsmap 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: mgrmap e13: y(active, since 45s) 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: overall HEALTH_OK 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:46.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:46.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: mon.a calling monitor election 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: mon.b calling monitor election 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: mon.c calling monitor election 2023-09-23T19:36:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: fsmap 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: mgrmap e13: y(active, since 45s) 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: overall HEALTH_OK 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: mon.a calling monitor election 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: mon.b calling monitor election 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: mon.c calling monitor election 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: fsmap 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: mgrmap e13: y(active, since 45s) 2023-09-23T19:36:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: overall HEALTH_OK 2023-09-23T19:36:46.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:46.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:47.161 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T19:36:47.162 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T19:36:47.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:47.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:47.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:47.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/1726403006' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:36:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[100251]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/1726403006' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:36:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[118715]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:47.528 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:47 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/1726403006' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:36:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:47 smithi144 ceph-mon[104994]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:36:48.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:48.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:48.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:48.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:36:48.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:48.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:48.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:48.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:49.612 INFO:teuthology.orchestra.run.smithi135.stdout:# minimal ceph.conf for 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:36:49.612 INFO:teuthology.orchestra.run.smithi135.stdout:[global] 2023-09-23T19:36:49.612 INFO:teuthology.orchestra.run.smithi135.stdout: fsid = 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:36:49.612 INFO:teuthology.orchestra.run.smithi135.stdout: mon_host = [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] [v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0] 2023-09-23T19:36:50.291 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:50 smithi135 ceph-mon[118715]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T19:36:50.292 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:50 smithi135 ceph-mon[118715]: Reconfiguring daemon mon.a on smithi135 2023-09-23T19:36:50.292 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:50 smithi135 ceph-mon[118715]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:50.292 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:50 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2276161971' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:50.292 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:50 smithi135 ceph-mon[100251]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T19:36:50.292 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:50 smithi135 ceph-mon[100251]: Reconfiguring daemon mon.a on smithi135 2023-09-23T19:36:50.292 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:50 smithi135 ceph-mon[100251]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:50.292 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:50 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2276161971' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:50 smithi144 ceph-mon[104994]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T19:36:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:50 smithi144 ceph-mon[104994]: Reconfiguring daemon mon.a on smithi135 2023-09-23T19:36:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:50 smithi144 ceph-mon[104994]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:50 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2276161971' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:50.511 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T19:36:50.511 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:36:50.511 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T19:36:50.540 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:36:50.540 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:36:50.607 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:36:50.607 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T19:36:50.643 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:36:50.643 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:36:50.713 INFO:tasks.cephadm:Adding mgr.y on smithi135 2023-09-23T19:36:50.714 INFO:tasks.cephadm:Adding mgr.x on smithi144 2023-09-23T19:36:50.714 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi135=y;smithi144=x' 2023-09-23T19:36:51.081 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:36:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: Reconfiguring mon.c (monmap changed)... 2023-09-23T19:36:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: Reconfiguring daemon mon.c on smithi135 2023-09-23T19:36:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:51.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:51.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: Reconfiguring mon.c (monmap changed)... 2023-09-23T19:36:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: Reconfiguring daemon mon.c on smithi135 2023-09-23T19:36:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:51.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:51.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: Reconfiguring mon.c (monmap changed)... 2023-09-23T19:36:51.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:51.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:51.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:51.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: Reconfiguring daemon mon.c on smithi135 2023-09-23T19:36:51.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:51.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:52.738 INFO:teuthology.orchestra.run.smithi144.stdout:Scheduled mgr update... 2023-09-23T19:36:53.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:52 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:52 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:52 smithi144 ceph-mon[104994]: Reconfiguring mon.b (monmap changed)... 2023-09-23T19:36:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:52 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:52 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:52 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:52 smithi144 ceph-mon[104994]: Reconfiguring daemon mon.b on smithi144 2023-09-23T19:36:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:52 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[100251]: Reconfiguring mon.b (monmap changed)... 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[100251]: Reconfiguring daemon mon.b on smithi144 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[118715]: Reconfiguring mon.b (monmap changed)... 2023-09-23T19:36:53.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:36:53.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:36:53.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:53.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[118715]: Reconfiguring daemon mon.b on smithi144 2023-09-23T19:36:53.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:52 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:53.605 DEBUG:teuthology.orchestra.run.smithi144:mgr.x> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T19:36:53.606 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T19:36:53.607 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:36:53.607 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T19:36:53.627 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T19:36:53.628 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_1 2023-09-23T19:36:53.635 INFO:journalctl@ceph.mgr.x.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:36:53.687 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T19:36:53.688 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T19:36:53.688 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 6h/6d Inode: 391062 Links: 1 2023-09-23T19:36:53.688 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T19:36:53.688 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T19:36:53.688 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2023-09-23 19:36:40.491328135 +0000 2023-09-23T19:36:53.688 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2023-09-23 19:28:13.273803181 +0000 2023-09-23T19:36:53.688 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2023-09-23 19:28:13.273803181 +0000 2023-09-23T19:36:53.688 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: - 2023-09-23T19:36:53.689 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T19:36:53.758 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2023-09-23T19:36:53.758 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2023-09-23T19:36:53.758 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000234525 s, 2.2 MB/s 2023-09-23T19:36:53.759 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T19:36:53.818 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_2 2023-09-23T19:36:53.876 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T19:36:53.876 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T19:36:53.876 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 6h/6d Inode: 391366 Links: 1 2023-09-23T19:36:53.876 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T19:36:53.876 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T19:36:53.876 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2023-09-23 19:36:40.491328135 +0000 2023-09-23T19:36:53.876 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2023-09-23 19:28:13.791791964 +0000 2023-09-23T19:36:53.876 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2023-09-23 19:28:13.791791964 +0000 2023-09-23T19:36:53.877 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: - 2023-09-23T19:36:53.877 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T19:36:53.942 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2023-09-23T19:36:53.942 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2023-09-23T19:36:53.942 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000231425 s, 2.2 MB/s 2023-09-23T19:36:53.943 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T19:36:54.001 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_3 2023-09-23T19:36:54.058 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T19:36:54.058 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T19:36:54.059 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 6h/6d Inode: 386810 Links: 1 2023-09-23T19:36:54.059 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T19:36:54.059 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T19:36:54.059 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2023-09-23 19:36:40.491328135 +0000 2023-09-23T19:36:54.059 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2023-09-23 19:28:14.283781311 +0000 2023-09-23T19:36:54.059 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2023-09-23 19:28:14.283781311 +0000 2023-09-23T19:36:54.059 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: - 2023-09-23T19:36:54.059 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T19:36:54.122 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi135=y;smithi144=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:54.122 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: Saving service mgr spec with placement smithi135=y;smithi144=x;count:2 2023-09-23T19:36:54.122 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:36:54.123 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi135=y;smithi144=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: Saving service mgr spec with placement smithi135=y;smithi144=x;count:2 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:36:54.124 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T19:36:54.125 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:36:54.125 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:54.126 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2023-09-23T19:36:54.126 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2023-09-23T19:36:54.126 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000219454 s, 2.3 MB/s 2023-09-23T19:36:54.126 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T19:36:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi135=y;smithi144=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:36:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: Saving service mgr spec with placement smithi135=y;smithi144=x;count:2 2023-09-23T19:36:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:36:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T19:36:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:36:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:54.184 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_4 2023-09-23T19:36:54.241 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T19:36:54.241 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T19:36:54.241 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 6h/6d Inode: 387696 Links: 1 2023-09-23T19:36:54.241 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T19:36:54.241 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T19:36:54.241 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2023-09-23 19:36:40.491328135 +0000 2023-09-23T19:36:54.241 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2023-09-23 19:28:14.772770722 +0000 2023-09-23T19:36:54.242 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2023-09-23 19:28:14.772770722 +0000 2023-09-23T19:36:54.242 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: - 2023-09-23T19:36:54.242 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T19:36:54.306 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2023-09-23T19:36:54.306 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2023-09-23T19:36:54.307 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000191255 s, 2.7 MB/s 2023-09-23T19:36:54.307 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T19:36:54.365 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:36:54.365 DEBUG:teuthology.orchestra.run.smithi144:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T19:36:54.383 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T19:36:54.383 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_1 2023-09-23T19:36:54.440 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T19:36:54.440 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T19:36:54.440 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 6h/6d Inode: 389557 Links: 1 2023-09-23T19:36:54.440 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T19:36:54.440 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T19:36:54.440 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2023-09-23 19:36:32.188396009 +0000 2023-09-23T19:36:54.440 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2023-09-23 19:28:13.267761072 +0000 2023-09-23T19:36:54.440 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2023-09-23 19:28:13.267761072 +0000 2023-09-23T19:36:54.441 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: - 2023-09-23T19:36:54.441 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T19:36:54.508 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2023-09-23T19:36:54.508 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2023-09-23T19:36:54.508 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000246592 s, 2.1 MB/s 2023-09-23T19:36:54.509 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T19:36:54.566 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_2 2023-09-23T19:36:54.624 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T19:36:54.624 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T19:36:54.624 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 6h/6d Inode: 386886 Links: 1 2023-09-23T19:36:54.624 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T19:36:54.624 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T19:36:54.624 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2023-09-23 19:36:32.188396009 +0000 2023-09-23T19:36:54.624 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2023-09-23 19:28:13.920749394 +0000 2023-09-23T19:36:54.625 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2023-09-23 19:28:13.920749394 +0000 2023-09-23T19:36:54.625 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: - 2023-09-23T19:36:54.625 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T19:36:54.689 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2023-09-23T19:36:54.689 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2023-09-23T19:36:54.689 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000213538 s, 2.4 MB/s 2023-09-23T19:36:54.690 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T19:36:54.747 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_3 2023-09-23T19:36:54.803 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T19:36:54.804 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T19:36:54.804 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 6h/6d Inode: 391152 Links: 1 2023-09-23T19:36:54.804 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T19:36:54.804 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T19:36:54.804 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2023-09-23 19:36:32.188396009 +0000 2023-09-23T19:36:54.804 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2023-09-23 19:28:14.428740309 +0000 2023-09-23T19:36:54.804 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2023-09-23 19:28:14.428740309 +0000 2023-09-23T19:36:54.804 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: - 2023-09-23T19:36:54.805 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T19:36:54.869 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2023-09-23T19:36:54.869 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2023-09-23T19:36:54.870 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000231289 s, 2.2 MB/s 2023-09-23T19:36:54.870 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T19:36:54.929 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_4 2023-09-23T19:36:54.996 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T19:36:54.997 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T19:36:54.997 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 6h/6d Inode: 381779 Links: 1 2023-09-23T19:36:54.997 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T19:36:54.997 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T19:36:54.997 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2023-09-23 19:36:32.188396009 +0000 2023-09-23T19:36:54.997 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2023-09-23 19:28:14.926731403 +0000 2023-09-23T19:36:54.997 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2023-09-23 19:28:14.926731403 +0000 2023-09-23T19:36:54.997 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: - 2023-09-23T19:36:54.997 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T19:36:55.070 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2023-09-23T19:36:55.070 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2023-09-23T19:36:55.070 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000237219 s, 2.2 MB/s 2023-09-23T19:36:55.071 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T19:36:55.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:55 smithi144 ceph-mon[104994]: Deploying daemon mgr.x on smithi144 2023-09-23T19:36:55.137 INFO:tasks.cephadm:Deploying osd.0 on smithi135 with /dev/vg_nvme/lv_4... 2023-09-23T19:36:55.137 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T19:36:55.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:55 smithi135 ceph-mon[100251]: Deploying daemon mgr.x on smithi144 2023-09-23T19:36:55.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:55 smithi135 ceph-mon[118715]: Deploying daemon mgr.x on smithi144 2023-09-23T19:36:55.479 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:55.632 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:55 smithi144 systemd[1]: Starting Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:36:56.046 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:55 smithi144 ceph-mgr[108251]: -- 172.21.15.144:0/915777039 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e15a1009c0 con 0x55e15a0d9000 2023-09-23T19:36:56.046 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:55 smithi144 bash[108185]: aadb0eba9c477537f1958b9164d1d17d422a8720f2cc7476af444c7d509d6fe9 2023-09-23T19:36:56.046 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:55 smithi144 systemd[1]: Started Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:36:56.046 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:55 smithi144 conmon[108226]: 2023-09-23T19:36:55.897+0000 7f48d4b0e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:36:56.046 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:55 smithi144 conmon[108226]: 2023-09-23T19:36:55.976+0000 7f48d4b0e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:36:56.357 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:56 smithi144 conmon[108226]: 2023-09-23T19:36:56.071+0000 7f48d4b0e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:36:56.357 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:56 smithi144 conmon[108226]: 2023-09-23T19:36:56.168+0000 7f48d4b0e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:36:56.357 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:56 smithi144 conmon[108226]: 2023-09-23T19:36:56.355+0000 7f48d4b0e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:36:56.357 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:56 smithi144 ceph-mon[104994]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:56.357 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:56 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.357 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:56 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.357 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:56 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.358 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:56 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.358 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:56 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:56.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[118715]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:56.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:56.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[100251]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:56.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:56.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:56 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:36:57.028 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:56 smithi144 conmon[108226]: 2023-09-23T19:36:56.703+0000 7f48d4b0e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:36:57.028 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:56 smithi144 conmon[108226]: 2023-09-23T19:36:56.794+0000 7f48d4b0e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:36:57.028 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:56 smithi144 conmon[108226]: 2023-09-23T19:36:56.927+0000 7f48d4b0e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:36:57.210 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T19:36:57.210 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T19:36:57.210 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2023-09-23T19:36:57.210 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2023-09-23T19:36:57.210 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125648 s, 835 MB/s 2023-09-23T19:36:57.210 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2023-09-23T19:36:57.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:57 smithi144 conmon[108226]: 2023-09-23T19:36:57.027+0000 7f48d4b0e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:36:57.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:57 smithi144 conmon[108226]: 2023-09-23T19:36:57.444+0000 7f48d4b0e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:36:57.932 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi135:vg_nvme/lv_4 2023-09-23T19:36:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[100251]: Reconfiguring daemon mgr.y on smithi135 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:58.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:58.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T19:36:58.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:36:58.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:36:58.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:58.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:36:57 smithi135 ceph-mon[118715]: Reconfiguring daemon mgr.y on smithi135 2023-09-23T19:36:58.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:58 smithi144 conmon[108226]: 2023-09-23T19:36:58.006+0000 7f48d4b0e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:36:58.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:58 smithi144 conmon[108226]: 2023-09-23T19:36:58.174+0000 7f48d4b0e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:36:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:36:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:36:57 smithi144 ceph-mon[104994]: Reconfiguring daemon mgr.y on smithi135 2023-09-23T19:36:58.555 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:36:59.184 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:58 smithi144 conmon[108226]: 2023-09-23T19:36:58.927+0000 7f48d4b0e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:36:59.185 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:59 smithi144 conmon[108226]: 2023-09-23T19:36:59.016+0000 7f48d4b0e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:36:59.459 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:59 smithi144 conmon[108226]: 2023-09-23T19:36:59.183+0000 7f48d4b0e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:36:59.839 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:59 smithi144 conmon[108226]: 2023-09-23T19:36:59.457+0000 7f48d4b0e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:36:59.839 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:59 smithi144 conmon[108226]: 2023-09-23T19:36:59.541+0000 7f48d4b0e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:37:00.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:36:59 smithi144 conmon[108226]: 2023-09-23T19:36:59.837+0000 7f48d4b0e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:37:00.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:37:00 smithi144 conmon[108226]: 2023-09-23T19:37:00.087+0000 7f48d4b0e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:37:00.268 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[100251]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:00.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:00.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:00.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:00.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:00.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:00.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:00.269 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[118715]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:00.269 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:00.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:00.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:00.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:00.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:00.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:00 smithi144 ceph-mon[104994]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:01.095 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:37:00 smithi144 conmon[108226]: 2023-09-23T19:37:00.781+0000 7f48d4b0e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:37:01.095 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:37:00 smithi144 conmon[108226]: 2023-09-23T19:37:00.910+0000 7f48d4b0e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:37:01.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:37:01 smithi144 conmon[108226]: 2023-09-23T19:37:01.093+0000 7f48d4b0e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:37:01.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:37:01 smithi144 conmon[108226]: 2023-09-23T19:37:01.205+0000 7f48d4b0e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:37:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:01.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:01.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:01.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:01.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:37:01 smithi144 conmon[108226]: 2023-09-23T19:37:01.416+0000 7f48d4b0e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:37:02.301 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:02 smithi135 ceph-mon[100251]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:02.301 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:02 smithi135 ceph-mon[100251]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:02.301 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:02 smithi135 ceph-mon[118715]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:02.302 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:02 smithi135 ceph-mon[118715]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:02 smithi144 ceph-mon[104994]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:02 smithi144 ceph-mon[104994]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:02.632 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:37:02 smithi144 conmon[108226]: 2023-09-23T19:37:02.283+0000 7f48d4b0e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:37:03.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: Standby manager daemon x started 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2325433169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]: dispatch 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]: dispatch 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]': finished 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T19:37:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: Standby manager daemon x started 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2325433169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]: dispatch 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]: dispatch 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]': finished 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T19:37:03.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:03 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: Standby manager daemon x started 2023-09-23T19:37:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:37:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:37:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:37:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/1791821485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:37:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2325433169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]: dispatch 2023-09-23T19:37:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]: dispatch 2023-09-23T19:37:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "369a8e37-43f6-4c41-a626-2b159656f305"}]': finished 2023-09-23T19:37:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T19:37:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:03 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:04.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:04 smithi135 ceph-mon[100251]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:04.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:04 smithi135 ceph-mon[100251]: mgrmap e14: y(active, since 62s), standbys: x 2023-09-23T19:37:04.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:37:04.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:04 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/917464513' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:04 smithi135 ceph-mon[118715]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:04 smithi135 ceph-mon[118715]: mgrmap e14: y(active, since 62s), standbys: x 2023-09-23T19:37:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:37:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:04 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/917464513' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:04 smithi144 ceph-mon[104994]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:04 smithi144 ceph-mon[104994]: mgrmap e14: y(active, since 62s), standbys: x 2023-09-23T19:37:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:37:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:04 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/917464513' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:06 smithi144 ceph-mon[104994]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:06.697 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:06 smithi135 ceph-mon[118715]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:06.698 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:06 smithi135 ceph-mon[100251]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:08.395 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:08 smithi135 ceph-mon[100251]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:08.396 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:08 smithi135 ceph-mon[118715]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:08 smithi144 ceph-mon[104994]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:10.449 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:10 smithi135 ceph-mon[100251]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:10.449 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:37:10.449 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:10.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:10 smithi135 ceph-mon[118715]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:10.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:37:10.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:10 smithi144 ceph-mon[104994]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:37:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:11.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:11 smithi135 ceph-mon[100251]: Deploying daemon osd.0 on smithi135 2023-09-23T19:37:11.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:11 smithi135 ceph-mon[118715]: Deploying daemon osd.0 on smithi135 2023-09-23T19:37:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:11 smithi144 ceph-mon[104994]: Deploying daemon osd.0 on smithi135 2023-09-23T19:37:12.450 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:12 smithi135 ceph-mon[100251]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:12.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:12 smithi135 ceph-mon[118715]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:12 smithi144 ceph-mon[104994]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:14 smithi144 ceph-mon[104994]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:14.633 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:14 smithi135 ceph-mon[100251]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:14.634 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:14 smithi135 ceph-mon[118715]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:15 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:15 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:15 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:15.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:15 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:15.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:15 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:15.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:15 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:15.687 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:15 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:15.687 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:15 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:15.687 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:15 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:16 smithi135 ceph-mon[118715]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:16 smithi135 ceph-mon[100251]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:16 smithi144 ceph-mon[104994]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:17.956 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:17.956 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:17.956 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[100251]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[118715]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:17.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:17 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:18.020 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 0 on host 'smithi135' 2023-09-23T19:37:18.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:17 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:18.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:17 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:18.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:17 smithi144 ceph-mon[104994]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:18.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:17 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:18.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:17 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:18.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:17 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:18.696 DEBUG:teuthology.orchestra.run.smithi135:osd.0> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T19:37:18.698 INFO:tasks.cephadm:Deploying osd.1 on smithi135 with /dev/vg_nvme/lv_3... 2023-09-23T19:37:18.698 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T19:37:18.727 INFO:journalctl@ceph.osd.0.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:37:18.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[100251]: from='osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:37:18.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[100251]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:37:18.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:18.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:18.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:18.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[118715]: from='osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:37:18.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[118715]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:37:18.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:18.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:18.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:18 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:19.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:18 smithi144 ceph-mon[104994]: from='osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:37:19.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:18 smithi144 ceph-mon[104994]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:37:19.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:18 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:19.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:18 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:19.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:18 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:19.767 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:19.970 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:37:19 smithi135 conmon[124960]: 2023-09-23T19:37:19.704+0000 7f3760084700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T19:37:19.970 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:37:19 smithi135 conmon[124960]: 2023-09-23T19:37:19.706+0000 7f375b680700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:37:19.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[100251]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:37:19.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[100251]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T19:37:19.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:19.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[100251]: from='osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:19.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[100251]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:19.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[100251]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:19.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[118715]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:37:19.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[118715]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T19:37:19.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:19.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[118715]: from='osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:19.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[118715]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:19.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:19 smithi135 ceph-mon[118715]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:19 smithi144 ceph-mon[104994]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:37:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:19 smithi144 ceph-mon[104994]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T19:37:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:20.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:19 smithi144 ceph-mon[104994]: from='osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:20.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:19 smithi144 ceph-mon[104994]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:20.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:19 smithi144 ceph-mon[104994]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:20.707 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:20 smithi135 ceph-mon[100251]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:37:20.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:20 smithi135 ceph-mon[100251]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T19:37:20.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:20 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:20.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:20 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:20.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:20 smithi135 ceph-mon[118715]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:37:20.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:20 smithi135 ceph-mon[118715]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T19:37:20.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:20 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:20.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:20 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:21.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:20 smithi144 ceph-mon[104994]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:37:21.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:20 smithi144 ceph-mon[104994]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T19:37:21.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:20 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:21.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:20 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:21.742 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T19:37:21.743 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T19:37:21.743 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2023-09-23T19:37:21.743 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2023-09-23T19:37:21.743 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0128236 s, 818 MB/s 2023-09-23T19:37:21.743 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2023-09-23T19:37:21.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[118715]: purged_snaps scrub starts 2023-09-23T19:37:21.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[118715]: purged_snaps scrub ok 2023-09-23T19:37:21.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[118715]: osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238] boot 2023-09-23T19:37:21.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[118715]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T19:37:21.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:21.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[118715]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:21.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[100251]: purged_snaps scrub starts 2023-09-23T19:37:21.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[100251]: purged_snaps scrub ok 2023-09-23T19:37:21.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[100251]: osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238] boot 2023-09-23T19:37:21.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[100251]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T19:37:21.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:21.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:21 smithi135 ceph-mon[100251]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:22.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:21 smithi144 ceph-mon[104994]: purged_snaps scrub starts 2023-09-23T19:37:22.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:21 smithi144 ceph-mon[104994]: purged_snaps scrub ok 2023-09-23T19:37:22.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:21 smithi144 ceph-mon[104994]: osd.0 [v2:172.21.15.135:6802/1288355238,v1:172.21.15.135:6803/1288355238] boot 2023-09-23T19:37:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:21 smithi144 ceph-mon[104994]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T19:37:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:21 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:37:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:21 smithi144 ceph-mon[104994]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T19:37:22.832 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:22 smithi135 ceph-mon[118715]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T19:37:22.832 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:22 smithi135 ceph-mon[100251]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T19:37:22.834 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi135:vg_nvme/lv_3 2023-09-23T19:37:23.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:22 smithi144 ceph-mon[104994]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T19:37:23.197 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:24.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:24.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: Detected new or changed devices on smithi135 2023-09-23T19:37:24.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:24.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi135 to 16144M 2023-09-23T19:37:24.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:24.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:24.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:24.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: Detected new or changed devices on smithi135 2023-09-23T19:37:24.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:24.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi135 to 16144M 2023-09-23T19:37:24.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:24.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:24.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:23 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: Detected new or changed devices on smithi135 2023-09-23T19:37:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:24.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi135 to 16144M 2023-09-23T19:37:24.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:24.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:24.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:24.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:23 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:25.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:25 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:25.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:25 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:25 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:25.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:25 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:25.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:25 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:25.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:25 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:25.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:25 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:25.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:25 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:25.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:25 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:26.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:26 smithi135 ceph-mon[118715]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:26.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:26 smithi135 ceph-mon[118715]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:26.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:26 smithi135 ceph-mon[100251]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:26.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:26 smithi135 ceph-mon[100251]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:26 smithi144 ceph-mon[104994]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:26 smithi144 ceph-mon[104994]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:28 smithi144 ceph-mon[104994]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:28 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1795710959' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]: dispatch 2023-09-23T19:37:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:28 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]: dispatch 2023-09-23T19:37:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:28 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]': finished 2023-09-23T19:37:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:28 smithi144 ceph-mon[104994]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T19:37:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:28 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:28 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2260500029' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[100251]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1795710959' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]: dispatch 2023-09-23T19:37:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]: dispatch 2023-09-23T19:37:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]': finished 2023-09-23T19:37:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[100251]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T19:37:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2260500029' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[118715]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1795710959' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]: dispatch 2023-09-23T19:37:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]: dispatch 2023-09-23T19:37:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ccf1949a-3059-45d2-a46a-07dfbbc68fb5"}]': finished 2023-09-23T19:37:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[118715]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T19:37:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:28 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2260500029' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:30 smithi144 ceph-mon[104994]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:30.440 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:30 smithi135 ceph-mon[100251]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:30.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:30 smithi135 ceph-mon[118715]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:32.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:32 smithi135 ceph-mon[100251]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:32.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:32 smithi135 ceph-mon[118715]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:32.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:32 smithi144 ceph-mon[104994]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:34.361 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:34 smithi135 ceph-mon[100251]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:34.361 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:34 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:37:34.361 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:34 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:34.361 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:34 smithi135 ceph-mon[118715]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:34.362 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:34 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:37:34.362 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:34 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:34.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:34 smithi144 ceph-mon[104994]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:34.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:34 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:37:34.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:34 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:35 smithi144 ceph-mon[104994]: Deploying daemon osd.1 on smithi135 2023-09-23T19:37:35.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:35 smithi135 ceph-mon[100251]: Deploying daemon osd.1 on smithi135 2023-09-23T19:37:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:35 smithi135 ceph-mon[118715]: Deploying daemon osd.1 on smithi135 2023-09-23T19:37:36.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:36 smithi135 ceph-mon[100251]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:36.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:36 smithi135 ceph-mon[118715]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:36 smithi144 ceph-mon[104994]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:38 smithi144 ceph-mon[104994]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:38 smithi135 ceph-mon[100251]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:38 smithi135 ceph-mon[118715]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:40.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:39 smithi135 ceph-mon[100251]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:40.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:39 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:40.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:39 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:40.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:39 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:39 smithi135 ceph-mon[118715]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:39 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:39 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:39 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:39 smithi144 ceph-mon[104994]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:39 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:39 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:39 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:42.059 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 1 on host 'smithi135' 2023-09-23T19:37:42.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[118715]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:42.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:42.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:42.270 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.270 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[100251]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:42.270 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:42.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:42.271 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:42 smithi144 ceph-mon[104994]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:42.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:42.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:42.700 DEBUG:teuthology.orchestra.run.smithi135:osd.1> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T19:37:42.702 INFO:tasks.cephadm:Deploying osd.2 on smithi135 with /dev/vg_nvme/lv_2... 2023-09-23T19:37:42.703 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T19:37:42.731 INFO:journalctl@ceph.osd.1.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:37:43.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:43 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:43.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:43 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:43.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:43 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:43.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:43 smithi144 ceph-mon[104994]: from='osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:37:43.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:43 smithi144 ceph-mon[104994]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:37:43.447 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:43.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:43.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:43.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:43.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[100251]: from='osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:37:43.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[100251]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:37:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:37:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[118715]: from='osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:37:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:43 smithi135 ceph-mon[118715]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:37:44.378 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:37:44 smithi135 conmon[134619]: 2023-09-23T19:37:44.032+0000 7f89d0499700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T19:37:44.378 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:37:44 smithi135 conmon[134619]: 2023-09-23T19:37:44.035+0000 7f89c9a91700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:37:44.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[100251]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:44.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[100251]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:37:44.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[100251]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T19:37:44.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:44.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[100251]: from='osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:44.379 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[100251]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:44.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[118715]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:44.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[118715]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:37:44.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[118715]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T19:37:44.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:44.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[118715]: from='osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:44.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:44 smithi135 ceph-mon[118715]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:44 smithi144 ceph-mon[104994]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:44 smithi144 ceph-mon[104994]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:37:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:44 smithi144 ceph-mon[104994]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T19:37:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:44 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:44 smithi144 ceph-mon[104994]: from='osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:44 smithi144 ceph-mon[104994]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:37:45.196 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:45 smithi135 ceph-mon[118715]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:37:45.196 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:45 smithi135 ceph-mon[118715]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T19:37:45.196 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:45 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:45.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:45 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:45.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:45 smithi135 ceph-mon[100251]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:37:45.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:45 smithi135 ceph-mon[100251]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T19:37:45.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:45 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:45.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:45 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:45 smithi144 ceph-mon[104994]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:37:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:45 smithi144 ceph-mon[104994]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T19:37:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:45 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:45 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:45.678 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T19:37:45.678 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T19:37:45.678 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2023-09-23T19:37:45.678 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2023-09-23T19:37:45.678 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0115127 s, 911 MB/s 2023-09-23T19:37:45.678 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2023-09-23T19:37:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:46 smithi144 ceph-mon[104994]: purged_snaps scrub starts 2023-09-23T19:37:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:46 smithi144 ceph-mon[104994]: purged_snaps scrub ok 2023-09-23T19:37:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:46 smithi144 ceph-mon[104994]: pgmap v54: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:46 smithi144 ceph-mon[104994]: osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379] boot 2023-09-23T19:37:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:46 smithi144 ceph-mon[104994]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T19:37:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:46.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[118715]: purged_snaps scrub starts 2023-09-23T19:37:46.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[118715]: purged_snaps scrub ok 2023-09-23T19:37:46.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[118715]: pgmap v54: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:46.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:46.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[118715]: osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379] boot 2023-09-23T19:37:46.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[118715]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T19:37:46.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:46.419 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[100251]: purged_snaps scrub starts 2023-09-23T19:37:46.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[100251]: purged_snaps scrub ok 2023-09-23T19:37:46.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[100251]: pgmap v54: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T19:37:46.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:46.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[100251]: osd.1 [v2:172.21.15.135:6810/460054379,v1:172.21.15.135:6811/460054379] boot 2023-09-23T19:37:46.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[100251]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T19:37:46.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:37:46.775 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi135:vg_nvme/lv_2 2023-09-23T19:37:47.059 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T19:37:47.059 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.059 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.060 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:47.060 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:47.060 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.060 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:47.060 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:47.060 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.127 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:37:47.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T19:37:47.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:47.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:47.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:47.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:47.311 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:47 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T19:37:47.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:47.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:37:47.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:47.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:47.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:37:47.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:47 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:37:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:48 smithi144 ceph-mon[104994]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:48 smithi144 ceph-mon[104994]: Detected new or changed devices on smithi135 2023-09-23T19:37:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:48 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi135 to 8072M 2023-09-23T19:37:48.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:48 smithi135 ceph-mon[118715]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:48.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:48 smithi135 ceph-mon[118715]: Detected new or changed devices on smithi135 2023-09-23T19:37:48.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:48 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi135 to 8072M 2023-09-23T19:37:48.395 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:48 smithi135 ceph-mon[100251]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:48.395 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:48 smithi135 ceph-mon[100251]: Detected new or changed devices on smithi135 2023-09-23T19:37:48.395 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:48 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi135 to 8072M 2023-09-23T19:37:49.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:49 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:49.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:49 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:49.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:49 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:49.343 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:49 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:49.343 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:49 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:49.343 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:49 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:49.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:49 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:37:49.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:49 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:37:49.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:49 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:50 smithi144 ceph-mon[104994]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:50 smithi144 ceph-mon[104994]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:50.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:50 smithi135 ceph-mon[118715]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:50.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:50 smithi135 ceph-mon[118715]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:50.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:50 smithi135 ceph-mon[100251]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:37:50.411 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:50 smithi135 ceph-mon[100251]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:52 smithi144 ceph-mon[104994]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:52 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3259364114' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]: dispatch 2023-09-23T19:37:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:52 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]: dispatch 2023-09-23T19:37:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:52 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]': finished 2023-09-23T19:37:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:52 smithi144 ceph-mon[104994]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T19:37:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:52 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:37:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:52 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2336992407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[100251]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3259364114' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]: dispatch 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]: dispatch 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]': finished 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[100251]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2336992407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[118715]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3259364114' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]: dispatch 2023-09-23T19:37:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]: dispatch 2023-09-23T19:37:52.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2a8e9267-6c97-4c1f-b56b-7ca1844a1334"}]': finished 2023-09-23T19:37:52.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[118715]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T19:37:52.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:37:52.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:52 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2336992407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:37:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:54 smithi144 ceph-mon[104994]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:54.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:54 smithi135 ceph-mon[100251]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:54.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:54 smithi135 ceph-mon[118715]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:56.260 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:56 smithi135 ceph-mon[118715]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:56.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:56 smithi135 ceph-mon[100251]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:56 smithi144 ceph-mon[104994]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:58 smithi144 ceph-mon[104994]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:58 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:37:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:58 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:37:58 smithi144 ceph-mon[104994]: Deploying daemon osd.2 on smithi135 2023-09-23T19:37:58.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:58 smithi135 ceph-mon[118715]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:58.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:58 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:37:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:58 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:37:58 smithi135 ceph-mon[118715]: Deploying daemon osd.2 on smithi135 2023-09-23T19:37:58.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:58 smithi135 ceph-mon[100251]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:37:58.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:58 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:37:58.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:58 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:37:58.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:37:58 smithi135 ceph-mon[100251]: Deploying daemon osd.2 on smithi135 2023-09-23T19:38:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:00 smithi144 ceph-mon[104994]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:00.408 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:00 smithi135 ceph-mon[100251]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:00.408 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:00 smithi135 ceph-mon[118715]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:02 smithi135 ceph-mon[100251]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:02 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:02.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:02 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:02 smithi135 ceph-mon[118715]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:02 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:02 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:02 smithi144 ceph-mon[104994]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:02 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:02 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:03.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:03 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:03 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:03 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:04.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[100251]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:04.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:04.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:04.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:04.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:04.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[118715]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:04.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:04.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:04.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:04.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:04 smithi144 ceph-mon[104994]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:04.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:04.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:04.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:05.085 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 2 on host 'smithi135' 2023-09-23T19:38:05.274 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:05 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:05 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:05 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:05 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:05.275 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:05 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.275 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:05 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.275 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:05 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.275 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:05 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:05.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:05 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:05 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:05 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:05.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:05 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:05.687 DEBUG:teuthology.orchestra.run.smithi135:osd.2> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T19:38:05.689 INFO:tasks.cephadm:Deploying osd.3 on smithi135 with /dev/vg_nvme/lv_1... 2023-09-23T19:38:05.689 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T19:38:05.718 INFO:journalctl@ceph.osd.2.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:38:06.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:06 smithi135 ceph-mon[100251]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:06.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:06 smithi135 ceph-mon[100251]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:06.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:06 smithi135 ceph-mon[118715]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:06.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:06 smithi135 ceph-mon[118715]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:06 smithi144 ceph-mon[104994]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:06 smithi144 ceph-mon[104994]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:38:06.712 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:07.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[100251]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:38:07.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[100251]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T19:38:07.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:07.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[100251]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:38:07.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[100251]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:38:07.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[100251]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T19:38:07.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:07.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:07.221 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:07 smithi135 conmon[143550]: 2023-09-23T19:38:07.104+0000 7f0194598700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T19:38:07.222 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:07 smithi135 conmon[143550]: 2023-09-23T19:38:07.110+0000 7f018f393700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:38:07.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[118715]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:38:07.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[118715]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T19:38:07.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:07.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[118715]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:38:07.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[118715]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:38:07.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[118715]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T19:38:07.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:07.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:07 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:07 smithi144 ceph-mon[104994]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:38:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:07 smithi144 ceph-mon[104994]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T19:38:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:07 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:07 smithi144 ceph-mon[104994]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:38:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:07 smithi144 ceph-mon[104994]: from='osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:38:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:07 smithi144 ceph-mon[104994]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T19:38:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:07 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:07 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:08.133 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:08 smithi135 ceph-mon[100251]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:08.133 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:08 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:08.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:08 smithi135 ceph-mon[118715]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:08.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:08 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:08 smithi144 ceph-mon[104994]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:08 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:08.900 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T19:38:08.900 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T19:38:08.900 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2023-09-23T19:38:08.900 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2023-09-23T19:38:08.900 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0131926 s, 795 MB/s 2023-09-23T19:38:08.900 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2023-09-23T19:38:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[118715]: purged_snaps scrub starts 2023-09-23T19:38:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[118715]: purged_snaps scrub ok 2023-09-23T19:38:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[118715]: osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382] boot 2023-09-23T19:38:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[118715]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T19:38:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:38:09.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[100251]: purged_snaps scrub starts 2023-09-23T19:38:09.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[100251]: purged_snaps scrub ok 2023-09-23T19:38:09.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[100251]: osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382] boot 2023-09-23T19:38:09.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[100251]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T19:38:09.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:09.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:09 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:38:09.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:09 smithi144 ceph-mon[104994]: purged_snaps scrub starts 2023-09-23T19:38:09.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:09 smithi144 ceph-mon[104994]: purged_snaps scrub ok 2023-09-23T19:38:09.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:09 smithi144 ceph-mon[104994]: osd.2 [v2:172.21.15.135:6818/1252775382,v1:172.21.15.135:6819/1252775382] boot 2023-09-23T19:38:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:09 smithi144 ceph-mon[104994]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T19:38:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:09 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:38:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:09 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T19:38:09.705 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi135:vg_nvme/lv_1 2023-09-23T19:38:10.049 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:38:10.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: pgmap v72: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:10.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T19:38:10.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T19:38:10.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:38:10.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.461 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.461 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.461 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.461 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.461 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:10.461 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:10.461 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.461 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: pgmap v72: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:10.463 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:10.463 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:10 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: pgmap v72: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T19:38:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:10 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:10.720 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:38:10 smithi135 sudo[149392]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T19:38:10.720 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:38:10 smithi135 sudo[149392]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:10.720 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:38:10 smithi135 sudo[149392]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:10.720 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:38:10 smithi135 sudo[149392]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:11.005 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:38:10 smithi135 sudo[149439]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T19:38:11.005 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:38:10 smithi135 sudo[149439]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:11.005 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:38:10 smithi135 sudo[149439]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:11.006 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:38:10 smithi135 sudo[149439]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:11.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[100251]: Detected new or changed devices on smithi135 2023-09-23T19:38:11.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi135 to 5381M 2023-09-23T19:38:11.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:38:11.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[100251]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T19:38:11.275 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:11.276 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[118715]: Detected new or changed devices on smithi135 2023-09-23T19:38:11.276 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi135 to 5381M 2023-09-23T19:38:11.276 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:38:11.276 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[118715]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T19:38:11.276 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:11 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:11.276 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149442]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T19:38:11.276 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149442]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:11.276 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149442]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:11.276 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149442]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:11.534 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149500]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T19:38:11.534 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149500]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:11.535 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149500]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:11.535 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149500]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:11 smithi144 ceph-mon[104994]: Detected new or changed devices on smithi135 2023-09-23T19:38:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:11 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi135 to 5381M 2023-09-23T19:38:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:11 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:38:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:11 smithi144 ceph-mon[104994]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T19:38:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:11 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:11.798 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149530]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T19:38:11.799 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149530]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:11.799 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149530]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:11.799 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149530]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:12.058 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149553]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T19:38:12.059 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149553]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:12.059 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149553]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:12.059 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:38:11 smithi135 sudo[149553]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:12.331 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 sudo[149633]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T19:38:12.331 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 sudo[149633]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:12.331 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 sudo[149633]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:12.331 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 sudo[149633]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:12.596 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 ceph-mon[104994]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 ceph-mon[104994]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 ceph-mon[104994]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 sudo[109134]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 sudo[109134]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 sudo[109134]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:12.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 sudo[109134]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:12.715 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[118715]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:12.715 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[118715]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T19:38:12.715 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:12.715 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:12.715 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:12.716 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[118715]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:12.716 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 sudo[149778]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T19:38:12.716 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 sudo[149778]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:12.716 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 sudo[149778]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:12.716 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 sudo[149778]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:12.716 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[100251]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:12.716 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[100251]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T19:38:12.716 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:12.717 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:12.717 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:12.717 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:12 smithi135 ceph-mon[100251]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 sudo[109137]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T19:38:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 sudo[109137]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 sudo[109137]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:12.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:12 smithi144 sudo[109137]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:13.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 sudo[149854]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T19:38:13.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 sudo[149854]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:13.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 sudo[149854]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:13.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 sudo[149854]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 sudo[149912]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 sudo[149912]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 sudo[149912]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 sudo[149912]: pam_unix(sudo:session): session closed for user root 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:13.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:38:13.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:38:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:14 smithi144 ceph-mon[104994]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:14 smithi144 ceph-mon[104994]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:14 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2197601016' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21fd337a-25fa-42ee-9f4c-130d8882de6c"}]: dispatch 2023-09-23T19:38:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:14 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2197601016' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21fd337a-25fa-42ee-9f4c-130d8882de6c"}]': finished 2023-09-23T19:38:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:14 smithi144 ceph-mon[104994]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T19:38:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:14 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:14.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[100251]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:14.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[100251]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2197601016' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21fd337a-25fa-42ee-9f4c-130d8882de6c"}]: dispatch 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2197601016' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21fd337a-25fa-42ee-9f4c-130d8882de6c"}]': finished 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[100251]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[118715]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[118715]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2197601016' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21fd337a-25fa-42ee-9f4c-130d8882de6c"}]: dispatch 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2197601016' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21fd337a-25fa-42ee-9f4c-130d8882de6c"}]': finished 2023-09-23T19:38:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[118715]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T19:38:14.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:14 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:15 smithi144 ceph-mon[104994]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:15 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3217197395' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:38:15.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:15 smithi135 ceph-mon[100251]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:15.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:15 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3217197395' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:38:15.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:15 smithi135 ceph-mon[118715]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:15.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:15 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3217197395' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:38:18.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:17 smithi144 ceph-mon[104994]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.2 KiB/s rd, 138 KiB/s wr, 9 op/s 2023-09-23T19:38:18.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:17 smithi135 ceph-mon[100251]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.2 KiB/s rd, 138 KiB/s wr, 9 op/s 2023-09-23T19:38:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:17 smithi135 ceph-mon[118715]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.2 KiB/s rd, 138 KiB/s wr, 9 op/s 2023-09-23T19:38:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:20 smithi144 ceph-mon[104994]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.3 KiB/s rd, 102 KiB/s wr, 7 op/s 2023-09-23T19:38:20.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:20 smithi135 ceph-mon[100251]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.3 KiB/s rd, 102 KiB/s wr, 7 op/s 2023-09-23T19:38:20.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:20 smithi135 ceph-mon[118715]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.3 KiB/s rd, 102 KiB/s wr, 7 op/s 2023-09-23T19:38:21.364 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:21 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:38:21.364 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:21 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:21.364 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:38:21.364 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:21 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:38:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:21 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:22 smithi144 ceph-mon[104994]: Deploying daemon osd.3 on smithi135 2023-09-23T19:38:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:22 smithi144 ceph-mon[104994]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.9 KiB/s rd, 81 KiB/s wr, 5 op/s 2023-09-23T19:38:22.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:22 smithi135 ceph-mon[100251]: Deploying daemon osd.3 on smithi135 2023-09-23T19:38:22.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:22 smithi135 ceph-mon[100251]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.9 KiB/s rd, 81 KiB/s wr, 5 op/s 2023-09-23T19:38:22.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:22 smithi135 ceph-mon[118715]: Deploying daemon osd.3 on smithi135 2023-09-23T19:38:22.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:22 smithi135 ceph-mon[118715]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.9 KiB/s rd, 81 KiB/s wr, 5 op/s 2023-09-23T19:38:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:24 smithi144 ceph-mon[104994]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T19:38:24.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:24 smithi135 ceph-mon[100251]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T19:38:24.422 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:24 smithi135 ceph-mon[118715]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T19:38:26.361 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:26 smithi135 ceph-mon[118715]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 74 KiB/s wr, 5 op/s 2023-09-23T19:38:26.361 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:26 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:26.362 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:26 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:26.362 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:26 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:26.362 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:26 smithi135 ceph-mon[100251]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 74 KiB/s wr, 5 op/s 2023-09-23T19:38:26.362 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:26.362 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:26.362 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:26 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:26 smithi144 ceph-mon[104994]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 74 KiB/s wr, 5 op/s 2023-09-23T19:38:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:26 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:26 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:26 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:28 smithi144 ceph-mon[104994]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T19:38:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:28 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:28 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:28 smithi135 ceph-mon[100251]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T19:38:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:28 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:28 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:28 smithi135 ceph-mon[118715]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T19:38:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:28 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:28 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:28.699 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 3 on host 'smithi135' 2023-09-23T19:38:29.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:29.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:29.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[100251]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:29 smithi135 ceph-mon[118715]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:38:29.351 DEBUG:teuthology.orchestra.run.smithi135:osd.3> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T19:38:29.353 INFO:tasks.cephadm:Deploying osd.4 on smithi144 with /dev/vg_nvme/lv_4... 2023-09-23T19:38:29.353 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T19:38:29.385 INFO:journalctl@ceph.osd.3.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:38:29.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:29 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:29.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:29 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:29.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:29 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:29 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:29 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:29.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:29 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:29.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:29 smithi144 ceph-mon[104994]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:38:29.694 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:38:30.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:30 smithi144 ceph-mon[104994]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:30.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:30 smithi144 ceph-mon[104994]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:38:30.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:30 smithi144 ceph-mon[104994]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T19:38:30.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:30 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:30.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:30 smithi144 ceph-mon[104994]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:38:30.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[118715]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[118715]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:38:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[118715]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T19:38:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[118715]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:38:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[100251]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[100251]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:38:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[100251]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T19:38:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:30.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:30 smithi135 ceph-mon[100251]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:38:30.790 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:38:30 smithi135 conmon[152525]: 2023-09-23T19:38:30.669+0000 7f65c43a4700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T19:38:30.791 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:38:30 smithi135 conmon[152525]: 2023-09-23T19:38:30.674+0000 7f65bf19f700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:38:31.509 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T19:38:31.509 INFO:teuthology.orchestra.run.smithi144.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T19:38:31.510 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10+0 records in 2023-09-23T19:38:31.510 INFO:teuthology.orchestra.run.smithi144.stderr:10+0 records out 2023-09-23T19:38:31.510 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112487 s, 932 MB/s 2023-09-23T19:38:31.510 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping successful for: 2023-09-23T19:38:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:31 smithi144 ceph-mon[104994]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:38:31.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:31 smithi144 ceph-mon[104994]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T19:38:31.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:31 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:31.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:31 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:31.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:31 smithi144 ceph-mon[104994]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:31.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[100251]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:38:31.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[100251]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T19:38:31.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:31.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:31.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[100251]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:31.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[118715]: from='osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-09-23T19:38:31.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[118715]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T19:38:31.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:31.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:31.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:31 smithi135 ceph-mon[118715]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T19:38:32.171 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi144:vg_nvme/lv_4 2023-09-23T19:38:32.542 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:38:32.927 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: purged_snaps scrub starts 2023-09-23T19:38:32.927 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: purged_snaps scrub ok 2023-09-23T19:38:32.927 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392] boot 2023-09-23T19:38:32.927 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T19:38:32.927 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:32.927 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.927 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.927 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.928 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.928 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.928 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.928 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.928 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:32.928 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:32.928 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:32 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: purged_snaps scrub starts 2023-09-23T19:38:32.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: purged_snaps scrub ok 2023-09-23T19:38:32.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392] boot 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:32.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: purged_snaps scrub starts 2023-09-23T19:38:32.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: purged_snaps scrub ok 2023-09-23T19:38:32.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: osd.3 [v2:172.21.15.135:6826/3652622392,v1:172.21.15.135:6827/3652622392] boot 2023-09-23T19:38:32.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T19:38:32.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:38:32.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:32 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:33.800 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:33 smithi144 ceph-mon[104994]: Detected new or changed devices on smithi135 2023-09-23T19:38:33.800 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:33 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi135 to 4036M 2023-09-23T19:38:33.800 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:33 smithi144 ceph-mon[104994]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T19:38:33.801 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:33 smithi144 ceph-mon[104994]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:33.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:33 smithi135 ceph-mon[100251]: Detected new or changed devices on smithi135 2023-09-23T19:38:33.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:33 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi135 to 4036M 2023-09-23T19:38:33.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:33 smithi135 ceph-mon[100251]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T19:38:33.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:33 smithi135 ceph-mon[100251]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:33.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:33 smithi135 ceph-mon[118715]: Detected new or changed devices on smithi135 2023-09-23T19:38:33.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:33 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi135 to 4036M 2023-09-23T19:38:33.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:33 smithi135 ceph-mon[118715]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T19:38:33.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:33 smithi135 ceph-mon[118715]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:35 smithi144 ceph-mon[104994]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:35 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:35 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:35 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:35 smithi135 ceph-mon[118715]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:35 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:35 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:35 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:35 smithi135 ceph-mon[100251]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:35 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:35 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:35 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:36 smithi144 ceph-mon[104994]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:36.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:36 smithi135 ceph-mon[100251]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:36 smithi135 ceph-mon[118715]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:37 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/4179597077' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]: dispatch 2023-09-23T19:38:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:37 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]: dispatch 2023-09-23T19:38:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:37 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]': finished 2023-09-23T19:38:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:37 smithi144 ceph-mon[104994]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T19:38:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:37 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:37 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/949312998' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:38:37.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/4179597077' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]: dispatch 2023-09-23T19:38:37.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]: dispatch 2023-09-23T19:38:37.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]': finished 2023-09-23T19:38:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[100251]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T19:38:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/949312998' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:38:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/4179597077' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]: dispatch 2023-09-23T19:38:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]: dispatch 2023-09-23T19:38:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de72ce1b-4c64-4f34-b1ba-563833e9fcf4"}]': finished 2023-09-23T19:38:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[118715]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T19:38:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:37 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/949312998' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:38:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:38 smithi144 ceph-mon[104994]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:38 smithi135 ceph-mon[100251]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:38 smithi135 ceph-mon[118715]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:40 smithi144 ceph-mon[104994]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:40 smithi135 ceph-mon[100251]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:40 smithi135 ceph-mon[118715]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:42.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:42 smithi144 ceph-mon[104994]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:42 smithi135 ceph-mon[118715]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:42.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:42 smithi135 ceph-mon[100251]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:43.434 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:43 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:38:43.435 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:43 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:43.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:43 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:38:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:43 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:43 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:38:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:43 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:44.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:44 smithi144 ceph-mon[104994]: Deploying daemon osd.4 on smithi144 2023-09-23T19:38:44.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:44 smithi144 ceph-mon[104994]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:44 smithi135 ceph-mon[100251]: Deploying daemon osd.4 on smithi144 2023-09-23T19:38:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:44 smithi135 ceph-mon[100251]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:44.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:44 smithi135 ceph-mon[118715]: Deploying daemon osd.4 on smithi144 2023-09-23T19:38:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:44 smithi135 ceph-mon[118715]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:46 smithi144 ceph-mon[104994]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:46.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:46 smithi135 ceph-mon[100251]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:46 smithi135 ceph-mon[118715]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:48 smithi135 ceph-mon[100251]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:48 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:48 smithi135 ceph-mon[118715]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:48 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:48.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:48 smithi144 ceph-mon[104994]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:48.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:48.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:48.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:48 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:50 smithi135 ceph-mon[100251]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:50 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:50 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:50 smithi135 ceph-mon[118715]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:50 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:50 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:50 smithi144 ceph-mon[104994]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:50 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:50 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:50.864 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 4 on host 'smithi144' 2023-09-23T19:38:51.412 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:51.412 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:51.412 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.412 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.413 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.413 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:51 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:51.413 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:51 smithi144 ceph-mon[104994]: from='osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:38:51.413 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:51 smithi144 ceph-mon[104994]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:38:51.414 DEBUG:teuthology.orchestra.run.smithi144:osd.4> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T19:38:51.416 INFO:tasks.cephadm:Deploying osd.5 on smithi144 with /dev/vg_nvme/lv_3... 2023-09-23T19:38:51.416 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T19:38:51.448 INFO:journalctl@ceph.osd.4.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:38:51.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:51.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:51.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:51.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[100251]: from='osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:38:51.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[100251]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:38:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:51.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:38:51.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[118715]: from='osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:38:51.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:51 smithi135 ceph-mon[118715]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T19:38:52.378 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:52 smithi144 ceph-mon[104994]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:52.378 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:52 smithi144 ceph-mon[104994]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T19:38:52.378 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:52 smithi144 ceph-mon[104994]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T19:38:52.378 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:52 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:52.379 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:52 smithi144 ceph-mon[104994]: from='osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:38:52.379 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:52 smithi144 ceph-mon[104994]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:38:52.391 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:38:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[100251]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[100251]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T19:38:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[100251]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T19:38:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[100251]: from='osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:38:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[100251]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:38:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[118715]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[118715]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T19:38:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[118715]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T19:38:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[118715]: from='osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:38:52.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:52 smithi135 ceph-mon[118715]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:38:52.969 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 19:38:52 smithi144 conmon[113051]: 2023-09-23T19:38:52.832+0000 7f609354d700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T19:38:52.969 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 19:38:52 smithi144 conmon[113051]: 2023-09-23T19:38:52.837+0000 7f608fb4b700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:38:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:53 smithi144 ceph-mon[104994]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:53 smithi144 ceph-mon[104994]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:38:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:53 smithi144 ceph-mon[104994]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T19:38:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:54.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[118715]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:54.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[118715]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:38:54.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[118715]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T19:38:54.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:54.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:54.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[100251]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T19:38:54.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[100251]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:38:54.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[100251]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T19:38:54.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:54.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:54.434 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T19:38:54.435 INFO:teuthology.orchestra.run.smithi144.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T19:38:54.435 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10+0 records in 2023-09-23T19:38:54.435 INFO:teuthology.orchestra.run.smithi144.stderr:10+0 records out 2023-09-23T19:38:54.435 INFO:teuthology.orchestra.run.smithi144.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110753 s, 947 MB/s 2023-09-23T19:38:54.435 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 2023-09-23T19:38:54.435 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping successful for: 2023-09-23T19:38:55.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: purged_snaps scrub starts 2023-09-23T19:38:55.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: purged_snaps scrub ok 2023-09-23T19:38:55.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977] boot 2023-09-23T19:38:55.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T19:38:55.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:55.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T19:38:55.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:55.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:55.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:54 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:55.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: purged_snaps scrub starts 2023-09-23T19:38:55.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: purged_snaps scrub ok 2023-09-23T19:38:55.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977] boot 2023-09-23T19:38:55.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: purged_snaps scrub starts 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: purged_snaps scrub ok 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: osd.4 [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977] boot 2023-09-23T19:38:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T19:38:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:38:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T19:38:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:54 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:38:55.233 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi144:vg_nvme/lv_3 2023-09-23T19:38:55.565 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:38:56.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:55 smithi144 ceph-mon[104994]: Detected new or changed devices on smithi144 2023-09-23T19:38:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:55 smithi144 ceph-mon[104994]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:38:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:55 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi144 to 17168M 2023-09-23T19:38:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:55 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:55 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:55 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:55 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:55 smithi144 ceph-mon[104994]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T19:38:56.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[100251]: Detected new or changed devices on smithi144 2023-09-23T19:38:56.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[100251]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:38:56.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi144 to 17168M 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[100251]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[118715]: Detected new or changed devices on smithi144 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[118715]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi144 to 17168M 2023-09-23T19:38:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:56.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:56.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:38:56.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:38:56.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:55 smithi135 ceph-mon[118715]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T19:38:58.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:57 smithi144 ceph-mon[104994]: pgmap v110: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T19:38:58.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:57 smithi144 ceph-mon[104994]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T19:38:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:57 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:58.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[118715]: pgmap v110: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T19:38:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[118715]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T19:38:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[100251]: pgmap v110: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T19:38:58.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[100251]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T19:38:58.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:38:58.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:38:58.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:57 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:38:58.937 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:58 smithi144 ceph-mon[104994]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:58.937 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:38:58 smithi144 ceph-mon[104994]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T19:38:59.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:58 smithi135 ceph-mon[100251]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:59.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:38:58 smithi135 ceph-mon[100251]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T19:38:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:58 smithi135 ceph-mon[118715]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:38:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:38:58 smithi135 ceph-mon[118715]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T19:39:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:00 smithi144 ceph-mon[104994]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 123 KiB/s, 0 objects/s recovering 2023-09-23T19:39:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:00 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/2168541104' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]: dispatch 2023-09-23T19:39:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:00 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]: dispatch 2023-09-23T19:39:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:00 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]': finished 2023-09-23T19:39:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:00 smithi144 ceph-mon[104994]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T19:39:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[100251]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 123 KiB/s, 0 objects/s recovering 2023-09-23T19:39:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/2168541104' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]: dispatch 2023-09-23T19:39:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]: dispatch 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]': finished 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[100251]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[118715]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 123 KiB/s, 0 objects/s recovering 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/2168541104' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]: dispatch 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]: dispatch 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bd0e910c-7c37-4ee6-97e8-031834dadc71"}]': finished 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[118715]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T19:39:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:01.129 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:39:00 smithi135 conmon[100683]: 2023-09-23T19:39:00.812+0000 7f7925df5700 -1 calc_pg_upmaps abort due to max <= 0 2023-09-23T19:39:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:01 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/984969525' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:01.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:01 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/984969525' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:01 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/984969525' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:02 smithi144 ceph-mon[104994]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T19:39:02.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:02 smithi135 ceph-mon[100251]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T19:39:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:02 smithi135 ceph-mon[118715]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T19:39:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:04 smithi144 ceph-mon[104994]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:04.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:04 smithi135 ceph-mon[100251]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:04.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:04 smithi135 ceph-mon[118715]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:06 smithi144 ceph-mon[104994]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:06 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:39:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:06 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:06.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:06 smithi135 ceph-mon[100251]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:06.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:06 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:39:06.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:06 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:06 smithi135 ceph-mon[118715]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:06 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:39:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:06 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:07.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:07 smithi135 ceph-mon[100251]: Deploying daemon osd.5 on smithi144 2023-09-23T19:39:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:07 smithi135 ceph-mon[118715]: Deploying daemon osd.5 on smithi144 2023-09-23T19:39:07.551 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:07 smithi144 ceph-mon[104994]: Deploying daemon osd.5 on smithi144 2023-09-23T19:39:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:08 smithi144 ceph-mon[104994]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:08.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:08 smithi135 ceph-mon[100251]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:08 smithi135 ceph-mon[118715]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:10 smithi135 ceph-mon[100251]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:10 smithi135 ceph-mon[118715]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:10 smithi144 ceph-mon[104994]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:12.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:11 smithi144 ceph-mon[104994]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:12.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:11 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:12.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:11 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:12.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:11 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:12.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:11 smithi135 ceph-mon[100251]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:12.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:11 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:12.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:11 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:12.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:11 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:12.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:11 smithi135 ceph-mon[118715]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:12.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:11 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:12.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:11 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:12.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:11 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:13.824 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 5 on host 'smithi144' 2023-09-23T19:39:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:13 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:13 smithi144 ceph-mon[104994]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:13.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[100251]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:13 smithi135 ceph-mon[118715]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:14.479 DEBUG:teuthology.orchestra.run.smithi144:osd.5> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T19:39:14.481 INFO:tasks.cephadm:Deploying osd.6 on smithi144 with /dev/vg_nvme/lv_2... 2023-09-23T19:39:14.482 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T19:39:14.511 INFO:journalctl@ceph.osd.5.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:39:15.068 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:39:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:14 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:14 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:14 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:14 smithi144 ceph-mon[104994]: from='osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:39:15.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:14 smithi144 ceph-mon[104994]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:39:15.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:15.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:15.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:15.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[100251]: from='osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:39:15.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[100251]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:39:15.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:15.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:15.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:15.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[118715]: from='osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:39:15.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:14 smithi135 ceph-mon[118715]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:39:16.381 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 19:39:16 smithi144 conmon[121943]: 2023-09-23T19:39:16.147+0000 7ff670e20700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T19:39:16.382 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 19:39:16 smithi144 conmon[121943]: 2023-09-23T19:39:16.153+0000 7ff66a418700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:39:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:16 smithi144 ceph-mon[104994]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:16 smithi144 ceph-mon[104994]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:39:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:16 smithi144 ceph-mon[104994]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T19:39:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:16 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:16 smithi144 ceph-mon[104994]: from='osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:16 smithi144 ceph-mon[104994]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:16.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[100251]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[100251]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[100251]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[100251]: from='osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[100251]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[118715]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[118715]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[118715]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[118715]: from='osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:16.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:16 smithi135 ceph-mon[118715]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:17.365 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T19:39:17.365 INFO:teuthology.orchestra.run.smithi144.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T19:39:17.365 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10+0 records in 2023-09-23T19:39:17.366 INFO:teuthology.orchestra.run.smithi144.stderr:10+0 records out 2023-09-23T19:39:17.366 INFO:teuthology.orchestra.run.smithi144.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124133 s, 845 MB/s 2023-09-23T19:39:17.366 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping successful for: 2023-09-23T19:39:17.413 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:17 smithi144 ceph-mon[104994]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:39:17.413 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:17 smithi144 ceph-mon[104994]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T19:39:17.414 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:17 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:17.414 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:17 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:17.414 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:17 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:17.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[118715]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:39:17.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[118715]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T19:39:17.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:17.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:17.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[100251]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:39:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[100251]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T19:39:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:17 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:18.320 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:18 smithi144 ceph-mon[104994]: purged_snaps scrub starts 2023-09-23T19:39:18.320 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:18 smithi144 ceph-mon[104994]: purged_snaps scrub ok 2023-09-23T19:39:18.320 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:18 smithi144 ceph-mon[104994]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:18.320 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:18 smithi144 ceph-mon[104994]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:39:18.320 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:18 smithi144 ceph-mon[104994]: osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223] boot 2023-09-23T19:39:18.320 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:18 smithi144 ceph-mon[104994]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T19:39:18.320 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:18 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:18.339 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi144:vg_nvme/lv_2 2023-09-23T19:39:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[100251]: purged_snaps scrub starts 2023-09-23T19:39:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[100251]: purged_snaps scrub ok 2023-09-23T19:39:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[100251]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[100251]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:39:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[100251]: osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223] boot 2023-09-23T19:39:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[100251]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T19:39:18.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[118715]: purged_snaps scrub starts 2023-09-23T19:39:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[118715]: purged_snaps scrub ok 2023-09-23T19:39:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[118715]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T19:39:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[118715]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:39:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[118715]: osd.5 [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223] boot 2023-09-23T19:39:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[118715]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T19:39:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:18 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:39:18.681 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:39:19.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T19:39:19.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:19.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:19.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:19.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:19.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:19 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T19:39:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:19.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:19.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:19 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:20.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:20 smithi135 ceph-mon[100251]: Detected new or changed devices on smithi144 2023-09-23T19:39:20.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:20 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi144 to 8584M 2023-09-23T19:39:20.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:20 smithi135 ceph-mon[100251]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:20.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:20 smithi135 ceph-mon[100251]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T19:39:20.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:20 smithi135 ceph-mon[118715]: Detected new or changed devices on smithi144 2023-09-23T19:39:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:20 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi144 to 8584M 2023-09-23T19:39:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:20 smithi135 ceph-mon[118715]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:20 smithi135 ceph-mon[118715]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T19:39:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:20 smithi144 ceph-mon[104994]: Detected new or changed devices on smithi144 2023-09-23T19:39:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:20 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi144 to 8584M 2023-09-23T19:39:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:20 smithi144 ceph-mon[104994]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:20 smithi144 ceph-mon[104994]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T19:39:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:21 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:39:21.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:21 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:39:21.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:21 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:21.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:21 smithi135 ceph-mon[118715]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T19:39:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:39:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:39:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:21 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:21.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:21 smithi135 ceph-mon[100251]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T19:39:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:21 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:39:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:21 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:39:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:21 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:21 smithi144 ceph-mon[104994]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T19:39:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:22 smithi135 ceph-mon[100251]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:39:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:22 smithi135 ceph-mon[100251]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:22 smithi135 ceph-mon[100251]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T19:39:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:22 smithi135 ceph-mon[118715]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:39:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:22 smithi135 ceph-mon[118715]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:22 smithi135 ceph-mon[118715]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T19:39:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:22 smithi144 ceph-mon[104994]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:39:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:22 smithi144 ceph-mon[104994]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:22 smithi144 ceph-mon[104994]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T19:39:23.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/4097354505' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]: dispatch 2023-09-23T19:39:23.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]: dispatch 2023-09-23T19:39:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]': finished 2023-09-23T19:39:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[100251]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T19:39:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/4097354505' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]: dispatch 2023-09-23T19:39:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]: dispatch 2023-09-23T19:39:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]': finished 2023-09-23T19:39:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[118715]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T19:39:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:23 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:23 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/4097354505' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]: dispatch 2023-09-23T19:39:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:23 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]: dispatch 2023-09-23T19:39:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:23 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84e8261b-0520-44c9-aeae-c25256356c3f"}]': finished 2023-09-23T19:39:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:23 smithi144 ceph-mon[104994]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T19:39:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:23 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:24.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:24 smithi135 ceph-mon[100251]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:24.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:24 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/4070755839' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:24 smithi135 ceph-mon[118715]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:24 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/4070755839' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:24 smithi144 ceph-mon[104994]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:24 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/4070755839' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:25.577 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:25 smithi144 ceph-mon[104994]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:39:25.577 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:25 smithi144 ceph-mon[104994]: Cluster is now healthy 2023-09-23T19:39:25.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:25 smithi135 ceph-mon[100251]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:39:25.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:25 smithi135 ceph-mon[100251]: Cluster is now healthy 2023-09-23T19:39:25.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:25 smithi135 ceph-mon[118715]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:39:25.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:25 smithi135 ceph-mon[118715]: Cluster is now healthy 2023-09-23T19:39:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:26 smithi144 ceph-mon[104994]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 102 KiB/s, 0 objects/s recovering 2023-09-23T19:39:26.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:26 smithi135 ceph-mon[100251]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 102 KiB/s, 0 objects/s recovering 2023-09-23T19:39:26.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:26 smithi135 ceph-mon[118715]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 102 KiB/s, 0 objects/s recovering 2023-09-23T19:39:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:28 smithi144 ceph-mon[104994]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-09-23T19:39:28.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:28 smithi135 ceph-mon[100251]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-09-23T19:39:28.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:28 smithi135 ceph-mon[118715]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-09-23T19:39:29.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:29 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:39:29.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:29 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:29.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:39:29.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:29 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:29.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:29 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:39:29.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:29 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:30.546 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:30 smithi144 ceph-mon[104994]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T19:39:30.547 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:30 smithi144 ceph-mon[104994]: Deploying daemon osd.6 on smithi144 2023-09-23T19:39:30.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:30 smithi135 ceph-mon[100251]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T19:39:30.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:30 smithi135 ceph-mon[100251]: Deploying daemon osd.6 on smithi144 2023-09-23T19:39:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:30 smithi135 ceph-mon[118715]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T19:39:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:30 smithi135 ceph-mon[118715]: Deploying daemon osd.6 on smithi144 2023-09-23T19:39:32.590 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:32 smithi144 ceph-mon[104994]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-09-23T19:39:32.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:32 smithi135 ceph-mon[100251]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-09-23T19:39:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:32 smithi135 ceph-mon[118715]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-09-23T19:39:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:34 smithi144 ceph-mon[104994]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-23T19:39:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:34 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:34 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:34 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:34.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:34 smithi135 ceph-mon[100251]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-23T19:39:34.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:34 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:34.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:34 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:34 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:34 smithi135 ceph-mon[118715]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-23T19:39:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:34 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:34 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:34 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:36 smithi144 ceph-mon[104994]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T19:39:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:36 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:36 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:36 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:36 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:36 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:36.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[100251]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T19:39:36.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:36.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:36.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:36.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:36.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[118715]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T19:39:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:36 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:37.109 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 6 on host 'smithi144' 2023-09-23T19:39:37.749 DEBUG:teuthology.orchestra.run.smithi144:osd.6> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T19:39:37.751 INFO:tasks.cephadm:Deploying osd.7 on smithi144 with /dev/vg_nvme/lv_1... 2023-09-23T19:39:37.752 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T19:39:37.780 INFO:journalctl@ceph.osd.6.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:39:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:38 smithi144 ceph-mon[104994]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:38 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:38 smithi144 ceph-mon[104994]: from='osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:39:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:38 smithi144 ceph-mon[104994]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:39:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[100251]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[100251]: from='osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:39:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[100251]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:39:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[118715]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[118715]: from='osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:39:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:38 smithi135 ceph-mon[118715]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T19:39:38.870 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:39:39.528 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:39:39 smithi144 conmon[131356]: 2023-09-23T19:39:39.260+0000 7f87f64ef700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T19:39:39.528 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:39:39 smithi144 conmon[131356]: 2023-09-23T19:39:39.267+0000 7f87f12ea700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:39:39.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:39 smithi144 ceph-mon[104994]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T19:39:39.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:39 smithi144 ceph-mon[104994]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T19:39:39.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:39 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:39.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:39 smithi144 ceph-mon[104994]: from='osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:39.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:39 smithi144 ceph-mon[104994]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:39.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[100251]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T19:39:39.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[100251]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T19:39:39.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:39.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[100251]: from='osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:39.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[100251]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:39.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[118715]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T19:39:39.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[118715]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T19:39:39.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:39.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[118715]: from='osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:39.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:39 smithi135 ceph-mon[118715]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:39:40.356 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:40 smithi144 ceph-mon[104994]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:40.356 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:40 smithi144 ceph-mon[104994]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:39:40.356 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:40 smithi144 ceph-mon[104994]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T19:39:40.356 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:40 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:40.356 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:40 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:40.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[100251]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:40.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[100251]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:39:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[100251]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T19:39:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:40.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[118715]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:40.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[118715]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:39:40.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[118715]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T19:39:40.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:40.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:40 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:40.834 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T19:39:40.834 INFO:teuthology.orchestra.run.smithi144.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T19:39:40.834 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10+0 records in 2023-09-23T19:39:40.834 INFO:teuthology.orchestra.run.smithi144.stderr:10+0 records out 2023-09-23T19:39:40.834 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0115333 s, 909 MB/s 2023-09-23T19:39:40.834 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping successful for: 2023-09-23T19:39:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:41 smithi144 ceph-mon[104994]: purged_snaps scrub starts 2023-09-23T19:39:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:41 smithi144 ceph-mon[104994]: purged_snaps scrub ok 2023-09-23T19:39:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:41 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:41 smithi144 ceph-mon[104994]: osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554] boot 2023-09-23T19:39:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:41 smithi144 ceph-mon[104994]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T19:39:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:41 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:41.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[118715]: purged_snaps scrub starts 2023-09-23T19:39:41.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[118715]: purged_snaps scrub ok 2023-09-23T19:39:41.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[118715]: osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554] boot 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[118715]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[100251]: purged_snaps scrub starts 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[100251]: purged_snaps scrub ok 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[100251]: osd.6 [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554] boot 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[100251]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T19:39:41.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:41 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:39:41.868 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi144:vg_nvme/lv_1 2023-09-23T19:39:42.206 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:39:42.590 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:42.590 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T19:39:42.590 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.590 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.590 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:42.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:42.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:42 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:42.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:39:42.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:42 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:43.429 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:43 smithi144 ceph-mon[104994]: Detected new or changed devices on smithi144 2023-09-23T19:39:43.429 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:43 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi144 to 5722M 2023-09-23T19:39:43.429 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:43 smithi144 ceph-mon[104994]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T19:39:43.429 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:43 smithi144 ceph-mon[104994]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:39:43.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:43 smithi135 ceph-mon[100251]: Detected new or changed devices on smithi144 2023-09-23T19:39:43.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:43 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi144 to 5722M 2023-09-23T19:39:43.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:43 smithi135 ceph-mon[100251]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T19:39:43.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:43 smithi135 ceph-mon[100251]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:39:43.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:43 smithi135 ceph-mon[118715]: Detected new or changed devices on smithi144 2023-09-23T19:39:43.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:43 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi144 to 5722M 2023-09-23T19:39:43.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:43 smithi135 ceph-mon[118715]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T19:39:43.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:43 smithi135 ceph-mon[118715]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:39:44.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:44 smithi144 ceph-mon[104994]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T19:39:44.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:44 smithi144 ceph-mon[104994]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:39:44.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:44 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:39:44.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:44 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:39:44.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:44 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:44.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[100251]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T19:39:44.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[100251]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:39:44.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:39:44.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:39:44.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:44.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[118715]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T19:39:44.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[118715]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:39:44.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T19:39:44.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T19:39:44.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:44 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:45.519 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:45 smithi144 ceph-mon[104994]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T19:39:45.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:45 smithi144 ceph-mon[104994]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:39:45.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:45 smithi135 ceph-mon[100251]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T19:39:45.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:45 smithi135 ceph-mon[100251]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:39:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:45 smithi135 ceph-mon[118715]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T19:39:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:45 smithi135 ceph-mon[118715]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:39:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:46 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/205178084' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]: dispatch 2023-09-23T19:39:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:46 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]: dispatch 2023-09-23T19:39:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:46 smithi144 ceph-mon[104994]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]': finished 2023-09-23T19:39:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:46 smithi144 ceph-mon[104994]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T19:39:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:46 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:39:46.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/205178084' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]: dispatch 2023-09-23T19:39:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]: dispatch 2023-09-23T19:39:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[118715]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]': finished 2023-09-23T19:39:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[118715]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T19:39:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:39:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/205178084' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]: dispatch 2023-09-23T19:39:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]: dispatch 2023-09-23T19:39:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[100251]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "030b85ad-503a-4291-a267-97a0e0f2db6e"}]': finished 2023-09-23T19:39:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[100251]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T19:39:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:46 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:39:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:47 smithi144 ceph-mon[104994]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-09-23T19:39:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:47 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/3961519324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:47.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:47 smithi135 ceph-mon[100251]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-09-23T19:39:47.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:47 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/3961519324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:47.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:47 smithi135 ceph-mon[118715]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-09-23T19:39:47.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:47 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/3961519324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T19:39:50.401 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:50 smithi144 ceph-mon[104994]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T19:39:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:50 smithi135 ceph-mon[100251]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T19:39:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:50 smithi135 ceph-mon[118715]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T19:39:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:52 smithi135 ceph-mon[100251]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-09-23T19:39:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:52 smithi135 ceph-mon[118715]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-09-23T19:39:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:52 smithi144 ceph-mon[104994]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-09-23T19:39:53.267 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:39:53.267 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:53 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:39:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:53 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:39:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:53 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:39:54.322 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:54 smithi144 ceph-mon[104994]: Deploying daemon osd.7 on smithi144 2023-09-23T19:39:54.323 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:54 smithi144 ceph-mon[104994]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2023-09-23T19:39:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:54 smithi135 ceph-mon[118715]: Deploying daemon osd.7 on smithi144 2023-09-23T19:39:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:54 smithi135 ceph-mon[118715]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2023-09-23T19:39:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:54 smithi135 ceph-mon[100251]: Deploying daemon osd.7 on smithi144 2023-09-23T19:39:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:54 smithi135 ceph-mon[100251]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2023-09-23T19:39:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:56 smithi144 ceph-mon[104994]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T19:39:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:56 smithi135 ceph-mon[100251]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T19:39:56.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:56 smithi135 ceph-mon[118715]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T19:39:58.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:58 smithi135 ceph-mon[100251]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-09-23T19:39:58.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:58 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:58.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:58 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:58.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:39:58 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:58 smithi135 ceph-mon[118715]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-09-23T19:39:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:58 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:58 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:39:58 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:39:58.624 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:58 smithi144 ceph-mon[104994]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-09-23T19:39:58.624 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:58 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:58.625 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:58 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:39:58.625 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:39:58 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:00.317 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 7 on host 'smithi144' 2023-09-23T19:40:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:00 smithi135 ceph-mon[100251]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:00 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:00 smithi135 ceph-mon[100251]: overall HEALTH_OK 2023-09-23T19:40:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:00 smithi135 ceph-mon[118715]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:00 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:00 smithi135 ceph-mon[118715]: overall HEALTH_OK 2023-09-23T19:40:00.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:00 smithi144 ceph-mon[104994]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:00.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:00.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:00 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:00.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:00 smithi144 ceph-mon[104994]: overall HEALTH_OK 2023-09-23T19:40:00.986 DEBUG:teuthology.orchestra.run.smithi144:osd.7> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T19:40:00.988 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T19:40:00.988 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T19:40:01.017 INFO:journalctl@ceph.osd.7.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:40:01.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:01.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:01.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:01 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:01.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:01 smithi144 ceph-mon[104994]: from='osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:40:01.251 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:01 smithi144 ceph-mon[104994]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:40:01.344 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[100251]: from='osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[100251]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:01.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:01.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[118715]: from='osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:40:01.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:01 smithi135 ceph-mon[118715]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:40:02.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:02 smithi144 ceph-mon[104994]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:02.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:02 smithi144 ceph-mon[104994]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:40:02.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:02 smithi144 ceph-mon[104994]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T19:40:02.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:02 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:02.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:02 smithi144 ceph-mon[104994]: from='osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:40:02.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:02 smithi144 ceph-mon[104994]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:40:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[118715]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[118715]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:40:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[118715]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T19:40:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[118715]: from='osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:40:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[118715]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:40:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[100251]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[100251]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:40:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[100251]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T19:40:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[100251]: from='osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:40:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:02 smithi135 ceph-mon[100251]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T19:40:02.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 19:40:02 smithi144 conmon[141003]: 2023-09-23T19:40:02.293+0000 7fad52633700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T19:40:02.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 19:40:02 smithi144 conmon[141003]: 2023-09-23T19:40:02.300+0000 7fad4dc2f700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:40:03.016 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:40:03.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:03 smithi144 ceph-mon[104994]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:40:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:03 smithi144 ceph-mon[104994]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T19:40:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:03 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:03 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:03 smithi144 ceph-mon[104994]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:03 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/486361665' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:40:03.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[100251]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:40:03.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[100251]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T19:40:03.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:03.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:03.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[100251]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:03.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/486361665' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:40:03.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[118715]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2023-09-23T19:40:03.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[118715]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T19:40:03.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:03.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:03.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[118715]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T19:40:03.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:03 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/486361665' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:40:03.772 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1695497980,"num_in_osds":8,"osd_in_since":1695497986,"num_remapped_pgs":0} 2023-09-23T19:40:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: purged_snaps scrub starts 2023-09-23T19:40:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: purged_snaps scrub ok 2023-09-23T19:40:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059] boot 2023-09-23T19:40:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: Detected new or changed devices on smithi144 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi144 to 4292M 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:04.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:04 smithi144 ceph-mon[104994]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T19:40:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: purged_snaps scrub starts 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: purged_snaps scrub ok 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059] boot 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: Detected new or changed devices on smithi144 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi144 to 4292M 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[118715]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: purged_snaps scrub starts 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: purged_snaps scrub ok 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: osd.7 [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059] boot 2023-09-23T19:40:04.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: Detected new or changed devices on smithi144 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi144 to 4292M 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:04.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:04.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:04.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:04 smithi135 ceph-mon[100251]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T19:40:04.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T19:40:05.126 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:06.432 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:06 smithi135 ceph-mon[118715]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:06.433 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:06 smithi135 ceph-mon[118715]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T19:40:06.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:06 smithi135 ceph-mon[100251]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:06.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:06 smithi135 ceph-mon[100251]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T19:40:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:06 smithi144 ceph-mon[104994]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:06 smithi144 ceph-mon[104994]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T19:40:06.813 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:40:07.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:07 smithi135 ceph-mon[100251]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T19:40:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:07 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4170850379' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:40:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:07 smithi135 ceph-mon[118715]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T19:40:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:07 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4170850379' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:40:07.522 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1695498003,"num_in_osds":8,"osd_in_since":1695497986,"num_remapped_pgs":0} 2023-09-23T19:40:07.522 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:40:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:07 smithi144 ceph-mon[104994]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T19:40:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:07 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4170850379' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T19:40:07.856 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:08.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:08 smithi135 ceph-mon[100251]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:08.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:08 smithi135 ceph-mon[100251]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T19:40:08.423 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:08 smithi135 ceph-mon[118715]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:08.423 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:08 smithi135 ceph-mon[118715]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T19:40:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:08 smithi144 ceph-mon[104994]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:08 smithi144 ceph-mon[104994]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T19:40:09.477 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:40:09.477 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":58,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T19:40:07.151528+0000","last_up_change":"2023-09-23T19:40:03.286566+0000","last_in_change":"2023-09-23T19:39:46.254783+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6803","nonce":1288355238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6805","nonce":1288355238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6809","nonce":1288355238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6807","nonce":1288355238}]},"public_addr":"172.21.15.135:6803/1288355238","cluster_addr":"172.21.15.135:6805/1288355238","heartbeat_back_addr":"172.21.15.135:6809/1288355238","heartbeat_front_addr":"172.21.15.135:6807/1288355238","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6811","nonce":460054379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6813","nonce":460054379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6817","nonce":460054379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6815","nonce":460054379}]},"public_addr":"172.21.15.135:6811/460054379","cluster_addr":"172.21.15.135:6813/460054379","heartbeat_back_addr":"172.21.15.135:6817/460054379","heartbeat_front_addr":"172.21.15.135:6815/460054379","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6819","nonce":1252775382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6821","nonce":1252775382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6825","nonce":1252775382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6823","nonce":1252775382}]},"public_addr":"172.21.15.135:6819/1252775382","cluster_addr":"172.21.15.135:6821/1252775382","heartbeat_back_addr":"172.21.15.135:6825/1252775382","heartbeat_front_addr":"172.21.15.135:6823/1252775382","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6827","nonce":3652622392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6829","nonce":3652622392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6833","nonce":3652622392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6831","nonce":3652622392}]},"public_addr":"172.21.15.135:6827/3652622392","cluster_addr":"172.21.15.135:6829/3652622392","heartbeat_back_addr":"172.21.15.135:6833/3652622392","heartbeat_front_addr":"172.21.15.135:6831/3652622392","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6817","nonce":1530810554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6819","nonce":1530810554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6823","nonce":1530810554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6821","nonce":1530810554}]},"public_addr":"172.21.15.144:6817/1530810554","cluster_addr":"172.21.15.144:6819/1530810554","heartbeat_back_addr":"172.21.15.144:6823/1530810554","heartbeat_front_addr":"172.21.15.144:6821/1530810554","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:40:10.172 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-23T19:38:09.111581+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-23T19:40:10.172 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-23T19:40:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:10 smithi135 ceph-mon[118715]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:10 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1257220573' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:10 smithi135 ceph-mon[100251]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:10 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1257220573' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:10.512 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:10 smithi144 ceph-mon[104994]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:10 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1257220573' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:12.155 INFO:teuthology.orchestra.run.smithi135.stdout:pg_num: 1 2023-09-23T19:40:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:12 smithi135 ceph-mon[100251]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T19:40:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:12 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1855844165' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T19:40:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:12 smithi135 ceph-mon[118715]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T19:40:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:12 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1855844165' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T19:40:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:12 smithi144 ceph-mon[104994]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T19:40:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:12 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1855844165' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T19:40:12.880 INFO:tasks.cephadm:Adding prometheus.a on smithi144 2023-09-23T19:40:12.880 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi144=a' 2023-09-23T19:40:13.232 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:14.440 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:14 smithi144 ceph-mon[104994]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-09-23T19:40:14.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:14 smithi135 ceph-mon[100251]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-09-23T19:40:14.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:14 smithi135 ceph-mon[118715]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-09-23T19:40:14.844 INFO:teuthology.orchestra.run.smithi144.stdout:Scheduled prometheus update... 2023-09-23T19:40:15.453 DEBUG:teuthology.orchestra.run.smithi144:prometheus.a> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T19:40:15.455 INFO:tasks.cephadm:Adding node-exporter.a on smithi135 2023-09-23T19:40:15.455 INFO:tasks.cephadm:Adding node-exporter.b on smithi144 2023-09-23T19:40:15.455 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi135=a;smithi144=b' 2023-09-23T19:40:15.486 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:40:15.818 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:16.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T19:40:16.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi144=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: Saving service prometheus spec with placement smithi144=a;count:1 2023-09-23T19:40:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:15 smithi144 ceph-mon[104994]: Deploying daemon prometheus.a on smithi144 2023-09-23T19:40:16.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T19:40:16.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi144=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: Saving service prometheus spec with placement smithi144=a;count:1 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[118715]: Deploying daemon prometheus.a on smithi144 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T19:40:16.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi144=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: Saving service prometheus spec with placement smithi144=a;count:1 2023-09-23T19:40:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:15 smithi135 ceph-mon[100251]: Deploying daemon prometheus.a on smithi144 2023-09-23T19:40:17.891 INFO:teuthology.orchestra.run.smithi144.stdout:Scheduled node-exporter update... 2023-09-23T19:40:18.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:17 smithi144 ceph-mon[104994]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T19:40:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:17 smithi135 ceph-mon[118715]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T19:40:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:17 smithi135 ceph-mon[100251]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T19:40:19.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:18 smithi135 ceph-mon[100251]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi135=a;smithi144=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:18 smithi135 ceph-mon[100251]: Saving service node-exporter spec with placement smithi135=a;smithi144=b;count:2 2023-09-23T19:40:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:18 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:18 smithi135 ceph-mon[118715]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi135=a;smithi144=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:18 smithi135 ceph-mon[118715]: Saving service node-exporter spec with placement smithi135=a;smithi144=b;count:2 2023-09-23T19:40:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:18 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:18 smithi144 ceph-mon[104994]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi135=a;smithi144=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:18 smithi144 ceph-mon[104994]: Saving service node-exporter spec with placement smithi135=a;smithi144=b;count:2 2023-09-23T19:40:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:18 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:19.502 DEBUG:teuthology.orchestra.run.smithi135:node-exporter.a> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T19:40:19.505 DEBUG:teuthology.orchestra.run.smithi144:node-exporter.b> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T19:40:19.507 INFO:tasks.cephadm:Adding alertmanager.a on smithi135 2023-09-23T19:40:19.507 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi135=a' 2023-09-23T19:40:19.538 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:40:19.541 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:40:20.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:20 smithi135 ceph-mon[118715]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T19:40:20.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:20 smithi135 ceph-mon[100251]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T19:40:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:20 smithi144 ceph-mon[104994]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T19:40:22.012 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:22 smithi135 ceph-mon[118715]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T19:40:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:22 smithi135 ceph-mon[100251]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T19:40:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:22 smithi144 ceph-mon[104994]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T19:40:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:24 smithi135 ceph-mon[118715]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:24 smithi135 ceph-mon[100251]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:24.493 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:24 smithi144 ceph-mon[104994]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:26.293 INFO:teuthology.orchestra.run.smithi144.stdout:Scheduled alertmanager update... 2023-09-23T19:40:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:26 smithi135 ceph-mon[100251]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:26 smithi135 ceph-mon[118715]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:26.519 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:26 smithi144 ceph-mon[104994]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:27.002 DEBUG:teuthology.orchestra.run.smithi135:alertmanager.a> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T19:40:27.005 INFO:tasks.cephadm:Adding grafana.a on smithi144 2023-09-23T19:40:27.005 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi144=a' 2023-09-23T19:40:27.037 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T19:40:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:27 smithi144 ceph-mon[104994]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi135=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:27 smithi144 ceph-mon[104994]: Saving service alertmanager spec with placement smithi135=a;count:1 2023-09-23T19:40:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:27 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:27 smithi144 ceph-mon[104994]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:27 smithi135 ceph-mon[118715]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi135=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:27 smithi135 ceph-mon[118715]: Saving service alertmanager spec with placement smithi135=a;count:1 2023-09-23T19:40:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:27 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:27.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:27 smithi135 ceph-mon[118715]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:27 smithi135 ceph-mon[100251]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi135=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:27 smithi135 ceph-mon[100251]: Saving service alertmanager spec with placement smithi135=a;count:1 2023-09-23T19:40:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:27 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:27 smithi135 ceph-mon[100251]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:28.053 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:29.631 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:40:29 smithi144 systemd[1]: Started Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:40:29.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:40:29 smithi144 conmon[148569]: ts=2023-09-23T19:40:29.468Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.588322ms db_storage=766ns remote_storage=1.103µs web_handler=399ns query_engine=1.056µs scrape=454.356µs scrape_sd=19.045µs notify=669ns notify_sd=1.413µs rules=14.938355ms tracing=5.839µs 2023-09-23T19:40:29.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:40:29 smithi144 conmon[148569]: ts=2023-09-23T19:40:29.468Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T19:40:29.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:40:29 smithi144 conmon[148569]: ts=2023-09-23T19:40:29.468Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T19:40:30.095 INFO:teuthology.orchestra.run.smithi144.stdout:Scheduled grafana update... 2023-09-23T19:40:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:30 smithi144 ceph-mon[104994]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:30 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:30 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:30 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:30 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T19:40:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:30 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[118715]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T19:40:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[100251]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T19:40:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' 2023-09-23T19:40:30.668 DEBUG:teuthology.orchestra.run.smithi144:grafana.a> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T19:40:30.670 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T19:40:30.671 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T19:40:30.700 INFO:journalctl@ceph.grafana.a.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T19:40:30.870 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:30 smithi144 conmon[108226]: ignoring --setuser ceph since I am not root 2023-09-23T19:40:30.870 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:30 smithi144 conmon[108226]: ignoring --setgroup ceph since I am not root 2023-09-23T19:40:30.870 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:30 smithi144 ceph-mgr[108251]: -- 172.21.15.144:0/3192797954 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559bd13109c0 con 0x559bd12e8800 2023-09-23T19:40:30.870 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:30 smithi144 conmon[108226]: 2023-09-23T19:40:30.707+0000 7f56eedd11c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:40:30.871 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:30 smithi144 conmon[108226]: 2023-09-23T19:40:30.784+0000 7f56eedd11c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:40:30.871 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:30 smithi144 conmon[108226]: 2023-09-23T19:40:30.870+0000 7f56eedd11c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:40:30.876 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:30 smithi135 conmon[100683]: ignoring --setuser ceph since I am not root 2023-09-23T19:40:30.877 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:30 smithi135 conmon[100683]: ignoring --setgroup ceph since I am not root 2023-09-23T19:40:30.877 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:30 smithi135 ceph-mgr[100707]: -- 172.21.15.135:0/977835446 <== mon.2 v2:172.21.15.135:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e9b746a9c0 con 0x55e9b7442c00 2023-09-23T19:40:30.877 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:30 smithi135 conmon[100683]: 2023-09-23T19:40:30.702+0000 7f41ac5321c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:40:30.877 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:30 smithi135 conmon[100683]: 2023-09-23T19:40:30.783+0000 7f41ac5321c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:40:30.877 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:30 smithi135 conmon[100683]: 2023-09-23T19:40:30.876+0000 7f41ac5321c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:40:31.033 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:31.125 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:30 smithi144 conmon[108226]: 2023-09-23T19:40:30.960+0000 7f56eedd11c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:40:31.144 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:30 smithi135 conmon[100683]: 2023-09-23T19:40:30.989+0000 7f41ac5321c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:40:31.145 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:31 smithi135 ceph-mon[100251]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi144=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:31.145 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:31 smithi135 ceph-mon[100251]: Saving service grafana spec with placement smithi144=a;count:1 2023-09-23T19:40:31.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:31 smithi144 conmon[108226]: 2023-09-23T19:40:31.124+0000 7f56eedd11c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:40:31.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:31 smithi144 ceph-mon[104994]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi144=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:31 smithi144 ceph-mon[104994]: Saving service grafana spec with placement smithi144=a;count:1 2023-09-23T19:40:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:31 smithi144 ceph-mon[104994]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T19:40:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:31 smithi144 ceph-mon[104994]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T19:40:31.431 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:31 smithi135 ceph-mon[100251]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T19:40:31.431 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:31 smithi135 ceph-mon[100251]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T19:40:31.432 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:31 smithi135 conmon[100683]: 2023-09-23T19:40:31.171+0000 7f41ac5321c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:40:31.432 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:31 smithi135 ceph-mon[118715]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi144=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:40:31.432 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:31 smithi135 ceph-mon[118715]: Saving service grafana spec with placement smithi144=a;count:1 2023-09-23T19:40:31.432 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:31 smithi135 ceph-mon[118715]: from='mgr.14152 172.21.15.135:0/685681097' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T19:40:31.432 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:31 smithi135 ceph-mon[118715]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T19:40:31.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:31 smithi135 conmon[100683]: 2023-09-23T19:40:31.502+0000 7f41ac5321c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:40:31.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:31 smithi135 conmon[100683]: 2023-09-23T19:40:31.595+0000 7f41ac5321c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:40:31.735 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:31 smithi144 conmon[108226]: 2023-09-23T19:40:31.430+0000 7f56eedd11c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:40:31.735 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:31 smithi144 conmon[108226]: 2023-09-23T19:40:31.516+0000 7f56eedd11c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:40:31.735 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:31 smithi144 conmon[108226]: 2023-09-23T19:40:31.643+0000 7f56eedd11c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:40:32.086 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:31 smithi135 conmon[100683]: 2023-09-23T19:40:31.724+0000 7f41ac5321c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:40:32.086 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:31 smithi135 conmon[100683]: 2023-09-23T19:40:31.815+0000 7f41ac5321c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:40:32.337 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:32 smithi135 conmon[100683]: 2023-09-23T19:40:32.252+0000 7f41ac5321c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:40:32.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:31 smithi144 conmon[108226]: 2023-09-23T19:40:31.735+0000 7f56eedd11c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:40:32.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:32 smithi144 conmon[108226]: 2023-09-23T19:40:32.149+0000 7f56eedd11c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:40:32.872 INFO:teuthology.orchestra.run.smithi135.stdout:[client.0] 2023-09-23T19:40:32.872 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAwPw9lf5HVMxAA/n4Ax2ykrK6nLazWlQaEOw== 2023-09-23T19:40:33.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:32 smithi144 conmon[108226]: 2023-09-23T19:40:32.703+0000 7f56eedd11c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:40:33.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:32 smithi144 conmon[108226]: 2023-09-23T19:40:32.871+0000 7f56eedd11c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:40:33.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:32 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3546222852' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:33.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:32 smithi144 ceph-mon[104994]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:33.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:32 smithi144 ceph-mon[104994]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:40:33.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:32 smithi135 conmon[100683]: 2023-09-23T19:40:32.837+0000 7f41ac5321c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:40:33.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:33 smithi135 conmon[100683]: 2023-09-23T19:40:33.014+0000 7f41ac5321c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:40:33.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:32 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3546222852' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:33.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:32 smithi135 ceph-mon[118715]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:33.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:32 smithi135 ceph-mon[118715]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:40:33.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:32 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3546222852' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:33.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:32 smithi135 ceph-mon[100251]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:33.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:32 smithi135 ceph-mon[100251]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:40:33.535 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:40:33.536 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T19:40:33.536 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T19:40:33.580 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T19:40:33.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:33 smithi144 conmon[108226]: 2023-09-23T19:40:33.630+0000 7f56eedd11c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:40:33.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:33 smithi144 conmon[108226]: 2023-09-23T19:40:33.725+0000 7f56eedd11c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:40:33.934 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:40:34.051 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:33 smithi135 conmon[100683]: 2023-09-23T19:40:33.795+0000 7f41ac5321c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:40:34.051 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:33 smithi135 conmon[100683]: 2023-09-23T19:40:33.885+0000 7f41ac5321c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:40:34.176 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:33 smithi144 conmon[108226]: 2023-09-23T19:40:33.916+0000 7f56eedd11c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:40:34.324 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:34 smithi135 conmon[100683]: 2023-09-23T19:40:34.050+0000 7f41ac5321c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:40:34.506 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:34 smithi144 conmon[108226]: 2023-09-23T19:40:34.210+0000 7f56eedd11c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:40:34.506 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:34 smithi144 conmon[108226]: 2023-09-23T19:40:34.298+0000 7f56eedd11c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:40:34.694 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:34 smithi135 conmon[100683]: 2023-09-23T19:40:34.323+0000 7f41ac5321c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:40:34.694 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:34 smithi135 conmon[100683]: 2023-09-23T19:40:34.409+0000 7f41ac5321c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:40:34.862 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:34 smithi144 conmon[108226]: 2023-09-23T19:40:34.604+0000 7f56eedd11c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:40:34.949 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:34 smithi135 conmon[100683]: 2023-09-23T19:40:34.693+0000 7f41ac5321c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:40:35.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:34 smithi144 conmon[108226]: 2023-09-23T19:40:34.861+0000 7f56eedd11c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:40:35.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:34 smithi135 conmon[100683]: 2023-09-23T19:40:34.949+0000 7f41ac5321c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:40:35.639 INFO:teuthology.orchestra.run.smithi144.stdout:[client.1] 2023-09-23T19:40:35.639 INFO:teuthology.orchestra.run.smithi144.stdout: key = AQAzPw9lwzbuJRAA3Wdcam9ZIuCsYiRHJAPNpQ== 2023-09-23T19:40:35.858 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:35 smithi144 ceph-mon[104994]: from='client.? 172.21.15.144:0/3890922476' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:35.858 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:35 smithi144 ceph-mon[104994]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:35.858 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:35 smithi144 ceph-mon[104994]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:40:35.858 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:35 smithi144 conmon[108226]: 2023-09-23T19:40:35.606+0000 7f56eedd11c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:40:35.858 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:35 smithi144 conmon[108226]: 2023-09-23T19:40:35.757+0000 7f56eedd11c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:40:35.969 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:35 smithi135 conmon[100683]: 2023-09-23T19:40:35.651+0000 7f41ac5321c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:40:35.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:35 smithi135 conmon[100683]: 2023-09-23T19:40:35.781+0000 7f41ac5321c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:40:35.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:35 smithi135 ceph-mon[118715]: from='client.? 172.21.15.144:0/3890922476' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:35.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:35 smithi135 ceph-mon[118715]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:35.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:35 smithi135 ceph-mon[118715]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:40:35.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:35 smithi135 ceph-mon[100251]: from='client.? 172.21.15.144:0/3890922476' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:35.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:35 smithi135 ceph-mon[100251]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T19:40:35.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:35 smithi135 ceph-mon[100251]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T19:40:36.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:35 smithi144 conmon[108226]: 2023-09-23T19:40:35.953+0000 7f56eedd11c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:40:36.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:36 smithi144 conmon[108226]: 2023-09-23T19:40:36.067+0000 7f56eedd11c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:40:36.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:35 smithi135 conmon[100683]: 2023-09-23T19:40:35.969+0000 7f41ac5321c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:40:36.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:36 smithi135 conmon[100683]: 2023-09-23T19:40:36.083+0000 7f41ac5321c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:40:36.250 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2023-09-23T19:40:36.251 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T19:40:36.251 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T19:40:36.291 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T19:40:36.291 DEBUG:teuthology.orchestra.run.smithi135:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T19:40:36.619 INFO:teuthology.misc.health.smithi135.stdout: 2023-09-23T19:40:36.619 INFO:teuthology.misc.health.smithi135.stdout:{"epoch":58,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T19:40:07.151528+0000","last_up_change":"2023-09-23T19:40:03.286566+0000","last_in_change":"2023-09-23T19:39:46.254783+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6803","nonce":1288355238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6805","nonce":1288355238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6809","nonce":1288355238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6807","nonce":1288355238}]},"public_addr":"172.21.15.135:6803/1288355238","cluster_addr":"172.21.15.135:6805/1288355238","heartbeat_back_addr":"172.21.15.135:6809/1288355238","heartbeat_front_addr":"172.21.15.135:6807/1288355238","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6811","nonce":460054379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6813","nonce":460054379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6817","nonce":460054379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6815","nonce":460054379}]},"public_addr":"172.21.15.135:6811/460054379","cluster_addr":"172.21.15.135:6813/460054379","heartbeat_back_addr":"172.21.15.135:6817/460054379","heartbeat_front_addr":"172.21.15.135:6815/460054379","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6819","nonce":1252775382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6821","nonce":1252775382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6825","nonce":1252775382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6823","nonce":1252775382}]},"public_addr":"172.21.15.135:6819/1252775382","cluster_addr":"172.21.15.135:6821/1252775382","heartbeat_back_addr":"172.21.15.135:6825/1252775382","heartbeat_front_addr":"172.21.15.135:6823/1252775382","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6827","nonce":3652622392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6829","nonce":3652622392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6833","nonce":3652622392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6831","nonce":3652622392}]},"public_addr":"172.21.15.135:6827/3652622392","cluster_addr":"172.21.15.135:6829/3652622392","heartbeat_back_addr":"172.21.15.135:6833/3652622392","heartbeat_front_addr":"172.21.15.135:6831/3652622392","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6817","nonce":1530810554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6819","nonce":1530810554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6823","nonce":1530810554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6821","nonce":1530810554}]},"public_addr":"172.21.15.144:6817/1530810554","cluster_addr":"172.21.15.144:6819/1530810554","heartbeat_back_addr":"172.21.15.144:6823/1530810554","heartbeat_front_addr":"172.21.15.144:6821/1530810554","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:40:36.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:36 smithi144 conmon[108226]: 2023-09-23T19:40:36.291+0000 7f56eedd11c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:40:36.632 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T19:40:36.632 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T19:40:36.632 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T19:40:36.655 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:36 smithi135 conmon[100683]: 2023-09-23T19:40:36.297+0000 7f41ac5321c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:40:36.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:36 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2178967215' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:36.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:36 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2178967215' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:36.989 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:37.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:36 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2178967215' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:37.425 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:37 smithi144 conmon[108226]: 2023-09-23T19:40:37.146+0000 7f56eedd11c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:40:37.425 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:37 smithi144 conmon[108226]: [23/Sep/2023:19:40:37] ENGINE Bus STARTING 2023-09-23T19:40:37.425 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:37 smithi144 conmon[108226]: CherryPy Checker: 2023-09-23T19:40:37.425 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:37 smithi144 conmon[108226]: The Application mounted at '' has an empty config. 2023-09-23T19:40:37.425 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:37 smithi144 conmon[108226]: 2023-09-23T19:40:37.426 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:37 smithi144 conmon[108226]: [23/Sep/2023:19:40:37] ENGINE Serving on http://:::9283 2023-09-23T19:40:37.426 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:37 smithi144 conmon[108226]: [23/Sep/2023:19:40:37] ENGINE Bus STARTED 2023-09-23T19:40:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:37 smithi135 conmon[100683]: 2023-09-23T19:40:37.217+0000 7f41ac5321c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:40:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:37 smithi135 conmon[100683]: [23/Sep/2023:19:40:37] ENGINE Bus STARTING 2023-09-23T19:40:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:37 smithi135 conmon[100683]: CherryPy Checker: 2023-09-23T19:40:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:37 smithi135 conmon[100683]: The Application mounted at '' has an empty config. 2023-09-23T19:40:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:37 smithi135 conmon[100683]: 2023-09-23T19:40:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:37 smithi135 conmon[100683]: [23/Sep/2023:19:40:37] ENGINE Serving on http://:::9283 2023-09-23T19:40:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:37 smithi135 conmon[100683]: [23/Sep/2023:19:40:37] ENGINE Bus STARTED 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: Standby manager daemon x restarted 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: Standby manager daemon x started 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: Active manager daemon y restarted 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: Activating manager daemon y 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T19:40:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: mgrmap e17: y(active, starting, since 0.00730622s), standbys: x 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:40:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: Manager daemon y is now available 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:40:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:40:37.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: Standby manager daemon x restarted 2023-09-23T19:40:37.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: Standby manager daemon x started 2023-09-23T19:40:37.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:40:37.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:40:37.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:40:37.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:40:37.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: Active manager daemon y restarted 2023-09-23T19:40:37.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: Activating manager daemon y 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: mgrmap e17: y(active, starting, since 0.00730622s), standbys: x 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:40:37.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: Manager daemon y is now available 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:40:37.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:40:38.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: Standby manager daemon x restarted 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: Standby manager daemon x started 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/386030926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: Active manager daemon y restarted 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: Activating manager daemon y 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: mgrmap e17: y(active, starting, since 0.00730622s), standbys: x 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:40:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:40:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: Manager daemon y is now available 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:40:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:40:39.247 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'rbd' created 2023-09-23T19:40:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:39 smithi144 ceph-mon[104994]: [23/Sep/2023:19:40:37] ENGINE Bus STARTING 2023-09-23T19:40:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:39 smithi144 ceph-mon[104994]: [23/Sep/2023:19:40:37] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:40:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:39 smithi144 ceph-mon[104994]: [23/Sep/2023:19:40:37] ENGINE Bus STARTED 2023-09-23T19:40:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:39 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:39 smithi144 ceph-mon[104994]: mgrmap e18: y(active, since 1.01298s), standbys: x 2023-09-23T19:40:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:39 smithi144 ceph-mon[104994]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:39.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[118715]: [23/Sep/2023:19:40:37] ENGINE Bus STARTING 2023-09-23T19:40:39.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[118715]: [23/Sep/2023:19:40:37] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:40:39.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[118715]: [23/Sep/2023:19:40:37] ENGINE Bus STARTED 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[118715]: mgrmap e18: y(active, since 1.01298s), standbys: x 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[118715]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[100251]: [23/Sep/2023:19:40:37] ENGINE Bus STARTING 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[100251]: [23/Sep/2023:19:40:37] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[100251]: [23/Sep/2023:19:40:37] ENGINE Bus STARTED 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[100251]: mgrmap e18: y(active, since 1.01298s), standbys: x 2023-09-23T19:40:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:39 smithi135 ceph-mon[100251]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:39.975 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T19:40:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2542080114' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T19:40:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2542080114' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2542080114' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2542080114' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2542080114' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T19:40:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2542080114' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T19:40:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:40:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:40.387 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi144 to 4036M 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi144 to 4036M 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:40:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T19:40:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T19:40:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:40:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:40:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi144 to 4036M 2023-09-23T19:40:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:42 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:40:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:42 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:40:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:42 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:40:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:42 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:40:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:42 smithi144 ceph-mon[104994]: Deploying daemon alertmanager.a on smithi135 2023-09-23T19:40:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:42 smithi144 ceph-mon[104994]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:42 smithi144 ceph-mon[104994]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T19:40:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:42 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1519522661' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:40:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:40:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:40:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:40:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[118715]: Deploying daemon alertmanager.a on smithi135 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[118715]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[118715]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1519522661' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[100251]: Deploying daemon alertmanager.a on smithi135 2023-09-23T19:40:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[100251]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:42.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[100251]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T19:40:42.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:42 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1519522661' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:40:43.223 INFO:teuthology.orchestra.run.smithi135.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T19:40:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:44 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1519522661' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:40:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:44 smithi144 ceph-mon[104994]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T19:40:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:44 smithi144 ceph-mon[104994]: pgmap v10: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:44.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:44 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1519522661' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:40:44.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:44 smithi135 ceph-mon[118715]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T19:40:44.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:44 smithi135 ceph-mon[118715]: pgmap v10: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:44.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:44 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1519522661' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:40:44.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:44 smithi135 ceph-mon[100251]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T19:40:44.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:44 smithi135 ceph-mon[100251]: pgmap v10: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:45.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:45 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:40:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:40:45.894 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T19:40:45.894 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T19:40:45.895 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T19:40:46.526 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:46 smithi144 ceph-mon[104994]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:46.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:46 smithi135 ceph-mon[118715]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:46.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:46 smithi135 ceph-mon[100251]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:48.549 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:48 smithi135 ceph-mon[100251]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:48.549 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:48 smithi135 ceph-mon[118715]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:48 smithi144 ceph-mon[104994]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:49.475 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:40:49.766 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:40:49 smithi135 systemd[1]: Started Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:40:49.766 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:40:49 smithi135 conmon[164272]: ts=2023-09-23T19:40:49.765Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:40:50.052 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:40:49 smithi135 conmon[164272]: ts=2023-09-23T19:40:49.766Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:40:50.052 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:40:49 smithi135 conmon[164272]: ts=2023-09-23T19:40:49.769Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T19:40:50.052 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:40:49 smithi135 conmon[164272]: ts=2023-09-23T19:40:49.769Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T19:40:50.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:50 smithi135 conmon[100683]: [23/Sep/2023:19:40:50] ENGINE Bus STOPPING 2023-09-23T19:40:50.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:50 smithi135 conmon[100683]: [23/Sep/2023:19:40:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:40:50.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:50 smithi135 conmon[100683]: [23/Sep/2023:19:40:50] ENGINE Bus STOPPED 2023-09-23T19:40:50.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:50 smithi135 conmon[100683]: [23/Sep/2023:19:40:50] ENGINE Bus STARTING 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:50 smithi135 conmon[100683]: [23/Sep/2023:19:40:50] ENGINE Serving on http://:::9283 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:40:50 smithi135 conmon[100683]: [23/Sep/2023:19:40:50] ENGINE Bus STARTED 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3462984857' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3462984857' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:40:50.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.512 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":19,"active_gid":24359,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":1642850725},{"type":"v1","addr":"172.21.15.135:6801","nonce":1642850725}]},"active_addr":"172.21.15.135:6801/1642850725","active_change":"2023-09-23T19:40:37.221091+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24364,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":3668294279}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":3693941440}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":3516425720}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":4238089144}]}]} 2023-09-23T19:40:50.516 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T19:40:50.516 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T19:40:50.516 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:40:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3462984857' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:40:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:40:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:50.864 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:51.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:51 smithi135 ceph-mon[100251]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:40:51.434 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:51 smithi135 ceph-mon[100251]: Deploying daemon grafana.a on smithi144 2023-09-23T19:40:51.434 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:51 smithi135 ceph-mon[118715]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:40:51.434 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:51 smithi135 ceph-mon[118715]: Deploying daemon grafana.a on smithi144 2023-09-23T19:40:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:51 smithi144 ceph-mon[104994]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T19:40:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:51 smithi144 ceph-mon[104994]: Deploying daemon grafana.a on smithi144 2023-09-23T19:40:52.156 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:40:51 smithi135 conmon[164272]: ts=2023-09-23T19:40:51.737Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000544865s 2023-09-23T19:40:52.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:52 smithi135 ceph-mon[100251]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:52 smithi135 ceph-mon[118715]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:52.544 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:40:52.545 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":63,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T19:40:43.215745+0000","last_up_change":"2023-09-23T19:40:03.286566+0000","last_in_change":"2023-09-23T19:39:46.254783+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:40:39.075611+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6803","nonce":1288355238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6805","nonce":1288355238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6809","nonce":1288355238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6807","nonce":1288355238}]},"public_addr":"172.21.15.135:6803/1288355238","cluster_addr":"172.21.15.135:6805/1288355238","heartbeat_back_addr":"172.21.15.135:6809/1288355238","heartbeat_front_addr":"172.21.15.135:6807/1288355238","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6811","nonce":460054379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6813","nonce":460054379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6817","nonce":460054379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6815","nonce":460054379}]},"public_addr":"172.21.15.135:6811/460054379","cluster_addr":"172.21.15.135:6813/460054379","heartbeat_back_addr":"172.21.15.135:6817/460054379","heartbeat_front_addr":"172.21.15.135:6815/460054379","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6819","nonce":1252775382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6821","nonce":1252775382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6825","nonce":1252775382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6823","nonce":1252775382}]},"public_addr":"172.21.15.135:6819/1252775382","cluster_addr":"172.21.15.135:6821/1252775382","heartbeat_back_addr":"172.21.15.135:6825/1252775382","heartbeat_front_addr":"172.21.15.135:6823/1252775382","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6827","nonce":3652622392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6829","nonce":3652622392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6833","nonce":3652622392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6831","nonce":3652622392}]},"public_addr":"172.21.15.135:6827/3652622392","cluster_addr":"172.21.15.135:6829/3652622392","heartbeat_back_addr":"172.21.15.135:6833/3652622392","heartbeat_front_addr":"172.21.15.135:6831/3652622392","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6817","nonce":1530810554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6819","nonce":1530810554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6823","nonce":1530810554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6821","nonce":1530810554}]},"public_addr":"172.21.15.144:6817/1530810554","cluster_addr":"172.21.15.144:6819/1530810554","heartbeat_back_addr":"172.21.15.144:6823/1530810554","heartbeat_front_addr":"172.21.15.144:6821/1530810554","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:6801/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1005337745":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/775791172":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/173145882":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/3361160768":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:40:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:52 smithi144 ceph-mon[104994]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:53.240 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T19:40:53.240 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:40:53.581 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:53 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4231404519' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:53 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4231404519' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:53.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:40:53.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:40:53.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:53 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4231404519' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:54 smithi144 ceph-mon[104994]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:54.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:54 smithi135 ceph-mon[118715]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:54.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:54 smithi135 ceph-mon[100251]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:55.446 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:40:55.447 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":63,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T19:40:43.215745+0000","last_up_change":"2023-09-23T19:40:03.286566+0000","last_in_change":"2023-09-23T19:39:46.254783+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:40:39.075611+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6803","nonce":1288355238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6805","nonce":1288355238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6809","nonce":1288355238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6807","nonce":1288355238}]},"public_addr":"172.21.15.135:6803/1288355238","cluster_addr":"172.21.15.135:6805/1288355238","heartbeat_back_addr":"172.21.15.135:6809/1288355238","heartbeat_front_addr":"172.21.15.135:6807/1288355238","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6811","nonce":460054379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6813","nonce":460054379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6817","nonce":460054379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6815","nonce":460054379}]},"public_addr":"172.21.15.135:6811/460054379","cluster_addr":"172.21.15.135:6813/460054379","heartbeat_back_addr":"172.21.15.135:6817/460054379","heartbeat_front_addr":"172.21.15.135:6815/460054379","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6819","nonce":1252775382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6821","nonce":1252775382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6825","nonce":1252775382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6823","nonce":1252775382}]},"public_addr":"172.21.15.135:6819/1252775382","cluster_addr":"172.21.15.135:6821/1252775382","heartbeat_back_addr":"172.21.15.135:6825/1252775382","heartbeat_front_addr":"172.21.15.135:6823/1252775382","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6827","nonce":3652622392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6829","nonce":3652622392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6833","nonce":3652622392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6831","nonce":3652622392}]},"public_addr":"172.21.15.135:6827/3652622392","cluster_addr":"172.21.15.135:6829/3652622392","heartbeat_back_addr":"172.21.15.135:6833/3652622392","heartbeat_front_addr":"172.21.15.135:6831/3652622392","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6817","nonce":1530810554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6819","nonce":1530810554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6823","nonce":1530810554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6821","nonce":1530810554}]},"public_addr":"172.21.15.144:6817/1530810554","cluster_addr":"172.21.15.144:6819/1530810554","heartbeat_back_addr":"172.21.15.144:6823/1530810554","heartbeat_front_addr":"172.21.15.144:6821/1530810554","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:6801/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1005337745":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/775791172":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/173145882":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/3361160768":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:40:55.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:40:55 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:40:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:40:56.165 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T19:40:56.165 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T19:40:56.165 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T19:40:56.166 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T19:40:56.166 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T19:40:56.167 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T19:40:56.167 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T19:40:56.167 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T19:40:56.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:56 smithi135 ceph-mon[118715]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:56.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:56 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/620836890' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:56.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:56 smithi135 ceph-mon[100251]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:56.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:56 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/620836890' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:56 smithi144 ceph-mon[104994]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:56 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/620836890' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:40:57.054 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:57.106 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:57.358 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:40:58.387 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:40:58 smithi135 ceph-mon[100251]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:58.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:40:58 smithi135 ceph-mon[118715]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:40:58 smithi144 ceph-mon[104994]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:40:59.224 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:00.153 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:40:59 smithi135 conmon[164272]: ts=2023-09-23T19:40:59.739Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002311196s 2023-09-23T19:41:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:00 smithi135 ceph-mon[118715]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:00 smithi135 ceph-mon[100251]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:00.583 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:00 smithi144 ceph-mon[104994]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:02.309 INFO:teuthology.orchestra.run.smithi135.stdout:77309411365 2023-09-23T19:41:02.310 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T19:41:02.586 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:02 smithi144 ceph-mon[104994]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:02.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:02 smithi135 ceph-mon[118715]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:02.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:02 smithi135 ceph-mon[100251]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:03.012 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:03.046 INFO:teuthology.orchestra.run.smithi135.stdout:197568495634 2023-09-23T19:41:03.046 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T19:41:03.208 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:04.399 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:04 smithi135 ceph-mon[118715]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:04.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:04 smithi135 ceph-mon[100251]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:04 smithi144 ceph-mon[104994]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:05.487 INFO:teuthology.orchestra.run.smithi135.stdout:163208757272 2023-09-23T19:41:05.488 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T19:41:05.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:41:05 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:41:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:41:06.446 INFO:teuthology.orchestra.run.smithi135.stdout:34359738414 2023-09-23T19:41:06.446 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T19:41:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:06 smithi135 ceph-mon[118715]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:06.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:06 smithi135 ceph-mon[100251]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:06 smithi144 ceph-mon[104994]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:07.528 INFO:teuthology.orchestra.run.smithi135.stdout:55834574890 2023-09-23T19:41:07.528 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T19:41:07.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:07 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:07.679 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:07.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:07 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:07 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:08 smithi144 ceph-mon[104994]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:08.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:08 smithi144 systemd[1]: Starting Ceph grafana.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:41:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:08 smithi135 ceph-mon[118715]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:08.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:08 smithi135 ceph-mon[100251]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:08.910 INFO:teuthology.orchestra.run.smithi135.stdout:231928233998 2023-09-23T19:41:08.910 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T19:41:09.166 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:09.193 INFO:teuthology.orchestra.run.smithi135.stdout:128849018908 2023-09-23T19:41:09.193 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T19:41:09.381 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 bash[151692]: 2ca98132db5dcb54eb5fc7917004d37fde6f48c503a5501a8e7eea9b2b636840 2023-09-23T19:41:09.381 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 systemd[1]: Started Ceph grafana.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:41:09.381 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250460035Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250639476Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250656605Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250662955Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250668967Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250675201Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250680647Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250686172Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250691769Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250697223Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T19:41:09.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250702744Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250707862Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250713612Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=settings t=2023-09-23T19:41:09.250719025Z level=info msg="App mode production" 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=sqlstore t=2023-09-23T19:41:09.250772633Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=sqlstore t=2023-09-23T19:41:09.250788566Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.251710033Z level=info msg="Starting DB migrations" 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.251784146Z level=info msg="Executing migration" id="create migration_log table" 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.315217451Z level=info msg="Executing migration" id="create user table" 2023-09-23T19:41:09.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.357040862Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-23T19:41:09.590 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:09.642 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.390639164Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-23T19:41:09.643 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.416010317Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-23T19:41:09.643 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.432976784Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-23T19:41:09.643 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.449706876Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-23T19:41:09.643 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.483350529Z level=info msg="Executing migration" id="create user table v2" 2023-09-23T19:41:09.643 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.500411721Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-23T19:41:09.644 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.517020497Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-23T19:41:09.644 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.53389686Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-23T19:41:09.644 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.550713615Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-23T19:41:09.644 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.575933251Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-23T19:41:09.645 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.60111011Z level=info msg="Executing migration" id="Update user table charset" 2023-09-23T19:41:09.645 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.643203193Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-23T19:41:09.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.668650139Z level=info msg="Executing migration" id="Add missing user data" 2023-09-23T19:41:09.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.685604671Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-23T19:41:09.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.702348719Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-23T19:41:09.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.769446776Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-23T19:41:09.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.786252549Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-23T19:41:09.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.811284412Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-23T19:41:09.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.828086811Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-23T19:41:09.896 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.844992755Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T19:41:09.896 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.861916904Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-23T19:41:09.896 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.895557415Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-23T19:41:10.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.920712314Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-23T19:41:10.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.937783313Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-23T19:41:10.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.954624701Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-23T19:41:10.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:09 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:09.971270492Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-23T19:41:10.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.004930689Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-23T19:41:10.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.021826519Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T19:41:10.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.038710155Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-23T19:41:10.148 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.080569377Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-23T19:41:10.148 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.122398431Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-23T19:41:10.148 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.147622295Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-23T19:41:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[118715]: Deploying daemon node-exporter.a on smithi135 2023-09-23T19:41:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[118715]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[100251]: Deploying daemon node-exporter.a on smithi135 2023-09-23T19:41:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:10 smithi135 ceph-mon[100251]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:10.415 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:10 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.415 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:10 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.415 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:10 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.416 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:10 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:10.416 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:10 smithi144 ceph-mon[104994]: Deploying daemon node-exporter.a on smithi135 2023-09-23T19:41:10.416 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:10 smithi144 ceph-mon[104994]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:10.416 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.23158133Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-23T19:41:10.416 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.256603445Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-23T19:41:10.416 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.281917237Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-23T19:41:10.416 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.323681497Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T19:41:10.416 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.348988626Z level=info msg="Executing migration" id="create star table" 2023-09-23T19:41:10.416 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.390894569Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-23T19:41:10.417 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.416041779Z level=info msg="Executing migration" id="create org table v1" 2023-09-23T19:41:10.683 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.466597698Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-23T19:41:10.684 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.491779938Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-23T19:41:10.684 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.516961984Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-23T19:41:10.684 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.533816478Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T19:41:10.684 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.550689102Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-23T19:41:10.684 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.575913986Z level=info msg="Executing migration" id="Update org table charset" 2023-09-23T19:41:10.684 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.600954985Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-23T19:41:10.684 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.684560084Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-23T19:41:10.721 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:10 smithi135 systemd[1]: Starting Ceph node-exporter.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:41:10.851 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:10.944 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.726718236Z level=info msg="Executing migration" id="create dashboard table" 2023-09-23T19:41:10.944 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.751783147Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-23T19:41:10.944 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.802396534Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-23T19:41:10.944 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.827449133Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-23T19:41:10.944 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.869377979Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T19:41:10.945 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.894611562Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T19:41:10.945 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.919687563Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T19:41:10.945 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.945069212Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-23T19:41:11.229 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:10 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:10.97879842Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-23T19:41:11.229 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.003940244Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T19:41:11.229 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.029051045Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-23T19:41:11.229 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.070926016Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-23T19:41:11.229 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.121145335Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-23T19:41:11.230 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.146239539Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-23T19:41:11.230 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.188110456Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-23T19:41:11.230 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.230046203Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-23T19:41:11.489 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.255462658Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-23T19:41:11.489 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.305660174Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-23T19:41:11.489 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.35607311Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-23T19:41:11.490 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.406205067Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-23T19:41:11.490 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.439787763Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-23T19:41:11.490 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.464914839Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-23T19:41:11.490 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.490117958Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-23T19:41:11.749 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.515440899Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-23T19:41:11.750 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.58232539Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-23T19:41:11.750 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.599214753Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-23T19:41:11.750 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.624381671Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-23T19:41:11.750 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.641222385Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-23T19:41:11.750 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.658110779Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-23T19:41:11.750 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.700092284Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-23T19:41:11.750 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.725378734Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T19:41:11.750 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.750429119Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-23T19:41:12.005 INFO:teuthology.orchestra.run.smithi135.stdout:107374182433 2023-09-23T19:41:12.005 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T19:41:12.043 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.79247706Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T19:41:12.043 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.817887855Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-23T19:41:12.044 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.843061509Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T19:41:12.044 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.884854893Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T19:41:12.044 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.909943045Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-23T19:41:12.044 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.935162546Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T19:41:12.044 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:11 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:11.960488631Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-23T19:41:12.044 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.044169283Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-23T19:41:12.311 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:12 smithi144 ceph-mon[104994]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:12.312 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.069212311Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-23T19:41:12.312 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.11117548Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-23T19:41:12.312 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.136252957Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-23T19:41:12.312 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.161507981Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-23T19:41:12.312 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.203369788Z level=info msg="Executing migration" id="create data_source table" 2023-09-23T19:41:12.312 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.22873651Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-23T19:41:12.312 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.270602919Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-23T19:41:12.565 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.31247111Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-23T19:41:12.565 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.362779957Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T19:41:12.565 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.404923657Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-23T19:41:12.566 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.480170392Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-23T19:41:12.566 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.505372631Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-23T19:41:12.566 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.530354345Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-23T19:41:12.566 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.547406712Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-23T19:41:12.566 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.564055203Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-23T19:41:12.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:12 smithi135 ceph-mon[100251]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:12.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:12 smithi135 ceph-mon[118715]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:12.824 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.589475594Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-23T19:41:12.824 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.606486636Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-23T19:41:12.824 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.623435418Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-23T19:41:12.824 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.648343202Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-23T19:41:12.824 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.682069588Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-23T19:41:12.824 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.698894971Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-23T19:41:12.825 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.715738642Z level=info msg="Executing migration" id="Add uid column" 2023-09-23T19:41:12.825 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.732528Z level=info msg="Executing migration" id="Update uid value" 2023-09-23T19:41:12.825 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.749381773Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-23T19:41:12.825 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.774591129Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-23T19:41:12.825 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.799827665Z level=info msg="Executing migration" id="create api_key table" 2023-09-23T19:41:12.825 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.825076845Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-23T19:41:13.075 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.875417263Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-23T19:41:13.076 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.942258841Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-23T19:41:13.076 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:12 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:12.984238162Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-23T19:41:13.076 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.009453794Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-23T19:41:13.076 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.034728335Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T19:41:13.076 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.076610378Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-23T19:41:13.306 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:13 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:13.307 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:13 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1489909819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:41:13.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:13 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:13.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:13 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1489909819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:41:13.364 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.102058955Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-23T19:41:13.364 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.135820845Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-23T19:41:13.364 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.177691264Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-23T19:41:13.364 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.202832297Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-23T19:41:13.364 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.244793267Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-23T19:41:13.364 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.269971047Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-23T19:41:13.364 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.29519939Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-23T19:41:13.442 INFO:teuthology.orchestra.run.smithi135.stdout:197568495636 2023-09-23T19:41:13.482 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:13.499 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:13.571 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:13 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:13.571 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:13 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1489909819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:41:13.631 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.364279332Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-23T19:41:13.631 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.412688362Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-23T19:41:13.631 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.437836607Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-23T19:41:13.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.463027045Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-23T19:41:13.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.488249133Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-23T19:41:13.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.52995803Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-23T19:41:13.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.555342192Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-23T19:41:13.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.580435741Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-23T19:41:13.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.605637923Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T19:41:13.932 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.649994352Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T19:41:13.932 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.689674134Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T19:41:13.932 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.714697671Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T19:41:13.932 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.739851849Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-23T19:41:13.932 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.765157759Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T19:41:13.932 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.840213111Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-23T19:41:13.932 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.86547135Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T19:41:13.932 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.890828815Z level=info msg="Executing migration" id="create quota table v1" 2023-09-23T19:41:13.933 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.932654348Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T19:41:13.970 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:13 smithi135 bash[170473]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T19:41:14.192 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:13 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:13.983240414Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-23T19:41:14.192 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.008359873Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-23T19:41:14.192 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.033378345Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T19:41:14.192 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.075508951Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-23T19:41:14.192 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.100549293Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-23T19:41:14.192 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.125773947Z level=info msg="Executing migration" id="create session table" 2023-09-23T19:41:14.193 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.150996771Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-23T19:41:14.193 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.192796735Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-23T19:41:14.210 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495634 got 197568495636 for osd.6 2023-09-23T19:41:14.210 DEBUG:teuthology.parallel:result is None 2023-09-23T19:41:14.452 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:14 smithi144 ceph-mon[104994]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:14.453 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.23491013Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-23T19:41:14.453 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.301814817Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-23T19:41:14.453 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.344064902Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-23T19:41:14.453 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.36085825Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-23T19:41:14.453 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.394621655Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-23T19:41:14.453 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.419653089Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-23T19:41:14.453 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.436415012Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-23T19:41:14.453 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.453428574Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-23T19:41:14.468 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:14 smithi135 ceph-mon[118715]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:14.468 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:14 smithi135 bash[170473]: Getting image source signatures 2023-09-23T19:41:14.468 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:14 smithi135 ceph-mon[100251]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:14.619 INFO:teuthology.orchestra.run.smithi135.stdout:163208757274 2023-09-23T19:41:14.720 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:14 smithi135 bash[170473]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T19:41:14.720 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:14 smithi135 bash[170473]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T19:41:14.720 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:14 smithi135 bash[170473]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T19:41:14.746 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.495456298Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-23T19:41:14.746 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.512323448Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-23T19:41:14.746 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.529090105Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-23T19:41:14.747 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.554357451Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-23T19:41:14.747 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.579469298Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-23T19:41:14.747 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.604740584Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-23T19:41:14.747 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.646698497Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-23T19:41:14.747 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.688612419Z level=info msg="Executing migration" id="create alert table v1" 2023-09-23T19:41:14.747 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.747146881Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-23T19:41:14.999 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:15.014 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.789144148Z level=info msg="Executing migration" id="add index alert state" 2023-09-23T19:41:15.014 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.822922834Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-23T19:41:15.014 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.856284879Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-23T19:41:15.014 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.889726116Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T19:41:15.014 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.940006066Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T19:41:15.015 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:14 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:14.97345655Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T19:41:15.015 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.015129144Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-23T19:41:15.195 INFO:teuthology.orchestra.run.smithi135.stdout:34359738416 2023-09-23T19:41:15.233 INFO:teuthology.orchestra.run.smithi135.stdout:77309411368 2023-09-23T19:41:15.266 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.065748016Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T19:41:15.266 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.09091261Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-23T19:41:15.266 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.116060979Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-23T19:41:15.266 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.199744105Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-23T19:41:15.266 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.24164199Z level=info msg="Executing migration" id="Add column is_default" 2023-09-23T19:41:15.266 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.266699237Z level=info msg="Executing migration" id="Add column frequency" 2023-09-23T19:41:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:15 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4252664985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:41:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:15 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2737565770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:41:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:15 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1643766748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:41:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:15 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4252664985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:41:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:15 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2737565770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:41:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:15 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1643766748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:41:15.526 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:41:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:41:15.526 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:15 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4252664985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:41:15.526 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:15 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2737565770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:41:15.526 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:15 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1643766748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:41:15.527 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.308684508Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-23T19:41:15.527 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.350653989Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-23T19:41:15.527 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.392458978Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-23T19:41:15.527 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.434444172Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-23T19:41:15.527 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.459599194Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-23T19:41:15.527 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.484852589Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-23T19:41:15.527 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.526675468Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T19:41:15.794 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.594285973Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-23T19:41:15.794 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.669477584Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-23T19:41:15.794 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.69465679Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T19:41:15.795 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.719647345Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-23T19:41:15.795 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.744872382Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-23T19:41:15.795 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.770036389Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-23T19:41:15.795 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.795295136Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-23T19:41:16.058 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757272 got 163208757274 for osd.5 2023-09-23T19:41:16.059 DEBUG:teuthology.parallel:result is None 2023-09-23T19:41:16.095 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.812227485Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-23T19:41:16.095 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.828955259Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-23T19:41:16.095 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.85425679Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-23T19:41:16.095 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.871023164Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T19:41:16.096 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.887904734Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T19:41:16.096 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.904652292Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-23T19:41:16.096 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.92165959Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-23T19:41:16.096 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.938358005Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-23T19:41:16.096 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:15 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:15.980242776Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-23T19:41:16.096 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.005573063Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-23T19:41:16.096 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.030813436Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-23T19:41:16.096 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.096177785Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-23T19:41:16.319 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:16 smithi135 bash[170473]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T19:41:16.319 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:16 smithi135 bash[170473]: Writing manifest to image destination 2023-09-23T19:41:16.319 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:16 smithi135 bash[170473]: Storing signatures 2023-09-23T19:41:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:16 smithi144 ceph-mon[104994]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:16.381 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.13142638Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-23T19:41:16.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.1563377Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-23T19:41:16.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.181596993Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-23T19:41:16.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.223526155Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-23T19:41:16.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.248744775Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-23T19:41:16.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.273889647Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T19:41:16.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.299051895Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T19:41:16.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.34095465Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T19:41:16.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:16 smithi135 ceph-mon[118715]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:16.572 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:16 smithi135 ceph-mon[100251]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:16.616 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:16.634 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.382846668Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-23T19:41:16.634 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.425204956Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T19:41:16.634 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.44209791Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-23T19:41:16.634 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.467364583Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-23T19:41:16.634 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.509149974Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-23T19:41:16.634 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.584305102Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-23T19:41:16.634 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.609523261Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-23T19:41:16.634 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.634783304Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-23T19:41:16.894 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.676938712Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-23T19:41:16.894 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.727457044Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-23T19:41:16.894 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.752533944Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-23T19:41:16.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.794386313Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-23T19:41:16.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.819549234Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-23T19:41:16.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.844722063Z level=info msg="Executing migration" id="Move region to single row" 2023-09-23T19:41:16.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.870011201Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-23T19:41:16.895 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.895159592Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T19:41:17.146 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.920550152Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.945770802Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:16 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:16.979383226Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.021267554Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.038030883Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.071670222Z level=info msg="Executing migration" id="create test_data table" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.088615665Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.113806493Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.130520076Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T19:41:17.147 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.147396725Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-23T19:41:17.415 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.189479287Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-23T19:41:17.415 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.214837789Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-23T19:41:17.415 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.25652683Z level=info msg="Executing migration" id="create team table" 2023-09-23T19:41:17.415 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.281832819Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-23T19:41:17.415 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.323562707Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-23T19:41:17.416 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.365667334Z level=info msg="Executing migration" id="create team member table" 2023-09-23T19:41:17.416 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.415804151Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-23T19:41:17.666 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.499598718Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T19:41:17.666 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.533195626Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-23T19:41:17.666 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.566663272Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-23T19:41:17.666 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.600054994Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-23T19:41:17.666 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.633411476Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-23T19:41:17.666 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.667030419Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-23T19:41:17.967 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.717303181Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-23T19:41:17.968 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.742540775Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T19:41:17.968 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.767733389Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T19:41:17.968 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.792967169Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-23T19:41:17.968 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.818020041Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-23T19:41:17.968 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.843242059Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-23T19:41:17.968 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.885098554Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-23T19:41:17.968 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:17 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:17.968448239Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-23T19:41:18.236 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.002524098Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-23T19:41:18.236 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.027769612Z level=info msg="Executing migration" id="create tag table" 2023-09-23T19:41:18.237 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.069740932Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-23T19:41:18.237 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.094780296Z level=info msg="Executing migration" id="create login attempt table" 2023-09-23T19:41:18.237 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.119884753Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-23T19:41:18.237 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.144999613Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-23T19:41:18.237 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.170325746Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T19:41:18.237 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.212213979Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-23T19:41:18.237 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.237341934Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-23T19:41:18.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:18 smithi135 ceph-mon[118715]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:18 smithi135 ceph-mon[100251]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:18.496 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:18 smithi144 ceph-mon[104994]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:18.497 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.287911001Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-23T19:41:18.497 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.3295913Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-23T19:41:18.497 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.355064652Z level=info msg="Executing migration" id="create user auth table" 2023-09-23T19:41:18.497 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.430244441Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T19:41:18.497 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.472134678Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-23T19:41:18.497 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.497348438Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-23T19:41:18.587 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411368 for osd.2 2023-09-23T19:41:18.588 DEBUG:teuthology.parallel:result is None 2023-09-23T19:41:18.781 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.522569342Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.547834465Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.573019943Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.598080234Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.623352363Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.648341885Z level=info msg="Executing migration" id="create server_lock table" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.690305438Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.715398315Z level=info msg="Executing migration" id="create user auth token table" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.74066694Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-23T19:41:18.782 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.782473757Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-23T19:41:19.033 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.859196135Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-23T19:41:19.033 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.908315249Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-23T19:41:19.033 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.933363896Z level=info msg="Executing migration" id="create cache_data table" 2023-09-23T19:41:19.034 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.958713994Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-23T19:41:19.034 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:18 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:18.983939684Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-23T19:41:19.034 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.009041495Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-23T19:41:19.034 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.034174253Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-23T19:41:19.055 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738416 for osd.0 2023-09-23T19:41:19.055 DEBUG:teuthology.parallel:result is None 2023-09-23T19:41:19.343 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.076061894Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-23T19:41:19.343 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.101177748Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-23T19:41:19.343 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.126431807Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-23T19:41:19.344 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.151635252Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-23T19:41:19.344 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.193389105Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T19:41:19.344 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.218601743Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-23T19:41:19.344 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.269057312Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-23T19:41:19.344 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.344250662Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-23T19:41:19.603 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.394818631Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T19:41:19.603 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.419982889Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-23T19:41:19.604 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.461796899Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-23T19:41:19.604 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.486977815Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-23T19:41:19.604 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.512478582Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-23T19:41:19.604 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.537421191Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T19:41:19.604 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.562437375Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T19:41:19.604 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.604339431Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T19:41:19.855 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.654933171Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-23T19:41:19.855 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.679984326Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-23T19:41:19.855 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.705269815Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T19:41:19.855 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.730239314Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T19:41:19.855 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.805663264Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-23T19:41:19.855 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.830616999Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T19:41:19.856 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.855913739Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-23T19:41:20.124 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.914610344Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T19:41:20.124 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.939927008Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-23T19:41:20.124 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.965041656Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T19:41:20.124 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:19 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:19.990131615Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-23T19:41:20.124 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.015353336Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-23T19:41:20.124 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.04044971Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-23T19:41:20.124 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.099021626Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-23T19:41:20.125 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.124182456Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-23T19:41:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:20 smithi144 ceph-mon[104994]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:20.381 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.149551498Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T19:41:20.381 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.199619678Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T19:41:20.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.274968411Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-23T19:41:20.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.308575829Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-23T19:41:20.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.358646714Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-23T19:41:20.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:20 smithi135 ceph-mon[118715]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:20.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:20 smithi135 ceph-mon[100251]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:20.727 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.417420239Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T19:41:20.727 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.459333813Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T19:41:20.727 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.484392903Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-23T19:41:20.727 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.509756447Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-23T19:41:20.727 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.551492031Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T19:41:20.727 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.576771565Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-23T19:41:20.727 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.601899684Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-23T19:41:20.727 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.643951481Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-23T19:41:20.728 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.727656338Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-23T19:41:20.794 INFO:teuthology.orchestra.run.smithi135.stdout:55834574893 2023-09-23T19:41:20.832 INFO:teuthology.orchestra.run.smithi135.stdout:231928234001 2023-09-23T19:41:20.987 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.753037751Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T19:41:20.987 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.794823283Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T19:41:20.987 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.820029142Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T19:41:20.987 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.845195379Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-23T19:41:20.987 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.870481291Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-23T19:41:20.988 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.912307779Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-23T19:41:20.988 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.937430392Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-23T19:41:20.988 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.962594798Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-23T19:41:20.988 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:20 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:20.987841256Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-23T19:41:21.247 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:21 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2032191941' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:41:21.247 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:21 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3118349437' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:41:21.247 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.029972486Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-23T19:41:21.247 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.071939564Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-23T19:41:21.247 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.097011867Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T19:41:21.248 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.122299112Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-23T19:41:21.248 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.197674534Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-23T19:41:21.248 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.222721622Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-23T19:41:21.248 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.247905704Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-23T19:41:21.429 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:21 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2032191941' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:41:21.429 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:21 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3118349437' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:41:21.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:21 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2032191941' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:41:21.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:21 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3118349437' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:41:21.430 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.002Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T19:41:21.430 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.002Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T19:41:21.430 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.002Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T19:41:21.430 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.002Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T19:41:21.430 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.002Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T19:41:21.430 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T19:41:21.430 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T19:41:21.430 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T19:41:21.431 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T19:41:21.432 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T19:41:21.433 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T19:41:21.434 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T19:41:21.435 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T19:41:21.435 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T19:41:21.435 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 conmon[171659]: ts=2023-09-23T19:41:21.003Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T19:41:21.507 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.298269559Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-23T19:41:21.507 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.340278255Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-23T19:41:21.507 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.381948638Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-23T19:41:21.507 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.424008081Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-23T19:41:21.507 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.465915289Z level=info msg="Executing migration" id="create alert_image table" 2023-09-23T19:41:21.507 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.491080217Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-23T19:41:21.507 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.507940254Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-23T19:41:21.767 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.541678701Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-23T19:41:21.767 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.558544807Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.59203154Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.592161468Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.659044115Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.675832644Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.701056076Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.701279251Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.701446651Z level=info msg="alerts found to migrate" alerts=0 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.743103974Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-23T19:41:21.768 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.768290249Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-23T19:41:21.843 INFO:teuthology.orchestra.run.smithi135.stdout:128849018911 2023-09-23T19:41:22.028 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.81897891Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-23T19:41:22.028 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.844375212Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T19:41:22.028 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.86945043Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-23T19:41:22.028 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.89468535Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-23T19:41:22.028 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.936469733Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-23T19:41:22.028 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:21 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:21.978351602Z level=info msg="Executing migration" id="create data_keys table" 2023-09-23T19:41:22.028 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.028648927Z level=info msg="Executing migration" id="create secrets table" 2023-09-23T19:41:22.160 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 bash[170473]: b3c82b1cd135126be94d69510af032d7ef9e752e199395bd3e063166a45739cd 2023-09-23T19:41:22.160 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:41:21 smithi135 systemd[1]: Started Ceph node-exporter.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:41:22.279 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.096274552Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-23T19:41:22.280 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.129145251Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-23T19:41:22.280 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.170992482Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-23T19:41:22.280 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.213009212Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-23T19:41:22.280 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:22 smithi144 ceph-mon[104994]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:22.280 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:22 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3167368337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:41:22.280 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:22 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.280 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:22 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.280 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:22 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.342 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233998 got 231928234001 for osd.7 2023-09-23T19:41:22.342 DEBUG:teuthology.parallel:result is None 2023-09-23T19:41:22.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[118715]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:22.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3167368337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:41:22.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[100251]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:22.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3167368337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:41:22.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:22 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:22.573 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.279907942Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-23T19:41:22.573 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.330462054Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-23T19:41:22.573 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.355689179Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-23T19:41:22.573 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.397931184Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T19:41:22.573 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.439728439Z level=info msg="Executing migration" id="create permission table" 2023-09-23T19:41:22.574 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.465070696Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-23T19:41:22.574 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.490050282Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-23T19:41:22.645 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574893 for osd.1 2023-09-23T19:41:22.645 DEBUG:teuthology.parallel:result is None 2023-09-23T19:41:22.841 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:22 smithi144 systemd[1]: Starting Ceph node-exporter.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.573869659Z level=info msg="Executing migration" id="create role table" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.624264418Z level=info msg="Executing migration" id="add column display_name" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.649406297Z level=info msg="Executing migration" id="add column group_name" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.691261561Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.716546208Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.741712459Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.76690596Z level=info msg="Executing migration" id="create team role table" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.792049544Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.81722766Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T19:41:22.842 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.842406626Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-23T19:41:23.110 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:22 smithi144 bash[152162]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T19:41:23.110 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.89298714Z level=info msg="Executing migration" id="create user role table" 2023-09-23T19:41:23.110 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:22 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:22.935037929Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-23T19:41:23.110 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.010150623Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T19:41:23.110 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.035304702Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-23T19:41:23.111 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.077156993Z level=info msg="Executing migration" id="create builtin role table" 2023-09-23T19:41:23.111 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.111050428Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-23T19:41:23.319 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018911 for osd.4 2023-09-23T19:41:23.320 DEBUG:teuthology.parallel:result is None 2023-09-23T19:41:23.370 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:23 smithi144 ceph-mon[104994]: Deploying daemon node-exporter.b on smithi144 2023-09-23T19:41:23.370 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:23 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:23.370 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.152948992Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-23T19:41:23.370 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.194980499Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-23T19:41:23.370 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.220007531Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-23T19:41:23.370 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.262153336Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T19:41:23.370 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.304033481Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-23T19:41:23.371 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.345760722Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-23T19:41:23.371 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.370934631Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-23T19:41:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:23 smithi135 ceph-mon[118715]: Deploying daemon node-exporter.b on smithi144 2023-09-23T19:41:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:23 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:23.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:23 smithi135 ceph-mon[100251]: Deploying daemon node-exporter.b on smithi144 2023-09-23T19:41:23.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:23 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:23.625 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:23 smithi144 bash[152162]: Getting image source signatures 2023-09-23T19:41:23.625 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:23 smithi144 bash[152162]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T19:41:23.625 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.396412532Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-23T19:41:23.625 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.471747624Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-23T19:41:23.625 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.488397591Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-23T19:41:23.625 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.52212542Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-23T19:41:23.625 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.584148765Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-23T19:41:23.679 INFO:teuthology.orchestra.run.smithi135.stdout:107374182435 2023-09-23T19:41:23.881 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:23 smithi144 bash[152162]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T19:41:23.881 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:23 smithi144 bash[152162]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T19:41:23.881 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.664354154Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-23T19:41:23.881 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.734144479Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-23T19:41:23.882 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.790388097Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-23T19:41:24.176 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.89619238Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-23T19:41:24.177 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:23 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:23.984172214Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-23T19:41:24.177 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.041536129Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-23T19:41:24.177 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.125229717Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-23T19:41:24.417 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182433 got 107374182435 for osd.3 2023-09-23T19:41:24.418 DEBUG:teuthology.parallel:result is None 2023-09-23T19:41:24.418 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T19:41:24.418 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:41:24.434 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:24 smithi144 bash[152162]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T19:41:24.434 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:24 smithi144 bash[152162]: Writing manifest to image destination 2023-09-23T19:41:24.434 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:24 smithi144 bash[152162]: Storing signatures 2023-09-23T19:41:24.435 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:24 smithi144 ceph-mon[104994]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:24.435 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:24 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1639811161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:41:24.435 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.204645455Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-23T19:41:24.435 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.275912853Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-23T19:41:24.435 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.384618938Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-23T19:41:24.436 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.435341421Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-23T19:41:24.444 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:24 smithi135 ceph-mon[118715]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:24.444 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:24 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1639811161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:41:24.445 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:24 smithi135 ceph-mon[100251]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:24.445 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:24 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1639811161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:41:24.688 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.461004986Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-23T19:41:24.688 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.503083757Z level=info msg="Executing migration" id="add correlation config column" 2023-09-23T19:41:24.688 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.579054709Z level=info msg="Executing migration" id="create entity_events table" 2023-09-23T19:41:24.688 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.612826542Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-23T19:41:24.688 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.646879851Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T19:41:24.688 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.646982945Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T19:41:24.688 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.68892353Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T19:41:24.762 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:24.955 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.689100194Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T19:41:24.955 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.74126825Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-23T19:41:24.955 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.813123226Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-23T19:41:24.955 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.8967133Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-23T19:41:24.955 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:24 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:24.955856243Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T19:41:25.274 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.014891821Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-23T19:41:25.275 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.05695232Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T19:41:25.275 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.09890199Z level=info msg="Executing migration" id="Drop public config table" 2023-09-23T19:41:25.275 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.140884279Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-23T19:41:25.275 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.191226583Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-23T19:41:25.594 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:41:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:41:25.594 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.308695212Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T19:41:25.595 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.368030078Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-23T19:41:25.595 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.443436706Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-23T19:41:25.595 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.519114822Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-23T19:41:25.881 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.595100844Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-23T19:41:25.881 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.679523684Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-23T19:41:25.881 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.771971143Z level=info msg="Executing migration" id="add share column" 2023-09-23T19:41:25.881 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.839075506Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-23T19:41:26.181 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.915259084Z level=info msg="Executing migration" id="create file table" 2023-09-23T19:41:26.181 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:25 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:25.990900391Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-23T19:41:26.181 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.049938524Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-23T19:41:26.182 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.117098973Z level=info msg="Executing migration" id="create file_meta table" 2023-09-23T19:41:26.182 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.159248536Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-23T19:41:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:26 smithi135 ceph-mon[118715]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:26 smithi135 ceph-mon[100251]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:26.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:26 smithi144 ceph-mon[104994]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:26.486 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.234798485Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-23T19:41:26.486 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.283474539Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-23T19:41:26.486 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.330950955Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-23T19:41:26.486 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.385893379Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-23T19:41:26.486 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.428040329Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-23T19:41:26.487 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.470131675Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-23T19:41:26.487 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.48687448Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-23T19:41:26.487 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.305Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T19:41:26.487 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.305Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T19:41:26.487 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.306Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T19:41:26.487 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.306Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T19:41:26.487 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T19:41:26.488 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T19:41:26.489 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T19:41:26.490 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.307Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T19:41:26.491 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.308Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T19:41:26.492 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[152239]: ts=2023-09-23T19:41:26.308Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T19:41:26.492 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 bash[152162]: d31dbba1772d837c35d669944934aaaa4b63d04aeb43dabd1f12d9092d058c49 2023-09-23T19:41:26.492 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:41:26 smithi144 systemd[1]: Started Ceph node-exporter.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:41:26.561 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:41:26.563 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:41:26.755 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.540938946Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-23T19:41:26.755 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.571405253Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-23T19:41:26.755 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.588257955Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-23T19:41:26.756 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.655288389Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-23T19:41:26.756 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.688916887Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-23T19:41:26.756 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.714084438Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-23T19:41:26.756 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.756080642Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-23T19:41:27.016 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.80652877Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-23T19:41:27.016 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.831653007Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-23T19:41:27.016 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.856868414Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-23T19:41:27.016 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.882070094Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-23T19:41:27.016 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.923971612Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-23T19:41:27.016 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.965915375Z level=info msg="Executing migration" id="create folder table" 2023-09-23T19:41:27.017 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:26 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:26.991146008Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-23T19:41:27.017 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:27.016699076Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-23T19:41:27.292 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":31,"stamp":"2023-09-23T19:41:25.242559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50356,"kb_used_data":6644,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910860,"statfs":{"total":767960285184,"available":767908720640,"internally_reserved":0,"allocated":6803456,"data_stored":3875182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002289"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775498+0000","last_change":"2023-09-23T19:40:40.249273+0000","last_active":"2023-09-23T19:40:41.775498+0000","last_peered":"2023-09-23T19:40:41.775498+0000","last_clean":"2023-09-23T19:40:41.775498+0000","last_became_active":"2023-09-23T19:40:40.248979+0000","last_became_peered":"2023-09-23T19:40:40.248979+0000","last_unstale":"2023-09-23T19:40:41.775498+0000","last_undegraded":"2023-09-23T19:40:41.775498+0000","last_fullsized":"2023-09-23T19:40:41.775498+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:25.089853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.250358+0000","last_change":"2023-09-23T19:40:40.251554+0000","last_active":"2023-09-23T19:40:41.250358+0000","last_peered":"2023-09-23T19:40:41.250358+0000","last_clean":"2023-09-23T19:40:41.250358+0000","last_became_active":"2023-09-23T19:40:40.251284+0000","last_became_peered":"2023-09-23T19:40:40.251284+0000","last_unstale":"2023-09-23T19:40:41.250358+0000","last_undegraded":"2023-09-23T19:40:41.250358+0000","last_fullsized":"2023-09-23T19:40:41.250358+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:40.680795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775425+0000","last_change":"2023-09-23T19:40:40.361547+0000","last_active":"2023-09-23T19:40:41.775425+0000","last_peered":"2023-09-23T19:40:41.775425+0000","last_clean":"2023-09-23T19:40:41.775425+0000","last_became_active":"2023-09-23T19:40:40.361206+0000","last_became_peered":"2023-09-23T19:40:40.361206+0000","last_unstale":"2023-09-23T19:40:41.775425+0000","last_undegraded":"2023-09-23T19:40:41.775425+0000","last_fullsized":"2023-09-23T19:40:41.775425+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:27.379089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775400+0000","last_change":"2023-09-23T19:40:40.361627+0000","last_active":"2023-09-23T19:40:41.775400+0000","last_peered":"2023-09-23T19:40:41.775400+0000","last_clean":"2023-09-23T19:40:41.775400+0000","last_became_active":"2023-09-23T19:40:40.361250+0000","last_became_peered":"2023-09-23T19:40:40.361250+0000","last_unstale":"2023-09-23T19:40:41.775400+0000","last_undegraded":"2023-09-23T19:40:41.775400+0000","last_fullsized":"2023-09-23T19:40:41.775400+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:04.232689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775501+0000","last_change":"2023-09-23T19:40:07.160277+0000","last_active":"2023-09-23T19:40:41.775501+0000","last_peered":"2023-09-23T19:40:41.775501+0000","last_clean":"2023-09-23T19:40:41.775501+0000","last_became_active":"2023-09-23T19:40:07.159949+0000","last_became_peered":"2023-09-23T19:40:07.159949+0000","last_unstale":"2023-09-23T19:40:41.775501+0000","last_undegraded":"2023-09-23T19:40:41.775501+0000","last_fullsized":"2023-09-23T19:40:41.775501+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:20.218507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775366+0000","last_change":"2023-09-23T19:40:40.249857+0000","last_active":"2023-09-23T19:40:41.775366+0000","last_peered":"2023-09-23T19:40:41.775366+0000","last_clean":"2023-09-23T19:40:41.775366+0000","last_became_active":"2023-09-23T19:40:40.249587+0000","last_became_peered":"2023-09-23T19:40:40.249587+0000","last_unstale":"2023-09-23T19:40:41.775366+0000","last_undegraded":"2023-09-23T19:40:41.775366+0000","last_fullsized":"2023-09-23T19:40:41.775366+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:37.624835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.249565+0000","last_change":"2023-09-23T19:40:40.361713+0000","last_active":"2023-09-23T19:40:41.249565+0000","last_peered":"2023-09-23T19:40:41.249565+0000","last_clean":"2023-09-23T19:40:41.249565+0000","last_became_active":"2023-09-23T19:40:40.361369+0000","last_became_peered":"2023-09-23T19:40:40.361369+0000","last_unstale":"2023-09-23T19:40:41.249565+0000","last_undegraded":"2023-09-23T19:40:41.249565+0000","last_fullsized":"2023-09-23T19:40:41.249565+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:04.414357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775453+0000","last_change":"2023-09-23T19:40:40.251667+0000","last_active":"2023-09-23T19:40:41.775453+0000","last_peered":"2023-09-23T19:40:41.775453+0000","last_clean":"2023-09-23T19:40:41.775453+0000","last_became_active":"2023-09-23T19:40:40.251366+0000","last_became_peered":"2023-09-23T19:40:40.251366+0000","last_unstale":"2023-09-23T19:40:41.775453+0000","last_undegraded":"2023-09-23T19:40:41.775453+0000","last_fullsized":"2023-09-23T19:40:41.775453+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:38.869721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.250491+0000","last_change":"2023-09-23T19:40:40.361619+0000","last_active":"2023-09-23T19:40:41.250491+0000","last_peered":"2023-09-23T19:40:41.250491+0000","last_clean":"2023-09-23T19:40:41.250491+0000","last_became_active":"2023-09-23T19:40:40.361237+0000","last_became_peered":"2023-09-23T19:40:40.361237+0000","last_unstale":"2023-09-23T19:40:41.250491+0000","last_undegraded":"2023-09-23T19:40:41.250491+0000","last_fullsized":"2023-09-23T19:40:41.250491+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:09.643497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234002,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6320,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738832,"statfs":{"total":95995035648,"available":95988563968,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.435}]},{"osd":2,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.48899999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.63100000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.72599999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6380,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738772,"statfs":{"total":95995035648,"available":95988502528,"internally_reserved":0,"allocated":1216512,"data_stored":853061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.54700000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.625}]},{"osd":3,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.64800000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.68000000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6512,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738640,"statfs":{"total":95995035648,"available":95988367360,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.33500000000000002},"min":{"1min":0.24199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.60099999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.36199999999999999,"15min":0.34399999999999997},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":0.55600000000000005,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.626}]},{"osd":2,"last update":"Sat Sep 23 19:41:13 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.38600000000000001,"15min":0.38800000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56100000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41899999999999998,"15min":0.42199999999999999},"min":{"1min":0.27800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.53000000000000003,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.67900000000000005,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.626,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.439}]},{"osd":4,"last update":"Sat Sep 23 19:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.58499999999999996,"15min":0.59299999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72099999999999997,"5min":0.87,"15min":0.87},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.59999999999999998,"15min":0.61099999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79600000000000004,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Sep 23 19:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.60099999999999998,"15min":0.60999999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76200000000000001,"5min":0.871,"15min":0.871},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81799999999999995,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.58899999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7152,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738000,"statfs":{"total":95995035648,"available":95987712000,"internally_reserved":0,"allocated":1220608,"data_stored":853326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.33300000000000002,"15min":0.313},"min":{"1min":0.26500000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.53900000000000003,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.36099999999999999,"15min":0.34100000000000003},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54000000000000004,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.38300000000000001,"15min":0.38100000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.45700000000000002,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39200000000000002,"15min":0.39100000000000001},"min":{"1min":0.27600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55200000000000005,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.38}]},{"osd":3,"last update":"Sat Sep 23 19:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.47499999999999998,"15min":0.48099999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.51300000000000001,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.47199999999999998,"15min":0.47899999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.57499999999999996,"5min":0.627,"15min":0.627},"last":0.29999999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56999999999999995,"15min":0.57499999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68400000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.505},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56999999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55100000000000005,"15min":0.55300000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.57699999999999996,"15min":0.57899999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.74099999999999999,"5min":0.749,"15min":0.749},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.54500000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.621}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6448,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738704,"statfs":{"total":95995035648,"available":95988432896,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.40400000000000003,"15min":0.35399999999999998},"min":{"1min":0.247,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.5629999999999997,"5min":5.5629999999999997,"15min":5.5629999999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.996,"5min":0.496,"15min":0.40699999999999997},"min":{"1min":0.28299999999999997,"5min":0.216,"15min":0.216},"max":{"1min":10.811,"5min":10.811,"15min":10.811},"last":0.36699999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:41:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.46400000000000002,"15min":0.41899999999999998},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.29,"5min":5.29,"15min":5.29},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":0.55300000000000005,"15min":0.44800000000000001},"min":{"1min":0.30599999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":11.986000000000001,"5min":11.986000000000001,"15min":11.986000000000001},"last":0.35899999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42999999999999999,"15min":0.432},"min":{"1min":0.316,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57299999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.34000000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.56399999999999995,"15min":0.56200000000000006},"min":{"1min":0.44400000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.63400000000000001,"15min":0.58999999999999997},"min":{"1min":0.40999999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":5.1340000000000003,"5min":5.1340000000000003,"15min":5.1340000000000003},"last":0.71199999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.64300000000000002,"15min":0.60899999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":5.1920000000000002,"5min":5.1920000000000002,"15min":5.1920000000000002},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.81599999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:40:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:41:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":5.2629999999999999,"5min":5.2629999999999999,"15min":5.2629999999999999},"last":0.69799999999999995}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.36399999999999999,"15min":0.36199999999999999},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.252},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.42099999999999999,"15min":0.42999999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.52300000000000002,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.224}]},{"osd":1,"last update":"Sat Sep 23 19:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.42799999999999999,"15min":0.432},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.434,"15min":0.439},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.56799999999999995,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.27400000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.48799999999999999,"15min":0.495},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58399999999999996,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.313},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.45900000000000002,"15min":0.46700000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.53400000000000003,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52300000000000002,"15min":0.52100000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53800000000000003,"15min":0.53500000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.39700000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.56599999999999995,"15min":0.56200000000000006},"min":{"1min":0.42899999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:41:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.47199999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5868,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739284,"statfs":{"total":95995035648,"available":95989026816,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48899999999999999,"15min":0.48999999999999999},"min":{"1min":0.28699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.61799999999999999,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.50700000000000001,"15min":0.50800000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.503,"15min":0.502},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.46899999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.40799999999999997,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.498,"15min":0.49399999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.52800000000000002,"15min":0.52200000000000002},"min":{"1min":0.38100000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.35599999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.40799999999999997,"15min":0.39700000000000002},"min":{"1min":0.30099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.76600000000000001,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.40899999999999997,"15min":0.39100000000000001},"min":{"1min":0.36199999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72299999999999998,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:40:40 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.626}]},{"osd":7,"last update":"Sat Sep 23 19:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.63900000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5804,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739348,"statfs":{"total":95995035648,"available":95989092352,"internally_reserved":0,"allocated":626688,"data_stored":262900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.54600000000000004,"15min":0.55500000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.72799999999999998,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.498,"15min":0.496},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.44800000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65800000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.32200000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.51300000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77400000000000002,"5min":0.874,"15min":0.874},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.40500000000000003,"15min":0.39200000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.41799999999999998,"15min":0.40600000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 19:40:40 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.496}]},{"osd":7,"last update":"Sat Sep 23 19:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.505}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:41:27.295 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:41:27.333 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:27.092259659Z level=info msg="Executing migration" id="Update folder title length" 2023-09-23T19:41:27.333 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:27.158742878Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-23T19:41:27.333 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=migrator t=2023-09-23T19:41:27.234056426Z level=info msg="migrations completed" performed=485 skipped=0 duration=17.982288524s 2023-09-23T19:41:27.333 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=sqlstore t=2023-09-23T19:41:27.234646962Z level=info msg="Created default organization" 2023-09-23T19:41:27.631 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=plugin.loader t=2023-09-23T19:41:27.3335615Z level=info msg="Plugin registered" pluginID=input 2023-09-23T19:41:27.631 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=plugin.loader t=2023-09-23T19:41:27.361914665Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T19:41:27.631 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=plugin.loader t=2023-09-23T19:41:27.361946397Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T19:41:27.631 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=secrets t=2023-09-23T19:41:27.361986095Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T19:41:27.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=query_data t=2023-09-23T19:41:27.363808641Z level=info msg="Query Service initialization" 2023-09-23T19:41:27.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:27 smithi144 conmon[151733]: logger=live.push_http t=2023-09-23T19:41:27.368151319Z level=info msg="Live Push Gateway initialization" 2023-09-23T19:41:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:27 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:27 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:27 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:27 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:27 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:27 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:27 smithi144 ceph-mon[104994]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:41:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:27 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[118715]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[100251]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:41:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:27 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:27.967 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:28.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:28 smithi144 ceph-mon[104994]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=infra.usagestats.collector t=2023-09-23T19:41:28.222406165Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=provisioning.datasources t=2023-09-23T19:41:28.222837187Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=provisioning.datasources t=2023-09-23T19:41:28.256519952Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=provisioning.datasources t=2023-09-23T19:41:28.339128419Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=provisioning.alerting t=2023-09-23T19:41:28.396124418Z level=info msg="starting to provision alerting" 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=provisioning.alerting t=2023-09-23T19:41:28.396154731Z level=info msg="finished to provision alerting" 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=ngalert.state.manager t=2023-09-23T19:41:28.396253918Z level=info msg="Warming state cache for startup" 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=grafanaStorageLogger t=2023-09-23T19:41:28.396271286Z level=info msg="storage starting" 2023-09-23T19:41:28.587 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=http.server t=2023-09-23T19:41:28.39896153Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T19:41:28.588 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=sqlstore.transactions t=2023-09-23T19:41:28.407079422Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T19:41:28.588 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=ngalert.state.manager t=2023-09-23T19:41:28.586982446Z level=info msg="State cache has been initialized" states=0 duration=190.7244ms 2023-09-23T19:41:28.588 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=ticker t=2023-09-23T19:41:28.587091853Z level=info msg=starting first_tick=2023-09-23T19:41:30Z 2023-09-23T19:41:28.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:28 smithi135 ceph-mon[118715]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:28.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:28 smithi135 ceph-mon[100251]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:28.881 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=ngalert.multiorg.alertmanager t=2023-09-23T19:41:28.587073171Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T19:41:28.881 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:28 smithi144 conmon[151733]: logger=sqlstore.transactions t=2023-09-23T19:41:28.726947366Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T19:41:29.381 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:41:29 smithi144 conmon[151733]: logger=sqlstore.transactions t=2023-09-23T19:41:29.061606707Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T19:41:29.783 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:41:29.785 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:41:30.811 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[118715]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:30.811 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[100251]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.812 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.813 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.813 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.813 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:30.813 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:30.813 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.826 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":33,"stamp":"2023-09-23T19:41:29.243177+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50356,"kb_used_data":6644,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910860,"statfs":{"total":767960285184,"available":767908720640,"internally_reserved":0,"allocated":6803456,"data_stored":3875182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002209"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775498+0000","last_change":"2023-09-23T19:40:40.249273+0000","last_active":"2023-09-23T19:40:41.775498+0000","last_peered":"2023-09-23T19:40:41.775498+0000","last_clean":"2023-09-23T19:40:41.775498+0000","last_became_active":"2023-09-23T19:40:40.248979+0000","last_became_peered":"2023-09-23T19:40:40.248979+0000","last_unstale":"2023-09-23T19:40:41.775498+0000","last_undegraded":"2023-09-23T19:40:41.775498+0000","last_fullsized":"2023-09-23T19:40:41.775498+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:25.089853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.250358+0000","last_change":"2023-09-23T19:40:40.251554+0000","last_active":"2023-09-23T19:40:41.250358+0000","last_peered":"2023-09-23T19:40:41.250358+0000","last_clean":"2023-09-23T19:40:41.250358+0000","last_became_active":"2023-09-23T19:40:40.251284+0000","last_became_peered":"2023-09-23T19:40:40.251284+0000","last_unstale":"2023-09-23T19:40:41.250358+0000","last_undegraded":"2023-09-23T19:40:41.250358+0000","last_fullsized":"2023-09-23T19:40:41.250358+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:40.680795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775425+0000","last_change":"2023-09-23T19:40:40.361547+0000","last_active":"2023-09-23T19:40:41.775425+0000","last_peered":"2023-09-23T19:40:41.775425+0000","last_clean":"2023-09-23T19:40:41.775425+0000","last_became_active":"2023-09-23T19:40:40.361206+0000","last_became_peered":"2023-09-23T19:40:40.361206+0000","last_unstale":"2023-09-23T19:40:41.775425+0000","last_undegraded":"2023-09-23T19:40:41.775425+0000","last_fullsized":"2023-09-23T19:40:41.775425+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:27.379089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775400+0000","last_change":"2023-09-23T19:40:40.361627+0000","last_active":"2023-09-23T19:40:41.775400+0000","last_peered":"2023-09-23T19:40:41.775400+0000","last_clean":"2023-09-23T19:40:41.775400+0000","last_became_active":"2023-09-23T19:40:40.361250+0000","last_became_peered":"2023-09-23T19:40:40.361250+0000","last_unstale":"2023-09-23T19:40:41.775400+0000","last_undegraded":"2023-09-23T19:40:41.775400+0000","last_fullsized":"2023-09-23T19:40:41.775400+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:04.232689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775501+0000","last_change":"2023-09-23T19:40:07.160277+0000","last_active":"2023-09-23T19:40:41.775501+0000","last_peered":"2023-09-23T19:40:41.775501+0000","last_clean":"2023-09-23T19:40:41.775501+0000","last_became_active":"2023-09-23T19:40:07.159949+0000","last_became_peered":"2023-09-23T19:40:07.159949+0000","last_unstale":"2023-09-23T19:40:41.775501+0000","last_undegraded":"2023-09-23T19:40:41.775501+0000","last_fullsized":"2023-09-23T19:40:41.775501+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:20.218507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775366+0000","last_change":"2023-09-23T19:40:40.249857+0000","last_active":"2023-09-23T19:40:41.775366+0000","last_peered":"2023-09-23T19:40:41.775366+0000","last_clean":"2023-09-23T19:40:41.775366+0000","last_became_active":"2023-09-23T19:40:40.249587+0000","last_became_peered":"2023-09-23T19:40:40.249587+0000","last_unstale":"2023-09-23T19:40:41.775366+0000","last_undegraded":"2023-09-23T19:40:41.775366+0000","last_fullsized":"2023-09-23T19:40:41.775366+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:37.624835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.249565+0000","last_change":"2023-09-23T19:40:40.361713+0000","last_active":"2023-09-23T19:40:41.249565+0000","last_peered":"2023-09-23T19:40:41.249565+0000","last_clean":"2023-09-23T19:40:41.249565+0000","last_became_active":"2023-09-23T19:40:40.361369+0000","last_became_peered":"2023-09-23T19:40:40.361369+0000","last_unstale":"2023-09-23T19:40:41.249565+0000","last_undegraded":"2023-09-23T19:40:41.249565+0000","last_fullsized":"2023-09-23T19:40:41.249565+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:04.414357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.775453+0000","last_change":"2023-09-23T19:40:40.251667+0000","last_active":"2023-09-23T19:40:41.775453+0000","last_peered":"2023-09-23T19:40:41.775453+0000","last_clean":"2023-09-23T19:40:41.775453+0000","last_became_active":"2023-09-23T19:40:40.251366+0000","last_became_peered":"2023-09-23T19:40:40.251366+0000","last_unstale":"2023-09-23T19:40:41.775453+0000","last_undegraded":"2023-09-23T19:40:41.775453+0000","last_fullsized":"2023-09-23T19:40:41.775453+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:38.869721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T19:40:41.250491+0000","last_change":"2023-09-23T19:40:40.361619+0000","last_active":"2023-09-23T19:40:41.250491+0000","last_peered":"2023-09-23T19:40:41.250491+0000","last_clean":"2023-09-23T19:40:41.250491+0000","last_became_active":"2023-09-23T19:40:40.361237+0000","last_became_peered":"2023-09-23T19:40:40.361237+0000","last_unstale":"2023-09-23T19:40:41.250491+0000","last_undegraded":"2023-09-23T19:40:41.250491+0000","last_fullsized":"2023-09-23T19:40:41.250491+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:09.643497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6320,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738832,"statfs":{"total":95995035648,"available":95988563968,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.40699999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.317}]},{"osd":4,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.373}]},{"osd":5,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.441}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6380,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738772,"statfs":{"total":95995035648,"available":95988502528,"internally_reserved":0,"allocated":1216512,"data_stored":853061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.439}]},{"osd":1,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.501},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.53400000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.314},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.38600000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.33900000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6512,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738640,"statfs":{"total":95995035648,"available":95988367360,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.33500000000000002},"min":{"1min":0.24199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.60099999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.36199999999999999,"15min":0.34399999999999997},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":0.55600000000000005,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:41:13 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.38600000000000001,"15min":0.38800000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56100000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41899999999999998,"15min":0.42199999999999999},"min":{"1min":0.27800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.53000000000000003,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.67900000000000005,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.626,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.39800000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.58499999999999996,"15min":0.59299999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72099999999999997,"5min":0.87,"15min":0.87},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.59999999999999998,"15min":0.61099999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79600000000000004,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.60099999999999998,"15min":0.60999999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76200000000000001,"5min":0.871,"15min":0.871},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81799999999999995,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.437}]},{"osd":7,"last update":"Sat Sep 23 19:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.59599999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7152,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738000,"statfs":{"total":95995035648,"available":95987712000,"internally_reserved":0,"allocated":1220608,"data_stored":853326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.33300000000000002,"15min":0.313},"min":{"1min":0.26500000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.53900000000000003,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.36099999999999999,"15min":0.34100000000000003},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54000000000000004,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.46500000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.38300000000000001,"15min":0.38100000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.45700000000000002,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39200000000000002,"15min":0.39100000000000001},"min":{"1min":0.27600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55200000000000005,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.47499999999999998,"15min":0.48099999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.51300000000000001,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.47199999999999998,"15min":0.47899999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.57499999999999996,"5min":0.627,"15min":0.627},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56999999999999995,"15min":0.57499999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68400000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.51900000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55100000000000005,"15min":0.55300000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.57699999999999996,"15min":0.57899999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.74099999999999999,"5min":0.749,"15min":0.749},"last":0.53300000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.58599999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6448,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738704,"statfs":{"total":95995035648,"available":95988432896,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.40400000000000003,"15min":0.35399999999999998},"min":{"1min":0.247,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.5629999999999997,"5min":5.5629999999999997,"15min":5.5629999999999997},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.996,"5min":0.496,"15min":0.40699999999999997},"min":{"1min":0.28299999999999997,"5min":0.216,"15min":0.216},"max":{"1min":10.811,"5min":10.811,"15min":10.811},"last":0.27800000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:41:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.46400000000000002,"15min":0.41899999999999998},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.29,"5min":5.29,"15min":5.29},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.1040000000000001,"5min":0.55300000000000005,"15min":0.44800000000000001},"min":{"1min":0.30599999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":11.986000000000001,"5min":11.986000000000001,"15min":11.986000000000001},"last":0.29599999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42999999999999999,"15min":0.432},"min":{"1min":0.316,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.317},{"interface":"front","average":{"1min":0.437,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57299999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.30399999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.56399999999999995,"15min":0.56200000000000006},"min":{"1min":0.44400000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.377},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.63400000000000001,"15min":0.58999999999999997},"min":{"1min":0.40999999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":5.1340000000000003,"5min":5.1340000000000003,"15min":5.1340000000000003},"last":0.433}]},{"osd":5,"last update":"Sat Sep 23 19:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.64300000000000002,"15min":0.60899999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":5.1920000000000002,"5min":5.1920000000000002,"15min":5.1920000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.81599999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:40:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:41:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":5.2629999999999999,"5min":5.2629999999999999,"15min":5.2629999999999999},"last":0.38500000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.36399999999999999,"15min":0.36199999999999999},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.42099999999999999,"15min":0.42999999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.52300000000000002,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.42799999999999999,"15min":0.432},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.434,"15min":0.439},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.56799999999999995,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.622}]},{"osd":2,"last update":"Sat Sep 23 19:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.48799999999999999,"15min":0.495},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58399999999999996,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.45900000000000002,"15min":0.46700000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.53400000000000003,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52300000000000002,"15min":0.52100000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53800000000000003,"15min":0.53500000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.56599999999999995,"15min":0.56200000000000006},"min":{"1min":0.42899999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:40:41 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:41:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.499},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.44800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5868,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739284,"statfs":{"total":95995035648,"available":95989026816,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48899999999999999,"15min":0.48999999999999999},"min":{"1min":0.28699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.61799999999999999,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.48399999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.50700000000000001,"15min":0.50800000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.503,"15min":0.502},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.47399999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.40799999999999997,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.435}]},{"osd":3,"last update":"Sat Sep 23 19:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.498,"15min":0.49399999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.52800000000000002,"15min":0.52200000000000002},"min":{"1min":0.38100000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.40799999999999997,"15min":0.39700000000000002},"min":{"1min":0.30099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.76600000000000001,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.40899999999999997,"15min":0.39100000000000001},"min":{"1min":0.36199999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72299999999999998,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.34200000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:40:40 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.52900000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757277,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5804,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739348,"statfs":{"total":95995035648,"available":95989092352,"internally_reserved":0,"allocated":626688,"data_stored":262900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.54600000000000004,"15min":0.55500000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.72799999999999998,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":1.111},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.498,"15min":0.496},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.94599999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":1.0720000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65800000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":1.089}]},{"osd":3,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.51300000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77400000000000002,"5min":0.874,"15min":0.874},"last":0.93700000000000006}]},{"osd":4,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.40500000000000003,"15min":0.39200000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.41799999999999998,"15min":0.40600000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.753}]},{"osd":6,"last update":"Sat Sep 23 19:40:40 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.81699999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.78300000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:41:30.828 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T19:41:30.828 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T19:41:30.828 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T19:41:30.828 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T19:41:30.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:30 smithi144 ceph-mon[104994]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:30.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:31 smithi144 ceph-mon[104994]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:41:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:31 smithi144 ceph-mon[104994]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:41:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:31 smithi144 ceph-mon[104994]: Reconfiguring daemon alertmanager.a on smithi135 2023-09-23T19:41:31.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:31 smithi135 ceph-mon[118715]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:41:31.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:31 smithi135 ceph-mon[118715]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:41:31.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:31 smithi135 ceph-mon[118715]: Reconfiguring daemon alertmanager.a on smithi135 2023-09-23T19:41:31.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:31 smithi135 ceph-mon[100251]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:41:31.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:31 smithi135 ceph-mon[100251]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:41:31.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:31 smithi135 ceph-mon[100251]: Reconfiguring daemon alertmanager.a on smithi135 2023-09-23T19:41:32.037 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:32.659 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:32 smithi135 ceph-mon[100251]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:32.659 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:32 smithi135 ceph-mon[118715]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:32.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:32 smithi144 ceph-mon[104994]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:34.082 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:33 smithi135 systemd[1]: Stopping Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:41:34.470 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:34 smithi135 conmon[164272]: ts=2023-09-23T19:41:34.082Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:41:34.470 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:34 smithi135 bash[174728]: 7f866f3f242106c953276e40ed57e993906c5965f5358ac7d824987251ff291e 2023-09-23T19:41:34.704 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:41:34.704 INFO:teuthology.orchestra.run.smithi135.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T19:41:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:34 smithi144 ceph-mon[104994]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:34.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:34 smithi135 ceph-mon[100251]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:34.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:34 smithi135 ceph-mon[118715]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:35.470 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:35 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T19:41:35.471 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:35 smithi135 systemd[1]: Stopped Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:41:35.471 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:35 smithi135 systemd[1]: Starting Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:41:35.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:35 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3767149025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:41:35.632 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:41:35 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:41:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:41:35.683 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T19:41:35.683 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T19:41:35.683 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T19:41:35.692 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi135.front.sepia.ceph.com 2023-09-23T19:41:35.693 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T19:41:35.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:35 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3767149025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:41:35.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:35 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3767149025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:41:36.370 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:36.584 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:36 smithi135 ceph-mon[118715]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:36.584 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:36 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:36.584 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:36 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:36.585 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 conmon[175280]: ts=2023-09-23T19:41:36.260Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T19:41:36.585 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 conmon[175280]: ts=2023-09-23T19:41:36.260Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T19:41:36.585 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 conmon[175280]: ts=2023-09-23T19:41:36.261Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.135 port=9094 2023-09-23T19:41:36.585 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 conmon[175280]: ts=2023-09-23T19:41:36.262Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T19:41:36.585 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 conmon[175280]: ts=2023-09-23T19:41:36.285Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:41:36.585 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 conmon[175280]: ts=2023-09-23T19:41:36.286Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:41:36.586 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 conmon[175280]: ts=2023-09-23T19:41:36.288Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T19:41:36.586 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 conmon[175280]: ts=2023-09-23T19:41:36.288Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T19:41:36.586 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 bash[175186]: ef1f5819ae6814f2f84dcd77198532a39815300b82ca9061567df1a34fc77107 2023-09-23T19:41:36.586 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:36 smithi135 systemd[1]: Started Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:41:36.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:36 smithi135 ceph-mon[100251]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:36.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:36 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:36.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:36 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:36 smithi144 ceph-mon[104994]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:36 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:36 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:37 smithi144 ceph-mon[104994]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:41:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:37 smithi144 ceph-mon[104994]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:41:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:37.819 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:37 smithi135 ceph-mon[118715]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:41:37.819 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:37 smithi135 ceph-mon[118715]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:41:37.819 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:37.820 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:37 smithi135 ceph-mon[100251]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:41:37.820 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:37 smithi135 ceph-mon[100251]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:41:37.820 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:38.513 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:38 smithi135 conmon[175280]: ts=2023-09-23T19:41:38.263Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000501786s 2023-09-23T19:41:38.748 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:38 smithi144 ceph-mon[104994]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:38.748 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:38 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:41:38.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:38 smithi135 ceph-mon[100251]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:38.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:38 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:41:38.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:38 smithi135 ceph-mon[118715]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:38 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T19:41:39.491 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 systemd[1]: Stopping Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:41:39.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:39 smithi144 ceph-mon[104994]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:41:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:39 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T19:41:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:39 smithi144 ceph-mon[104994]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T19:41:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:39 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:41:39.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.490Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:41:39.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.491Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T19:41:39.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.491Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T19:41:39.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.491Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T19:41:39.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.491Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T19:41:39.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.491Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T19:41:39.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.491Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T19:41:39.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.491Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T19:41:39.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.491Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T19:41:39.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.492Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T19:41:39.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.492Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T19:41:39.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:39 smithi144 conmon[148569]: ts=2023-09-23T19:41:39.492Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T19:41:39.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:39 smithi135 ceph-mon[118715]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:41:39.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:39 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T19:41:39.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:39 smithi135 ceph-mon[118715]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T19:41:39.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:39 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:41:39.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:39 smithi135 conmon[100227]: 2023-09-23T19:41:39.533+0000 7f001ed4b700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T19:41:39.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:39 smithi135 ceph-mon[100251]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:41:39.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:39 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T19:41:39.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:39 smithi135 ceph-mon[100251]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T19:41:39.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:39 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T19:41:40.381 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 bash[153774]: eef0f7c4275c0ec37c3dc088709385f19c760aa5e72073a0f272b76c3695ac8b 2023-09-23T19:41:40.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T19:41:40.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 systemd[1]: Stopped Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:41:40.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 systemd[1]: Starting Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:41:40.397 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T19:41:40.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:40.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: fsmap foo:0 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: Saving service mds.foo spec with placement count:2 2023-09-23T19:41:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: fsmap foo:0 2023-09-23T19:41:40.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: Saving service mds.foo spec with placement count:2 2023-09-23T19:41:40.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:40 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:40.757 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:40.847 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: fsmap foo:0 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: Saving service mds.foo spec with placement count:2 2023-09-23T19:41:40.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:40 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.847Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T19:41:41.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.847Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.847Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi144 (none))" 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.847Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.848Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.849Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.849Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.852Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.852Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.853Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.853Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.267µs 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.853Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T19:41:41.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.853Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.853Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.853Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.92µs wal_replay_duration=587.506µs wbl_replay_duration=91ns total_replay_duration=627.6µs 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.854Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.854Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.854Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.870Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.09629ms db_storage=826ns remote_storage=970ns web_handler=349ns query_engine=809ns scrape=569.289µs scrape_sd=28.989µs notify=18.911µs notify_sd=5.312µs rules=14.273716ms tracing=5.431µs 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.870Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 conmon[154137]: ts=2023-09-23T19:41:40.870Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 bash[154096]: c5d581813489c64cbe3792260e6debd1190ee9f6c7f39bd1b6efac044be23d3c 2023-09-23T19:41:41.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:41:40 smithi144 systemd[1]: Started Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:41:41.184 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:40 smithi135 conmon[100683]: [23/Sep/2023:19:41:40] ENGINE Bus STOPPING 2023-09-23T19:41:41.184 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:40 smithi135 conmon[100683]: [23/Sep/2023:19:41:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:40 smithi135 conmon[100683]: [23/Sep/2023:19:41:40] ENGINE Bus STOPPED 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:40 smithi135 conmon[100683]: [23/Sep/2023:19:41:40] ENGINE Bus STARTING 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Serving on http://:::9283 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STARTED 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STOPPING 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STOPPED 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STARTING 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Serving on http://:::9283 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STARTED 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STOPPING 2023-09-23T19:41:41.185 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:41:41.186 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STOPPED 2023-09-23T19:41:41.186 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STARTING 2023-09-23T19:41:41.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Serving on http://:::9283 2023-09-23T19:41:41.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:41 smithi135 conmon[100683]: [23/Sep/2023:19:41:41] ENGINE Bus STARTED 2023-09-23T19:41:41.839 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T19:41:41.839 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.839 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.839 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:41:41.839 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135:9093"}]: dispatch 2023-09-23T19:41:41.839 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.839 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.840 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135:9093"}]: dispatch 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:41 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:41.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T19:41:41.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135:9093"}]: dispatch 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:41 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:42.810 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[118715]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:41:42.810 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[118715]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135:9093"}]: dispatch 2023-09-23T19:41:42.810 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[118715]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:41:42.810 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[118715]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[118715]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[118715]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[118715]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[118715]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[100251]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[100251]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135:9093"}]: dispatch 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[100251]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[100251]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[100251]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:41:42.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[100251]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:41:42.812 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[100251]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:42.812 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:42 smithi135 ceph-mon[100251]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T19:41:42.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:42 smithi144 ceph-mon[104994]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:41:42.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:42 smithi144 ceph-mon[104994]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135:9093"}]: dispatch 2023-09-23T19:41:42.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:42 smithi144 ceph-mon[104994]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:41:42.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:42 smithi144 ceph-mon[104994]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:41:42.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:42 smithi144 ceph-mon[104994]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:41:42.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:42 smithi144 ceph-mon[104994]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T19:41:42.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:42 smithi144 ceph-mon[104994]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:42.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:42 smithi144 ceph-mon[104994]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T19:41:43.313 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T19:41:43.705 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:44.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:44 smithi144 ceph-mon[104994]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:44.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:44 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:44.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:44 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:44.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:44 smithi135 ceph-mon[118715]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:44.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:44 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:44.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:44 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:44.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:44 smithi135 ceph-mon[100251]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:44.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:44 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:44.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:44 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.446 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:41:46 smithi135 conmon[175280]: ts=2023-09-23T19:41:46.264Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001844959s 2023-09-23T19:41:46.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:46.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:41:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:41:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:41:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:41:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:41:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:41:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:46.910 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T19:41:46.919 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T19:41:46.920 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T19:41:46.920 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T19:41:47.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:47 smithi135 ceph-mon[118715]: Deploying daemon mds.foo.smithi135.srojfl on smithi135 2023-09-23T19:41:47.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:47 smithi135 ceph-mon[100251]: Deploying daemon mds.foo.smithi135.srojfl on smithi135 2023-09-23T19:41:47.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:47 smithi144 ceph-mon[104994]: Deploying daemon mds.foo.smithi135.srojfl on smithi135 2023-09-23T19:41:48.141 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:48.640 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:48 smithi135 ceph-mon[118715]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:48.640 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:48 smithi135 ceph-mon[100251]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:48.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:48 smithi144 ceph-mon[104994]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:50.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[100251]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:50.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:41:50.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:41:50.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:50.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[118715]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:50.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:41:50.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:41:50.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:50 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:50.421 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:41:50.488 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:50 smithi144 ceph-mon[104994]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:50.488 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.488 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.488 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:50.488 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:41:50.488 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T19:41:50.488 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:50 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:51.114 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":19,"active_gid":24359,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":1642850725},{"type":"v1","addr":"172.21.15.135:6801","nonce":1642850725}]},"active_addr":"172.21.15.135:6801/1642850725","active_change":"2023-09-23T19:40:37.221091+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24364,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":3668294279}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":3693941440}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":3516425720}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":4238089144}]}]} 2023-09-23T19:41:51.119 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T19:41:51.119 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T19:41:51.119 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:41:51.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: Deploying daemon mds.foo.smithi144.plbefj on smithi144 2023-09-23T19:41:51.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:boot 2023-09-23T19:41:51.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: daemon mds.foo.smithi135.srojfl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T19:41:51.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: Cluster is now healthy 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: fsmap foo:0 1 up:standby 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: fsmap foo:1 {0=foo.smithi135.srojfl=up:creating} 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: daemon mds.foo.smithi135.srojfl is now active in filesystem foo as rank 0 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4099070116' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: Deploying daemon mds.foo.smithi144.plbefj on smithi144 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:boot 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: daemon mds.foo.smithi135.srojfl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T19:41:51.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T19:41:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T19:41:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: Cluster is now healthy 2023-09-23T19:41:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: fsmap foo:0 1 up:standby 2023-09-23T19:41:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:41:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: fsmap foo:1 {0=foo.smithi135.srojfl=up:creating} 2023-09-23T19:41:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: daemon mds.foo.smithi135.srojfl is now active in filesystem foo as rank 0 2023-09-23T19:41:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:51 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4099070116' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:41:51.481 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:51.514 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: Deploying daemon mds.foo.smithi144.plbefj on smithi144 2023-09-23T19:41:51.514 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:boot 2023-09-23T19:41:51.514 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: daemon mds.foo.smithi135.srojfl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T19:41:51.514 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T19:41:51.515 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T19:41:51.515 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: Cluster is now healthy 2023-09-23T19:41:51.515 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: fsmap foo:0 1 up:standby 2023-09-23T19:41:51.515 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:41:51.515 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: fsmap foo:1 {0=foo.smithi135.srojfl=up:creating} 2023-09-23T19:41:51.515 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: daemon mds.foo.smithi135.srojfl is now active in filesystem foo as rank 0 2023-09-23T19:41:51.515 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:51 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4099070116' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:41:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:52 smithi135 ceph-mon[118715]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:active 2023-09-23T19:41:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:52 smithi135 ceph-mon[118715]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 2023-09-23T19:41:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:52 smithi135 ceph-mon[118715]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:52 smithi135 ceph-mon[100251]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:active 2023-09-23T19:41:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:52 smithi135 ceph-mon[100251]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 2023-09-23T19:41:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:52 smithi135 ceph-mon[100251]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:52 smithi144 ceph-mon[104994]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:active 2023-09-23T19:41:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:52 smithi144 ceph-mon[104994]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 2023-09-23T19:41:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:52 smithi144 ceph-mon[104994]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:41:53.412 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:41:53.412 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":68,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T19:41:41.550183+0000","last_up_change":"2023-09-23T19:40:03.286566+0000","last_in_change":"2023-09-23T19:39:46.254783+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:40:39.075611+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:41:38.466408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:41:38.762420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6803","nonce":1288355238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6805","nonce":1288355238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6809","nonce":1288355238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6807","nonce":1288355238}]},"public_addr":"172.21.15.135:6803/1288355238","cluster_addr":"172.21.15.135:6805/1288355238","heartbeat_back_addr":"172.21.15.135:6809/1288355238","heartbeat_front_addr":"172.21.15.135:6807/1288355238","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6811","nonce":460054379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6813","nonce":460054379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6817","nonce":460054379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6815","nonce":460054379}]},"public_addr":"172.21.15.135:6811/460054379","cluster_addr":"172.21.15.135:6813/460054379","heartbeat_back_addr":"172.21.15.135:6817/460054379","heartbeat_front_addr":"172.21.15.135:6815/460054379","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6819","nonce":1252775382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6821","nonce":1252775382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6825","nonce":1252775382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6823","nonce":1252775382}]},"public_addr":"172.21.15.135:6819/1252775382","cluster_addr":"172.21.15.135:6821/1252775382","heartbeat_back_addr":"172.21.15.135:6825/1252775382","heartbeat_front_addr":"172.21.15.135:6823/1252775382","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6827","nonce":3652622392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6829","nonce":3652622392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6833","nonce":3652622392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6831","nonce":3652622392}]},"public_addr":"172.21.15.135:6827/3652622392","cluster_addr":"172.21.15.135:6829/3652622392","heartbeat_back_addr":"172.21.15.135:6833/3652622392","heartbeat_front_addr":"172.21.15.135:6831/3652622392","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6817","nonce":1530810554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6819","nonce":1530810554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6823","nonce":1530810554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6821","nonce":1530810554}]},"public_addr":"172.21.15.144:6817/1530810554","cluster_addr":"172.21.15.144:6819/1530810554","heartbeat_back_addr":"172.21.15.144:6823/1530810554","heartbeat_front_addr":"172.21.15.144:6821/1530810554","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:6801/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1005337745":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/775791172":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/173145882":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/3361160768":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:41:53.606 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.606 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: mds.? [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] up:boot 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:41:53.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[118715]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:53.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: mds.? [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] up:boot 2023-09-23T19:41:53.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:53.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:41:53.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:53 smithi135 ceph-mon[100251]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:41:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: mds.? [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] up:boot 2023-09-23T19:41:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:41:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:53 smithi144 ceph-mon[104994]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:41:53 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:41:53] "GET /metrics HTTP/1.1" 200 238073 "" "Prometheus/2.43.0" 2023-09-23T19:41:54.335 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T19:41:54.335 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:41:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:54 smithi144 ceph-mon[104994]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:54 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/390371797' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:41:54.718 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:54.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:54 smithi135 ceph-mon[118715]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:54.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:54 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/390371797' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:41:54.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:54 smithi135 ceph-mon[100251]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:54.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:54 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/390371797' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:41:55.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:41:55 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:41:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:41:56.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[100251]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:active 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[100251]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[100251]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[118715]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:active 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[118715]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:56 smithi135 ceph-mon[118715]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:56 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:56 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:56 smithi144 ceph-mon[104994]: mds.? [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] up:active 2023-09-23T19:41:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:56 smithi144 ceph-mon[104994]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:56 smithi144 ceph-mon[104994]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:41:56.828 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":68,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T19:41:41.550183+0000","last_up_change":"2023-09-23T19:40:03.286566+0000","last_in_change":"2023-09-23T19:39:46.254783+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:40:39.075611+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:41:38.466408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:41:38.762420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6803","nonce":1288355238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6805","nonce":1288355238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6809","nonce":1288355238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1288355238},{"type":"v1","addr":"172.21.15.135:6807","nonce":1288355238}]},"public_addr":"172.21.15.135:6803/1288355238","cluster_addr":"172.21.15.135:6805/1288355238","heartbeat_back_addr":"172.21.15.135:6809/1288355238","heartbeat_front_addr":"172.21.15.135:6807/1288355238","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6811","nonce":460054379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6813","nonce":460054379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6817","nonce":460054379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":460054379},{"type":"v1","addr":"172.21.15.135:6815","nonce":460054379}]},"public_addr":"172.21.15.135:6811/460054379","cluster_addr":"172.21.15.135:6813/460054379","heartbeat_back_addr":"172.21.15.135:6817/460054379","heartbeat_front_addr":"172.21.15.135:6815/460054379","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6819","nonce":1252775382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6821","nonce":1252775382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6825","nonce":1252775382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1252775382},{"type":"v1","addr":"172.21.15.135:6823","nonce":1252775382}]},"public_addr":"172.21.15.135:6819/1252775382","cluster_addr":"172.21.15.135:6821/1252775382","heartbeat_back_addr":"172.21.15.135:6825/1252775382","heartbeat_front_addr":"172.21.15.135:6823/1252775382","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6827","nonce":3652622392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6829","nonce":3652622392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6833","nonce":3652622392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":3652622392},{"type":"v1","addr":"172.21.15.135:6831","nonce":3652622392}]},"public_addr":"172.21.15.135:6827/3652622392","cluster_addr":"172.21.15.135:6829/3652622392","heartbeat_back_addr":"172.21.15.135:6833/3652622392","heartbeat_front_addr":"172.21.15.135:6831/3652622392","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6817","nonce":1530810554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6819","nonce":1530810554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6823","nonce":1530810554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6821","nonce":1530810554}]},"public_addr":"172.21.15.144:6817/1530810554","cluster_addr":"172.21.15.144:6819/1530810554","heartbeat_back_addr":"172.21.15.144:6823/1530810554","heartbeat_front_addr":"172.21.15.144:6821/1530810554","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:6801/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1005337745":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/775791172":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/173145882":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/3361160768":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:41:57.629 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T19:41:57.630 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T19:41:57.630 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T19:41:57.631 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T19:41:57.631 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T19:41:57.632 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T19:41:57.632 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T19:41:57.633 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T19:41:57.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1075192613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: mds.? [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] up:standby 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[118715]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:57.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1075192613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: mds.? [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] up:standby 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:41:57 smithi135 ceph-mon[100251]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:58.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:58.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:41:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:41:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:41:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1075192613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:41:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:41:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: mds.? [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] up:standby 2023-09-23T19:41:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:41:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:41:57 smithi144 ceph-mon[104994]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:41:59.216 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:59.362 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:59.428 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:59.429 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:59.516 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:41:59.703 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:00.318 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:00 smithi135 ceph-mon[118715]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:00 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:00 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:00 smithi135 ceph-mon[100251]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:00 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:00 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:00 smithi144 ceph-mon[104994]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:00 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:00 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:02.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:02 smithi135 ceph-mon[118715]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:02.573 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:02 smithi135 ceph-mon[100251]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:02 smithi144 ceph-mon[104994]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:03.487 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:03.924 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:03 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:03] "GET /metrics HTTP/1.1" 200 238073 "" "Prometheus/2.43.0" 2023-09-23T19:42:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:04 smithi144 ceph-mon[104994]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:04.682 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:04 smithi135 ceph-mon[118715]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:04.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:04 smithi135 ceph-mon[100251]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T19:42:05.630 INFO:teuthology.orchestra.run.smithi135.stdout:55834574903 2023-09-23T19:42:05.630 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T19:42:05.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:42:05 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:42:06.325 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:06 smithi135 ceph-mon[118715]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:06.326 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:06 smithi135 ceph-mon[100251]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:06 smithi144 ceph-mon[104994]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:07.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:07 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:07.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:07 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:07 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:08.309 INFO:teuthology.orchestra.run.smithi135.stdout:197568495648 2023-09-23T19:42:08.309 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T19:42:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:08 smithi144 ceph-mon[104994]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:08 smithi135 ceph-mon[118715]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:08.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:08 smithi135 ceph-mon[100251]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:08.756 INFO:teuthology.orchestra.run.smithi135.stdout:231928234011 2023-09-23T19:42:08.756 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T19:42:09.091 INFO:teuthology.orchestra.run.smithi135.stdout:163208757285 2023-09-23T19:42:09.091 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T19:42:09.394 INFO:teuthology.orchestra.run.smithi135.stdout:107374182445 2023-09-23T19:42:09.394 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T19:42:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:10 smithi135 ceph-mon[100251]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:10 smithi135 ceph-mon[118715]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:10 smithi144 ceph-mon[104994]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T19:42:11.451 INFO:teuthology.orchestra.run.smithi135.stdout:128849018922 2023-09-23T19:42:11.452 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T19:42:11.687 INFO:teuthology.orchestra.run.smithi135.stdout:34359738428 2023-09-23T19:42:11.687 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T19:42:11.722 INFO:teuthology.orchestra.run.smithi135.stdout:77309411380 2023-09-23T19:42:11.722 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T19:42:11.995 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:12.121 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:12.248 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:12.599 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[100251]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[118715]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:12.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:12.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:12 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:12 smithi144 ceph-mon[104994]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:12 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:12 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:12 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:12 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:12 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:13.151 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:13.730 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:14.008 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:13 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:13] "GET /metrics HTTP/1.1" 200 238659 "" "Prometheus/2.43.0" 2023-09-23T19:42:14.548 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:14 smithi135 ceph-mon[118715]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:14.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:14 smithi135 ceph-mon[100251]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:14 smithi144 ceph-mon[104994]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:14.786 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:15.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:42:15 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:42:15.927 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:15.929 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:16.016 INFO:teuthology.orchestra.run.smithi135.stdout:55834574905 2023-09-23T19:42:16.061 INFO:teuthology.orchestra.run.smithi135.stdout:231928234013 2023-09-23T19:42:16.474 INFO:teuthology.orchestra.run.smithi135.stdout:197568495650 2023-09-23T19:42:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:16 smithi144 ceph-mon[104994]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:16 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/88153925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:42:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:16 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3456286442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:42:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:16 smithi135 ceph-mon[118715]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:16 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/88153925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:42:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:16 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3456286442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:42:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:16 smithi135 ceph-mon[100251]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:16 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/88153925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:42:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:16 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3456286442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:42:17.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:17 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4059018191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:42:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:17 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4059018191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:42:17.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:17 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4059018191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:42:18.201 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574903 got 55834574905 for osd.1 2023-09-23T19:42:18.201 DEBUG:teuthology.parallel:result is None 2023-09-23T19:42:18.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:18 smithi135 ceph-mon[118715]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:18.440 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:18 smithi135 ceph-mon[100251]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:18.441 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495648 got 197568495650 for osd.6 2023-09-23T19:42:18.441 DEBUG:teuthology.parallel:result is None 2023-09-23T19:42:18.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:18 smithi144 ceph-mon[104994]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:18.709 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234011 got 231928234013 for osd.7 2023-09-23T19:42:18.709 DEBUG:teuthology.parallel:result is None 2023-09-23T19:42:20.160 INFO:teuthology.orchestra.run.smithi135.stdout:163208757287 2023-09-23T19:42:20.172 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:20 smithi135 ceph-mon[100251]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:20.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:20 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/347563887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:42:20.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:20 smithi135 ceph-mon[118715]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:20.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:20 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/347563887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:42:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:20 smithi144 ceph-mon[104994]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:20 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/347563887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:42:20.806 INFO:teuthology.orchestra.run.smithi135.stdout:107374182448 2023-09-23T19:42:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:21 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4157456255' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:42:21.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:21 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4157456255' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:42:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:21 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4157456255' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:42:21.670 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757285 got 163208757287 for osd.5 2023-09-23T19:42:21.671 DEBUG:teuthology.parallel:result is None 2023-09-23T19:42:22.243 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182445 got 107374182448 for osd.3 2023-09-23T19:42:22.243 DEBUG:teuthology.parallel:result is None 2023-09-23T19:42:22.285 INFO:teuthology.orchestra.run.smithi135.stdout:128849018924 2023-09-23T19:42:22.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:22 smithi135 ceph-mon[118715]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:22.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:22 smithi135 ceph-mon[100251]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:22.586 INFO:teuthology.orchestra.run.smithi135.stdout:34359738430 2023-09-23T19:42:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:22 smithi144 ceph-mon[104994]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:23.342 INFO:teuthology.orchestra.run.smithi135.stdout:77309411382 2023-09-23T19:42:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:23 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3828217843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:42:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:23 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:23 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2820768347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:42:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:23 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2658805194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:42:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:23 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3828217843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:42:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:23 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:23 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2820768347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:42:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:23 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2658805194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:42:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:23 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3828217843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:42:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:23 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:23 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2820768347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:42:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:23 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2658805194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:42:23.892 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738428 got 34359738430 for osd.0 2023-09-23T19:42:23.892 DEBUG:teuthology.parallel:result is None 2023-09-23T19:42:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:23 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:23] "GET /metrics HTTP/1.1" 200 238623 "" "Prometheus/2.43.0" 2023-09-23T19:42:24.092 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018922 got 128849018924 for osd.4 2023-09-23T19:42:24.093 DEBUG:teuthology.parallel:result is None 2023-09-23T19:42:24.116 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411380 got 77309411382 for osd.2 2023-09-23T19:42:24.116 DEBUG:teuthology.parallel:result is None 2023-09-23T19:42:24.116 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T19:42:24.116 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:42:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:24 smithi135 ceph-mon[118715]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:24.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:24 smithi135 ceph-mon[100251]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:24.505 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:24 smithi144 ceph-mon[104994]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:25.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:42:25 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:42:26.185 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:42:26.186 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:42:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:26 smithi135 ceph-mon[118715]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:26 smithi135 ceph-mon[100251]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:26 smithi144 ceph-mon[104994]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:26.851 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":66,"stamp":"2023-09-23T19:42:25.255369+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52404,"kb_used_data":7220,"kb_used_omap":0,"kb_used_meta":45120,"kb_avail":749908812,"statfs":{"total":767960285184,"available":767906623488,"internally_reserved":0,"allocated":7393280,"data_stored":4172772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46202880},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002839"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551378+0000","last_change":"2023-09-23T19:41:40.334173+0000","last_active":"2023-09-23T19:41:40.551378+0000","last_peered":"2023-09-23T19:41:40.551378+0000","last_clean":"2023-09-23T19:41:40.551378+0000","last_became_active":"2023-09-23T19:41:40.333894+0000","last_became_peered":"2023-09-23T19:41:40.333894+0000","last_unstale":"2023-09-23T19:41:40.551378+0000","last_undegraded":"2023-09-23T19:41:40.551378+0000","last_fullsized":"2023-09-23T19:41:40.551378+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:15.497357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.184077+0000","last_change":"2023-09-23T19:41:39.540478+0000","last_active":"2023-09-23T19:41:50.184077+0000","last_peered":"2023-09-23T19:41:50.184077+0000","last_clean":"2023-09-23T19:41:50.184077+0000","last_became_active":"2023-09-23T19:41:39.540258+0000","last_became_peered":"2023-09-23T19:41:39.540258+0000","last_unstale":"2023-09-23T19:41:50.184077+0000","last_undegraded":"2023-09-23T19:41:50.184077+0000","last_fullsized":"2023-09-23T19:41:50.184077+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:08.609351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.562717+0000","last_change":"2023-09-23T19:41:40.559640+0000","last_active":"2023-09-23T19:41:40.562717+0000","last_peered":"2023-09-23T19:41:40.562717+0000","last_clean":"2023-09-23T19:41:40.562717+0000","last_became_active":"2023-09-23T19:41:40.559390+0000","last_became_peered":"2023-09-23T19:41:40.559390+0000","last_unstale":"2023-09-23T19:41:40.562717+0000","last_undegraded":"2023-09-23T19:41:40.562717+0000","last_fullsized":"2023-09-23T19:41:40.562717+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:32.021313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.567746+0000","last_change":"2023-09-23T19:41:39.546388+0000","last_active":"2023-09-23T19:41:41.567746+0000","last_peered":"2023-09-23T19:41:41.567746+0000","last_clean":"2023-09-23T19:41:41.567746+0000","last_became_active":"2023-09-23T19:41:39.546009+0000","last_became_peered":"2023-09-23T19:41:39.546009+0000","last_unstale":"2023-09-23T19:41:41.567746+0000","last_undegraded":"2023-09-23T19:41:41.567746+0000","last_fullsized":"2023-09-23T19:41:41.567746+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:17.348379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551526+0000","last_change":"2023-09-23T19:41:40.425778+0000","last_active":"2023-09-23T19:41:40.551526+0000","last_peered":"2023-09-23T19:41:40.551526+0000","last_clean":"2023-09-23T19:41:40.551526+0000","last_became_active":"2023-09-23T19:41:40.425410+0000","last_became_peered":"2023-09-23T19:41:40.425410+0000","last_unstale":"2023-09-23T19:41:40.551526+0000","last_undegraded":"2023-09-23T19:41:40.551526+0000","last_fullsized":"2023-09-23T19:41:40.551526+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:53.615465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.185472+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:41:50.185472+0000","last_peered":"2023-09-23T19:41:50.185472+0000","last_clean":"2023-09-23T19:41:50.185472+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:41:50.185472+0000","last_undegraded":"2023-09-23T19:41:50.185472+0000","last_fullsized":"2023-09-23T19:41:50.185472+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551819+0000","last_change":"2023-09-23T19:41:40.425722+0000","last_active":"2023-09-23T19:41:40.551819+0000","last_peered":"2023-09-23T19:41:40.551819+0000","last_clean":"2023-09-23T19:41:40.551819+0000","last_became_active":"2023-09-23T19:41:40.425351+0000","last_became_peered":"2023-09-23T19:41:40.425351+0000","last_unstale":"2023-09-23T19:41:40.551819+0000","last_undegraded":"2023-09-23T19:41:40.551819+0000","last_fullsized":"2023-09-23T19:41:40.551819+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:17.608671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.185477+0000","last_change":"2023-09-23T19:41:39.543292+0000","last_active":"2023-09-23T19:41:50.185477+0000","last_peered":"2023-09-23T19:41:50.185477+0000","last_clean":"2023-09-23T19:41:50.185477+0000","last_became_active":"2023-09-23T19:41:39.543047+0000","last_became_peered":"2023-09-23T19:41:39.543047+0000","last_unstale":"2023-09-23T19:41:50.185477+0000","last_undegraded":"2023-09-23T19:41:50.185477+0000","last_fullsized":"2023-09-23T19:41:50.185477+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:14.795347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.564012+0000","last_change":"2023-09-23T19:41:40.557784+0000","last_active":"2023-09-23T19:41:40.564012+0000","last_peered":"2023-09-23T19:41:40.564012+0000","last_clean":"2023-09-23T19:41:40.564012+0000","last_became_active":"2023-09-23T19:41:40.557467+0000","last_became_peered":"2023-09-23T19:41:40.557467+0000","last_unstale":"2023-09-23T19:41:40.564012+0000","last_undegraded":"2023-09-23T19:41:40.564012+0000","last_fullsized":"2023-09-23T19:41:40.564012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:07.877469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.291265+0000","last_change":"2023-09-23T19:41:39.545236+0000","last_active":"2023-09-23T19:41:42.291265+0000","last_peered":"2023-09-23T19:41:42.291265+0000","last_clean":"2023-09-23T19:41:42.291265+0000","last_became_active":"2023-09-23T19:41:39.544931+0000","last_became_peered":"2023-09-23T19:41:39.544931+0000","last_unstale":"2023-09-23T19:41:42.291265+0000","last_undegraded":"2023-09-23T19:41:42.291265+0000","last_fullsized":"2023-09-23T19:41:42.291265+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:08.229073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551547+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:41:40.551547+0000","last_peered":"2023-09-23T19:41:40.551547+0000","last_clean":"2023-09-23T19:41:40.551547+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:41:40.551547+0000","last_undegraded":"2023-09-23T19:41:40.551547+0000","last_fullsized":"2023-09-23T19:41:40.551547+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.291326+0000","last_change":"2023-09-23T19:41:39.546450+0000","last_active":"2023-09-23T19:41:42.291326+0000","last_peered":"2023-09-23T19:41:42.291326+0000","last_clean":"2023-09-23T19:41:42.291326+0000","last_became_active":"2023-09-23T19:41:39.546151+0000","last_became_peered":"2023-09-23T19:41:39.546151+0000","last_unstale":"2023-09-23T19:41:42.291326+0000","last_undegraded":"2023-09-23T19:41:42.291326+0000","last_fullsized":"2023-09-23T19:41:42.291326+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:21.171118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556349+0000","last_change":"2023-09-23T19:40:40.249273+0000","last_active":"2023-09-23T19:41:40.556349+0000","last_peered":"2023-09-23T19:41:40.556349+0000","last_clean":"2023-09-23T19:41:40.556349+0000","last_became_active":"2023-09-23T19:40:40.248979+0000","last_became_peered":"2023-09-23T19:40:40.248979+0000","last_unstale":"2023-09-23T19:41:40.556349+0000","last_undegraded":"2023-09-23T19:41:40.556349+0000","last_fullsized":"2023-09-23T19:41:40.556349+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:25.089853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598700+0000","last_change":"2023-09-23T19:41:40.597923+0000","last_active":"2023-09-23T19:41:40.598700+0000","last_peered":"2023-09-23T19:41:40.598700+0000","last_clean":"2023-09-23T19:41:40.598700+0000","last_became_active":"2023-09-23T19:41:40.597629+0000","last_became_peered":"2023-09-23T19:41:40.597629+0000","last_unstale":"2023-09-23T19:41:40.598700+0000","last_undegraded":"2023-09-23T19:41:40.598700+0000","last_fullsized":"2023-09-23T19:41:40.598700+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:55.670197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.553338+0000","last_change":"2023-09-23T19:40:40.251554+0000","last_active":"2023-09-23T19:41:40.553338+0000","last_peered":"2023-09-23T19:41:40.553338+0000","last_clean":"2023-09-23T19:41:40.553338+0000","last_became_active":"2023-09-23T19:40:40.251284+0000","last_became_peered":"2023-09-23T19:40:40.251284+0000","last_unstale":"2023-09-23T19:41:40.553338+0000","last_undegraded":"2023-09-23T19:41:40.553338+0000","last_fullsized":"2023-09-23T19:41:40.553338+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:40.680795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.184368+0000","last_change":"2023-09-23T19:41:39.550097+0000","last_active":"2023-09-23T19:41:50.184368+0000","last_peered":"2023-09-23T19:41:50.184368+0000","last_clean":"2023-09-23T19:41:50.184368+0000","last_became_active":"2023-09-23T19:41:39.549819+0000","last_became_peered":"2023-09-23T19:41:39.549819+0000","last_unstale":"2023-09-23T19:41:50.184368+0000","last_undegraded":"2023-09-23T19:41:50.184368+0000","last_fullsized":"2023-09-23T19:41:50.184368+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:42.084440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.596885+0000","last_change":"2023-09-23T19:41:40.596692+0000","last_active":"2023-09-23T19:41:40.596885+0000","last_peered":"2023-09-23T19:41:40.596885+0000","last_clean":"2023-09-23T19:41:40.596885+0000","last_became_active":"2023-09-23T19:41:40.596304+0000","last_became_peered":"2023-09-23T19:41:40.596304+0000","last_unstale":"2023-09-23T19:41:40.596885+0000","last_undegraded":"2023-09-23T19:41:40.596885+0000","last_fullsized":"2023-09-23T19:41:40.596885+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:07.852948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552981+0000","last_change":"2023-09-23T19:41:39.538994+0000","last_active":"2023-09-23T19:41:40.552981+0000","last_peered":"2023-09-23T19:41:40.552981+0000","last_clean":"2023-09-23T19:41:40.552981+0000","last_became_active":"2023-09-23T19:41:39.538696+0000","last_became_peered":"2023-09-23T19:41:39.538696+0000","last_unstale":"2023-09-23T19:41:40.552981+0000","last_undegraded":"2023-09-23T19:41:40.552981+0000","last_fullsized":"2023-09-23T19:41:40.552981+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:12.235516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.594956+0000","last_change":"2023-09-23T19:40:40.361547+0000","last_active":"2023-09-23T19:41:40.594956+0000","last_peered":"2023-09-23T19:41:40.594956+0000","last_clean":"2023-09-23T19:41:40.594956+0000","last_became_active":"2023-09-23T19:40:40.361206+0000","last_became_peered":"2023-09-23T19:40:40.361206+0000","last_unstale":"2023-09-23T19:41:40.594956+0000","last_undegraded":"2023-09-23T19:41:40.594956+0000","last_fullsized":"2023-09-23T19:41:40.594956+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:27.379089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.565563+0000","last_change":"2023-09-23T19:41:40.565030+0000","last_active":"2023-09-23T19:41:40.565563+0000","last_peered":"2023-09-23T19:41:40.565563+0000","last_clean":"2023-09-23T19:41:40.565563+0000","last_became_active":"2023-09-23T19:41:40.564684+0000","last_became_peered":"2023-09-23T19:41:40.564684+0000","last_unstale":"2023-09-23T19:41:40.565563+0000","last_undegraded":"2023-09-23T19:41:40.565563+0000","last_fullsized":"2023-09-23T19:41:40.565563+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:25.266861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558708+0000","last_change":"2023-09-23T19:41:39.545896+0000","last_active":"2023-09-23T19:41:40.558708+0000","last_peered":"2023-09-23T19:41:40.558708+0000","last_clean":"2023-09-23T19:41:40.558708+0000","last_became_active":"2023-09-23T19:41:39.545217+0000","last_became_peered":"2023-09-23T19:41:39.545217+0000","last_unstale":"2023-09-23T19:41:40.558708+0000","last_undegraded":"2023-09-23T19:41:40.558708+0000","last_fullsized":"2023-09-23T19:41:40.558708+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:12.382093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.594661+0000","last_change":"2023-09-23T19:40:40.361627+0000","last_active":"2023-09-23T19:41:40.594661+0000","last_peered":"2023-09-23T19:41:40.594661+0000","last_clean":"2023-09-23T19:41:40.594661+0000","last_became_active":"2023-09-23T19:40:40.361250+0000","last_became_peered":"2023-09-23T19:40:40.361250+0000","last_unstale":"2023-09-23T19:41:40.594661+0000","last_undegraded":"2023-09-23T19:41:40.594661+0000","last_fullsized":"2023-09-23T19:41:40.594661+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:04.232689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598709+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:41:40.598709+0000","last_peered":"2023-09-23T19:41:40.598709+0000","last_clean":"2023-09-23T19:41:40.598709+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:41:40.598709+0000","last_undegraded":"2023-09-23T19:41:40.598709+0000","last_fullsized":"2023-09-23T19:41:40.598709+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556180+0000","last_change":"2023-09-23T19:41:39.539226+0000","last_active":"2023-09-23T19:41:40.556180+0000","last_peered":"2023-09-23T19:41:40.556180+0000","last_clean":"2023-09-23T19:41:40.556180+0000","last_became_active":"2023-09-23T19:41:39.538953+0000","last_became_peered":"2023-09-23T19:41:39.538953+0000","last_unstale":"2023-09-23T19:41:40.556180+0000","last_undegraded":"2023-09-23T19:41:40.556180+0000","last_fullsized":"2023-09-23T19:41:40.556180+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:44.360143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":274,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.594449+0000","last_change":"2023-09-23T19:40:07.160277+0000","last_active":"2023-09-23T19:41:40.594449+0000","last_peered":"2023-09-23T19:41:40.594449+0000","last_clean":"2023-09-23T19:41:40.594449+0000","last_became_active":"2023-09-23T19:40:07.159949+0000","last_became_peered":"2023-09-23T19:40:07.159949+0000","last_unstale":"2023-09-23T19:41:40.594449+0000","last_undegraded":"2023-09-23T19:41:40.594449+0000","last_fullsized":"2023-09-23T19:41:40.594449+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:20.218507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555571+0000","last_change":"2023-09-23T19:40:40.249857+0000","last_active":"2023-09-23T19:41:40.555571+0000","last_peered":"2023-09-23T19:41:40.555571+0000","last_clean":"2023-09-23T19:41:40.555571+0000","last_became_active":"2023-09-23T19:40:40.249587+0000","last_became_peered":"2023-09-23T19:40:40.249587+0000","last_unstale":"2023-09-23T19:41:40.555571+0000","last_undegraded":"2023-09-23T19:41:40.555571+0000","last_fullsized":"2023-09-23T19:41:40.555571+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:37.624835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.565655+0000","last_change":"2023-09-23T19:41:40.565135+0000","last_active":"2023-09-23T19:41:40.565655+0000","last_peered":"2023-09-23T19:41:40.565655+0000","last_clean":"2023-09-23T19:41:40.565655+0000","last_became_active":"2023-09-23T19:41:40.564825+0000","last_became_peered":"2023-09-23T19:41:40.564825+0000","last_unstale":"2023-09-23T19:41:40.565655+0000","last_undegraded":"2023-09-23T19:41:40.565655+0000","last_fullsized":"2023-09-23T19:41:40.565655+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:55.848621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.553695+0000","last_change":"2023-09-23T19:41:39.539162+0000","last_active":"2023-09-23T19:41:40.553695+0000","last_peered":"2023-09-23T19:41:40.553695+0000","last_clean":"2023-09-23T19:41:40.553695+0000","last_became_active":"2023-09-23T19:41:39.538874+0000","last_became_peered":"2023-09-23T19:41:39.538874+0000","last_unstale":"2023-09-23T19:41:40.553695+0000","last_undegraded":"2023-09-23T19:41:40.553695+0000","last_fullsized":"2023-09-23T19:41:40.553695+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:35.739627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551130+0000","last_change":"2023-09-23T19:40:40.361713+0000","last_active":"2023-09-23T19:41:40.551130+0000","last_peered":"2023-09-23T19:41:40.551130+0000","last_clean":"2023-09-23T19:41:40.551130+0000","last_became_active":"2023-09-23T19:40:40.361369+0000","last_became_peered":"2023-09-23T19:40:40.361369+0000","last_unstale":"2023-09-23T19:41:40.551130+0000","last_undegraded":"2023-09-23T19:41:40.551130+0000","last_fullsized":"2023-09-23T19:41:40.551130+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:04.414357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558775+0000","last_change":"2023-09-23T19:41:40.558096+0000","last_active":"2023-09-23T19:41:40.558775+0000","last_peered":"2023-09-23T19:41:40.558775+0000","last_clean":"2023-09-23T19:41:40.558775+0000","last_became_active":"2023-09-23T19:41:40.557808+0000","last_became_peered":"2023-09-23T19:41:40.557808+0000","last_unstale":"2023-09-23T19:41:40.558775+0000","last_undegraded":"2023-09-23T19:41:40.558775+0000","last_fullsized":"2023-09-23T19:41:40.558775+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:23:58.943551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551750+0000","last_change":"2023-09-23T19:41:39.545431+0000","last_active":"2023-09-23T19:41:40.551750+0000","last_peered":"2023-09-23T19:41:40.551750+0000","last_clean":"2023-09-23T19:41:40.551750+0000","last_became_active":"2023-09-23T19:41:39.545150+0000","last_became_peered":"2023-09-23T19:41:39.545150+0000","last_unstale":"2023-09-23T19:41:40.551750+0000","last_undegraded":"2023-09-23T19:41:40.551750+0000","last_fullsized":"2023-09-23T19:41:40.551750+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:10.130116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555745+0000","last_change":"2023-09-23T19:40:40.251667+0000","last_active":"2023-09-23T19:41:40.555745+0000","last_peered":"2023-09-23T19:41:40.555745+0000","last_clean":"2023-09-23T19:41:40.555745+0000","last_became_active":"2023-09-23T19:40:40.251366+0000","last_became_peered":"2023-09-23T19:40:40.251366+0000","last_unstale":"2023-09-23T19:41:40.555745+0000","last_undegraded":"2023-09-23T19:41:40.555745+0000","last_fullsized":"2023-09-23T19:41:40.555745+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:38.869721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.561274+0000","last_change":"2023-09-23T19:41:40.559609+0000","last_active":"2023-09-23T19:41:40.561274+0000","last_peered":"2023-09-23T19:41:40.561274+0000","last_clean":"2023-09-23T19:41:40.561274+0000","last_became_active":"2023-09-23T19:41:40.558615+0000","last_became_peered":"2023-09-23T19:41:40.558615+0000","last_unstale":"2023-09-23T19:41:40.561274+0000","last_undegraded":"2023-09-23T19:41:40.561274+0000","last_fullsized":"2023-09-23T19:41:40.561274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:18.052507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556329+0000","last_change":"2023-09-23T19:41:39.546473+0000","last_active":"2023-09-23T19:41:40.556329+0000","last_peered":"2023-09-23T19:41:40.556329+0000","last_clean":"2023-09-23T19:41:40.556329+0000","last_became_active":"2023-09-23T19:41:39.546208+0000","last_became_peered":"2023-09-23T19:41:39.546208+0000","last_unstale":"2023-09-23T19:41:40.556329+0000","last_undegraded":"2023-09-23T19:41:40.556329+0000","last_fullsized":"2023-09-23T19:41:40.556329+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:20.849754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552867+0000","last_change":"2023-09-23T19:40:40.361619+0000","last_active":"2023-09-23T19:41:40.552867+0000","last_peered":"2023-09-23T19:41:40.552867+0000","last_clean":"2023-09-23T19:41:40.552867+0000","last_became_active":"2023-09-23T19:40:40.361237+0000","last_became_peered":"2023-09-23T19:40:40.361237+0000","last_unstale":"2023-09-23T19:41:40.552867+0000","last_undegraded":"2023-09-23T19:41:40.552867+0000","last_fullsized":"2023-09-23T19:41:40.552867+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:09.643497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558945+0000","last_change":"2023-09-23T19:41:40.558310+0000","last_active":"2023-09-23T19:41:40.558945+0000","last_peered":"2023-09-23T19:41:40.558945+0000","last_clean":"2023-09-23T19:41:40.558945+0000","last_became_active":"2023-09-23T19:41:40.558033+0000","last_became_peered":"2023-09-23T19:41:40.558033+0000","last_unstale":"2023-09-23T19:41:40.558945+0000","last_undegraded":"2023-09-23T19:41:40.558945+0000","last_fullsized":"2023-09-23T19:41:40.558945+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.736832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556080+0000","last_change":"2023-09-23T19:41:39.542335+0000","last_active":"2023-09-23T19:41:40.556080+0000","last_peered":"2023-09-23T19:41:40.556080+0000","last_clean":"2023-09-23T19:41:40.556080+0000","last_became_active":"2023-09-23T19:41:39.540848+0000","last_became_peered":"2023-09-23T19:41:39.540848+0000","last_unstale":"2023-09-23T19:41:40.556080+0000","last_undegraded":"2023-09-23T19:41:40.556080+0000","last_fullsized":"2023-09-23T19:41:40.556080+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:07.239425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552186+0000","last_change":"2023-09-23T19:41:40.334887+0000","last_active":"2023-09-23T19:41:40.552186+0000","last_peered":"2023-09-23T19:41:40.552186+0000","last_clean":"2023-09-23T19:41:40.552186+0000","last_became_active":"2023-09-23T19:41:40.334628+0000","last_became_peered":"2023-09-23T19:41:40.334628+0000","last_unstale":"2023-09-23T19:41:40.552186+0000","last_undegraded":"2023-09-23T19:41:40.552186+0000","last_fullsized":"2023-09-23T19:41:40.552186+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:09.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551561+0000","last_change":"2023-09-23T19:41:39.540507+0000","last_active":"2023-09-23T19:41:40.551561+0000","last_peered":"2023-09-23T19:41:40.551561+0000","last_clean":"2023-09-23T19:41:40.551561+0000","last_became_active":"2023-09-23T19:41:39.540001+0000","last_became_peered":"2023-09-23T19:41:39.540001+0000","last_unstale":"2023-09-23T19:41:40.551561+0000","last_undegraded":"2023-09-23T19:41:40.551561+0000","last_fullsized":"2023-09-23T19:41:40.551561+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:44.405007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552124+0000","last_change":"2023-09-23T19:41:39.554902+0000","last_active":"2023-09-23T19:41:40.552124+0000","last_peered":"2023-09-23T19:41:40.552124+0000","last_clean":"2023-09-23T19:41:40.552124+0000","last_became_active":"2023-09-23T19:41:39.554558+0000","last_became_peered":"2023-09-23T19:41:39.554558+0000","last_unstale":"2023-09-23T19:41:40.552124+0000","last_undegraded":"2023-09-23T19:41:40.552124+0000","last_fullsized":"2023-09-23T19:41:40.552124+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:11.716842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555889+0000","last_change":"2023-09-23T19:41:39.540408+0000","last_active":"2023-09-23T19:41:40.555889+0000","last_peered":"2023-09-23T19:41:40.555889+0000","last_clean":"2023-09-23T19:41:40.555889+0000","last_became_active":"2023-09-23T19:41:39.540127+0000","last_became_peered":"2023-09-23T19:41:39.540127+0000","last_unstale":"2023-09-23T19:41:40.555889+0000","last_undegraded":"2023-09-23T19:41:40.555889+0000","last_fullsized":"2023-09-23T19:41:40.555889+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:05.728760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552008+0000","last_change":"2023-09-23T19:41:40.426167+0000","last_active":"2023-09-23T19:41:40.552008+0000","last_peered":"2023-09-23T19:41:40.552008+0000","last_clean":"2023-09-23T19:41:40.552008+0000","last_became_active":"2023-09-23T19:41:40.425797+0000","last_became_peered":"2023-09-23T19:41:40.425797+0000","last_unstale":"2023-09-23T19:41:40.552008+0000","last_undegraded":"2023-09-23T19:41:40.552008+0000","last_fullsized":"2023-09-23T19:41:40.552008+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:19.716310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.185836+0000","last_change":"2023-09-23T19:41:39.546538+0000","last_active":"2023-09-23T19:41:50.185836+0000","last_peered":"2023-09-23T19:41:50.185836+0000","last_clean":"2023-09-23T19:41:50.185836+0000","last_became_active":"2023-09-23T19:41:39.546214+0000","last_became_peered":"2023-09-23T19:41:39.546214+0000","last_unstale":"2023-09-23T19:41:50.185836+0000","last_undegraded":"2023-09-23T19:41:50.185836+0000","last_fullsized":"2023-09-23T19:41:50.185836+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:59.036343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.564443+0000","last_change":"2023-09-23T19:41:40.564174+0000","last_active":"2023-09-23T19:41:40.564443+0000","last_peered":"2023-09-23T19:41:40.564443+0000","last_clean":"2023-09-23T19:41:40.564443+0000","last_became_active":"2023-09-23T19:41:40.563856+0000","last_became_peered":"2023-09-23T19:41:40.563856+0000","last_unstale":"2023-09-23T19:41:40.564443+0000","last_undegraded":"2023-09-23T19:41:40.564443+0000","last_fullsized":"2023-09-23T19:41:40.564443+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:08.391644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555908+0000","last_change":"2023-09-23T19:41:39.542111+0000","last_active":"2023-09-23T19:41:40.555908+0000","last_peered":"2023-09-23T19:41:40.555908+0000","last_clean":"2023-09-23T19:41:40.555908+0000","last_became_active":"2023-09-23T19:41:39.541682+0000","last_became_peered":"2023-09-23T19:41:39.541682+0000","last_unstale":"2023-09-23T19:41:40.555908+0000","last_undegraded":"2023-09-23T19:41:40.555908+0000","last_fullsized":"2023-09-23T19:41:40.555908+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:10.032934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.595277+0000","last_change":"2023-09-23T19:41:40.594907+0000","last_active":"2023-09-23T19:41:40.595277+0000","last_peered":"2023-09-23T19:41:40.595277+0000","last_clean":"2023-09-23T19:41:40.595277+0000","last_became_active":"2023-09-23T19:41:40.594574+0000","last_became_peered":"2023-09-23T19:41:40.594574+0000","last_unstale":"2023-09-23T19:41:40.595277+0000","last_undegraded":"2023-09-23T19:41:40.595277+0000","last_fullsized":"2023-09-23T19:41:40.595277+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:22.485793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.468526+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:41:42.468526+0000","last_peered":"2023-09-23T19:41:42.468526+0000","last_clean":"2023-09-23T19:41:42.468526+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:41:42.468526+0000","last_undegraded":"2023-09-23T19:41:42.468526+0000","last_fullsized":"2023-09-23T19:41:42.468526+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551464+0000","last_change":"2023-09-23T19:41:40.425667+0000","last_active":"2023-09-23T19:41:40.551464+0000","last_peered":"2023-09-23T19:41:40.551464+0000","last_clean":"2023-09-23T19:41:40.551464+0000","last_became_active":"2023-09-23T19:41:40.425346+0000","last_became_peered":"2023-09-23T19:41:40.425346+0000","last_unstale":"2023-09-23T19:41:40.551464+0000","last_undegraded":"2023-09-23T19:41:40.551464+0000","last_fullsized":"2023-09-23T19:41:40.551464+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:54.159658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.183268+0000","last_change":"2023-09-23T19:41:39.540550+0000","last_active":"2023-09-23T19:41:50.183268+0000","last_peered":"2023-09-23T19:41:50.183268+0000","last_clean":"2023-09-23T19:41:50.183268+0000","last_became_active":"2023-09-23T19:41:39.540282+0000","last_became_peered":"2023-09-23T19:41:39.540282+0000","last_unstale":"2023-09-23T19:41:50.183268+0000","last_undegraded":"2023-09-23T19:41:50.183268+0000","last_fullsized":"2023-09-23T19:41:50.183268+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:19.128796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.596821+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:41:40.596821+0000","last_peered":"2023-09-23T19:41:40.596821+0000","last_clean":"2023-09-23T19:41:40.596821+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:41:40.596821+0000","last_undegraded":"2023-09-23T19:41:40.596821+0000","last_fullsized":"2023-09-23T19:41:40.596821+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.468511+0000","last_change":"2023-09-23T19:41:39.546086+0000","last_active":"2023-09-23T19:41:42.468511+0000","last_peered":"2023-09-23T19:41:42.468511+0000","last_clean":"2023-09-23T19:41:42.468511+0000","last_became_active":"2023-09-23T19:41:39.545818+0000","last_became_peered":"2023-09-23T19:41:39.545818+0000","last_unstale":"2023-09-23T19:41:42.468511+0000","last_undegraded":"2023-09-23T19:41:42.468511+0000","last_fullsized":"2023-09-23T19:41:42.468511+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:36.470729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.596822+0000","last_change":"2023-09-23T19:41:40.596504+0000","last_active":"2023-09-23T19:41:40.596822+0000","last_peered":"2023-09-23T19:41:40.596822+0000","last_clean":"2023-09-23T19:41:40.596822+0000","last_became_active":"2023-09-23T19:41:40.596198+0000","last_became_peered":"2023-09-23T19:41:40.596198+0000","last_unstale":"2023-09-23T19:41:40.596822+0000","last_undegraded":"2023-09-23T19:41:40.596822+0000","last_fullsized":"2023-09-23T19:41:40.596822+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:04.764297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.186274+0000","last_change":"2023-09-23T19:41:39.537846+0000","last_active":"2023-09-23T19:41:50.186274+0000","last_peered":"2023-09-23T19:41:50.186274+0000","last_clean":"2023-09-23T19:41:50.186274+0000","last_became_active":"2023-09-23T19:41:39.537296+0000","last_became_peered":"2023-09-23T19:41:39.537296+0000","last_unstale":"2023-09-23T19:41:50.186274+0000","last_undegraded":"2023-09-23T19:41:50.186274+0000","last_fullsized":"2023-09-23T19:41:50.186274+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:29.430699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.599001+0000","last_change":"2023-09-23T19:41:40.598346+0000","last_active":"2023-09-23T19:41:40.599001+0000","last_peered":"2023-09-23T19:41:40.599001+0000","last_clean":"2023-09-23T19:41:40.599001+0000","last_became_active":"2023-09-23T19:41:40.597877+0000","last_became_peered":"2023-09-23T19:41:40.597877+0000","last_unstale":"2023-09-23T19:41:40.599001+0000","last_undegraded":"2023-09-23T19:41:40.599001+0000","last_fullsized":"2023-09-23T19:41:40.599001+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:12.163854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.594426+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:41:40.594426+0000","last_peered":"2023-09-23T19:41:40.594426+0000","last_clean":"2023-09-23T19:41:40.594426+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:41:40.594426+0000","last_undegraded":"2023-09-23T19:41:40.594426+0000","last_fullsized":"2023-09-23T19:41:40.594426+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598708+0000","last_change":"2023-09-23T19:41:40.598132+0000","last_active":"2023-09-23T19:41:40.598708+0000","last_peered":"2023-09-23T19:41:40.598708+0000","last_clean":"2023-09-23T19:41:40.598708+0000","last_became_active":"2023-09-23T19:41:40.597692+0000","last_became_peered":"2023-09-23T19:41:40.597692+0000","last_unstale":"2023-09-23T19:41:40.598708+0000","last_undegraded":"2023-09-23T19:41:40.598708+0000","last_fullsized":"2023-09-23T19:41:40.598708+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:06.861045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558146+0000","last_change":"2023-09-23T19:41:39.542640+0000","last_active":"2023-09-23T19:41:40.558146+0000","last_peered":"2023-09-23T19:41:40.558146+0000","last_clean":"2023-09-23T19:41:40.558146+0000","last_became_active":"2023-09-23T19:41:39.542371+0000","last_became_peered":"2023-09-23T19:41:39.542371+0000","last_unstale":"2023-09-23T19:41:40.558146+0000","last_undegraded":"2023-09-23T19:41:40.558146+0000","last_fullsized":"2023-09-23T19:41:40.558146+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:55.308862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.595072+0000","last_change":"2023-09-23T19:41:40.594835+0000","last_active":"2023-09-23T19:41:40.595072+0000","last_peered":"2023-09-23T19:41:40.595072+0000","last_clean":"2023-09-23T19:41:40.595072+0000","last_became_active":"2023-09-23T19:41:40.594493+0000","last_became_peered":"2023-09-23T19:41:40.594493+0000","last_unstale":"2023-09-23T19:41:40.595072+0000","last_undegraded":"2023-09-23T19:41:40.595072+0000","last_fullsized":"2023-09-23T19:41:40.595072+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:03.135867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.184074+0000","last_change":"2023-09-23T19:41:39.544179+0000","last_active":"2023-09-23T19:41:50.184074+0000","last_peered":"2023-09-23T19:41:50.184074+0000","last_clean":"2023-09-23T19:41:50.184074+0000","last_became_active":"2023-09-23T19:41:39.543928+0000","last_became_peered":"2023-09-23T19:41:39.543928+0000","last_unstale":"2023-09-23T19:41:50.184074+0000","last_undegraded":"2023-09-23T19:41:50.184074+0000","last_fullsized":"2023-09-23T19:41:50.184074+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:04.486013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551601+0000","last_change":"2023-09-23T19:41:40.334429+0000","last_active":"2023-09-23T19:41:40.551601+0000","last_peered":"2023-09-23T19:41:40.551601+0000","last_clean":"2023-09-23T19:41:40.551601+0000","last_became_active":"2023-09-23T19:41:40.334122+0000","last_became_peered":"2023-09-23T19:41:40.334122+0000","last_unstale":"2023-09-23T19:41:40.551601+0000","last_undegraded":"2023-09-23T19:41:40.551601+0000","last_fullsized":"2023-09-23T19:41:40.551601+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.729985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.186987+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:41:50.186987+0000","last_peered":"2023-09-23T19:41:50.186987+0000","last_clean":"2023-09-23T19:41:50.186987+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:41:50.186987+0000","last_undegraded":"2023-09-23T19:41:50.186987+0000","last_fullsized":"2023-09-23T19:41:50.186987+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.561294+0000","last_change":"2023-09-23T19:41:40.559776+0000","last_active":"2023-09-23T19:41:40.561294+0000","last_peered":"2023-09-23T19:41:40.561294+0000","last_clean":"2023-09-23T19:41:40.561294+0000","last_became_active":"2023-09-23T19:41:40.559365+0000","last_became_peered":"2023-09-23T19:41:40.559365+0000","last_unstale":"2023-09-23T19:41:40.561294+0000","last_undegraded":"2023-09-23T19:41:40.561294+0000","last_fullsized":"2023-09-23T19:41:40.561294+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:06.462587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.184194+0000","last_change":"2023-09-23T19:41:39.545954+0000","last_active":"2023-09-23T19:41:50.184194+0000","last_peered":"2023-09-23T19:41:50.184194+0000","last_clean":"2023-09-23T19:41:50.184194+0000","last_became_active":"2023-09-23T19:41:39.545690+0000","last_became_peered":"2023-09-23T19:41:39.545690+0000","last_unstale":"2023-09-23T19:41:50.184194+0000","last_undegraded":"2023-09-23T19:41:50.184194+0000","last_fullsized":"2023-09-23T19:41:50.184194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:58.159061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598805+0000","last_change":"2023-09-23T19:41:40.598168+0000","last_active":"2023-09-23T19:41:40.598805+0000","last_peered":"2023-09-23T19:41:40.598805+0000","last_clean":"2023-09-23T19:41:40.598805+0000","last_became_active":"2023-09-23T19:41:40.597174+0000","last_became_peered":"2023-09-23T19:41:40.597174+0000","last_unstale":"2023-09-23T19:41:40.598805+0000","last_undegraded":"2023-09-23T19:41:40.598805+0000","last_fullsized":"2023-09-23T19:41:40.598805+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:14.395482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555514+0000","last_change":"2023-09-23T19:41:39.542457+0000","last_active":"2023-09-23T19:41:40.555514+0000","last_peered":"2023-09-23T19:41:40.555514+0000","last_clean":"2023-09-23T19:41:40.555514+0000","last_became_active":"2023-09-23T19:41:39.542148+0000","last_became_peered":"2023-09-23T19:41:39.542148+0000","last_unstale":"2023-09-23T19:41:40.555514+0000","last_undegraded":"2023-09-23T19:41:40.555514+0000","last_fullsized":"2023-09-23T19:41:40.555514+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:42.959334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.562795+0000","last_change":"2023-09-23T19:41:40.558606+0000","last_active":"2023-09-23T19:41:40.562795+0000","last_peered":"2023-09-23T19:41:40.562795+0000","last_clean":"2023-09-23T19:41:40.562795+0000","last_became_active":"2023-09-23T19:41:40.558355+0000","last_became_peered":"2023-09-23T19:41:40.558355+0000","last_unstale":"2023-09-23T19:41:40.562795+0000","last_undegraded":"2023-09-23T19:41:40.562795+0000","last_fullsized":"2023-09-23T19:41:40.562795+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:04.750882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.560574+0000","last_change":"2023-09-23T19:41:39.544822+0000","last_active":"2023-09-23T19:41:40.560574+0000","last_peered":"2023-09-23T19:41:40.560574+0000","last_clean":"2023-09-23T19:41:40.560574+0000","last_became_active":"2023-09-23T19:41:39.542108+0000","last_became_peered":"2023-09-23T19:41:39.542108+0000","last_unstale":"2023-09-23T19:41:40.560574+0000","last_undegraded":"2023-09-23T19:41:40.560574+0000","last_fullsized":"2023-09-23T19:41:40.560574+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.478529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.559588+0000","last_change":"2023-09-23T19:41:40.559379+0000","last_active":"2023-09-23T19:41:40.559588+0000","last_peered":"2023-09-23T19:41:40.559588+0000","last_clean":"2023-09-23T19:41:40.559588+0000","last_became_active":"2023-09-23T19:41:40.559146+0000","last_became_peered":"2023-09-23T19:41:40.559146+0000","last_unstale":"2023-09-23T19:41:40.559588+0000","last_undegraded":"2023-09-23T19:41:40.559588+0000","last_fullsized":"2023-09-23T19:41:40.559588+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:15.503413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556181+0000","last_change":"2023-09-23T19:41:39.547073+0000","last_active":"2023-09-23T19:41:40.556181+0000","last_peered":"2023-09-23T19:41:40.556181+0000","last_clean":"2023-09-23T19:41:40.556181+0000","last_became_active":"2023-09-23T19:41:39.546648+0000","last_became_peered":"2023-09-23T19:41:39.546648+0000","last_unstale":"2023-09-23T19:41:40.556181+0000","last_undegraded":"2023-09-23T19:41:40.556181+0000","last_fullsized":"2023-09-23T19:41:40.556181+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:45.969145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598842+0000","last_change":"2023-09-23T19:41:40.598150+0000","last_active":"2023-09-23T19:41:40.598842+0000","last_peered":"2023-09-23T19:41:40.598842+0000","last_clean":"2023-09-23T19:41:40.598842+0000","last_became_active":"2023-09-23T19:41:40.597573+0000","last_became_peered":"2023-09-23T19:41:40.597573+0000","last_unstale":"2023-09-23T19:41:40.598842+0000","last_undegraded":"2023-09-23T19:41:40.598842+0000","last_fullsized":"2023-09-23T19:41:40.598842+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:05.362995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552985+0000","last_change":"2023-09-23T19:41:39.538925+0000","last_active":"2023-09-23T19:41:40.552985+0000","last_peered":"2023-09-23T19:41:40.552985+0000","last_clean":"2023-09-23T19:41:40.552985+0000","last_became_active":"2023-09-23T19:41:39.538713+0000","last_became_peered":"2023-09-23T19:41:39.538713+0000","last_unstale":"2023-09-23T19:41:40.552985+0000","last_undegraded":"2023-09-23T19:41:40.552985+0000","last_fullsized":"2023-09-23T19:41:40.552985+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:41.173401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558704+0000","last_change":"2023-09-23T19:41:40.557696+0000","last_active":"2023-09-23T19:41:40.558704+0000","last_peered":"2023-09-23T19:41:40.558704+0000","last_clean":"2023-09-23T19:41:40.558704+0000","last_became_active":"2023-09-23T19:41:40.557427+0000","last_became_peered":"2023-09-23T19:41:40.557427+0000","last_unstale":"2023-09-23T19:41:40.558704+0000","last_undegraded":"2023-09-23T19:41:40.558704+0000","last_fullsized":"2023-09-23T19:41:40.558704+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:59.491319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.168337+0000","last_change":"2023-09-23T19:41:39.541190+0000","last_active":"2023-09-23T19:41:42.168337+0000","last_peered":"2023-09-23T19:41:42.168337+0000","last_clean":"2023-09-23T19:41:42.168337+0000","last_became_active":"2023-09-23T19:41:39.540754+0000","last_became_peered":"2023-09-23T19:41:39.540754+0000","last_unstale":"2023-09-23T19:41:42.168337+0000","last_undegraded":"2023-09-23T19:41:42.168337+0000","last_fullsized":"2023-09-23T19:41:42.168337+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:59.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234015,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6536,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93738616,"statfs":{"total":95995035648,"available":95988342784,"internally_reserved":0,"allocated":1310720,"data_stored":891354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.498,"15min":0.5},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.48199999999999998,"15min":0.47799999999999998},"min":{"1min":0.35599999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.51300000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.498,"5min":0.47199999999999998,"15min":0.46800000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.58799999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.496,"15min":0.48699999999999999},"min":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.51700000000000002,"15min":0.501},"min":{"1min":0.35699999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.819,"5min":1.819,"15min":1.819},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.48699999999999999,"15min":0.47999999999999998},"min":{"1min":0.38500000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.51300000000000001,"15min":0.50900000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.45100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.44900000000000001,"15min":0.44700000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.48099999999999998,"15min":0.47599999999999998},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.49099999999999999,"15min":0.48899999999999999},"min":{"1min":0.36099999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.503,"15min":0.495},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.72099999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.38600000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79600000000000004,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56100000000000005,"15min":0.55800000000000005},"min":{"1min":0.41399999999999998,"5min":0.371,"15min":0.371},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.70399999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495652,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6648,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93738504,"statfs":{"total":95995035648,"available":95988228096,"internally_reserved":0,"allocated":1294336,"data_stored":891001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.53800000000000003,"15min":0.54300000000000004},"min":{"1min":0.379,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.65400000000000003,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.52500000000000002,"15min":0.52300000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72999999999999998,"5min":0.871,"15min":0.871},"last":0.55800000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.36099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.748,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.376,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.48799999999999999,"15min":0.48599999999999999},"min":{"1min":0.40100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.60999999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.55600000000000005,"15min":0.54100000000000004},"min":{"1min":0.46300000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.54200000000000004,"15min":0.53600000000000003},"min":{"1min":0.33900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.81200000000000006,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.55900000000000005,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.0579999999999998,"5min":2.0579999999999998,"15min":2.0579999999999998},"last":0.57399999999999995}]},{"osd":4,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.45300000000000001,"15min":0.436},"min":{"1min":0.32000000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.48099999999999998,"15min":0.46500000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.45900000000000002,"15min":0.45200000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.51800000000000002,"15min":0.505},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56200000000000006,"15min":0.56499999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81200000000000006,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.626},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.59299999999999997,"15min":0.59699999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.83499999999999996,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.61199999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6764,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738388,"statfs":{"total":95995035648,"available":95988109312,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.35299999999999998,"15min":0.33500000000000002},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.5,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.36699999999999999,"15min":0.34599999999999997},"min":{"1min":0.25,"5min":0.222,"15min":0.222},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.34399999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:42:16 2023","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.374,"15min":0.38400000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.50800000000000001,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.40799999999999997,"15min":0.41799999999999998},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.46400000000000002,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.29599999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.45900000000000002,"15min":0.47099999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.54900000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.46500000000000002,"15min":0.47899999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.54000000000000004,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.307}]},{"osd":4,"last update":"Sat Sep 23 19:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58099999999999996,"15min":0.59099999999999997},"min":{"1min":0.434,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72899999999999998,"5min":0.87,"15min":0.87},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59299999999999997,"15min":0.60899999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.51500000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:42:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59399999999999997,"15min":0.60799999999999998},"min":{"1min":0.39400000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.755,"5min":0.871,"15min":0.871},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.60299999999999998,"15min":0.61899999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80400000000000005,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.70699999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:41:42 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.46600000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.288,"5min":1.288,"15min":1.288},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61499999999999999,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.64600000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59799999999999998,"15min":0.59899999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73999999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.58399999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7412,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737740,"statfs":{"total":95995035648,"available":95987445760,"internally_reserved":0,"allocated":1290240,"data_stored":890236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.33900000000000002,"15min":0.315},"min":{"1min":0.22600000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.45600000000000002,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.37,"5min":0.36799999999999999,"15min":0.34399999999999997},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.435,"15min":0.39800000000000002},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.5910000000000002,"5min":4.5910000000000002,"15min":4.5910000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39300000000000002,"15min":0.39100000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.27400000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:41:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.45600000000000002,"15min":0.47399999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.51500000000000001,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.378,"5min":0.45100000000000001,"15min":0.47199999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.57799999999999996,"5min":0.627,"15min":0.627},"last":0.379}]},{"osd":4,"last update":"Sat Sep 23 19:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.44800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55600000000000005,"15min":0.55300000000000005},"min":{"1min":0.41199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.81399999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.55300000000000005,"15min":0.55400000000000005},"min":{"1min":0.433,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.58899999999999997,"15min":0.58299999999999996},"min":{"1min":0.46500000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:41:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.60699999999999998,"15min":0.60999999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.745,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.59299999999999997,"15min":0.57899999999999996},"min":{"1min":0.497,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60399999999999998,"15min":0.60199999999999998},"min":{"1min":0.45600000000000002,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83499999999999996,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.55500000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411383,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6644,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738508,"statfs":{"total":95995035648,"available":95988232192,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:42:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.44700000000000001,"15min":0.36799999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.0369999999999999,"5min":5.5629999999999997,"15min":5.5629999999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53900000000000003,"15min":0.42099999999999999},"min":{"1min":0.247,"5min":0.216,"15min":0.216},"max":{"1min":4.4379999999999997,"5min":10.811,"15min":10.811},"last":0.56299999999999994}]},{"osd":1,"last update":"Sat Sep 23 19:42:17 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.46400000000000002,"15min":0.41899999999999998},"min":{"1min":0.254,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.72399999999999998,"5min":5.29,"15min":5.29},"last":0.624},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.58899999999999997,"15min":0.46000000000000002},"min":{"1min":0.26500000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.9540000000000002,"5min":11.986000000000001,"15min":11.986000000000001},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:41:35 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.41799999999999998,"15min":0.42799999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55500000000000005,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.417,"5min":0.65100000000000002,"15min":0.52800000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":15.945,"5min":15.945,"15min":15.945},"last":0.58099999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.66600000000000004,"15min":0.60099999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.8359999999999999,"5min":5.1340000000000003,"15min":5.1340000000000003},"last":0.82699999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.67400000000000004,"15min":0.61899999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.8919999999999999,"5min":5.1920000000000002,"15min":5.1920000000000002},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60099999999999998,"15min":0.60599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.85399999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.68899999999999995,"15min":0.66400000000000003},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.2199999999999998,"5min":5.2199999999999998,"15min":5.2199999999999998},"last":0.752},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.629,"15min":0.64100000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.746,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.91000000000000003}]},{"osd":7,"last update":"Sat Sep 23 19:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.40899999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.77400000000000002,"15min":0.80800000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.89000000000000001,"5min":5.2629999999999999,"15min":5.2629999999999999},"last":0.77200000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6128,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739024,"statfs":{"total":95995035648,"available":95988760576,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.35799999999999998,"15min":0.35999999999999999},"min":{"1min":0.252,"5min":0.189,"15min":0.189},"max":{"1min":0.46400000000000002,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40500000000000003,"15min":0.42499999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.53200000000000003,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.32800000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.434,"15min":0.434},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.6909999999999998,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.41899999999999998,"15min":0.434},"min":{"1min":0.26100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.311}]},{"osd":2,"last update":"Sat Sep 23 19:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.46800000000000003,"15min":0.48799999999999999},"min":{"1min":0.313,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60199999999999998,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.442,"15min":0.46100000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.61399999999999999,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:42:01 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.52600000000000002,"15min":0.52200000000000002},"min":{"1min":0.39700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54100000000000004,"15min":0.53600000000000003},"min":{"1min":0.39700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.56599999999999995,"15min":0.56200000000000006},"min":{"1min":0.42899999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56899999999999995,"15min":0.57199999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.71799999999999997,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.59899999999999998,"15min":0.60599999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.497}]},{"osd":7,"last update":"Sat Sep 23 19:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.60499999999999998,"15min":0.60399999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.59199999999999997,"15min":0.58999999999999997},"min":{"1min":0.44800000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.51700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6208,"kb_used_data":696,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738944,"statfs":{"total":95995035648,"available":95988678656,"internally_reserved":0,"allocated":712704,"data_stored":300974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.49199999999999999,"15min":0.49099999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.50800000000000001,"15min":0.51000000000000001},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.63400000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33200000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.50700000000000001,"15min":0.503},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.42499999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.52600000000000002,"15min":0.52100000000000002},"min":{"1min":0.38700000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.52300000000000002,"15min":0.51300000000000001},"min":{"1min":0.39200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.50900000000000001,"15min":0.497},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53100000000000003,"15min":0.52300000000000002},"min":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.42199999999999999,"15min":0.40200000000000002},"min":{"1min":0.32000000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.621,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.43099999999999999,"15min":0.39800000000000002},"min":{"1min":0.34000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:41:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.45100000000000001,"15min":0.44},"min":{"1min":0.33000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.49199999999999999,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.315},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.56000000000000005,"15min":0.56799999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68300000000000005,"5min":0.878,"15min":0.878},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.42799999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.67000000000000004,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.64500000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757289,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6064,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739088,"statfs":{"total":95995035648,"available":95988826112,"internally_reserved":0,"allocated":696320,"data_stored":299858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.54600000000000004,"15min":0.55500000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.72799999999999998,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.498,"15min":0.496},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.45800000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.503},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.55100000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65800000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.64600000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.51300000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77400000000000002,"5min":0.874,"15min":0.874},"last":0.60999999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.40500000000000003,"15min":0.39200000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.623},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.41799999999999998,"15min":0.40600000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.69599999999999995}]},{"osd":6,"last update":"Sat Sep 23 19:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.45200000000000001,"15min":0.438},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.499,"15min":0.48499999999999999},"min":{"1min":0.36399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.72599999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84799999999999998,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.56299999999999994,"15min":0.55500000000000005},"min":{"1min":0.43099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.67400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:42:26.856 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:42:27.204 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:27.607 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:27 smithi135 ceph-mon[118715]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:27.607 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:27 smithi135 ceph-mon[100251]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:27 smithi144 ceph-mon[104994]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:28.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:28 smithi135 ceph-mon[118715]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:28.553 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:28 smithi135 ceph-mon[100251]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:28 smithi144 ceph-mon[104994]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:28.924 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:42:28.926 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:42:29.575 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":67,"stamp":"2023-09-23T19:42:27.255770+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52404,"kb_used_data":7220,"kb_used_omap":0,"kb_used_meta":45120,"kb_avail":749908812,"statfs":{"total":767960285184,"available":767906623488,"internally_reserved":0,"allocated":7393280,"data_stored":4172772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46202880},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002772"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551378+0000","last_change":"2023-09-23T19:41:40.334173+0000","last_active":"2023-09-23T19:41:40.551378+0000","last_peered":"2023-09-23T19:41:40.551378+0000","last_clean":"2023-09-23T19:41:40.551378+0000","last_became_active":"2023-09-23T19:41:40.333894+0000","last_became_peered":"2023-09-23T19:41:40.333894+0000","last_unstale":"2023-09-23T19:41:40.551378+0000","last_undegraded":"2023-09-23T19:41:40.551378+0000","last_fullsized":"2023-09-23T19:41:40.551378+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:15.497357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.184077+0000","last_change":"2023-09-23T19:41:39.540478+0000","last_active":"2023-09-23T19:41:50.184077+0000","last_peered":"2023-09-23T19:41:50.184077+0000","last_clean":"2023-09-23T19:41:50.184077+0000","last_became_active":"2023-09-23T19:41:39.540258+0000","last_became_peered":"2023-09-23T19:41:39.540258+0000","last_unstale":"2023-09-23T19:41:50.184077+0000","last_undegraded":"2023-09-23T19:41:50.184077+0000","last_fullsized":"2023-09-23T19:41:50.184077+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:08.609351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.562717+0000","last_change":"2023-09-23T19:41:40.559640+0000","last_active":"2023-09-23T19:41:40.562717+0000","last_peered":"2023-09-23T19:41:40.562717+0000","last_clean":"2023-09-23T19:41:40.562717+0000","last_became_active":"2023-09-23T19:41:40.559390+0000","last_became_peered":"2023-09-23T19:41:40.559390+0000","last_unstale":"2023-09-23T19:41:40.562717+0000","last_undegraded":"2023-09-23T19:41:40.562717+0000","last_fullsized":"2023-09-23T19:41:40.562717+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:32.021313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:41.567746+0000","last_change":"2023-09-23T19:41:39.546388+0000","last_active":"2023-09-23T19:41:41.567746+0000","last_peered":"2023-09-23T19:41:41.567746+0000","last_clean":"2023-09-23T19:41:41.567746+0000","last_became_active":"2023-09-23T19:41:39.546009+0000","last_became_peered":"2023-09-23T19:41:39.546009+0000","last_unstale":"2023-09-23T19:41:41.567746+0000","last_undegraded":"2023-09-23T19:41:41.567746+0000","last_fullsized":"2023-09-23T19:41:41.567746+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:17.348379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551526+0000","last_change":"2023-09-23T19:41:40.425778+0000","last_active":"2023-09-23T19:41:40.551526+0000","last_peered":"2023-09-23T19:41:40.551526+0000","last_clean":"2023-09-23T19:41:40.551526+0000","last_became_active":"2023-09-23T19:41:40.425410+0000","last_became_peered":"2023-09-23T19:41:40.425410+0000","last_unstale":"2023-09-23T19:41:40.551526+0000","last_undegraded":"2023-09-23T19:41:40.551526+0000","last_fullsized":"2023-09-23T19:41:40.551526+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:53.615465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.185472+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:41:50.185472+0000","last_peered":"2023-09-23T19:41:50.185472+0000","last_clean":"2023-09-23T19:41:50.185472+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:41:50.185472+0000","last_undegraded":"2023-09-23T19:41:50.185472+0000","last_fullsized":"2023-09-23T19:41:50.185472+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551819+0000","last_change":"2023-09-23T19:41:40.425722+0000","last_active":"2023-09-23T19:41:40.551819+0000","last_peered":"2023-09-23T19:41:40.551819+0000","last_clean":"2023-09-23T19:41:40.551819+0000","last_became_active":"2023-09-23T19:41:40.425351+0000","last_became_peered":"2023-09-23T19:41:40.425351+0000","last_unstale":"2023-09-23T19:41:40.551819+0000","last_undegraded":"2023-09-23T19:41:40.551819+0000","last_fullsized":"2023-09-23T19:41:40.551819+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:17.608671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.185477+0000","last_change":"2023-09-23T19:41:39.543292+0000","last_active":"2023-09-23T19:41:50.185477+0000","last_peered":"2023-09-23T19:41:50.185477+0000","last_clean":"2023-09-23T19:41:50.185477+0000","last_became_active":"2023-09-23T19:41:39.543047+0000","last_became_peered":"2023-09-23T19:41:39.543047+0000","last_unstale":"2023-09-23T19:41:50.185477+0000","last_undegraded":"2023-09-23T19:41:50.185477+0000","last_fullsized":"2023-09-23T19:41:50.185477+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:14.795347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.564012+0000","last_change":"2023-09-23T19:41:40.557784+0000","last_active":"2023-09-23T19:41:40.564012+0000","last_peered":"2023-09-23T19:41:40.564012+0000","last_clean":"2023-09-23T19:41:40.564012+0000","last_became_active":"2023-09-23T19:41:40.557467+0000","last_became_peered":"2023-09-23T19:41:40.557467+0000","last_unstale":"2023-09-23T19:41:40.564012+0000","last_undegraded":"2023-09-23T19:41:40.564012+0000","last_fullsized":"2023-09-23T19:41:40.564012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:07.877469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.291265+0000","last_change":"2023-09-23T19:41:39.545236+0000","last_active":"2023-09-23T19:41:42.291265+0000","last_peered":"2023-09-23T19:41:42.291265+0000","last_clean":"2023-09-23T19:41:42.291265+0000","last_became_active":"2023-09-23T19:41:39.544931+0000","last_became_peered":"2023-09-23T19:41:39.544931+0000","last_unstale":"2023-09-23T19:41:42.291265+0000","last_undegraded":"2023-09-23T19:41:42.291265+0000","last_fullsized":"2023-09-23T19:41:42.291265+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:08.229073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551547+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:41:40.551547+0000","last_peered":"2023-09-23T19:41:40.551547+0000","last_clean":"2023-09-23T19:41:40.551547+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:41:40.551547+0000","last_undegraded":"2023-09-23T19:41:40.551547+0000","last_fullsized":"2023-09-23T19:41:40.551547+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.291326+0000","last_change":"2023-09-23T19:41:39.546450+0000","last_active":"2023-09-23T19:41:42.291326+0000","last_peered":"2023-09-23T19:41:42.291326+0000","last_clean":"2023-09-23T19:41:42.291326+0000","last_became_active":"2023-09-23T19:41:39.546151+0000","last_became_peered":"2023-09-23T19:41:39.546151+0000","last_unstale":"2023-09-23T19:41:42.291326+0000","last_undegraded":"2023-09-23T19:41:42.291326+0000","last_fullsized":"2023-09-23T19:41:42.291326+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:21.171118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556349+0000","last_change":"2023-09-23T19:40:40.249273+0000","last_active":"2023-09-23T19:41:40.556349+0000","last_peered":"2023-09-23T19:41:40.556349+0000","last_clean":"2023-09-23T19:41:40.556349+0000","last_became_active":"2023-09-23T19:40:40.248979+0000","last_became_peered":"2023-09-23T19:40:40.248979+0000","last_unstale":"2023-09-23T19:41:40.556349+0000","last_undegraded":"2023-09-23T19:41:40.556349+0000","last_fullsized":"2023-09-23T19:41:40.556349+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:25.089853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598700+0000","last_change":"2023-09-23T19:41:40.597923+0000","last_active":"2023-09-23T19:41:40.598700+0000","last_peered":"2023-09-23T19:41:40.598700+0000","last_clean":"2023-09-23T19:41:40.598700+0000","last_became_active":"2023-09-23T19:41:40.597629+0000","last_became_peered":"2023-09-23T19:41:40.597629+0000","last_unstale":"2023-09-23T19:41:40.598700+0000","last_undegraded":"2023-09-23T19:41:40.598700+0000","last_fullsized":"2023-09-23T19:41:40.598700+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:55.670197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.553338+0000","last_change":"2023-09-23T19:40:40.251554+0000","last_active":"2023-09-23T19:41:40.553338+0000","last_peered":"2023-09-23T19:41:40.553338+0000","last_clean":"2023-09-23T19:41:40.553338+0000","last_became_active":"2023-09-23T19:40:40.251284+0000","last_became_peered":"2023-09-23T19:40:40.251284+0000","last_unstale":"2023-09-23T19:41:40.553338+0000","last_undegraded":"2023-09-23T19:41:40.553338+0000","last_fullsized":"2023-09-23T19:41:40.553338+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:40.680795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.184368+0000","last_change":"2023-09-23T19:41:39.550097+0000","last_active":"2023-09-23T19:41:50.184368+0000","last_peered":"2023-09-23T19:41:50.184368+0000","last_clean":"2023-09-23T19:41:50.184368+0000","last_became_active":"2023-09-23T19:41:39.549819+0000","last_became_peered":"2023-09-23T19:41:39.549819+0000","last_unstale":"2023-09-23T19:41:50.184368+0000","last_undegraded":"2023-09-23T19:41:50.184368+0000","last_fullsized":"2023-09-23T19:41:50.184368+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:42.084440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.596885+0000","last_change":"2023-09-23T19:41:40.596692+0000","last_active":"2023-09-23T19:41:40.596885+0000","last_peered":"2023-09-23T19:41:40.596885+0000","last_clean":"2023-09-23T19:41:40.596885+0000","last_became_active":"2023-09-23T19:41:40.596304+0000","last_became_peered":"2023-09-23T19:41:40.596304+0000","last_unstale":"2023-09-23T19:41:40.596885+0000","last_undegraded":"2023-09-23T19:41:40.596885+0000","last_fullsized":"2023-09-23T19:41:40.596885+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:07.852948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552981+0000","last_change":"2023-09-23T19:41:39.538994+0000","last_active":"2023-09-23T19:41:40.552981+0000","last_peered":"2023-09-23T19:41:40.552981+0000","last_clean":"2023-09-23T19:41:40.552981+0000","last_became_active":"2023-09-23T19:41:39.538696+0000","last_became_peered":"2023-09-23T19:41:39.538696+0000","last_unstale":"2023-09-23T19:41:40.552981+0000","last_undegraded":"2023-09-23T19:41:40.552981+0000","last_fullsized":"2023-09-23T19:41:40.552981+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:12.235516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.594956+0000","last_change":"2023-09-23T19:40:40.361547+0000","last_active":"2023-09-23T19:41:40.594956+0000","last_peered":"2023-09-23T19:41:40.594956+0000","last_clean":"2023-09-23T19:41:40.594956+0000","last_became_active":"2023-09-23T19:40:40.361206+0000","last_became_peered":"2023-09-23T19:40:40.361206+0000","last_unstale":"2023-09-23T19:41:40.594956+0000","last_undegraded":"2023-09-23T19:41:40.594956+0000","last_fullsized":"2023-09-23T19:41:40.594956+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:27.379089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.565563+0000","last_change":"2023-09-23T19:41:40.565030+0000","last_active":"2023-09-23T19:41:40.565563+0000","last_peered":"2023-09-23T19:41:40.565563+0000","last_clean":"2023-09-23T19:41:40.565563+0000","last_became_active":"2023-09-23T19:41:40.564684+0000","last_became_peered":"2023-09-23T19:41:40.564684+0000","last_unstale":"2023-09-23T19:41:40.565563+0000","last_undegraded":"2023-09-23T19:41:40.565563+0000","last_fullsized":"2023-09-23T19:41:40.565563+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:25.266861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558708+0000","last_change":"2023-09-23T19:41:39.545896+0000","last_active":"2023-09-23T19:41:40.558708+0000","last_peered":"2023-09-23T19:41:40.558708+0000","last_clean":"2023-09-23T19:41:40.558708+0000","last_became_active":"2023-09-23T19:41:39.545217+0000","last_became_peered":"2023-09-23T19:41:39.545217+0000","last_unstale":"2023-09-23T19:41:40.558708+0000","last_undegraded":"2023-09-23T19:41:40.558708+0000","last_fullsized":"2023-09-23T19:41:40.558708+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:12.382093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.594661+0000","last_change":"2023-09-23T19:40:40.361627+0000","last_active":"2023-09-23T19:41:40.594661+0000","last_peered":"2023-09-23T19:41:40.594661+0000","last_clean":"2023-09-23T19:41:40.594661+0000","last_became_active":"2023-09-23T19:40:40.361250+0000","last_became_peered":"2023-09-23T19:40:40.361250+0000","last_unstale":"2023-09-23T19:41:40.594661+0000","last_undegraded":"2023-09-23T19:41:40.594661+0000","last_fullsized":"2023-09-23T19:41:40.594661+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:04.232689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598709+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:41:40.598709+0000","last_peered":"2023-09-23T19:41:40.598709+0000","last_clean":"2023-09-23T19:41:40.598709+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:41:40.598709+0000","last_undegraded":"2023-09-23T19:41:40.598709+0000","last_fullsized":"2023-09-23T19:41:40.598709+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556180+0000","last_change":"2023-09-23T19:41:39.539226+0000","last_active":"2023-09-23T19:41:40.556180+0000","last_peered":"2023-09-23T19:41:40.556180+0000","last_clean":"2023-09-23T19:41:40.556180+0000","last_became_active":"2023-09-23T19:41:39.538953+0000","last_became_peered":"2023-09-23T19:41:39.538953+0000","last_unstale":"2023-09-23T19:41:40.556180+0000","last_undegraded":"2023-09-23T19:41:40.556180+0000","last_fullsized":"2023-09-23T19:41:40.556180+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:44.360143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":274,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.594449+0000","last_change":"2023-09-23T19:40:07.160277+0000","last_active":"2023-09-23T19:41:40.594449+0000","last_peered":"2023-09-23T19:41:40.594449+0000","last_clean":"2023-09-23T19:41:40.594449+0000","last_became_active":"2023-09-23T19:40:07.159949+0000","last_became_peered":"2023-09-23T19:40:07.159949+0000","last_unstale":"2023-09-23T19:41:40.594449+0000","last_undegraded":"2023-09-23T19:41:40.594449+0000","last_fullsized":"2023-09-23T19:41:40.594449+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:20.218507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555571+0000","last_change":"2023-09-23T19:40:40.249857+0000","last_active":"2023-09-23T19:41:40.555571+0000","last_peered":"2023-09-23T19:41:40.555571+0000","last_clean":"2023-09-23T19:41:40.555571+0000","last_became_active":"2023-09-23T19:40:40.249587+0000","last_became_peered":"2023-09-23T19:40:40.249587+0000","last_unstale":"2023-09-23T19:41:40.555571+0000","last_undegraded":"2023-09-23T19:41:40.555571+0000","last_fullsized":"2023-09-23T19:41:40.555571+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:37.624835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.565655+0000","last_change":"2023-09-23T19:41:40.565135+0000","last_active":"2023-09-23T19:41:40.565655+0000","last_peered":"2023-09-23T19:41:40.565655+0000","last_clean":"2023-09-23T19:41:40.565655+0000","last_became_active":"2023-09-23T19:41:40.564825+0000","last_became_peered":"2023-09-23T19:41:40.564825+0000","last_unstale":"2023-09-23T19:41:40.565655+0000","last_undegraded":"2023-09-23T19:41:40.565655+0000","last_fullsized":"2023-09-23T19:41:40.565655+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:55.848621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.553695+0000","last_change":"2023-09-23T19:41:39.539162+0000","last_active":"2023-09-23T19:41:40.553695+0000","last_peered":"2023-09-23T19:41:40.553695+0000","last_clean":"2023-09-23T19:41:40.553695+0000","last_became_active":"2023-09-23T19:41:39.538874+0000","last_became_peered":"2023-09-23T19:41:39.538874+0000","last_unstale":"2023-09-23T19:41:40.553695+0000","last_undegraded":"2023-09-23T19:41:40.553695+0000","last_fullsized":"2023-09-23T19:41:40.553695+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:35.739627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551130+0000","last_change":"2023-09-23T19:40:40.361713+0000","last_active":"2023-09-23T19:41:40.551130+0000","last_peered":"2023-09-23T19:41:40.551130+0000","last_clean":"2023-09-23T19:41:40.551130+0000","last_became_active":"2023-09-23T19:40:40.361369+0000","last_became_peered":"2023-09-23T19:40:40.361369+0000","last_unstale":"2023-09-23T19:41:40.551130+0000","last_undegraded":"2023-09-23T19:41:40.551130+0000","last_fullsized":"2023-09-23T19:41:40.551130+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:04.414357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558775+0000","last_change":"2023-09-23T19:41:40.558096+0000","last_active":"2023-09-23T19:41:40.558775+0000","last_peered":"2023-09-23T19:41:40.558775+0000","last_clean":"2023-09-23T19:41:40.558775+0000","last_became_active":"2023-09-23T19:41:40.557808+0000","last_became_peered":"2023-09-23T19:41:40.557808+0000","last_unstale":"2023-09-23T19:41:40.558775+0000","last_undegraded":"2023-09-23T19:41:40.558775+0000","last_fullsized":"2023-09-23T19:41:40.558775+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:23:58.943551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551750+0000","last_change":"2023-09-23T19:41:39.545431+0000","last_active":"2023-09-23T19:41:40.551750+0000","last_peered":"2023-09-23T19:41:40.551750+0000","last_clean":"2023-09-23T19:41:40.551750+0000","last_became_active":"2023-09-23T19:41:39.545150+0000","last_became_peered":"2023-09-23T19:41:39.545150+0000","last_unstale":"2023-09-23T19:41:40.551750+0000","last_undegraded":"2023-09-23T19:41:40.551750+0000","last_fullsized":"2023-09-23T19:41:40.551750+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:10.130116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555745+0000","last_change":"2023-09-23T19:40:40.251667+0000","last_active":"2023-09-23T19:41:40.555745+0000","last_peered":"2023-09-23T19:41:40.555745+0000","last_clean":"2023-09-23T19:41:40.555745+0000","last_became_active":"2023-09-23T19:40:40.251366+0000","last_became_peered":"2023-09-23T19:40:40.251366+0000","last_unstale":"2023-09-23T19:41:40.555745+0000","last_undegraded":"2023-09-23T19:41:40.555745+0000","last_fullsized":"2023-09-23T19:41:40.555745+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:38.869721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.561274+0000","last_change":"2023-09-23T19:41:40.559609+0000","last_active":"2023-09-23T19:41:40.561274+0000","last_peered":"2023-09-23T19:41:40.561274+0000","last_clean":"2023-09-23T19:41:40.561274+0000","last_became_active":"2023-09-23T19:41:40.558615+0000","last_became_peered":"2023-09-23T19:41:40.558615+0000","last_unstale":"2023-09-23T19:41:40.561274+0000","last_undegraded":"2023-09-23T19:41:40.561274+0000","last_fullsized":"2023-09-23T19:41:40.561274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:18.052507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556329+0000","last_change":"2023-09-23T19:41:39.546473+0000","last_active":"2023-09-23T19:41:40.556329+0000","last_peered":"2023-09-23T19:41:40.556329+0000","last_clean":"2023-09-23T19:41:40.556329+0000","last_became_active":"2023-09-23T19:41:39.546208+0000","last_became_peered":"2023-09-23T19:41:39.546208+0000","last_unstale":"2023-09-23T19:41:40.556329+0000","last_undegraded":"2023-09-23T19:41:40.556329+0000","last_fullsized":"2023-09-23T19:41:40.556329+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:20.849754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552867+0000","last_change":"2023-09-23T19:40:40.361619+0000","last_active":"2023-09-23T19:41:40.552867+0000","last_peered":"2023-09-23T19:41:40.552867+0000","last_clean":"2023-09-23T19:41:40.552867+0000","last_became_active":"2023-09-23T19:40:40.361237+0000","last_became_peered":"2023-09-23T19:40:40.361237+0000","last_unstale":"2023-09-23T19:41:40.552867+0000","last_undegraded":"2023-09-23T19:41:40.552867+0000","last_fullsized":"2023-09-23T19:41:40.552867+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:09.643497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558945+0000","last_change":"2023-09-23T19:41:40.558310+0000","last_active":"2023-09-23T19:41:40.558945+0000","last_peered":"2023-09-23T19:41:40.558945+0000","last_clean":"2023-09-23T19:41:40.558945+0000","last_became_active":"2023-09-23T19:41:40.558033+0000","last_became_peered":"2023-09-23T19:41:40.558033+0000","last_unstale":"2023-09-23T19:41:40.558945+0000","last_undegraded":"2023-09-23T19:41:40.558945+0000","last_fullsized":"2023-09-23T19:41:40.558945+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.736832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556080+0000","last_change":"2023-09-23T19:41:39.542335+0000","last_active":"2023-09-23T19:41:40.556080+0000","last_peered":"2023-09-23T19:41:40.556080+0000","last_clean":"2023-09-23T19:41:40.556080+0000","last_became_active":"2023-09-23T19:41:39.540848+0000","last_became_peered":"2023-09-23T19:41:39.540848+0000","last_unstale":"2023-09-23T19:41:40.556080+0000","last_undegraded":"2023-09-23T19:41:40.556080+0000","last_fullsized":"2023-09-23T19:41:40.556080+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:07.239425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552186+0000","last_change":"2023-09-23T19:41:40.334887+0000","last_active":"2023-09-23T19:41:40.552186+0000","last_peered":"2023-09-23T19:41:40.552186+0000","last_clean":"2023-09-23T19:41:40.552186+0000","last_became_active":"2023-09-23T19:41:40.334628+0000","last_became_peered":"2023-09-23T19:41:40.334628+0000","last_unstale":"2023-09-23T19:41:40.552186+0000","last_undegraded":"2023-09-23T19:41:40.552186+0000","last_fullsized":"2023-09-23T19:41:40.552186+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:09.701900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551561+0000","last_change":"2023-09-23T19:41:39.540507+0000","last_active":"2023-09-23T19:41:40.551561+0000","last_peered":"2023-09-23T19:41:40.551561+0000","last_clean":"2023-09-23T19:41:40.551561+0000","last_became_active":"2023-09-23T19:41:39.540001+0000","last_became_peered":"2023-09-23T19:41:39.540001+0000","last_unstale":"2023-09-23T19:41:40.551561+0000","last_undegraded":"2023-09-23T19:41:40.551561+0000","last_fullsized":"2023-09-23T19:41:40.551561+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:44.405007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552124+0000","last_change":"2023-09-23T19:41:39.554902+0000","last_active":"2023-09-23T19:41:40.552124+0000","last_peered":"2023-09-23T19:41:40.552124+0000","last_clean":"2023-09-23T19:41:40.552124+0000","last_became_active":"2023-09-23T19:41:39.554558+0000","last_became_peered":"2023-09-23T19:41:39.554558+0000","last_unstale":"2023-09-23T19:41:40.552124+0000","last_undegraded":"2023-09-23T19:41:40.552124+0000","last_fullsized":"2023-09-23T19:41:40.552124+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:11.716842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555889+0000","last_change":"2023-09-23T19:41:39.540408+0000","last_active":"2023-09-23T19:41:40.555889+0000","last_peered":"2023-09-23T19:41:40.555889+0000","last_clean":"2023-09-23T19:41:40.555889+0000","last_became_active":"2023-09-23T19:41:39.540127+0000","last_became_peered":"2023-09-23T19:41:39.540127+0000","last_unstale":"2023-09-23T19:41:40.555889+0000","last_undegraded":"2023-09-23T19:41:40.555889+0000","last_fullsized":"2023-09-23T19:41:40.555889+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:05.728760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552008+0000","last_change":"2023-09-23T19:41:40.426167+0000","last_active":"2023-09-23T19:41:40.552008+0000","last_peered":"2023-09-23T19:41:40.552008+0000","last_clean":"2023-09-23T19:41:40.552008+0000","last_became_active":"2023-09-23T19:41:40.425797+0000","last_became_peered":"2023-09-23T19:41:40.425797+0000","last_unstale":"2023-09-23T19:41:40.552008+0000","last_undegraded":"2023-09-23T19:41:40.552008+0000","last_fullsized":"2023-09-23T19:41:40.552008+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:19.716310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.185836+0000","last_change":"2023-09-23T19:41:39.546538+0000","last_active":"2023-09-23T19:41:50.185836+0000","last_peered":"2023-09-23T19:41:50.185836+0000","last_clean":"2023-09-23T19:41:50.185836+0000","last_became_active":"2023-09-23T19:41:39.546214+0000","last_became_peered":"2023-09-23T19:41:39.546214+0000","last_unstale":"2023-09-23T19:41:50.185836+0000","last_undegraded":"2023-09-23T19:41:50.185836+0000","last_fullsized":"2023-09-23T19:41:50.185836+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:59.036343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.564443+0000","last_change":"2023-09-23T19:41:40.564174+0000","last_active":"2023-09-23T19:41:40.564443+0000","last_peered":"2023-09-23T19:41:40.564443+0000","last_clean":"2023-09-23T19:41:40.564443+0000","last_became_active":"2023-09-23T19:41:40.563856+0000","last_became_peered":"2023-09-23T19:41:40.563856+0000","last_unstale":"2023-09-23T19:41:40.564443+0000","last_undegraded":"2023-09-23T19:41:40.564443+0000","last_fullsized":"2023-09-23T19:41:40.564443+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:08.391644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555908+0000","last_change":"2023-09-23T19:41:39.542111+0000","last_active":"2023-09-23T19:41:40.555908+0000","last_peered":"2023-09-23T19:41:40.555908+0000","last_clean":"2023-09-23T19:41:40.555908+0000","last_became_active":"2023-09-23T19:41:39.541682+0000","last_became_peered":"2023-09-23T19:41:39.541682+0000","last_unstale":"2023-09-23T19:41:40.555908+0000","last_undegraded":"2023-09-23T19:41:40.555908+0000","last_fullsized":"2023-09-23T19:41:40.555908+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:10.032934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.595277+0000","last_change":"2023-09-23T19:41:40.594907+0000","last_active":"2023-09-23T19:41:40.595277+0000","last_peered":"2023-09-23T19:41:40.595277+0000","last_clean":"2023-09-23T19:41:40.595277+0000","last_became_active":"2023-09-23T19:41:40.594574+0000","last_became_peered":"2023-09-23T19:41:40.594574+0000","last_unstale":"2023-09-23T19:41:40.595277+0000","last_undegraded":"2023-09-23T19:41:40.595277+0000","last_fullsized":"2023-09-23T19:41:40.595277+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:22.485793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.468526+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:41:42.468526+0000","last_peered":"2023-09-23T19:41:42.468526+0000","last_clean":"2023-09-23T19:41:42.468526+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:41:42.468526+0000","last_undegraded":"2023-09-23T19:41:42.468526+0000","last_fullsized":"2023-09-23T19:41:42.468526+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551464+0000","last_change":"2023-09-23T19:41:40.425667+0000","last_active":"2023-09-23T19:41:40.551464+0000","last_peered":"2023-09-23T19:41:40.551464+0000","last_clean":"2023-09-23T19:41:40.551464+0000","last_became_active":"2023-09-23T19:41:40.425346+0000","last_became_peered":"2023-09-23T19:41:40.425346+0000","last_unstale":"2023-09-23T19:41:40.551464+0000","last_undegraded":"2023-09-23T19:41:40.551464+0000","last_fullsized":"2023-09-23T19:41:40.551464+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:54.159658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.183268+0000","last_change":"2023-09-23T19:41:39.540550+0000","last_active":"2023-09-23T19:41:50.183268+0000","last_peered":"2023-09-23T19:41:50.183268+0000","last_clean":"2023-09-23T19:41:50.183268+0000","last_became_active":"2023-09-23T19:41:39.540282+0000","last_became_peered":"2023-09-23T19:41:39.540282+0000","last_unstale":"2023-09-23T19:41:50.183268+0000","last_undegraded":"2023-09-23T19:41:50.183268+0000","last_fullsized":"2023-09-23T19:41:50.183268+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:19.128796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.596821+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:41:40.596821+0000","last_peered":"2023-09-23T19:41:40.596821+0000","last_clean":"2023-09-23T19:41:40.596821+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:41:40.596821+0000","last_undegraded":"2023-09-23T19:41:40.596821+0000","last_fullsized":"2023-09-23T19:41:40.596821+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.468511+0000","last_change":"2023-09-23T19:41:39.546086+0000","last_active":"2023-09-23T19:41:42.468511+0000","last_peered":"2023-09-23T19:41:42.468511+0000","last_clean":"2023-09-23T19:41:42.468511+0000","last_became_active":"2023-09-23T19:41:39.545818+0000","last_became_peered":"2023-09-23T19:41:39.545818+0000","last_unstale":"2023-09-23T19:41:42.468511+0000","last_undegraded":"2023-09-23T19:41:42.468511+0000","last_fullsized":"2023-09-23T19:41:42.468511+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:36.470729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.596822+0000","last_change":"2023-09-23T19:41:40.596504+0000","last_active":"2023-09-23T19:41:40.596822+0000","last_peered":"2023-09-23T19:41:40.596822+0000","last_clean":"2023-09-23T19:41:40.596822+0000","last_became_active":"2023-09-23T19:41:40.596198+0000","last_became_peered":"2023-09-23T19:41:40.596198+0000","last_unstale":"2023-09-23T19:41:40.596822+0000","last_undegraded":"2023-09-23T19:41:40.596822+0000","last_fullsized":"2023-09-23T19:41:40.596822+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:04.764297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.186274+0000","last_change":"2023-09-23T19:41:39.537846+0000","last_active":"2023-09-23T19:41:50.186274+0000","last_peered":"2023-09-23T19:41:50.186274+0000","last_clean":"2023-09-23T19:41:50.186274+0000","last_became_active":"2023-09-23T19:41:39.537296+0000","last_became_peered":"2023-09-23T19:41:39.537296+0000","last_unstale":"2023-09-23T19:41:50.186274+0000","last_undegraded":"2023-09-23T19:41:50.186274+0000","last_fullsized":"2023-09-23T19:41:50.186274+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:29.430699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.599001+0000","last_change":"2023-09-23T19:41:40.598346+0000","last_active":"2023-09-23T19:41:40.599001+0000","last_peered":"2023-09-23T19:41:40.599001+0000","last_clean":"2023-09-23T19:41:40.599001+0000","last_became_active":"2023-09-23T19:41:40.597877+0000","last_became_peered":"2023-09-23T19:41:40.597877+0000","last_unstale":"2023-09-23T19:41:40.599001+0000","last_undegraded":"2023-09-23T19:41:40.599001+0000","last_fullsized":"2023-09-23T19:41:40.599001+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:12.163854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.594426+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:41:40.594426+0000","last_peered":"2023-09-23T19:41:40.594426+0000","last_clean":"2023-09-23T19:41:40.594426+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:41:40.594426+0000","last_undegraded":"2023-09-23T19:41:40.594426+0000","last_fullsized":"2023-09-23T19:41:40.594426+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598708+0000","last_change":"2023-09-23T19:41:40.598132+0000","last_active":"2023-09-23T19:41:40.598708+0000","last_peered":"2023-09-23T19:41:40.598708+0000","last_clean":"2023-09-23T19:41:40.598708+0000","last_became_active":"2023-09-23T19:41:40.597692+0000","last_became_peered":"2023-09-23T19:41:40.597692+0000","last_unstale":"2023-09-23T19:41:40.598708+0000","last_undegraded":"2023-09-23T19:41:40.598708+0000","last_fullsized":"2023-09-23T19:41:40.598708+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:06.861045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558146+0000","last_change":"2023-09-23T19:41:39.542640+0000","last_active":"2023-09-23T19:41:40.558146+0000","last_peered":"2023-09-23T19:41:40.558146+0000","last_clean":"2023-09-23T19:41:40.558146+0000","last_became_active":"2023-09-23T19:41:39.542371+0000","last_became_peered":"2023-09-23T19:41:39.542371+0000","last_unstale":"2023-09-23T19:41:40.558146+0000","last_undegraded":"2023-09-23T19:41:40.558146+0000","last_fullsized":"2023-09-23T19:41:40.558146+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:55.308862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.595072+0000","last_change":"2023-09-23T19:41:40.594835+0000","last_active":"2023-09-23T19:41:40.595072+0000","last_peered":"2023-09-23T19:41:40.595072+0000","last_clean":"2023-09-23T19:41:40.595072+0000","last_became_active":"2023-09-23T19:41:40.594493+0000","last_became_peered":"2023-09-23T19:41:40.594493+0000","last_unstale":"2023-09-23T19:41:40.595072+0000","last_undegraded":"2023-09-23T19:41:40.595072+0000","last_fullsized":"2023-09-23T19:41:40.595072+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:03.135867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.184074+0000","last_change":"2023-09-23T19:41:39.544179+0000","last_active":"2023-09-23T19:41:50.184074+0000","last_peered":"2023-09-23T19:41:50.184074+0000","last_clean":"2023-09-23T19:41:50.184074+0000","last_became_active":"2023-09-23T19:41:39.543928+0000","last_became_peered":"2023-09-23T19:41:39.543928+0000","last_unstale":"2023-09-23T19:41:50.184074+0000","last_undegraded":"2023-09-23T19:41:50.184074+0000","last_fullsized":"2023-09-23T19:41:50.184074+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:04.486013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.551601+0000","last_change":"2023-09-23T19:41:40.334429+0000","last_active":"2023-09-23T19:41:40.551601+0000","last_peered":"2023-09-23T19:41:40.551601+0000","last_clean":"2023-09-23T19:41:40.551601+0000","last_became_active":"2023-09-23T19:41:40.334122+0000","last_became_peered":"2023-09-23T19:41:40.334122+0000","last_unstale":"2023-09-23T19:41:40.551601+0000","last_undegraded":"2023-09-23T19:41:40.551601+0000","last_fullsized":"2023-09-23T19:41:40.551601+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.729985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.186987+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:41:50.186987+0000","last_peered":"2023-09-23T19:41:50.186987+0000","last_clean":"2023-09-23T19:41:50.186987+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:41:50.186987+0000","last_undegraded":"2023-09-23T19:41:50.186987+0000","last_fullsized":"2023-09-23T19:41:50.186987+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.561294+0000","last_change":"2023-09-23T19:41:40.559776+0000","last_active":"2023-09-23T19:41:40.561294+0000","last_peered":"2023-09-23T19:41:40.561294+0000","last_clean":"2023-09-23T19:41:40.561294+0000","last_became_active":"2023-09-23T19:41:40.559365+0000","last_became_peered":"2023-09-23T19:41:40.559365+0000","last_unstale":"2023-09-23T19:41:40.561294+0000","last_undegraded":"2023-09-23T19:41:40.561294+0000","last_fullsized":"2023-09-23T19:41:40.561294+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:06.462587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:50.184194+0000","last_change":"2023-09-23T19:41:39.545954+0000","last_active":"2023-09-23T19:41:50.184194+0000","last_peered":"2023-09-23T19:41:50.184194+0000","last_clean":"2023-09-23T19:41:50.184194+0000","last_became_active":"2023-09-23T19:41:39.545690+0000","last_became_peered":"2023-09-23T19:41:39.545690+0000","last_unstale":"2023-09-23T19:41:50.184194+0000","last_undegraded":"2023-09-23T19:41:50.184194+0000","last_fullsized":"2023-09-23T19:41:50.184194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:58.159061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598805+0000","last_change":"2023-09-23T19:41:40.598168+0000","last_active":"2023-09-23T19:41:40.598805+0000","last_peered":"2023-09-23T19:41:40.598805+0000","last_clean":"2023-09-23T19:41:40.598805+0000","last_became_active":"2023-09-23T19:41:40.597174+0000","last_became_peered":"2023-09-23T19:41:40.597174+0000","last_unstale":"2023-09-23T19:41:40.598805+0000","last_undegraded":"2023-09-23T19:41:40.598805+0000","last_fullsized":"2023-09-23T19:41:40.598805+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:14.395482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.555514+0000","last_change":"2023-09-23T19:41:39.542457+0000","last_active":"2023-09-23T19:41:40.555514+0000","last_peered":"2023-09-23T19:41:40.555514+0000","last_clean":"2023-09-23T19:41:40.555514+0000","last_became_active":"2023-09-23T19:41:39.542148+0000","last_became_peered":"2023-09-23T19:41:39.542148+0000","last_unstale":"2023-09-23T19:41:40.555514+0000","last_undegraded":"2023-09-23T19:41:40.555514+0000","last_fullsized":"2023-09-23T19:41:40.555514+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:42.959334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.562795+0000","last_change":"2023-09-23T19:41:40.558606+0000","last_active":"2023-09-23T19:41:40.562795+0000","last_peered":"2023-09-23T19:41:40.562795+0000","last_clean":"2023-09-23T19:41:40.562795+0000","last_became_active":"2023-09-23T19:41:40.558355+0000","last_became_peered":"2023-09-23T19:41:40.558355+0000","last_unstale":"2023-09-23T19:41:40.562795+0000","last_undegraded":"2023-09-23T19:41:40.562795+0000","last_fullsized":"2023-09-23T19:41:40.562795+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:04.750882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.560574+0000","last_change":"2023-09-23T19:41:39.544822+0000","last_active":"2023-09-23T19:41:40.560574+0000","last_peered":"2023-09-23T19:41:40.560574+0000","last_clean":"2023-09-23T19:41:40.560574+0000","last_became_active":"2023-09-23T19:41:39.542108+0000","last_became_peered":"2023-09-23T19:41:39.542108+0000","last_unstale":"2023-09-23T19:41:40.560574+0000","last_undegraded":"2023-09-23T19:41:40.560574+0000","last_fullsized":"2023-09-23T19:41:40.560574+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.478529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.559588+0000","last_change":"2023-09-23T19:41:40.559379+0000","last_active":"2023-09-23T19:41:40.559588+0000","last_peered":"2023-09-23T19:41:40.559588+0000","last_clean":"2023-09-23T19:41:40.559588+0000","last_became_active":"2023-09-23T19:41:40.559146+0000","last_became_peered":"2023-09-23T19:41:40.559146+0000","last_unstale":"2023-09-23T19:41:40.559588+0000","last_undegraded":"2023-09-23T19:41:40.559588+0000","last_fullsized":"2023-09-23T19:41:40.559588+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:15.503413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.556181+0000","last_change":"2023-09-23T19:41:39.547073+0000","last_active":"2023-09-23T19:41:40.556181+0000","last_peered":"2023-09-23T19:41:40.556181+0000","last_clean":"2023-09-23T19:41:40.556181+0000","last_became_active":"2023-09-23T19:41:39.546648+0000","last_became_peered":"2023-09-23T19:41:39.546648+0000","last_unstale":"2023-09-23T19:41:40.556181+0000","last_undegraded":"2023-09-23T19:41:40.556181+0000","last_fullsized":"2023-09-23T19:41:40.556181+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:45.969145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.598842+0000","last_change":"2023-09-23T19:41:40.598150+0000","last_active":"2023-09-23T19:41:40.598842+0000","last_peered":"2023-09-23T19:41:40.598842+0000","last_clean":"2023-09-23T19:41:40.598842+0000","last_became_active":"2023-09-23T19:41:40.597573+0000","last_became_peered":"2023-09-23T19:41:40.597573+0000","last_unstale":"2023-09-23T19:41:40.598842+0000","last_undegraded":"2023-09-23T19:41:40.598842+0000","last_fullsized":"2023-09-23T19:41:40.598842+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:05.362995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.552985+0000","last_change":"2023-09-23T19:41:39.538925+0000","last_active":"2023-09-23T19:41:40.552985+0000","last_peered":"2023-09-23T19:41:40.552985+0000","last_clean":"2023-09-23T19:41:40.552985+0000","last_became_active":"2023-09-23T19:41:39.538713+0000","last_became_peered":"2023-09-23T19:41:39.538713+0000","last_unstale":"2023-09-23T19:41:40.552985+0000","last_undegraded":"2023-09-23T19:41:40.552985+0000","last_fullsized":"2023-09-23T19:41:40.552985+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:41.173401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T19:41:40.558704+0000","last_change":"2023-09-23T19:41:40.557696+0000","last_active":"2023-09-23T19:41:40.558704+0000","last_peered":"2023-09-23T19:41:40.558704+0000","last_clean":"2023-09-23T19:41:40.558704+0000","last_became_active":"2023-09-23T19:41:40.557427+0000","last_became_peered":"2023-09-23T19:41:40.557427+0000","last_unstale":"2023-09-23T19:41:40.558704+0000","last_undegraded":"2023-09-23T19:41:40.558704+0000","last_fullsized":"2023-09-23T19:41:40.558704+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:59.491319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T19:41:42.168337+0000","last_change":"2023-09-23T19:41:39.541190+0000","last_active":"2023-09-23T19:41:42.168337+0000","last_peered":"2023-09-23T19:41:42.168337+0000","last_clean":"2023-09-23T19:41:42.168337+0000","last_became_active":"2023-09-23T19:41:39.540754+0000","last_became_peered":"2023-09-23T19:41:39.540754+0000","last_unstale":"2023-09-23T19:41:42.168337+0000","last_undegraded":"2023-09-23T19:41:42.168337+0000","last_fullsized":"2023-09-23T19:41:42.168337+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:59.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234015,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6536,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93738616,"statfs":{"total":95995035648,"available":95988342784,"internally_reserved":0,"allocated":1310720,"data_stored":891354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.498,"15min":0.5},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.48199999999999998,"15min":0.47799999999999998},"min":{"1min":0.35599999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.51300000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.498,"5min":0.47199999999999998,"15min":0.46800000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.58799999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.496,"15min":0.48699999999999999},"min":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.51700000000000002,"15min":0.501},"min":{"1min":0.35699999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.819,"5min":1.819,"15min":1.819},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.48699999999999999,"15min":0.47999999999999998},"min":{"1min":0.38500000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.51300000000000001,"15min":0.50900000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.45100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.44900000000000001,"15min":0.44700000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.48099999999999998,"15min":0.47599999999999998},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.49099999999999999,"15min":0.48899999999999999},"min":{"1min":0.36099999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.503,"15min":0.495},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.72099999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.38600000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79600000000000004,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56100000000000005,"15min":0.55800000000000005},"min":{"1min":0.41399999999999998,"5min":0.371,"15min":0.371},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.70399999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495652,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6648,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93738504,"statfs":{"total":95995035648,"available":95988228096,"internally_reserved":0,"allocated":1294336,"data_stored":891001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.53800000000000003,"15min":0.54300000000000004},"min":{"1min":0.379,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.65400000000000003,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.52500000000000002,"15min":0.52300000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72999999999999998,"5min":0.871,"15min":0.871},"last":0.55800000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.36099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.748,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.376,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.48799999999999999,"15min":0.48599999999999999},"min":{"1min":0.40100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.60999999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.55600000000000005,"15min":0.54100000000000004},"min":{"1min":0.46300000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.54200000000000004,"15min":0.53600000000000003},"min":{"1min":0.33900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.81200000000000006,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.55900000000000005,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.0579999999999998,"5min":2.0579999999999998,"15min":2.0579999999999998},"last":0.57399999999999995}]},{"osd":4,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.45300000000000001,"15min":0.436},"min":{"1min":0.32000000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.48099999999999998,"15min":0.46500000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.45900000000000002,"15min":0.45200000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.51800000000000002,"15min":0.505},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56200000000000006,"15min":0.56499999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81200000000000006,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.626},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.59299999999999997,"15min":0.59699999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.83499999999999996,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.61199999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6764,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738388,"statfs":{"total":95995035648,"available":95988109312,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.35299999999999998,"15min":0.33500000000000002},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.5,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.36699999999999999,"15min":0.34599999999999997},"min":{"1min":0.25,"5min":0.222,"15min":0.222},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.34399999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:42:16 2023","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.374,"15min":0.38400000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.50800000000000001,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.40799999999999997,"15min":0.41799999999999998},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.46400000000000002,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.29599999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.45900000000000002,"15min":0.47099999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.54900000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.46500000000000002,"15min":0.47899999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.54000000000000004,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.307}]},{"osd":4,"last update":"Sat Sep 23 19:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58099999999999996,"15min":0.59099999999999997},"min":{"1min":0.434,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72899999999999998,"5min":0.87,"15min":0.87},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59299999999999997,"15min":0.60899999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.51500000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:42:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59399999999999997,"15min":0.60799999999999998},"min":{"1min":0.39400000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.755,"5min":0.871,"15min":0.871},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.60299999999999998,"15min":0.61899999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80400000000000005,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.70699999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:41:42 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.46600000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.288,"5min":1.288,"15min":1.288},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61499999999999999,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.64600000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59799999999999998,"15min":0.59899999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73999999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.58399999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7412,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737740,"statfs":{"total":95995035648,"available":95987445760,"internally_reserved":0,"allocated":1290240,"data_stored":890236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.33900000000000002,"15min":0.315},"min":{"1min":0.22600000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.45600000000000002,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.37,"5min":0.36799999999999999,"15min":0.34399999999999997},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.435,"15min":0.39800000000000002},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.5910000000000002,"5min":4.5910000000000002,"15min":4.5910000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39300000000000002,"15min":0.39100000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.27400000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:41:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.45600000000000002,"15min":0.47399999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.51500000000000001,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.378,"5min":0.45100000000000001,"15min":0.47199999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.57799999999999996,"5min":0.627,"15min":0.627},"last":0.379}]},{"osd":4,"last update":"Sat Sep 23 19:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.44800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55600000000000005,"15min":0.55300000000000005},"min":{"1min":0.41199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.81399999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.55300000000000005,"15min":0.55400000000000005},"min":{"1min":0.433,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.58899999999999997,"15min":0.58299999999999996},"min":{"1min":0.46500000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:41:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.60699999999999998,"15min":0.60999999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.745,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.59299999999999997,"15min":0.57899999999999996},"min":{"1min":0.497,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60399999999999998,"15min":0.60199999999999998},"min":{"1min":0.45600000000000002,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83499999999999996,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.55500000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411383,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6644,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738508,"statfs":{"total":95995035648,"available":95988232192,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:42:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.44700000000000001,"15min":0.36799999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.0369999999999999,"5min":5.5629999999999997,"15min":5.5629999999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53900000000000003,"15min":0.42099999999999999},"min":{"1min":0.247,"5min":0.216,"15min":0.216},"max":{"1min":4.4379999999999997,"5min":10.811,"15min":10.811},"last":0.56299999999999994}]},{"osd":1,"last update":"Sat Sep 23 19:42:17 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.46400000000000002,"15min":0.41899999999999998},"min":{"1min":0.254,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.72399999999999998,"5min":5.29,"15min":5.29},"last":0.624},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.58899999999999997,"15min":0.46000000000000002},"min":{"1min":0.26500000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.9540000000000002,"5min":11.986000000000001,"15min":11.986000000000001},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:41:35 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.41799999999999998,"15min":0.42799999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55500000000000005,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.417,"5min":0.65100000000000002,"15min":0.52800000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":15.945,"5min":15.945,"15min":15.945},"last":0.58099999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.66600000000000004,"15min":0.60099999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.8359999999999999,"5min":5.1340000000000003,"15min":5.1340000000000003},"last":0.82699999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.67400000000000004,"15min":0.61899999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.8919999999999999,"5min":5.1920000000000002,"15min":5.1920000000000002},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60099999999999998,"15min":0.60599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.85399999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.68899999999999995,"15min":0.66400000000000003},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.2199999999999998,"5min":5.2199999999999998,"15min":5.2199999999999998},"last":0.752},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.629,"15min":0.64100000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.746,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.91000000000000003}]},{"osd":7,"last update":"Sat Sep 23 19:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.40899999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.77400000000000002,"15min":0.80800000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.89000000000000001,"5min":5.2629999999999999,"15min":5.2629999999999999},"last":0.77200000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6128,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739024,"statfs":{"total":95995035648,"available":95988760576,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.35799999999999998,"15min":0.35999999999999999},"min":{"1min":0.252,"5min":0.189,"15min":0.189},"max":{"1min":0.46400000000000002,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40500000000000003,"15min":0.42499999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.53200000000000003,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.32800000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.434,"15min":0.434},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.6909999999999998,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.41899999999999998,"15min":0.434},"min":{"1min":0.26100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.311}]},{"osd":2,"last update":"Sat Sep 23 19:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.46800000000000003,"15min":0.48799999999999999},"min":{"1min":0.313,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60199999999999998,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.442,"15min":0.46100000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.61399999999999999,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:42:01 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.52600000000000002,"15min":0.52200000000000002},"min":{"1min":0.39700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54100000000000004,"15min":0.53600000000000003},"min":{"1min":0.39700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.56599999999999995,"15min":0.56200000000000006},"min":{"1min":0.42899999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56899999999999995,"15min":0.57199999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.71799999999999997,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.59899999999999998,"15min":0.60599999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.497}]},{"osd":7,"last update":"Sat Sep 23 19:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.60499999999999998,"15min":0.60399999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.59199999999999997,"15min":0.58999999999999997},"min":{"1min":0.44800000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.51700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6208,"kb_used_data":696,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738944,"statfs":{"total":95995035648,"available":95988678656,"internally_reserved":0,"allocated":712704,"data_stored":300974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.49199999999999999,"15min":0.49099999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.50800000000000001,"15min":0.51000000000000001},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.63400000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33200000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.50700000000000001,"15min":0.503},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.42499999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.52600000000000002,"15min":0.52100000000000002},"min":{"1min":0.38700000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.52300000000000002,"15min":0.51300000000000001},"min":{"1min":0.39200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.50900000000000001,"15min":0.497},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53100000000000003,"15min":0.52300000000000002},"min":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.42199999999999999,"15min":0.40200000000000002},"min":{"1min":0.32000000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.621,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.43099999999999999,"15min":0.39800000000000002},"min":{"1min":0.34000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:41:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.45100000000000001,"15min":0.44},"min":{"1min":0.33000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.49199999999999999,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.315},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.56000000000000005,"15min":0.56799999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68300000000000005,"5min":0.878,"15min":0.878},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.42799999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.67000000000000004,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.64500000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757289,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6064,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739088,"statfs":{"total":95995035648,"available":95988826112,"internally_reserved":0,"allocated":696320,"data_stored":299858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.54600000000000004,"15min":0.55500000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.72799999999999998,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.498,"15min":0.496},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.45800000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.503},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.55100000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65800000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.64600000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.51300000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77400000000000002,"5min":0.874,"15min":0.874},"last":0.60999999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.40500000000000003,"15min":0.39200000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.623},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.41799999999999998,"15min":0.40600000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.69599999999999995}]},{"osd":6,"last update":"Sat Sep 23 19:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.45200000000000001,"15min":0.438},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.499,"15min":0.48499999999999999},"min":{"1min":0.36399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.72599999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:42:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84799999999999998,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.56299999999999994,"15min":0.55500000000000005},"min":{"1min":0.43099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.67400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:42:29.581 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T19:42:29.582 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T19:42:29.582 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T19:42:29.582 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T19:42:29.926 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:30.467 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:30 smithi135 ceph-mon[118715]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:30.468 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:30 smithi135 ceph-mon[118715]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:30.468 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:30 smithi135 ceph-mon[100251]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:30.468 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:30 smithi135 ceph-mon[100251]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:30 smithi144 ceph-mon[104994]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:42:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:30 smithi144 ceph-mon[104994]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:31.626 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:42:31.626 INFO:teuthology.orchestra.run.smithi135.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T19:42:32.351 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T19:42:32.351 INFO:teuthology.run_tasks:Running task print... 2023-09-23T19:42:32.360 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T19:42:32.361 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T19:42:32.377 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T19:42:32.377 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T19:42:32.378 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T19:42:32.381 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T19:42:32.381 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T19:42:32.381 DEBUG:teuthology.orchestra.run.smithi135:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:42:32.382 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T19:42:32.382 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T19:42:32.382 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi135.front.sepia.ceph.com 2023-09-23T19:42:32.382 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T19:42:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:42:32.400 INFO:teuthology.orchestra.run.smithi135.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T19:42:32.400 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:42:32.456 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T19:42:32.456 DEBUG:teuthology.orchestra.run.smithi135:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T19:42:32.512 INFO:tasks.workunit:timeout=3h 2023-09-23T19:42:32.512 INFO:tasks.workunit:cleanup=True 2023-09-23T19:42:32.513 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T19:42:32.570 INFO:tasks.workunit.client.0.smithi135.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T19:42:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:32 smithi144 ceph-mon[104994]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:32 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1749586664' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:42:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:32 smithi135 ceph-mon[118715]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:32 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1749586664' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:42:32.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:32 smithi135 ceph-mon[100251]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:32 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1749586664' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:42:32.730 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:34.086 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:33 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:33] "GET /metrics HTTP/1.1" 200 238623 "" "Prometheus/2.43.0" 2023-09-23T19:42:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:34 smithi135 ceph-mon[118715]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:34.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:34 smithi135 ceph-mon[100251]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:34.493 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:34 smithi135 conmon[100683]: [23/Sep/2023:19:42:34] ENGINE Bus STOPPING 2023-09-23T19:42:34.493 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:34 smithi135 conmon[100683]: [23/Sep/2023:19:42:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:42:34.493 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:34 smithi135 conmon[100683]: [23/Sep/2023:19:42:34] ENGINE Bus STOPPED 2023-09-23T19:42:34.493 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:34 smithi135 conmon[100683]: [23/Sep/2023:19:42:34] ENGINE Bus STARTING 2023-09-23T19:42:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:34 smithi144 ceph-mon[104994]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:34.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:34 smithi135 conmon[100683]: [23/Sep/2023:19:42:34] ENGINE Serving on http://:::9283 2023-09-23T19:42:34.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:34 smithi135 conmon[100683]: [23/Sep/2023:19:42:34] ENGINE Bus STARTED 2023-09-23T19:42:35.242 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T19:42:35.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:42:35 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:42:35.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:35 smithi144 ceph-mon[104994]: from='client.? ' entity='client.admin' 2023-09-23T19:42:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:35 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:35 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:35 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:35 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:35.679 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[118715]: from='client.? ' entity='client.admin' 2023-09-23T19:42:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:35.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[100251]: from='client.? ' entity='client.admin' 2023-09-23T19:42:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:35 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:36 smithi144 ceph-mon[104994]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:36.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:36 smithi135 ceph-mon[100251]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:36.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:36 smithi135 ceph-mon[118715]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:37.298 INFO:tasks.workunit.client.0.smithi135.stderr:Updating files: 90% (10442/11502) Updating files: 91% (10467/11502) Updating files: 92% (10582/11502) Updating files: 93% (10697/11502) Updating files: 94% (10812/11502) Updating files: 95% (10927/11502) Updating files: 96% (11042/11502) Updating files: 97% (11157/11502) Updating files: 98% (11272/11502) Updating files: 99% (11387/11502) Updating files: 100% (11502/11502) Updating files: 100% (11502/11502), done. 2023-09-23T19:42:37.319 DEBUG:teuthology.orchestra.run.smithi135:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T19:42:37.461 INFO:tasks.workunit.client.0.smithi135.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T19:42:37.464 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T19:42:37.464 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T19:42:37.856 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T19:42:37.856 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-09-23T19:42:37.857 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T19:42:37.857 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:37.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T19:42:37.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-09-23T19:42:37.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T19:42:37.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T19:42:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-09-23T19:42:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T19:42:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:38.478 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T19:42:38.544 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T19:42:38.581 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T19:42:38.583 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T19:42:38.583 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T19:42:38.622 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T19:42:38.625 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T19:42:38.625 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T19:42:38.643 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T19:42:38.644 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T19:42:38.645 DEBUG:teuthology.orchestra.run.smithi135:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T19:42:38.704 INFO:tasks.workunit.client.0.smithi135.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T19:42:38.704 INFO:tasks.workunit.client.0.smithi135.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T19:42:38.704 INFO:tasks.workunit.client.0.smithi135.stderr:+ TMPDIR=/tmp/rbd_import_export_190031 2023-09-23T19:42:38.704 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm -rf /tmp/rbd_import_export_190031 2023-09-23T19:42:38.704 INFO:tasks.workunit.client.0.smithi135.stderr:+ mkdir /tmp/rbd_import_export_190031 2023-09-23T19:42:38.705 INFO:tasks.workunit.client.0.smithi135.stderr:+ trap 'rm -rf /tmp/rbd_import_export_190031' INT TERM EXIT 2023-09-23T19:42:38.705 INFO:tasks.workunit.client.0.smithi135.stderr:+ mkdir foo.190031 2023-09-23T19:42:38.706 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import foo.190031 foo.dir 2023-09-23T19:42:38.779 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: cannot import a directory 2023-09-23T19:42:38.779 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 0% complete...failed. 2023-09-23T19:42:38.780 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: import failed: (21) Is a directory 2023-09-23T19:42:38.782 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T19:42:38.782 INFO:tasks.workunit.client.0.smithi135.stderr:+ rmdir foo.190031 2023-09-23T19:42:38.808 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_190031/img bs=1k count=1 seek=10 2023-09-23T19:42:38.808 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T19:42:38.808 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T19:42:38.809 INFO:tasks.workunit.client.0.smithi135.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.7097e-05 s, 21.7 MB/s 2023-09-23T19:42:38.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[118715]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:38.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T19:42:38.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-09-23T19:42:38.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T19:42:38.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[118715]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T19:42:38.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[100251]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:38.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T19:42:38.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-09-23T19:42:38.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T19:42:38.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:38 smithi135 ceph-mon[100251]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_190031/img bs=1k count=10 seek=100 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:10+0 records in 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:10+0 records out 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.2085e-05 s, 165 MB/s 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_190031/img bs=1k count=100 seek=1000 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:70+1 records in 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:70+1 records out 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000175135 s, 411 MB/s 2023-09-23T19:42:38.811 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_190031/img bs=1k seek=10000 2023-09-23T19:42:38.812 INFO:tasks.workunit.client.0.smithi135.stderr:139+1 records in 2023-09-23T19:42:38.812 INFO:tasks.workunit.client.0.smithi135.stderr:139+1 records out 2023-09-23T19:42:38.812 INFO:tasks.workunit.client.0.smithi135.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000318095 s, 450 MB/s 2023-09-23T19:42:38.812 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_190031/img bs=1k seek=100000 2023-09-23T19:42:38.823 INFO:tasks.workunit.client.0.smithi135.stderr:70+1 records in 2023-09-23T19:42:38.823 INFO:tasks.workunit.client.0.smithi135.stderr:70+1 records out 2023-09-23T19:42:38.823 INFO:tasks.workunit.client.0.smithi135.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0109683 s, 6.6 MB/s 2023-09-23T19:42:38.823 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_190031/img bs=1k seek=1000000 2023-09-23T19:42:38.824 INFO:tasks.workunit.client.0.smithi135.stderr:189+1 records in 2023-09-23T19:42:38.824 INFO:tasks.workunit.client.0.smithi135.stderr:189+1 records out 2023-09-23T19:42:38.824 INFO:tasks.workunit.client.0.smithi135.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000417584 s, 464 MB/s 2023-09-23T19:42:38.824 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm testimg 2023-09-23T19:42:38.861 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 0% complete...failed. 2023-09-23T19:42:38.861 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T19:42:38.864 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T19:42:38.864 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format /tmp/rbd_import_export_190031/img testimg 2023-09-23T19:42:38.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:38 smithi144 ceph-mon[104994]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:38.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:38 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T19:42:38.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:38 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-09-23T19:42:38.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:38 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T19:42:38.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:38 smithi144 ceph-mon[104994]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T19:42:39.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:39 smithi144 ceph-mon[104994]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T19:42:39.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:39 smithi135 ceph-mon[100251]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T19:42:39.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:39 smithi135 ceph-mon[118715]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T19:42:40.678 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T19:42:40.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:40 smithi144 ceph-mon[104994]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:40.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:40 smithi144 ceph-mon[104994]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T19:42:40.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:40 smithi135 ceph-mon[100251]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:40.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:40 smithi135 ceph-mon[100251]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T19:42:40.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:40 smithi135 ceph-mon[118715]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:40.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:40 smithi135 ceph-mon[118715]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T19:42:41.078 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:41.124 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T19:42:41.124 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:42:41.127 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg /tmp/rbd_import_export_190031/img2 2023-09-23T19:42:41.412 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:41.417 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg - 2023-09-23T19:42:41.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:41 smithi144 ceph-mon[104994]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T19:42:41.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:41 smithi135 ceph-mon[118715]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T19:42:41.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:41 smithi135 ceph-mon[100251]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T19:42:42.162 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:42.165 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm testimg 2023-09-23T19:42:42.255 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T19:42:42.258 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp /tmp/rbd_import_export_190031/img /tmp/rbd_import_export_190031/img2 2023-09-23T19:42:42.877 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp /tmp/rbd_import_export_190031/img /tmp/rbd_import_export_190031/img3 2023-09-23T19:42:42.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:42 smithi144 ceph-mon[104994]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:42.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:42 smithi135 ceph-mon[100251]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:42.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:42 smithi135 ceph-mon[118715]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:43.285 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img2 /tmp/rbd_import_export_190031/img3 2023-09-23T19:42:44.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:43 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:43] "GET /metrics HTTP/1.1" 200 238618 "" "Prometheus/2.43.0" 2023-09-23T19:42:44.654 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format - testimg 2023-09-23T19:42:44.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:44 smithi144 ceph-mon[104994]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 207 KiB/s rd, 89 KiB/s wr, 53 op/s 2023-09-23T19:42:44.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:44 smithi144 ceph-mon[104994]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T19:42:44.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:44 smithi135 ceph-mon[118715]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 207 KiB/s rd, 89 KiB/s wr, 53 op/s 2023-09-23T19:42:44.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:44 smithi135 ceph-mon[118715]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T19:42:44.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:44 smithi135 ceph-mon[100251]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 207 KiB/s rd, 89 KiB/s wr, 53 op/s 2023-09-23T19:42:44.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:44 smithi135 ceph-mon[100251]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T19:42:45.234 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:42:45.235 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:42:45.238 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg /tmp/rbd_import_export_190031/img2 2023-09-23T19:42:45.541 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:45.544 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg - 2023-09-23T19:42:45.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:42:45 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:42:45.988 INFO:teuthology.orchestra.run.smithi135.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:42:46.292 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:46.295 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm testimg 2023-09-23T19:42:46.384 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T19:42:46.387 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp /tmp/rbd_import_export_190031/img /tmp/rbd_import_export_190031/img2 2023-09-23T19:42:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:46 smithi144 ceph-mon[104994]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T19:42:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:46 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:46.961 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp /tmp/rbd_import_export_190031/img /tmp/rbd_import_export_190031/img3 2023-09-23T19:42:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:46 smithi135 ceph-mon[118715]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T19:42:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:46 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:46.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:46 smithi135 ceph-mon[100251]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T19:42:46.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:46.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:46 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:42:47.374 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img /tmp/rbd_import_export_190031/img2 /tmp/rbd_import_export_190031/img3 2023-09-23T19:42:47.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:47 smithi144 ceph-mon[104994]: from='client.14877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:42:47.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:47 smithi144 ceph-mon[104994]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:42:47.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:47 smithi135 ceph-mon[118715]: from='client.14877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:42:47.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:47 smithi135 ceph-mon[118715]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:42:47.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:47 smithi135 ceph-mon[100251]: from='client.14877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:42:47.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:47 smithi135 ceph-mon[100251]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:42:48.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:48 smithi144 ceph-mon[104994]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 66 KiB/s wr, 39 op/s 2023-09-23T19:42:48.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:48 smithi135 ceph-mon[100251]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 66 KiB/s wr, 39 op/s 2023-09-23T19:42:48.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:48 smithi135 ceph-mon[118715]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 66 KiB/s wr, 39 op/s 2023-09-23T19:42:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:50 smithi135 ceph-mon[100251]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 290 KiB/s rd, 119 KiB/s wr, 92 op/s 2023-09-23T19:42:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:50 smithi135 ceph-mon[118715]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 290 KiB/s rd, 119 KiB/s wr, 92 op/s 2023-09-23T19:42:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:50 smithi144 ceph-mon[104994]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 290 KiB/s rd, 119 KiB/s wr, 92 op/s 2023-09-23T19:42:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:52 smithi144 ceph-mon[104994]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-23T19:42:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:52 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:52.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:52 smithi135 ceph-mon[100251]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-23T19:42:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:52 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:52 smithi135 ceph-mon[118715]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-09-23T19:42:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:52 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:42:52.804 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd help export 2023-09-23T19:42:52.804 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -q export-format 2023-09-23T19:42:52.824 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_190031/img bs=1k count=10 seek=100 2023-09-23T19:42:52.825 INFO:tasks.workunit.client.0.smithi135.stderr:10+0 records in 2023-09-23T19:42:52.825 INFO:tasks.workunit.client.0.smithi135.stderr:10+0 records out 2023-09-23T19:42:52.825 INFO:tasks.workunit.client.0.smithi135.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.3144e-05 s, 162 MB/s 2023-09-23T19:42:52.826 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format /tmp/rbd_import_export_190031/img testimg 2023-09-23T19:42:52.882 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:42:52.883 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:42:52.885 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap create testimg@snap 2023-09-23T19:42:53.332 INFO:tasks.workunit.client.0.smithi135.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T19:42:53.336 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:42:53.339 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T19:42:53.389 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd image-meta list testimg 2023-09-23T19:42:53.431 INFO:tasks.workunit.client.0.smithi135.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T19:42:53.431 INFO:tasks.workunit.client.0.smithi135.stderr: 2023-09-23T19:42:53.431 INFO:tasks.workunit.client.0.smithi135.stderr:Key Value 2023-09-23T19:42:53.431 INFO:tasks.workunit.client.0.smithi135.stderr:key1 value1' 2023-09-23T19:42:53.431 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190031/img_v2 2023-09-23T19:42:53.473 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:53.476 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190031/img_v2 testimg_import 2023-09-23T19:42:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:53.716 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:53.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:42:53.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:42:53.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:42:54.149 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:42:53 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:53] "GET /metrics HTTP/1.1" 200 239118 "" "Prometheus/2.43.0" 2023-09-23T19:42:54.344 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:42:54.347 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import 2023-09-23T19:42:54.381 INFO:tasks.workunit.client.0.smithi135.stdout:rbd image 'testimg_import': 2023-09-23T19:42:54.381 INFO:tasks.workunit.client.0.smithi135.stdout: size 110 KiB in 1 objects 2023-09-23T19:42:54.381 INFO:tasks.workunit.client.0.smithi135.stdout: order 22 (4 MiB objects) 2023-09-23T19:42:54.381 INFO:tasks.workunit.client.0.smithi135.stdout: snapshot_count: 1 2023-09-23T19:42:54.381 INFO:tasks.workunit.client.0.smithi135.stdout: id: 3a4798eb4da9 2023-09-23T19:42:54.381 INFO:tasks.workunit.client.0.smithi135.stdout: block_name_prefix: rbd_data.3a4798eb4da9 2023-09-23T19:42:54.382 INFO:tasks.workunit.client.0.smithi135.stdout: format: 2 2023-09-23T19:42:54.382 INFO:tasks.workunit.client.0.smithi135.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T19:42:54.382 INFO:tasks.workunit.client.0.smithi135.stdout: op_features: 2023-09-23T19:42:54.382 INFO:tasks.workunit.client.0.smithi135.stdout: flags: 2023-09-23T19:42:54.382 INFO:tasks.workunit.client.0.smithi135.stdout: create_timestamp: Sat Sep 23 19:42:53 2023 2023-09-23T19:42:54.382 INFO:tasks.workunit.client.0.smithi135.stdout: access_timestamp: Sat Sep 23 19:42:53 2023 2023-09-23T19:42:54.382 INFO:tasks.workunit.client.0.smithi135.stdout: modify_timestamp: Sat Sep 23 19:42:53 2023 2023-09-23T19:42:54.384 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import@snap 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout:rbd image 'testimg_import': 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: size 110 KiB in 1 objects 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: order 22 (4 MiB objects) 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: snapshot_count: 1 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: id: 3a4798eb4da9 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: block_name_prefix: rbd_data.3a4798eb4da9 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: format: 2 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: op_features: 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: flags: 2023-09-23T19:42:54.419 INFO:tasks.workunit.client.0.smithi135.stdout: create_timestamp: Sat Sep 23 19:42:53 2023 2023-09-23T19:42:54.420 INFO:tasks.workunit.client.0.smithi135.stdout: access_timestamp: Sat Sep 23 19:42:53 2023 2023-09-23T19:42:54.420 INFO:tasks.workunit.client.0.smithi135.stdout: modify_timestamp: Sat Sep 23 19:42:53 2023 2023-09-23T19:42:54.420 INFO:tasks.workunit.client.0.smithi135.stdout: protected: False 2023-09-23T19:42:54.422 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd image-meta list testimg_import 2023-09-23T19:42:54.460 INFO:tasks.workunit.client.0.smithi135.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr: 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr:Key Value 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr:key1 value1' 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr: 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr:Key Value 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr: 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr:Key Value 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr:key1 value1' ']' 2023-09-23T19:42:54.461 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg_import /tmp/rbd_import_export_190031/img_import 2023-09-23T19:42:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:54 smithi135 ceph-mon[118715]: pgmap v85: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 140 KiB/s rd, 54 KiB/s wr, 59 op/s 2023-09-23T19:42:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:54 smithi135 ceph-mon[118715]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T19:42:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:54 smithi135 ceph-mon[118715]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:42:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:54 smithi135 ceph-mon[100251]: pgmap v85: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 140 KiB/s rd, 54 KiB/s wr, 59 op/s 2023-09-23T19:42:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:54 smithi135 ceph-mon[100251]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T19:42:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:54 smithi135 ceph-mon[100251]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:42:54.495 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete...done. 2023-09-23T19:42:54.498 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190031/img /tmp/rbd_import_export_190031/img_import 2023-09-23T19:42:54.498 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_190031/img /tmp/rbd_import_export_190031/img_import 2023-09-23T19:42:54.499 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/img --format %b 2023-09-23T19:42:54.500 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=24 2023-09-23T19:42:54.500 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/img_import --format %b 2023-09-23T19:42:54.501 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=224 2023-09-23T19:42:54.501 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=200 2023-09-23T19:42:54.501 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=200 2023-09-23T19:42:54.501 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T19:42:54.501 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_190031/img_snap 2023-09-23T19:42:54.537 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete...done. 2023-09-23T19:42:54.540 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_190031/img_snap_import 2023-09-23T19:42:54.581 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete...done. 2023-09-23T19:42:54.584 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190031/img_snap /tmp/rbd_import_export_190031/img_snap_import 2023-09-23T19:42:54.585 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_190031/img_snap /tmp/rbd_import_export_190031/img_snap_import 2023-09-23T19:42:54.585 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/img_snap --format %b 2023-09-23T19:42:54.586 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=224 2023-09-23T19:42:54.586 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/img_snap_import --format %b 2023-09-23T19:42:54.587 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=224 2023-09-23T19:42:54.588 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:42:54.588 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:42:54.588 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:42:54.588 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img_v2 2023-09-23T19:42:54.588 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img_import 2023-09-23T19:42:54.589 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img_snap 2023-09-23T19:42:54.590 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img_snap_import 2023-09-23T19:42:54.590 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T19:42:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:54 smithi144 ceph-mon[104994]: pgmap v85: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 140 KiB/s rd, 54 KiB/s wr, 59 op/s 2023-09-23T19:42:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:54 smithi144 ceph-mon[104994]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T19:42:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:54 smithi144 ceph-mon[104994]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:42:55.349 INFO:tasks.workunit.client.0.smithi135.stderr: Removing snap: 100% complete...done. 2023-09-23T19:42:55.355 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T19:42:55.451 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:42:55.454 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap rm testimg@snap 2023-09-23T19:42:55.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:55 smithi135 ceph-mon[118715]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T19:42:55.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:55 smithi135 ceph-mon[100251]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T19:42:55.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:42:55 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:42:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:42:55.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:55 smithi144 ceph-mon[104994]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T19:42:55.678 INFO:teuthology.orchestra.run.smithi135.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:42:55.678 INFO:teuthology.orchestra.run.smithi135.stdout:alertmanager.a smithi135 *:9093,9094 running (79s) 43s ago 2m 17.9M - 0.25.0 c8568f914cd2 ef1f5819ae68 2023-09-23T19:42:55.678 INFO:teuthology.orchestra.run.smithi135.stdout:grafana.a smithi144 *:3000 running (106s) 56s ago 106s 87.7M - 9.4.7 2c41d148cca3 2ca98132db5d 2023-09-23T19:42:55.679 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi135.srojfl smithi135 running (66s) 43s ago 65s 14.9M - 17.2.6-1123-g61431f6b 0db14aa01094 37783bce2e06 2023-09-23T19:42:55.679 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi144.plbefj smithi144 running (63s) 56s ago 63s 13.3M - 17.2.6-1123-g61431f6b 0db14aa01094 af9c7e8247c3 2023-09-23T19:42:55.679 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.x smithi144 *:8443 running (6m) 56s ago 5m 438M - 17.2.6-1123-g61431f6b 0db14aa01094 aadb0eba9c47 2023-09-23T19:42:55.679 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.y smithi135 *:9283 running (7m) 43s ago 7m 493M - 17.2.6-1123-g61431f6b 0db14aa01094 7c00c1a62330 2023-09-23T19:42:55.679 INFO:teuthology.orchestra.run.smithi135.stdout:mon.a smithi135 running (7m) 43s ago 7m 80.3M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 1e28ea7cdc97 2023-09-23T19:42:55.679 INFO:teuthology.orchestra.run.smithi135.stdout:mon.b smithi144 running (6m) 56s ago 6m 57.5M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 40abc2a02dd4 2023-09-23T19:42:55.680 INFO:teuthology.orchestra.run.smithi135.stdout:mon.c smithi135 running (6m) 43s ago 6m 56.5M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 3ddce49c9396 2023-09-23T19:42:55.680 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.a smithi135 *:9100 running (96s) 43s ago 93s 15.2M - 1.5.0 0da6a335fe13 b3c82b1cd135 2023-09-23T19:42:55.680 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.b smithi144 *:9100 running (89s) 56s ago 89s 7646k - 1.5.0 0da6a335fe13 d31dbba1772d 2023-09-23T19:42:55.680 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 smithi135 running (5m) 43s ago 5m 59.1M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 af561f634ba3 2023-09-23T19:42:55.680 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1 smithi135 running (5m) 43s ago 5m 59.4M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 791e31e1318a 2023-09-23T19:42:55.680 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 smithi135 running (4m) 43s ago 4m 50.5M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 0ee5fdb5fb5e 2023-09-23T19:42:55.681 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 smithi135 running (4m) 43s ago 4m 48.9M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 0fd086d2e07f 2023-09-23T19:42:55.681 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 smithi144 running (4m) 56s ago 4m 49.6M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 076c88452d16 2023-09-23T19:42:55.681 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 smithi144 running (3m) 56s ago 3m 52.3M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 30c9f1e12bc8 2023-09-23T19:42:55.681 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 smithi144 running (3m) 56s ago 3m 48.2M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 d04b0ed25c0f 2023-09-23T19:42:55.681 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 smithi144 running (2m) 56s ago 2m 52.8M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 4b2a00379dc8 2023-09-23T19:42:55.681 INFO:teuthology.orchestra.run.smithi135.stdout:prometheus.a smithi144 *:9095 running (75s) 56s ago 2m 34.7M - 2.43.0 a07b618ecd1d c5d581813489 2023-09-23T19:42:56.348 INFO:tasks.workunit.client.0.smithi135.stderr: Removing snap: 100% complete...done. 2023-09-23T19:42:56.355 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm testimg 2023-09-23T19:42:56.453 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:42:56.456 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --order 20 /tmp/rbd_import_export_190031/img testimg 2023-09-23T19:42:56.509 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:42:56.510 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:42:56.512 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190031/img_v2 2023-09-23T19:42:56.548 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:56.551 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190031/img_v2 testimg_import 2023-09-23T19:42:56.613 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:42:56.617 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import 2023-09-23T19:42:56.618 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep order 2023-09-23T19:42:56.618 INFO:tasks.workunit.client.0.smithi135.stderr:+ awk '{print $2}' 2023-09-23T19:42:56.618 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 20 2023-09-23T19:42:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:56 smithi144 ceph-mon[104994]: pgmap v88: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 68 KiB/s wr, 74 op/s 2023-09-23T19:42:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:56 smithi144 ceph-mon[104994]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:42:56.637 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:56 smithi135 ceph-mon[118715]: pgmap v88: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 68 KiB/s wr, 74 op/s 2023-09-23T19:42:56.637 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:56 smithi135 ceph-mon[118715]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:42:56.639 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:56 smithi135 ceph-mon[100251]: pgmap v88: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 68 KiB/s wr, 74 op/s 2023-09-23T19:42:56.639 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:56 smithi135 ceph-mon[100251]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T19:42:56.661 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T19:42:56.664 INFO:tasks.workunit.client.0.smithi135.stdout:20 2023-09-23T19:42:56.664 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img_v2 2023-09-23T19:42:56.665 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T19:42:56.767 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:42:56.771 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg 2023-09-23T19:42:56.866 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:42:56.867 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_190031/img testimg 2023-09-23T19:42:56.918 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:42:56.922 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd info testimg 2023-09-23T19:42:56.922 INFO:tasks.workunit.client.0.smithi135.stderr:++ grep features 2023-09-23T19:42:56.970 INFO:tasks.workunit.client.0.smithi135.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T19:42:56.970 INFO:tasks.workunit.client.0.smithi135.stderr: op_features: ' 2023-09-23T19:42:56.970 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190031/img_v2 2023-09-23T19:42:57.015 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:57.019 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190031/img_v2 testimg_import 2023-09-23T19:42:57.081 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:42:57.085 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd info testimg_import 2023-09-23T19:42:57.085 INFO:tasks.workunit.client.0.smithi135.stderr:++ grep features 2023-09-23T19:42:57.101 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:42:57.132 INFO:tasks.workunit.client.0.smithi135.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T19:42:57.132 INFO:tasks.workunit.client.0.smithi135.stderr: op_features: ' 2023-09-23T19:42:57.132 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' ' features: layering 2023-09-23T19:42:57.132 INFO:tasks.workunit.client.0.smithi135.stderr: op_features: ' '!=' ' features: layering 2023-09-23T19:42:57.132 INFO:tasks.workunit.client.0.smithi135.stderr: op_features: ' ']' 2023-09-23T19:42:57.132 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img_v2 2023-09-23T19:42:57.133 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T19:42:57.207 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:42:57.210 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg 2023-09-23T19:42:57.284 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:42:57.287 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_190031/img testimg 2023-09-23T19:42:57.370 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:42:57.372 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190031/img_v2 2023-09-23T19:42:57.413 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:57.426 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190031/img_v2 testimg_import 2023-09-23T19:42:57.481 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:42:57.484 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import 2023-09-23T19:42:57.484 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 'stripe unit' 2023-09-23T19:42:57.484 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T19:42:57.532 INFO:tasks.workunit.client.0.smithi135.stdout: stripe unit: 4 KiB 2023-09-23T19:42:57.532 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import 2023-09-23T19:42:57.533 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 'stripe count' 2023-09-23T19:42:57.533 INFO:tasks.workunit.client.0.smithi135.stderr:+ awk '{print $3}' 2023-09-23T19:42:57.533 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 1000 2023-09-23T19:42:57.582 INFO:tasks.workunit.client.0.smithi135.stdout:1000 2023-09-23T19:42:57.583 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/img_v2 2023-09-23T19:42:57.583 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T19:42:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:57 smithi144 ceph-mon[104994]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:42:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:57 smithi144 ceph-mon[104994]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T19:42:57.664 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:57 smithi135 ceph-mon[100251]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:42:57.664 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:57 smithi135 ceph-mon[100251]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T19:42:57.665 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:57 smithi135 ceph-mon[118715]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:42:57.665 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:57 smithi135 ceph-mon[118715]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T19:42:57.677 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T19:42:57.680 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg 2023-09-23T19:42:57.772 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T19:42:57.775 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_190031/img testimg 2023-09-23T19:42:57.830 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:42:57.833 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap create testimg@snap1 2023-09-23T19:42:58.358 INFO:tasks.workunit.client.0.smithi135.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T19:42:58.364 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap create testimg@snap2 2023-09-23T19:42:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:58 smithi144 ceph-mon[104994]: pgmap v91: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:58.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:58 smithi135 ceph-mon[118715]: pgmap v91: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:58 smithi135 ceph-mon[100251]: pgmap v91: 73 pgs: 73 active+clean; 689 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:42:58.928 INFO:teuthology.orchestra.run.smithi135.stdout:wait for minority of mons to upgrade 2023-09-23T19:42:59.368 INFO:tasks.workunit.client.0.smithi135.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T19:42:59.376 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap protect testimg@snap2 2023-09-23T19:42:59.419 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190031/snap_protect 2023-09-23T19:42:59.462 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:42:59.465 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190031/snap_protect testimg_import 2023-09-23T19:42:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:42:59 smithi144 ceph-mon[104994]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T19:42:59.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:42:59 smithi135 ceph-mon[118715]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T19:42:59.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:42:59 smithi135 ceph-mon[100251]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T19:42:59.812 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T19:43:00.236 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:43:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:00 smithi144 ceph-mon[104994]: pgmap v93: 73 pgs: 73 active+clean; 689 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 395 KiB/s rd, 42 KiB/s wr, 517 op/s 2023-09-23T19:43:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:00 smithi144 ceph-mon[104994]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T19:43:00.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:00 smithi135 ceph-mon[118715]: pgmap v93: 73 pgs: 73 active+clean; 689 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 395 KiB/s rd, 42 KiB/s wr, 517 op/s 2023-09-23T19:43:00.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:00 smithi135 ceph-mon[118715]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T19:43:00.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:00 smithi135 ceph-mon[100251]: pgmap v93: 73 pgs: 73 active+clean; 689 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 395 KiB/s rd, 42 KiB/s wr, 517 op/s 2023-09-23T19:43:00.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:00 smithi135 ceph-mon[100251]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T19:43:01.399 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:43:01.402 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import@snap1 2023-09-23T19:43:01.402 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 'protected: False' 2023-09-23T19:43:01.442 INFO:tasks.workunit.client.0.smithi135.stdout: protected: False 2023-09-23T19:43:01.442 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import@snap2 2023-09-23T19:43:01.442 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 'protected: True' 2023-09-23T19:43:01.481 INFO:tasks.workunit.client.0.smithi135.stdout: protected: True 2023-09-23T19:43:01.482 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/snap_protect 2023-09-23T19:43:01.483 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T19:43:01.533 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T19:43:01.595 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap purge testimg 2023-09-23T19:43:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:01 smithi144 ceph-mon[104994]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T19:43:01.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:01 smithi135 ceph-mon[118715]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T19:43:01.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:01 smithi135 ceph-mon[100251]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T19:43:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:02 smithi144 ceph-mon[104994]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 396 KiB/s rd, 42 KiB/s wr, 519 op/s 2023-09-23T19:43:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:02 smithi144 ceph-mon[104994]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T19:43:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:02 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3386554063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:02.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:02 smithi135 ceph-mon[118715]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 396 KiB/s rd, 42 KiB/s wr, 519 op/s 2023-09-23T19:43:02.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:02 smithi135 ceph-mon[118715]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T19:43:02.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:02 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3386554063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:02.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:02 smithi135 ceph-mon[100251]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 396 KiB/s rd, 42 KiB/s wr, 519 op/s 2023-09-23T19:43:02.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:02 smithi135 ceph-mon[100251]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T19:43:02.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:02 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3386554063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:03.395 INFO:tasks.workunit.client.0.smithi135.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T19:43:03.402 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap purge testimg_import 2023-09-23T19:43:03.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:03 smithi135 ceph-mon[118715]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T19:43:03.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:03 smithi135 ceph-mon[100251]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T19:43:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:03 smithi144 ceph-mon[104994]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T19:43:04.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:43:03 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:03] "GET /metrics HTTP/1.1" 200 239118 "" "Prometheus/2.43.0" 2023-09-23T19:43:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:04 smithi135 ceph-mon[118715]: pgmap v99: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2023-09-23T19:43:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:04 smithi135 ceph-mon[118715]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T19:43:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:04 smithi135 ceph-mon[118715]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T19:43:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:04 smithi135 ceph-mon[100251]: pgmap v99: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2023-09-23T19:43:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:04 smithi135 ceph-mon[100251]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T19:43:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:04 smithi135 ceph-mon[100251]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T19:43:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:04 smithi144 ceph-mon[104994]: pgmap v99: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2023-09-23T19:43:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:04 smithi144 ceph-mon[104994]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T19:43:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:04 smithi144 ceph-mon[104994]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T19:43:05.195 INFO:tasks.workunit.client.0.smithi135.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T19:43:05.203 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg 2023-09-23T19:43:05.300 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:43:05.304 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T19:43:05.395 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:43:05.399 INFO:tasks.workunit.client.0.smithi135.stderr:+ tiered=0 2023-09-23T19:43:05.399 INFO:tasks.workunit.client.0.smithi135.stderr:+ ceph osd dump 2023-09-23T19:43:05.400 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep '^pool' 2023-09-23T19:43:05.400 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep ''\''rbd'\''' 2023-09-23T19:43:05.400 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep tier 2023-09-23T19:43:05.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:43:05 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:43:05.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:05 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2735733973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:05.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:05 smithi144 ceph-mon[104994]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T19:43:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:05 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2735733973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:05 smithi135 ceph-mon[118715]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T19:43:05.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:05 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2735733973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:05.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:05 smithi135 ceph-mon[100251]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T19:43:05.919 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_190031/sparse1 2023-09-23T19:43:05.925 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T19:43:05.925 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T19:43:05.925 INFO:tasks.workunit.client.0.smithi135.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0053388 s, 196 MB/s 2023-09-23T19:43:05.925 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_190031/sparse2 2023-09-23T19:43:05.931 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T19:43:05.931 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T19:43:05.931 INFO:tasks.workunit.client.0.smithi135.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00503671 s, 208 MB/s 2023-09-23T19:43:05.931 INFO:tasks.workunit.client.0.smithi135.stderr:+ truncate /tmp/rbd_import_export_190031/sparse2 -s 2M 2023-09-23T19:43:05.950 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse1 2023-09-23T19:43:05.989 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 0% complete...failed. 2023-09-23T19:43:05.990 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T19:43:05.992 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T19:43:05.992 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_190031/sparse1 2023-09-23T19:43:06.056 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T19:43:06.057 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:43:06.057 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:43:06.061 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd ls -l 2023-09-23T19:43:06.062 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep sparse1 2023-09-23T19:43:06.062 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T19:43:06.099 INFO:tasks.workunit.client.0.smithi135.stdout:sparse1 2 MiB 2 2023-09-23T19:43:06.099 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse1 2023-09-23T19:43:06.099 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse1 2023-09-23T19:43:06.100 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse1 2023-09-23T19:43:06.100 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T19:43:06.100 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T19:43:06.139 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.3b6dce1cd421 2023-09-23T19:43:06.140 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.3b6dce1cd421 2023-09-23T19:43:06.140 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.3b6dce1cd421\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T19:43:06.140 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse1 2023-09-23T19:43:06.140 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse1 2023-09-23T19:43:06.141 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T19:43:06.146 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse1 2023-09-23T19:43:06.147 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T19:43:06.147 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T19:43:06.177 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T19:43:06.178 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T19:43:06.178 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T19:43:06.178 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T19:43:06.178 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T19:43:06.211 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=1 2023-09-23T19:43:06.212 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 1 2023-09-23T19:43:06.212 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T19:43:06.212 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse1 /tmp/rbd_import_export_190031/sparse1.out 2023-09-23T19:43:06.247 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T19:43:06.250 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190031/sparse1 /tmp/rbd_import_export_190031/sparse1.out 2023-09-23T19:43:06.250 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_190031/sparse1 /tmp/rbd_import_export_190031/sparse1.out 2023-09-23T19:43:06.252 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse1 --format %b 2023-09-23T19:43:06.253 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=2048 2023-09-23T19:43:06.253 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse1.out --format %b 2023-09-23T19:43:06.254 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=2048 2023-09-23T19:43:06.254 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:43:06.254 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:43:06.255 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:43:06.255 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/sparse1.out 2023-09-23T19:43:06.255 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse1 2023-09-23T19:43:06.344 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T19:43:06.348 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse2 2023-09-23T19:43:06.385 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 0% complete...failed. 2023-09-23T19:43:06.385 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T19:43:06.388 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T19:43:06.388 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_190031/sparse2 2023-09-23T19:43:06.456 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T19:43:06.457 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:43:06.457 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:43:06.460 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd ls -l 2023-09-23T19:43:06.460 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep sparse2 2023-09-23T19:43:06.460 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T19:43:06.505 INFO:tasks.workunit.client.0.smithi135.stdout:sparse2 2 MiB 2 2023-09-23T19:43:06.506 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse2 2023-09-23T19:43:06.506 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse2 2023-09-23T19:43:06.506 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse2 2023-09-23T19:43:06.506 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T19:43:06.506 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T19:43:06.542 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.6242e6b11919 2023-09-23T19:43:06.542 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.6242e6b11919 2023-09-23T19:43:06.543 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.6242e6b11919\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T19:43:06.543 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse2 2023-09-23T19:43:06.543 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse2 2023-09-23T19:43:06.543 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T19:43:06.543 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse2 2023-09-23T19:43:06.544 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T19:43:06.544 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T19:43:06.580 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T19:43:06.580 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T19:43:06.580 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T19:43:06.580 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T19:43:06.580 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T19:43:06.612 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=0 2023-09-23T19:43:06.612 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 0 2023-09-23T19:43:06.612 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T19:43:06.612 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse2 /tmp/rbd_import_export_190031/sparse2.out 2023-09-23T19:43:06.647 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T19:43:06.651 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190031/sparse2 /tmp/rbd_import_export_190031/sparse2.out 2023-09-23T19:43:06.651 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_190031/sparse2 /tmp/rbd_import_export_190031/sparse2.out 2023-09-23T19:43:06.653 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse2 --format %b 2023-09-23T19:43:06.654 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=2048 2023-09-23T19:43:06.654 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse2.out --format %b 2023-09-23T19:43:06.655 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=2048 2023-09-23T19:43:06.655 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:43:06.655 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:43:06.655 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:43:06.655 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/sparse2.out 2023-09-23T19:43:06.656 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse2 2023-09-23T19:43:06.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:06 smithi135 ceph-mon[100251]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 12 KiB/s wr, 173 op/s 2023-09-23T19:43:06.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:06 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4093603623' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:43:06.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:06 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4093603623' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:43:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:06 smithi135 ceph-mon[118715]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 12 KiB/s wr, 173 op/s 2023-09-23T19:43:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:06 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4093603623' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:43:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:06 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4093603623' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:43:06.739 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T19:43:06.743 INFO:tasks.workunit.client.0.smithi135.stderr:+ truncate /tmp/rbd_import_export_190031/sparse1 -s 10M 2023-09-23T19:43:06.743 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T19:43:06.830 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:43:06.830 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:43:06.830 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:43:06.833 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd ls -l 2023-09-23T19:43:06.833 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep sparse1 2023-09-23T19:43:06.833 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T19:43:06.870 INFO:tasks.workunit.client.0.smithi135.stdout:sparse1 10 MiB 2 2023-09-23T19:43:06.871 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse1 2023-09-23T19:43:06.871 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse1 2023-09-23T19:43:06.871 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse1 2023-09-23T19:43:06.871 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T19:43:06.871 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T19:43:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:06 smithi144 ceph-mon[104994]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 12 KiB/s wr, 173 op/s 2023-09-23T19:43:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:06 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4093603623' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:43:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:06 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4093603623' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:43:06.911 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.3bcd3cd2d433 2023-09-23T19:43:06.911 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.3bcd3cd2d433 2023-09-23T19:43:06.911 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse1 2023-09-23T19:43:06.912 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.3bcd3cd2d433\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T19:43:06.912 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse1 2023-09-23T19:43:06.912 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T19:43:06.914 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse1 2023-09-23T19:43:06.915 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T19:43:06.915 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T19:43:06.951 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T19:43:06.951 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T19:43:06.951 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T19:43:06.951 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T19:43:06.951 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T19:43:06.986 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=1 2023-09-23T19:43:06.986 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 1 2023-09-23T19:43:06.986 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T19:43:06.986 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse1 /tmp/rbd_import_export_190031/sparse1.out 2023-09-23T19:43:07.030 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T19:43:07.034 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190031/sparse1 /tmp/rbd_import_export_190031/sparse1.out 2023-09-23T19:43:07.035 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_190031/sparse1 /tmp/rbd_import_export_190031/sparse1.out 2023-09-23T19:43:07.041 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse1 --format %b 2023-09-23T19:43:07.042 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=2048 2023-09-23T19:43:07.042 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse1.out --format %b 2023-09-23T19:43:07.043 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=2048 2023-09-23T19:43:07.043 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:43:07.043 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:43:07.044 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:43:07.044 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/sparse1.out 2023-09-23T19:43:07.044 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse1 2023-09-23T19:43:07.128 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T19:43:07.131 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_190031/sparse2 oflag=append conv=notrunc 2023-09-23T19:43:07.144 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T19:43:07.144 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T19:43:07.144 INFO:tasks.workunit.client.0.smithi135.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0117866 s, 178 MB/s 2023-09-23T19:43:07.144 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T19:43:07.229 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:43:07.229 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:43:07.230 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:43:07.232 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd ls -l 2023-09-23T19:43:07.233 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep sparse2 2023-09-23T19:43:07.233 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T19:43:07.270 INFO:tasks.workunit.client.0.smithi135.stdout:sparse2 4 MiB 2 2023-09-23T19:43:07.271 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse2 2023-09-23T19:43:07.271 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse2 2023-09-23T19:43:07.272 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse2 2023-09-23T19:43:07.272 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T19:43:07.272 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T19:43:07.310 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.62a29bc01cab 2023-09-23T19:43:07.311 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.62a29bc01cab 2023-09-23T19:43:07.311 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse2 2023-09-23T19:43:07.311 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse2 2023-09-23T19:43:07.312 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.62a29bc01cab\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T19:43:07.312 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T19:43:07.312 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse2 2023-09-23T19:43:07.312 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T19:43:07.312 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T19:43:07.355 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T19:43:07.355 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T19:43:07.355 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T19:43:07.355 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T19:43:07.356 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T19:43:07.391 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects='0 2023-09-23T19:43:07.391 INFO:tasks.workunit.client.0.smithi135.stderr:2 2023-09-23T19:43:07.391 INFO:tasks.workunit.client.0.smithi135.stderr:3' 2023-09-23T19:43:07.391 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 0 2 3 2023-09-23T19:43:07.391 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T19:43:07.391 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse2 /tmp/rbd_import_export_190031/sparse2.out 2023-09-23T19:43:07.432 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T19:43:07.436 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190031/sparse2 /tmp/rbd_import_export_190031/sparse2.out 2023-09-23T19:43:07.436 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_190031/sparse2 /tmp/rbd_import_export_190031/sparse2.out 2023-09-23T19:43:07.439 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse2 --format %b 2023-09-23T19:43:07.440 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=6144 2023-09-23T19:43:07.440 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse2.out --format %b 2023-09-23T19:43:07.441 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=6144 2023-09-23T19:43:07.441 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:43:07.441 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T19:43:07.441 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:43:07.441 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/sparse2.out 2023-09-23T19:43:07.442 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse2 2023-09-23T19:43:07.527 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T19:43:07.530 INFO:tasks.workunit.client.0.smithi135.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T19:43:07.531 INFO:tasks.workunit.client.0.smithi135.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T19:43:07.531 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_190031/sparse1 sparse 2023-09-23T19:43:07.598 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T19:43:07.599 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:43:07.599 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:43:07.602 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse 2023-09-23T19:43:07.602 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse 2023-09-23T19:43:07.602 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse 2023-09-23T19:43:07.602 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T19:43:07.603 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T19:43:07.639 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.62cfb6dd4249 2023-09-23T19:43:07.640 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.62cfb6dd4249 2023-09-23T19:43:07.640 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse 2023-09-23T19:43:07.640 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse 2023-09-23T19:43:07.640 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.62cfb6dd4249\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T19:43:07.640 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T19:43:07.640 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse 2023-09-23T19:43:07.641 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T19:43:07.642 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T19:43:07.677 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T19:43:07.677 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T19:43:07.678 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T19:43:07.678 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T19:43:07.678 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T19:43:07.710 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=1 2023-09-23T19:43:07.710 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 1 2023-09-23T19:43:07.711 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T19:43:07.711 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse 2023-09-23T19:43:07.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:07 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2550551681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:07.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:07 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:07.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:07 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2550551681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:07.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:07 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:07.798 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T19:43:07.800 INFO:tasks.workunit.client.0.smithi135.stdout:zeros import through stdin to sparse image 2023-09-23T19:43:07.801 INFO:tasks.workunit.client.0.smithi135.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T19:43:07.801 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T19:43:07.801 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format - sparse 2023-09-23T19:43:07.850 INFO:tasks.workunit.client.0.smithi135.stderr:4+0 records in 2023-09-23T19:43:07.850 INFO:tasks.workunit.client.0.smithi135.stderr:4+0 records out 2023-09-23T19:43:07.850 INFO:tasks.workunit.client.0.smithi135.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0487386 s, 86.1 MB/s 2023-09-23T19:43:07.860 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T19:43:07.861 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:43:07.864 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse 2023-09-23T19:43:07.864 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse 2023-09-23T19:43:07.864 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse 2023-09-23T19:43:07.864 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T19:43:07.865 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T19:43:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:07 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2550551681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:07 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:07.900 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.3c4819e3e98b 2023-09-23T19:43:07.901 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.3c4819e3e98b 2023-09-23T19:43:07.901 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse 2023-09-23T19:43:07.901 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.3c4819e3e98b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T19:43:07.901 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse 2023-09-23T19:43:07.901 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T19:43:07.902 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse 2023-09-23T19:43:07.902 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T19:43:07.902 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T19:43:07.937 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T19:43:07.937 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T19:43:07.937 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T19:43:07.938 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T19:43:07.938 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T19:43:07.970 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects= 2023-09-23T19:43:07.970 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 2023-09-23T19:43:07.970 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T19:43:07.970 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse 2023-09-23T19:43:08.051 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:43:08.055 INFO:tasks.workunit.client.0.smithi135.stdout:zeros export to sparse file 2023-09-23T19:43:08.055 INFO:tasks.workunit.client.0.smithi135.stderr:+ echo 'zeros export to sparse file' 2023-09-23T19:43:08.055 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T19:43:08.095 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:43:08.098 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd info sparse 2023-09-23T19:43:08.098 INFO:tasks.workunit.client.0.smithi135.stderr:++ grep block_name_prefix 2023-09-23T19:43:08.098 INFO:tasks.workunit.client.0.smithi135.stderr:++ awk '{print $NF;}' 2023-09-23T19:43:08.134 INFO:tasks.workunit.client.0.smithi135.stderr:+ prefix=rbd_data.3c698e34f021 2023-09-23T19:43:08.135 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T19:43:08.135 INFO:tasks.workunit.client.0.smithi135.stderr:++ get_image_data_pool sparse 2023-09-23T19:43:08.136 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse 2023-09-23T19:43:08.136 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse 2023-09-23T19:43:08.136 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep 'data_pool: ' 2023-09-23T19:43:08.136 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T19:43:08.171 INFO:tasks.workunit.client.0.smithi135.stderr:++ data_pool= 2023-09-23T19:43:08.171 INFO:tasks.workunit.client.0.smithi135.stderr:++ '[' -z ']' 2023-09-23T19:43:08.171 INFO:tasks.workunit.client.0.smithi135.stderr:++ data_pool=rbd 2023-09-23T19:43:08.171 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo rbd 2023-09-23T19:43:08.171 INFO:tasks.workunit.client.0.smithi135.stderr:+ rados -p rbd put rbd_data.3c698e34f021.000000000000 - 2023-09-23T19:43:08.201 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T19:43:08.202 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T19:43:08.202 INFO:tasks.workunit.client.0.smithi135.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0660734 s, 63.5 MB/s 2023-09-23T19:43:08.218 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse 2023-09-23T19:43:08.219 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse 2023-09-23T19:43:08.219 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse 2023-09-23T19:43:08.219 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T19:43:08.219 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T19:43:08.254 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.3c698e34f021 2023-09-23T19:43:08.255 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.3c698e34f021 2023-09-23T19:43:08.255 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse 2023-09-23T19:43:08.255 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.3c698e34f021\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T19:43:08.255 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse 2023-09-23T19:43:08.255 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T19:43:08.255 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse 2023-09-23T19:43:08.255 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T19:43:08.256 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T19:43:08.291 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T19:43:08.291 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T19:43:08.291 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T19:43:08.291 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T19:43:08.291 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T19:43:08.323 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=0 2023-09-23T19:43:08.323 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 0 2023-09-23T19:43:08.324 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T19:43:08.324 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/sparse 2023-09-23T19:43:08.324 INFO:tasks.workunit.client.0.smithi135.stderr:rm: cannot remove '/tmp/rbd_import_export_190031/sparse': No such file or directory 2023-09-23T19:43:08.324 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T19:43:08.325 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse /tmp/rbd_import_export_190031/sparse 2023-09-23T19:43:08.358 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete...done. 2023-09-23T19:43:08.362 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_190031/sparse --format=%b 2023-09-23T19:43:08.363 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 = 0 ']' 2023-09-23T19:43:08.363 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse 2023-09-23T19:43:08.448 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T19:43:08.451 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_190031/sparse /tmp/rbd_import_export_190031/sparse1 /tmp/rbd_import_export_190031/sparse2 /tmp/rbd_import_export_190031/sparse3 2023-09-23T19:43:08.453 INFO:tasks.workunit.client.0.smithi135.stderr:rm: cannot remove '/tmp/rbd_import_export_190031/sparse3': No such file or directory 2023-09-23T19:43:08.454 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T19:43:08.454 INFO:tasks.workunit.client.0.smithi135.stdout:OK 2023-09-23T19:43:08.455 INFO:tasks.workunit.client.0.smithi135.stderr:+ echo OK 2023-09-23T19:43:08.455 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm -rf /tmp/rbd_import_export_190031 2023-09-23T19:43:08.455 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T19:43:08.455 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T19:43:08.488 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T19:43:08.488 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T19:43:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:08 smithi135 ceph-mon[118715]: pgmap v104: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 9.6 KiB/s wr, 137 op/s 2023-09-23T19:43:08.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:08 smithi135 ceph-mon[100251]: pgmap v104: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 9.6 KiB/s wr, 137 op/s 2023-09-23T19:43:08.772 DEBUG:teuthology.parallel:result is None 2023-09-23T19:43:08.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:43:08.800 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:43:08.800 DEBUG:teuthology.orchestra.run.smithi135:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:43:08.856 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:43:08.856 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T19:43:08.856 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T19:43:08.856 DEBUG:teuthology.parallel:result is None 2023-09-23T19:43:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:08 smithi144 ceph-mon[104994]: pgmap v104: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 9.6 KiB/s wr, 137 op/s 2023-09-23T19:43:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:10 smithi135 ceph-mon[118715]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 417 op/s 2023-09-23T19:43:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:10 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4218875964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:10 smithi135 ceph-mon[100251]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 417 op/s 2023-09-23T19:43:10.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:10 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4218875964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:10 smithi144 ceph-mon[104994]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 417 op/s 2023-09-23T19:43:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:10 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4218875964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:12 smithi144 ceph-mon[104994]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 317 op/s 2023-09-23T19:43:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:12 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3163647498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:12.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:12 smithi135 ceph-mon[118715]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 317 op/s 2023-09-23T19:43:12.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:12 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3163647498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:12.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:12 smithi135 ceph-mon[100251]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 72 MiB used, 715 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 317 op/s 2023-09-23T19:43:12.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:12 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3163647498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:14.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:43:13 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:13] "GET /metrics HTTP/1.1" 200 239418 "" "Prometheus/2.43.0" 2023-09-23T19:43:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:14 smithi144 ceph-mon[104994]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 899 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2023-09-23T19:43:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:14 smithi144 ceph-mon[104994]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T19:43:14.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:14 smithi135 ceph-mon[100251]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 899 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2023-09-23T19:43:14.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:14 smithi135 ceph-mon[100251]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T19:43:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:14 smithi135 ceph-mon[118715]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 899 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2023-09-23T19:43:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:14 smithi135 ceph-mon[118715]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T19:43:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:15 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/709939506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:15.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:43:15 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:43:15.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:15 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/709939506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:15.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:15 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/709939506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:16 smithi144 ceph-mon[104994]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2023-09-23T19:43:16.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:16 smithi135 ceph-mon[100251]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2023-09-23T19:43:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:16 smithi135 ceph-mon[118715]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2023-09-23T19:43:17.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:17 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/407039701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:17.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:17 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/407039701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:17.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:17 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/407039701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:18.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:18 smithi144 ceph-mon[104994]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2023-09-23T19:43:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:18 smithi135 ceph-mon[118715]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2023-09-23T19:43:18.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:18 smithi135 ceph-mon[100251]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2023-09-23T19:43:20.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:20 smithi135 ceph-mon[100251]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 307 B/s wr, 20 op/s 2023-09-23T19:43:20.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:20 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/141371127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:20.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:20 smithi135 ceph-mon[118715]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 307 B/s wr, 20 op/s 2023-09-23T19:43:20.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:20 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/141371127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:20 smithi144 ceph-mon[104994]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 307 B/s wr, 20 op/s 2023-09-23T19:43:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:20 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/141371127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:22 smithi144 ceph-mon[104994]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 307 B/s wr, 20 op/s 2023-09-23T19:43:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:22 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2872637208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:22 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:22.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:22 smithi135 ceph-mon[100251]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 307 B/s wr, 20 op/s 2023-09-23T19:43:22.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:22 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2872637208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:22.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:22 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:22 smithi135 ceph-mon[118715]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 307 B/s wr, 20 op/s 2023-09-23T19:43:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:22 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2872637208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:22 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:24.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:43:23 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:23] "GET /metrics HTTP/1.1" 200 239479 "" "Prometheus/2.43.0" 2023-09-23T19:43:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:24 smithi144 ceph-mon[104994]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:24 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2662547039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:24.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:24 smithi135 ceph-mon[118715]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:24.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:24 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2662547039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:24.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:24 smithi135 ceph-mon[100251]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:24.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:24 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2662547039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:25.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:43:25 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:43:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:26 smithi144 ceph-mon[104994]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:26.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:26 smithi135 ceph-mon[118715]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:26.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:26 smithi135 ceph-mon[100251]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:27 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/582416991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:27.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:27 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/582416991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:27 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/582416991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:28 smithi144 ceph-mon[104994]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:28.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:28 smithi135 ceph-mon[118715]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:28.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:28 smithi135 ceph-mon[100251]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:29.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:29 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3245171507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:29.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:29 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3245171507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:29.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:29 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3245171507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:30 smithi144 ceph-mon[104994]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:43:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:43:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:30 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:43:30.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:30 smithi135 ceph-mon[118715]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:30.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:43:30.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:43:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:30 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:43:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:30 smithi135 ceph-mon[100251]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:43:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:43:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:30 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:43:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:31 smithi144 ceph-mon[104994]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T19:43:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:31 smithi144 ceph-mon[104994]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T19:43:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:31 smithi144 ceph-mon[104994]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T19:43:31.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:31 smithi144 ceph-mon[104994]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi144 2023-09-23T19:43:31.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:31 smithi135 ceph-mon[118715]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T19:43:31.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:31 smithi135 ceph-mon[118715]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T19:43:31.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:31 smithi135 ceph-mon[118715]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T19:43:31.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:31 smithi135 ceph-mon[118715]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi144 2023-09-23T19:43:31.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:31 smithi135 ceph-mon[100251]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T19:43:31.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:31 smithi135 ceph-mon[100251]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T19:43:31.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:31 smithi135 ceph-mon[100251]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T19:43:31.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:31 smithi135 ceph-mon[100251]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi144 2023-09-23T19:43:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:32 smithi144 ceph-mon[104994]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:32 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3566842528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:32 smithi135 ceph-mon[118715]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:32 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3566842528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:32.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:32 smithi135 ceph-mon[100251]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:32 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3566842528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:34.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:43:33 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:33] "GET /metrics HTTP/1.1" 200 239479 "" "Prometheus/2.43.0" 2023-09-23T19:43:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:34 smithi144 ceph-mon[104994]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:34 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1425429602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:34 smithi135 ceph-mon[118715]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:34 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1425429602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:34 smithi135 ceph-mon[100251]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:34 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1425429602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:35.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:43:35 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:43:36.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:36 smithi144 ceph-mon[104994]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:36.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:36 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4201328303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:36.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:36 smithi135 ceph-mon[118715]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:36.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:36 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4201328303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:36.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:36 smithi135 ceph-mon[100251]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:36.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:36 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4201328303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:37 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:37.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:37 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:37 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:38.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:38 smithi144 ceph-mon[104994]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:38.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:38 smithi135 ceph-mon[118715]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:38.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:38 smithi135 ceph-mon[100251]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:39 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1506214854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:39.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:39 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1506214854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:39.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:39 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1506214854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:40 smithi144 ceph-mon[104994]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:40.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:40 smithi135 ceph-mon[118715]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:40 smithi135 ceph-mon[100251]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:41 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/186931919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:41.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:41 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/186931919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:41.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:41 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/186931919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:42 smithi144 ceph-mon[104994]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:42 smithi135 ceph-mon[118715]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:42.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:42 smithi135 ceph-mon[100251]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:43.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:43 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1916195128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:43.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:43 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1916195128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:43.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:43 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1916195128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:44.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:43:43 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:43] "GET /metrics HTTP/1.1" 200 239440 "" "Prometheus/2.43.0" 2023-09-23T19:43:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:44 smithi144 ceph-mon[104994]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:44.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:44 smithi135 ceph-mon[118715]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:44.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:44 smithi135 ceph-mon[100251]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:45.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:43:45 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:43:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:46 smithi144 ceph-mon[104994]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:46 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4077291966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:46.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:46 smithi135 ceph-mon[100251]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:46.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:46 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4077291966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:46 smithi135 ceph-mon[118715]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:46 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4077291966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:48 smithi144 ceph-mon[104994]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:48 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2173334753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:48.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:48 smithi135 ceph-mon[118715]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:48.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:48 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2173334753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:48 smithi135 ceph-mon[100251]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:48 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2173334753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:50 smithi135 ceph-mon[118715]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:50 smithi135 ceph-mon[100251]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:50 smithi144 ceph-mon[104994]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:51.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:51 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1279045036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:51.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:51 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1279045036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:51 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1279045036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:52 smithi135 ceph-mon[118715]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:52 smithi135 ceph-mon[100251]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:52 smithi144 ceph-mon[104994]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:53 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:53 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3729067131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:53 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:53 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3729067131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:53 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:43:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:53 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3729067131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:54.208 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:43:53 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:53] "GET /metrics HTTP/1.1" 200 239393 "" "Prometheus/2.43.0" 2023-09-23T19:43:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:54 smithi135 ceph-mon[100251]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:54.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:54 smithi135 ceph-mon[118715]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:54 smithi144 ceph-mon[104994]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:55.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:55 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4092803385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:55.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:55 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4092803385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:55.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:55 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4092803385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:55.632 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:43:55 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:43:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:43:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:56 smithi135 ceph-mon[100251]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:56.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:56 smithi135 ceph-mon[118715]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:56 smithi144 ceph-mon[104994]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:58 smithi144 ceph-mon[104994]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:43:58 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/279578281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:58.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:58 smithi135 ceph-mon[118715]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:58.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:43:58 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/279578281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:43:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:58 smithi135 ceph-mon[100251]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:43:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:43:58 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/279578281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:00 smithi135 ceph-mon[118715]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:00 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1077918731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:00 smithi135 ceph-mon[100251]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:00 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1077918731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:00 smithi144 ceph-mon[104994]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:00 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1077918731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:02 smithi144 ceph-mon[104994]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:02 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3549614784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:02.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:02 smithi135 ceph-mon[118715]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:02.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:02 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3549614784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:02.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:02 smithi135 ceph-mon[100251]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:02.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:02 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3549614784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:04.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:03 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:03] "GET /metrics HTTP/1.1" 200 239393 "" "Prometheus/2.43.0" 2023-09-23T19:44:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:04 smithi144 ceph-mon[104994]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:04.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:04 smithi135 ceph-mon[100251]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:04 smithi135 ceph-mon[118715]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:05.571 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:05 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2726015818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:05.572 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:05 smithi144 conmon[108226]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:44:05.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:05 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2726015818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:05 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2726015818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:06.473 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:06 smithi144 ceph-mon[104994]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:06.473 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:06 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:06.473 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:06 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:06.473 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:06 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:44:06.473 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:06 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[118715]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:44:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[100251]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:44:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:06 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:07.350 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:07 smithi144 systemd[1]: Stopping Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:07 smithi144 ceph-mon[104994]: Upgrade: Updating mgr.x 2023-09-23T19:44:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:07 smithi144 ceph-mon[104994]: Deploying daemon mgr.x on smithi144 2023-09-23T19:44:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:07 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2441931034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:07 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:07.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:07 smithi135 ceph-mon[100251]: Upgrade: Updating mgr.x 2023-09-23T19:44:07.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:07 smithi135 ceph-mon[100251]: Deploying daemon mgr.x on smithi144 2023-09-23T19:44:07.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:07 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2441931034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:07.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:07 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:07 smithi135 ceph-mon[118715]: Upgrade: Updating mgr.x 2023-09-23T19:44:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:07 smithi135 ceph-mon[118715]: Deploying daemon mgr.x on smithi144 2023-09-23T19:44:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:07 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2441931034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:07 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:08.195 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:07 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:44:08.195 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:07 smithi144 bash[159065]: aadb0eba9c477537f1958b9164d1d17d422a8720f2cc7476af444c7d509d6fe9 2023-09-23T19:44:08.195 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:08 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T19:44:08.195 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:08 smithi144 systemd[1]: Stopped Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:08 smithi144 ceph-mon[104994]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:08.632 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:08 smithi144 systemd[1]: Starting Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:08 smithi135 ceph-mon[118715]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:08.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:08 smithi135 ceph-mon[100251]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:09.098 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:08 smithi144 ceph-mgr[159486]: -- 172.21.15.144:0/1628086295 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557dcb6731e0 con 0x557dcb64ac00 2023-09-23T19:44:09.124 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:08 smithi144 bash[159421]: 7aa64de85678e036b0c2d0822dc31db95f0b39d5649dd94e3db25144d43aad27 2023-09-23T19:44:09.124 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:08 smithi144 systemd[1]: Started Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:09.124 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:08 smithi144 conmon[159462]: 2023-09-23T19:44:08.919+0000 7fc2af8e7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:44:09.124 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:09 smithi144 conmon[159462]: 2023-09-23T19:44:09.003+0000 7fc2af8e7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:44:09.124 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:09 smithi144 conmon[159462]: 2023-09-23T19:44:09.097+0000 7fc2af8e7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:44:09.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:09 smithi144 conmon[159462]: 2023-09-23T19:44:09.206+0000 7fc2af8e7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:44:09.817 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:09 smithi144 conmon[159462]: 2023-09-23T19:44:09.424+0000 7fc2af8e7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:44:10.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:09 smithi144 conmon[159462]: 2023-09-23T19:44:09.815+0000 7fc2af8e7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:44:10.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:09 smithi144 conmon[159462]: 2023-09-23T19:44:09.963+0000 7fc2af8e7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:44:10.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:10 smithi144 conmon[159462]: 2023-09-23T19:44:10.057+0000 7fc2af8e7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:44:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:09 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:09 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:09 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:09 smithi144 ceph-mon[104994]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:09 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2770315066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[118715]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2770315066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:10.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:10.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:10.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[100251]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:09 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2770315066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:10.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:10 smithi144 conmon[159462]: 2023-09-23T19:44:10.159+0000 7fc2af8e7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:44:11.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:10 smithi144 conmon[159462]: 2023-09-23T19:44:10.648+0000 7fc2af8e7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:44:11.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:11 smithi144 conmon[159462]: 2023-09-23T19:44:11.389+0000 7fc2af8e7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:44:11.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:11 smithi144 conmon[159462]: 2023-09-23T19:44:11.491+0000 7fc2af8e7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:44:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:12 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:12 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:12 smithi144 ceph-mon[104994]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:12 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2614443049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:12.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:12 smithi144 conmon[159462]: 2023-09-23T19:44:12.350+0000 7fc2af8e7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:44:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:12 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:12 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:12 smithi135 ceph-mon[100251]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:12 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2614443049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:12 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:12 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:12 smithi135 ceph-mon[118715]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:12 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2614443049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:12.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:12 smithi144 conmon[159462]: 2023-09-23T19:44:12.448+0000 7fc2af8e7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:44:12.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:12 smithi144 conmon[159462]: 2023-09-23T19:44:12.637+0000 7fc2af8e7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:44:13.195 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:12 smithi144 conmon[159462]: 2023-09-23T19:44:12.941+0000 7fc2af8e7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:44:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:13 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:13 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:13 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:13 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.632 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:13 smithi144 conmon[159462]: 2023-09-23T19:44:13.195+0000 7fc2af8e7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:44:13.632 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:13 smithi144 conmon[159462]: 2023-09-23T19:44:13.364+0000 7fc2af8e7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:44:13.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:13 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:13 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:13 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:13 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:13 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:13 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:13 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:13 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:13.996 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:13 smithi144 conmon[159462]: 2023-09-23T19:44:13.664+0000 7fc2af8e7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:44:14.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:13 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:13] "GET /metrics HTTP/1.1" 200 239385 "" "Prometheus/2.43.0" 2023-09-23T19:44:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:14 smithi144 ceph-mon[104994]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:14.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:14 smithi135 ceph-mon[100251]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:14.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:14 smithi135 ceph-mon[118715]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:14.751 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:14 smithi144 conmon[159462]: 2023-09-23T19:44:14.402+0000 7fc2af8e7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:44:14.751 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:14 smithi144 conmon[159462]: 2023-09-23T19:44:14.537+0000 7fc2af8e7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:44:15.120 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:14 smithi144 conmon[159462]: 2023-09-23T19:44:14.750+0000 7fc2af8e7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:44:15.121 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:14 smithi144 conmon[159462]: 2023-09-23T19:44:14.873+0000 7fc2af8e7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:44:15.121 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:15 smithi144 conmon[159462]: 2023-09-23T19:44:15.120+0000 7fc2af8e7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:44:15.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:15 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1328829599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:15.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:15 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1328829599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:15.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:15 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1328829599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:16.171 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.063+0000 7fc2af8e7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:44:16.172 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE Bus STARTING 2023-09-23T19:44:16.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:16 smithi144 ceph-mon[104994]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:16.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:16 smithi144 ceph-mon[104994]: Standby manager daemon x restarted 2023-09-23T19:44:16.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:16 smithi144 ceph-mon[104994]: Standby manager daemon x started 2023-09-23T19:44:16.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:16 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:44:16.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:16 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:16.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:16 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:44:16.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:16 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:16.586 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: CherryPy Checker: 2023-09-23T19:44:16.586 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: The Application mounted at '' has an empty config. 2023-09-23T19:44:16.586 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.586 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE Serving on http://:::9283 2023-09-23T19:44:16.586 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE Bus STARTED 2023-09-23T19:44:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[118715]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[118715]: Standby manager daemon x restarted 2023-09-23T19:44:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[118715]: Standby manager daemon x started 2023-09-23T19:44:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:44:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[100251]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[100251]: Standby manager daemon x restarted 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[100251]: Standby manager daemon x started 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:44:16.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:16.862 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE Bus STOPPING 2023-09-23T19:44:16.862 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:44:16.862 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE Bus STOPPED 2023-09-23T19:44:16.862 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE Bus STARTING 2023-09-23T19:44:17.062 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:16 smithi135 conmon[100683]: 2023-09-23T19:44:16.729+0000 7f41a1a40700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T19:44:17.062 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:16 smithi135 conmon[100683]: ignoring --setuser ceph since I am not root 2023-09-23T19:44:17.062 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:16 smithi135 conmon[100683]: ignoring --setgroup ceph since I am not root 2023-09-23T19:44:17.062 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:16 smithi135 ceph-mgr[100707]: -- 172.21.15.135:0/2947400078 <== mon.0 v2:172.21.15.135:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55822eed89c0 con 0x55822eeb1000 2023-09-23T19:44:17.062 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:16 smithi135 conmon[100683]: 2023-09-23T19:44:16.894+0000 7fdd2ee551c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:44:17.062 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:16 smithi135 conmon[100683]: 2023-09-23T19:44:16.975+0000 7fdd2ee551c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:44:17.063 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:17 smithi135 conmon[100683]: 2023-09-23T19:44:17.061+0000 7fdd2ee551c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:44:17.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc20bd57700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc20bd57700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc20bd57700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc208d51700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc20bd57700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc208d51700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc20bd57700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc208d51700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc208d51700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: 2023-09-23T19:44:16.882+0000 7fc208d51700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE Serving on http://:::9283 2023-09-23T19:44:17.132 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:16 smithi144 conmon[159462]: [23/Sep/2023:19:44:16] ENGINE Bus STARTED 2023-09-23T19:44:17.341 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:17 smithi135 conmon[100683]: 2023-09-23T19:44:17.161+0000 7fdd2ee551c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:44:17.341 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T19:44:17.341 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.341 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.341 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3338969993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: mgrmap e21: x(active, starting, since 0.0051876s) 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:44:17.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:44:17.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: Manager daemon x is now available 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:17.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T19:44:17.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:17.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:17.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3338969993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: mgrmap e21: x(active, starting, since 0.0051876s) 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:44:17.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: Manager daemon x is now available 2023-09-23T19:44:17.347 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:17 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3338969993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T19:44:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.24359 172.21.15.135:0/3114983891' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: mgrmap e21: x(active, starting, since 0.0051876s) 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:44:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: Manager daemon x is now available 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:17.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:17.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:17.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:17.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:17.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:17 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:17.953 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:17 smithi135 conmon[100683]: 2023-09-23T19:44:17.340+0000 7fdd2ee551c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:44:17.953 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:17 smithi135 conmon[100683]: 2023-09-23T19:44:17.645+0000 7fdd2ee551c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:44:17.953 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:17 smithi135 conmon[100683]: 2023-09-23T19:44:17.732+0000 7fdd2ee551c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:44:17.954 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:17 smithi135 conmon[100683]: 2023-09-23T19:44:17.861+0000 7fdd2ee551c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:44:18.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:17 smithi135 conmon[100683]: 2023-09-23T19:44:17.953+0000 7fdd2ee551c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:44:18.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:18 smithi135 conmon[100683]: 2023-09-23T19:44:18.370+0000 7fdd2ee551c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:44:19.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:18 smithi144 ceph-mon[104994]: Deploying cephadm binary to smithi144 2023-09-23T19:44:19.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:18 smithi144 ceph-mon[104994]: Deploying cephadm binary to smithi135 2023-09-23T19:44:19.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:18 smithi144 ceph-mon[104994]: mgrmap e22: x(active, since 1.01146s) 2023-09-23T19:44:19.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:18 smithi144 ceph-mon[104994]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:19.165 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:18 smithi135 conmon[100683]: 2023-09-23T19:44:18.965+0000 7fdd2ee551c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:44:19.165 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:18 smithi135 ceph-mon[118715]: Deploying cephadm binary to smithi144 2023-09-23T19:44:19.166 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:18 smithi135 ceph-mon[118715]: Deploying cephadm binary to smithi135 2023-09-23T19:44:19.166 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:18 smithi135 ceph-mon[118715]: mgrmap e22: x(active, since 1.01146s) 2023-09-23T19:44:19.166 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:18 smithi135 ceph-mon[118715]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:19.166 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:18 smithi135 ceph-mon[100251]: Deploying cephadm binary to smithi144 2023-09-23T19:44:19.166 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:18 smithi135 ceph-mon[100251]: Deploying cephadm binary to smithi135 2023-09-23T19:44:19.166 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:18 smithi135 ceph-mon[100251]: mgrmap e22: x(active, since 1.01146s) 2023-09-23T19:44:19.166 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:18 smithi135 ceph-mon[100251]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:19.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:19 smithi135 conmon[100683]: 2023-09-23T19:44:19.164+0000 7fdd2ee551c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:44:20.044 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[100251]: [23/Sep/2023:19:44:17] ENGINE Bus STARTING 2023-09-23T19:44:20.044 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[100251]: [23/Sep/2023:19:44:17] ENGINE Serving on http://172.21.15.144:8765 2023-09-23T19:44:20.044 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[100251]: [23/Sep/2023:19:44:18] ENGINE Serving on https://172.21.15.144:7150 2023-09-23T19:44:20.044 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[100251]: [23/Sep/2023:19:44:18] ENGINE Bus STARTED 2023-09-23T19:44:20.044 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[100251]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[100251]: mgrmap e23: x(active, since 2s) 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3548098016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[118715]: [23/Sep/2023:19:44:17] ENGINE Bus STARTING 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[118715]: [23/Sep/2023:19:44:17] ENGINE Serving on http://172.21.15.144:8765 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[118715]: [23/Sep/2023:19:44:18] ENGINE Serving on https://172.21.15.144:7150 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[118715]: [23/Sep/2023:19:44:18] ENGINE Bus STARTED 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[118715]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[118715]: mgrmap e23: x(active, since 2s) 2023-09-23T19:44:20.045 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:19 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3548098016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:19 smithi144 ceph-mon[104994]: [23/Sep/2023:19:44:17] ENGINE Bus STARTING 2023-09-23T19:44:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:19 smithi144 ceph-mon[104994]: [23/Sep/2023:19:44:17] ENGINE Serving on http://172.21.15.144:8765 2023-09-23T19:44:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:19 smithi144 ceph-mon[104994]: [23/Sep/2023:19:44:18] ENGINE Serving on https://172.21.15.144:7150 2023-09-23T19:44:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:19 smithi144 ceph-mon[104994]: [23/Sep/2023:19:44:18] ENGINE Bus STARTED 2023-09-23T19:44:20.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:19 smithi144 ceph-mon[104994]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:20.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:19 smithi144 ceph-mon[104994]: mgrmap e23: x(active, since 2s) 2023-09-23T19:44:20.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:19 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3548098016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:20.335 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:20 smithi135 conmon[100683]: 2023-09-23T19:44:20.043+0000 7fdd2ee551c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:44:20.335 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:20 smithi135 conmon[100683]: 2023-09-23T19:44:20.153+0000 7fdd2ee551c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:44:20.645 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:20 smithi135 conmon[100683]: 2023-09-23T19:44:20.335+0000 7fdd2ee551c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:44:20.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:20 smithi135 conmon[100683]: 2023-09-23T19:44:20.644+0000 7fdd2ee551c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:44:20.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:20 smithi135 conmon[100683]: 2023-09-23T19:44:20.739+0000 7fdd2ee551c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:44:21.354 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:21 smithi135 conmon[100683]: 2023-09-23T19:44:21.045+0000 7fdd2ee551c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:44:21.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:21 smithi135 conmon[100683]: 2023-09-23T19:44:21.353+0000 7fdd2ee551c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:44:22.097 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[118715]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:22.097 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.098 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.098 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/640173217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:22.098 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.098 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.098 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[100251]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:22.098 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.098 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.099 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/640173217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:22.099 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.099 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:21 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:21 smithi144 ceph-mon[104994]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:22.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:21 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:21 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:21 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/640173217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:21 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:21 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:22.418 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:22 smithi135 conmon[100683]: 2023-09-23T19:44:22.096+0000 7fdd2ee551c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:44:22.418 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:22 smithi135 conmon[100683]: 2023-09-23T19:44:22.227+0000 7fdd2ee551c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:44:22.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:22 smithi135 conmon[100683]: 2023-09-23T19:44:22.418+0000 7fdd2ee551c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:44:22.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:22 smithi135 conmon[100683]: 2023-09-23T19:44:22.533+0000 7fdd2ee551c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:44:23.063 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.063 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.063 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.063 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.063 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.063 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.064 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.064 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.064 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.064 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.064 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi135 to 2500M 2023-09-23T19:44:23.064 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.064 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.064 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:22 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:22 smithi135 conmon[100683]: 2023-09-23T19:44:22.751+0000 7fdd2ee551c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:44:23.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi135 to 2500M 2023-09-23T19:44:23.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:23.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi135 to 2500M 2023-09-23T19:44:23.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:22 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:23.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[118715]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:23.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[118715]: Standby manager daemon y started 2023-09-23T19:44:23.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:23.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:23.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:44:23.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:23.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[100251]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:23.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[100251]: Standby manager daemon y started 2023-09-23T19:44:23.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:23.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:23.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:44:23.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:23 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:23.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:23 smithi135 conmon[100683]: 2023-09-23T19:44:23.618+0000 7fdd2ee551c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:44:23.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:23 smithi135 conmon[100683]: [23/Sep/2023:19:44:23] ENGINE Bus STARTING 2023-09-23T19:44:23.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:23 smithi135 conmon[100683]: CherryPy Checker: 2023-09-23T19:44:23.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:23 smithi135 conmon[100683]: The Application mounted at '' has an empty config. 2023-09-23T19:44:23.974 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:23 smithi135 conmon[100683]: 2023-09-23T19:44:23.974 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:23 smithi135 conmon[100683]: [23/Sep/2023:19:44:23] ENGINE Serving on http://:::9283 2023-09-23T19:44:23.974 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:23 smithi135 conmon[100683]: [23/Sep/2023:19:44:23] ENGINE Bus STARTED 2023-09-23T19:44:23.974 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:23 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:44:24.095 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:23 smithi144 ceph-mon[104994]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:24.095 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:23 smithi144 ceph-mon[104994]: Standby manager daemon y started 2023-09-23T19:44:24.095 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:23 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:24.095 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:23 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:24.095 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:23 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:44:24.095 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:23 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.135:0/2390070957' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:25.028 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:24 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3834672884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:25.029 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:24 smithi144 ceph-mon[104994]: mgrmap e24: x(active, since 7s), standbys: y 2023-09-23T19:44:25.029 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:24 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:44:25.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:24 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3834672884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:25.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:24 smithi135 ceph-mon[100251]: mgrmap e24: x(active, since 7s), standbys: y 2023-09-23T19:44:25.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:24 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:44:25.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:24 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3834672884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:25.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:24 smithi135 ceph-mon[118715]: mgrmap e24: x(active, since 7s), standbys: y 2023-09-23T19:44:25.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:24 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:44:25.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:25 smithi144 conmon[159462]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:25] "GET /metrics HTTP/1.1" 200 30886 "" "Prometheus/2.43.0" 2023-09-23T19:44:26.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:25 smithi144 ceph-mon[104994]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:26.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:25 smithi135 ceph-mon[118715]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:26.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:25 smithi135 ceph-mon[100251]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:27.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Adjusting osd_memory_target on smithi144 to 2500M 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:44:27.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1478360486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:26 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Adjusting osd_memory_target on smithi144 to 2500M 2023-09-23T19:44:27.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1478360486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:44:27.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Adjusting osd_memory_target on smithi144 to 2500M 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:44:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1478360486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:26 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:27 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:27 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:27 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:27 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:27 smithi144 ceph-mon[104994]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:44:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:27 smithi144 ceph-mon[104994]: Reconfiguring daemon alertmanager.a on smithi135 2023-09-23T19:44:28.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[118715]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[118715]: Reconfiguring daemon alertmanager.a on smithi135 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:28.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[100251]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T19:44:28.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:27 smithi135 ceph-mon[100251]: Reconfiguring daemon alertmanager.a on smithi135 2023-09-23T19:44:29.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:28 smithi135 ceph-mon[100251]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:29.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:28 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3506508617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:29.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:28 smithi135 ceph-mon[118715]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:28 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3506508617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:29.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:28 smithi144 ceph-mon[104994]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:29.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:28 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3506508617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:30.151 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:29 smithi135 systemd[1]: Stopping Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:30.470 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:30 smithi135 conmon[175280]: ts=2023-09-23T19:44:30.151Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:44:31.220 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:30 smithi135 bash[199186]: ef1f5819ae6814f2f84dcd77198532a39815300b82ca9061567df1a34fc77107 2023-09-23T19:44:31.220 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T19:44:31.220 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 systemd[1]: Stopped Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:31.221 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 systemd[1]: Starting Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:32.110 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:31 smithi135 ceph-mon[118715]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:32.110 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:31 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/343489511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:32.110 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:31 smithi135 ceph-mon[100251]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:32.111 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:31 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/343489511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:32.111 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 conmon[199576]: ts=2023-09-23T19:44:31.745Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T19:44:32.111 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 conmon[199576]: ts=2023-09-23T19:44:31.745Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T19:44:32.111 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 conmon[199576]: ts=2023-09-23T19:44:31.746Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.135 port=9094 2023-09-23T19:44:32.111 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 conmon[199576]: ts=2023-09-23T19:44:31.746Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T19:44:32.111 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 bash[199535]: 22e51759a65e52bf00d0348418b5f8654d9f519f952ca85c94179bab8d98f768 2023-09-23T19:44:32.111 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 systemd[1]: Started Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:32.111 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 conmon[199576]: ts=2023-09-23T19:44:31.770Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:44:32.112 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 conmon[199576]: ts=2023-09-23T19:44:31.770Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T19:44:32.112 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 conmon[199576]: ts=2023-09-23T19:44:31.773Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T19:44:32.112 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:31 smithi135 conmon[199576]: ts=2023-09-23T19:44:31.773Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T19:44:32.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:31 smithi144 ceph-mon[104994]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:32.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:31 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/343489511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:32.468 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:32 smithi135 systemd[1]: Stopping Ceph node-exporter.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:33.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:32 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:33.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:32 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:33.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:32 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:33.144 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:32 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:33.144 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:32 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:33.144 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:32 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:33.144 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:32 smithi135 bash[199806]: b3c82b1cd135126be94d69510af032d7ef9e752e199395bd3e063166a45739cd 2023-09-23T19:44:33.144 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:32 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:44:33.145 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:32 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:33.145 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:32 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:33.145 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:32 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:33.470 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T19:44:33.470 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 systemd[1]: Stopped Ceph node-exporter.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:33.470 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 systemd[1]: Starting Ceph node-exporter.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:34.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:33 smithi144 ceph-mon[104994]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T19:44:34.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:33 smithi144 ceph-mon[104994]: Reconfiguring daemon node-exporter.a on smithi135 2023-09-23T19:44:34.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:33 smithi144 ceph-mon[104994]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:34.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:33 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3344972870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:34.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:33 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:34.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:33 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[100251]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T19:44:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[100251]: Reconfiguring daemon node-exporter.a on smithi135 2023-09-23T19:44:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[100251]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3344972870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:34.222 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[100683]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:44:34.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[118715]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T19:44:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[118715]: Reconfiguring daemon node-exporter.a on smithi135 2023-09-23T19:44:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[118715]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3344972870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:33 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:34.223 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.728Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T19:44:34.224 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.728Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T19:44:34.224 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.728Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T19:44:34.224 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.728Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T19:44:34.224 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T19:44:34.224 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T19:44:34.224 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T19:44:34.224 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T19:44:34.224 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T19:44:34.225 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T19:44:34.226 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T19:44:34.226 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T19:44:34.226 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T19:44:34.226 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T19:44:34.226 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[199576]: ts=2023-09-23T19:44:33.748Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001034639s 2023-09-23T19:44:34.226 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T19:44:34.227 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T19:44:34.228 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T19:44:34.229 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.729Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T19:44:34.229 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.730Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T19:44:34.229 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 conmon[200192]: ts=2023-09-23T19:44:33.730Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T19:44:34.229 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 bash[200151]: caa9af75261e5f75e88c2c53173cb1ecb03a964ae5f232b013a43ea089586a36 2023-09-23T19:44:34.229 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 19:44:33 smithi135 systemd[1]: Started Ceph node-exporter.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:35 smithi135 ceph-mon[118715]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:44:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:35 smithi135 ceph-mon[118715]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:44:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:35 smithi135 ceph-mon[118715]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:35 smithi135 ceph-mon[100251]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:44:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:35 smithi135 ceph-mon[100251]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:44:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:35 smithi135 ceph-mon[100251]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:35.632 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:35 smithi144 conmon[159462]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:35] "GET /metrics HTTP/1.1" 200 33784 "" "Prometheus/2.43.0" 2023-09-23T19:44:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:35 smithi144 ceph-mon[104994]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:44:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:35 smithi144 ceph-mon[104994]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:44:35.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:35 smithi144 ceph-mon[104994]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:36 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3394128347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:36 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3394128347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:36.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:36 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3394128347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:36.734 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 systemd[1]: Stopping Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:37.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:44:37.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T19:44:37.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T19:44:37.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T19:44:37.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T19:44:37.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T19:44:37.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T19:44:37.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T19:44:37.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.734Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T19:44:37.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.735Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T19:44:37.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.735Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T19:44:37.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:36 smithi144 conmon[154137]: ts=2023-09-23T19:44:36.735Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T19:44:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:37 smithi135 ceph-mon[118715]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:37 smithi135 ceph-mon[100251]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:37 smithi144 ceph-mon[104994]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:37.631 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:37 smithi144 bash[172552]: c5d581813489c64cbe3792260e6debd1190ee9f6c7f39bd1b6efac044be23d3c 2023-09-23T19:44:37.631 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:37 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T19:44:37.631 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:37 smithi144 systemd[1]: Stopped Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:37.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:37 smithi144 systemd[1]: Starting Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:38.381 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.049Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T19:44:38.381 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.049Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T19:44:38.381 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 bash[172883]: 400efc42515fc13d2b3cd6cf0b582a08a0bbdbe962d3f38651d83f97929174c5 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.049Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi144 (none))" 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.049Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.049Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.051Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.051Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.052Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.053Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 systemd[1]: Started Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.054Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.054Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.807µs 2023-09-23T19:44:38.382 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.054Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T19:44:38.383 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.055Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T19:44:38.383 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.072Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T19:44:38.383 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.072Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T19:44:38.383 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.072Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.599µs wal_replay_duration=17.871812ms wbl_replay_duration=120ns total_replay_duration=17.92882ms 2023-09-23T19:44:38.383 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.075Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T19:44:38.384 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.075Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T19:44:38.384 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.075Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T19:44:38.384 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.093Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.009091ms db_storage=780ns remote_storage=1.23µs web_handler=475ns query_engine=737ns scrape=282.537µs scrape_sd=89.457µs notify=15.23µs notify_sd=17.04µs rules=17.349297ms tracing=6.126µs 2023-09-23T19:44:38.384 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.093Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T19:44:38.384 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:38 smithi144 conmon[172924]: ts=2023-09-23T19:44:38.093Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T19:44:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:39 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:39 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:39 smithi144 ceph-mon[104994]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:44:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:39 smithi144 ceph-mon[104994]: Reconfiguring daemon grafana.a on smithi144 2023-09-23T19:44:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:39 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4288771394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:39 smithi144 ceph-mon[104994]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[118715]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:44:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[118715]: Reconfiguring daemon grafana.a on smithi144 2023-09-23T19:44:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4288771394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[118715]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[100251]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T19:44:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[100251]: Reconfiguring daemon grafana.a on smithi144 2023-09-23T19:44:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4288771394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:39 smithi135 ceph-mon[100251]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:40.815 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:40 smithi144 systemd[1]: Stopping Ceph grafana.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:41.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:40 smithi144 ceph-mon[104994]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:41.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:40 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/100359407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:41.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:40 smithi135 ceph-mon[118715]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:41.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:40 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/100359407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:40 smithi135 ceph-mon[100251]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:40 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/100359407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:41.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:41 smithi144 bash[173400]: 2ca98132db5dcb54eb5fc7917004d37fde6f48c503a5501a8e7eea9b2b636840 2023-09-23T19:44:41.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:41 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:44:41.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:41 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T19:44:41.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:41 smithi144 systemd[1]: Stopped Ceph grafana.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:41.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:41 smithi144 systemd[1]: Starting Ceph grafana.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:42.220 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:41 smithi135 conmon[199576]: ts=2023-09-23T19:44:41.749Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002947691s 2023-09-23T19:44:42.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 bash[173724]: 902bff181c8509e46c48b7724c84d8c06e55d53f5a5aaea09119e79aa3b5a3dc 2023-09-23T19:44:42.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 systemd[1]: Started Ceph grafana.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:42.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162358044Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T19:44:42.382 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162536337Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162554296Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162561576Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162568202Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162574402Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162580228Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162586115Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.16259199Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162598649Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162604112Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162610296Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T19:44:42.383 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162615949Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=settings t=2023-09-23T19:44:42.162621711Z level=info msg="App mode production" 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=sqlstore t=2023-09-23T19:44:42.162672188Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=sqlstore t=2023-09-23T19:44:42.162688844Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=migrator t=2023-09-23T19:44:42.176937931Z level=info msg="Starting DB migrations" 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=migrator t=2023-09-23T19:44:42.181481368Z level=info msg="migrations completed" performed=0 skipped=481 duration=416.125µs 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=sqlstore t=2023-09-23T19:44:42.181843127Z level=info msg="Created default organization" 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=plugin.loader t=2023-09-23T19:44:42.240636946Z level=info msg="Plugin registered" pluginID=input 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=plugin.loader t=2023-09-23T19:44:42.268475742Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=plugin.loader t=2023-09-23T19:44:42.268526034Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=secrets t=2023-09-23T19:44:42.268565113Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T19:44:42.384 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=query_data t=2023-09-23T19:44:42.270260939Z level=info msg="Query Service initialization" 2023-09-23T19:44:42.385 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:42 smithi144 conmon[173765]: logger=live.push_http t=2023-09-23T19:44:42.27272892Z level=info msg="Live Push Gateway initialization" 2023-09-23T19:44:42.783 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:42 smithi144 systemd[1]: Stopping Ceph node-exporter.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:43.293 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=infra.usagestats.collector t=2023-09-23T19:44:43.037388483Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T19:44:43.294 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=provisioning.datasources t=2023-09-23T19:44:43.215880628Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-23T19:44:43.294 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=provisioning.datasources t=2023-09-23T19:44:43.215912718Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-23T19:44:43.294 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=provisioning.datasources t=2023-09-23T19:44:43.227100216Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T19:44:43.294 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:43 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:43.294 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:43 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:43.294 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:43 smithi144 ceph-mon[104994]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:44:43.294 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:43 smithi144 ceph-mon[104994]: Reconfiguring daemon node-exporter.b on smithi144 2023-09-23T19:44:43.294 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:43 smithi144 ceph-mon[104994]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[118715]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:44:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[118715]: Reconfiguring daemon node-exporter.b on smithi144 2023-09-23T19:44:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[118715]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[100251]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T19:44:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[100251]: Reconfiguring daemon node-exporter.b on smithi144 2023-09-23T19:44:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:43 smithi135 ceph-mon[100251]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:43.631 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:43 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:44:43.631 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:43 smithi144 bash[173999]: d31dbba1772d837c35d669944934aaaa4b63d04aeb43dabd1f12d9092d058c49 2023-09-23T19:44:43.631 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:43 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T19:44:43.631 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:43 smithi144 systemd[1]: Stopped Ceph node-exporter.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:43.632 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:43 smithi144 systemd[1]: Starting Ceph node-exporter.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:43.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=provisioning.alerting t=2023-09-23T19:44:43.426652178Z level=info msg="starting to provision alerting" 2023-09-23T19:44:43.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=provisioning.alerting t=2023-09-23T19:44:43.42668975Z level=info msg="finished to provision alerting" 2023-09-23T19:44:43.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=ngalert.state.manager t=2023-09-23T19:44:43.426797284Z level=info msg="Warming state cache for startup" 2023-09-23T19:44:43.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=grafanaStorageLogger t=2023-09-23T19:44:43.426895854Z level=info msg="storage starting" 2023-09-23T19:44:43.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=http.server t=2023-09-23T19:44:43.428661702Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T19:44:43.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=ngalert.state.manager t=2023-09-23T19:44:43.470395474Z level=info msg="State cache has been initialized" states=0 duration=43.595143ms 2023-09-23T19:44:43.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=ticker t=2023-09-23T19:44:43.470485451Z level=info msg=starting first_tick=2023-09-23T19:44:50Z 2023-09-23T19:44:43.632 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:44:43 smithi144 conmon[173765]: logger=ngalert.multiorg.alertmanager t=2023-09-23T19:44:43.470479191Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T19:44:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:44 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1210637199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STOPPING 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STOPPED 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STARTING 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Serving on http://:::9283 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STARTED 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STOPPING 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STOPPED 2023-09-23T19:44:44.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STARTING 2023-09-23T19:44:44.383 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.061Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T19:44:44.383 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.061Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T19:44:44.383 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.061Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T19:44:44.383 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.061Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T19:44:44.383 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T19:44:44.383 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T19:44:44.383 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T19:44:44.384 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T19:44:44.385 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.062Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T19:44:44.386 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T19:44:44.387 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[174370]: ts=2023-09-23T19:44:44.063Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T19:44:44.388 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 bash[174329]: a8b1ed1aeea77b07fec36a14302a64287c8393793a2e043b353610a175fada7f 2023-09-23T19:44:44.388 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 19:44:44 smithi144 systemd[1]: Started Ceph node-exporter.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:44.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:44 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1210637199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:44 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1210637199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:44.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Serving on http://:::9283 2023-09-23T19:44:44.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STARTED 2023-09-23T19:44:44.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STOPPING 2023-09-23T19:44:44.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:44:44.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STOPPED 2023-09-23T19:44:44.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STARTING 2023-09-23T19:44:44.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Serving on http://:::9283 2023-09-23T19:44:44.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:44 smithi144 conmon[159462]: [23/Sep/2023:19:44:44] ENGINE Bus STARTED 2023-09-23T19:44:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144:9095"}]: dispatch 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144:9095"}]: dispatch 2023-09-23T19:44:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144:3000"}]: dispatch 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144:3000"}]: dispatch 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:44:45.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:45 smithi144 ceph-mon[104994]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:45.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144:9095"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144:9095"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:44:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144:3000"}]: dispatch 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144:3000"}]: dispatch 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[118715]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi135.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144:9095"}]: dispatch 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144:9095"}]: dispatch 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:44:45.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T19:44:45.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144:3000"}]: dispatch 2023-09-23T19:44:45.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144:3000"}]: dispatch 2023-09-23T19:44:45.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:45.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:45.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:44:45.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T19:44:45.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:45 smithi135 ceph-mon[100251]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:46.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:45 smithi144 conmon[159462]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:45] "GET /metrics HTTP/1.1" 200 33784 "" "Prometheus/2.43.0" 2023-09-23T19:44:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2970664404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:44:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2970664404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:44:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:46 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:46 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2970664404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:46 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:46 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:46 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:44:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:46 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:44:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:46 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:47.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:47 smithi135 ceph-mon[100251]: Upgrade: Updating mgr.y 2023-09-23T19:44:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:47 smithi135 ceph-mon[100251]: Deploying daemon mgr.y on smithi135 2023-09-23T19:44:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:47 smithi135 ceph-mon[100251]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:47 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:47 smithi135 ceph-mon[118715]: Upgrade: Updating mgr.y 2023-09-23T19:44:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:47 smithi135 ceph-mon[118715]: Deploying daemon mgr.y on smithi135 2023-09-23T19:44:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:47 smithi135 ceph-mon[118715]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:47 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:47 smithi144 ceph-mon[104994]: Upgrade: Updating mgr.y 2023-09-23T19:44:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:47 smithi144 ceph-mon[104994]: Deploying daemon mgr.y on smithi135 2023-09-23T19:44:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:47 smithi144 ceph-mon[104994]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:47 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:47.898 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:47 smithi135 systemd[1]: Stopping Ceph mgr.y for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:48.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:48 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/845597744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:48.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:48 smithi135 bash[200954]: 7c00c1a623308619c8d0c5941281f0a796425f78245d05c4ea698ccc322e5e31 2023-09-23T19:44:48.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:48 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:44:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:48 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/845597744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:48 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/845597744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:48.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:48 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T19:44:48.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:48 smithi135 systemd[1]: Stopped Ceph mgr.y for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:48.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:48 smithi135 systemd[1]: Starting Ceph mgr.y for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:44:49.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:48 smithi144 conmon[172924]: ts=2023-09-23T19:44:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:44:49.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:49 smithi135 ceph-mon[118715]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:49.441 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:49 smithi135 bash[201326]: eb6afb9206f5972fd6db3bb7a42ab572e818773676ecd929e37e9907d02d2c6a 2023-09-23T19:44:49.441 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:49 smithi135 ceph-mgr[201390]: -- 172.21.15.135:0/2449355664 <== mon.0 v2:172.21.15.135:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563ac27931e0 con 0x563ac276b000 2023-09-23T19:44:49.441 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:49 smithi135 systemd[1]: Started Ceph mgr.y for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:44:49.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:49 smithi135 ceph-mon[100251]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:49 smithi144 ceph-mon[104994]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:49.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:49 smithi135 conmon[201366]: 2023-09-23T19:44:49.440+0000 7fb032951200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:44:49.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:49 smithi135 conmon[201366]: 2023-09-23T19:44:49.523+0000 7fb032951200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:44:49.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:49 smithi135 conmon[201366]: 2023-09-23T19:44:49.623+0000 7fb032951200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:44:50.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:49 smithi135 conmon[201366]: 2023-09-23T19:44:49.736+0000 7fb032951200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:44:50.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:49 smithi135 conmon[201366]: 2023-09-23T19:44:49.933+0000 7fb032951200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:44:50.598 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:50 smithi135 conmon[201366]: 2023-09-23T19:44:50.331+0000 7fb032951200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:44:50.598 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:50 smithi135 conmon[201366]: 2023-09-23T19:44:50.491+0000 7fb032951200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:44:50.598 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:50 smithi135 conmon[201366]: 2023-09-23T19:44:50.596+0000 7fb032951200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:44:50.598 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:50 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:50.599 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:50 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:50.599 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:50 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:50.599 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:50 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:50.599 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:50 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:50.599 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:50 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:50 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:50 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:50 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:50.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:50 smithi135 conmon[201366]: 2023-09-23T19:44:50.697+0000 7fb032951200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:44:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:51 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2357356120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:51 smithi144 ceph-mon[104994]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:51.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:51 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2357356120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:51.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:51 smithi135 ceph-mon[100251]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:51 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2357356120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:51 smithi135 ceph-mon[118715]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:51.721 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:51 smithi135 conmon[201366]: 2023-09-23T19:44:51.222+0000 7fb032951200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:44:52.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:51 smithi135 conmon[201366]: 2023-09-23T19:44:51.946+0000 7fb032951200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:44:52.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:52 smithi135 conmon[201366]: 2023-09-23T19:44:52.035+0000 7fb032951200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:44:53.212 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:52 smithi135 conmon[201366]: 2023-09-23T19:44:52.906+0000 7fb032951200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:44:53.212 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:53 smithi135 conmon[201366]: 2023-09-23T19:44:53.011+0000 7fb032951200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:44:53.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[118715]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:53.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2337548189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:53.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[100251]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:53.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2337548189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:53.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:53 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.469 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:53 smithi135 conmon[201366]: 2023-09-23T19:44:53.210+0000 7fb032951200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:44:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:53 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:53 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:53 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:53 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:53 smithi144 ceph-mon[104994]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:53 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2337548189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:53 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:53 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:53.766 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:53 smithi135 conmon[201366]: 2023-09-23T19:44:53.522+0000 7fb032951200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:44:53.766 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:53 smithi135 conmon[199576]: ts=2023-09-23T19:44:53.509Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi135.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.135:8443: connect: connection refused" 2023-09-23T19:44:54.194 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:53 smithi135 conmon[201366]: 2023-09-23T19:44:53.763+0000 7fb032951200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:44:54.194 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:53 smithi135 conmon[201366]: 2023-09-23T19:44:53.922+0000 7fb032951200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:44:54.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:54 smithi135 conmon[201366]: 2023-09-23T19:44:54.193+0000 7fb032951200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:44:55.207 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:54 smithi135 conmon[201366]: 2023-09-23T19:44:54.954+0000 7fb032951200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:44:55.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:55 smithi135 conmon[201366]: 2023-09-23T19:44:55.104+0000 7fb032951200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:44:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:55 smithi135 ceph-mon[100251]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:55 smithi135 ceph-mon[118715]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:55.459 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:55 smithi135 conmon[201366]: 2023-09-23T19:44:55.322+0000 7fb032951200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:44:55.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:55 smithi144 ceph-mon[104994]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:55.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:55 smithi135 conmon[201366]: 2023-09-23T19:44:55.458+0000 7fb032951200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:44:55.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:55 smithi135 conmon[201366]: 2023-09-23T19:44:55.704+0000 7fb032951200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:44:56.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:55 smithi144 conmon[159462]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:44:55] "GET /metrics HTTP/1.1" 200 33783 "" "Prometheus/2.43.0" 2023-09-23T19:44:56.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:56 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3957428466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:56 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3957428466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:56.550 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:56 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3957428466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:56.859 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.625+0000 7fb032951200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:44:56.859 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE Bus STARTING 2023-09-23T19:44:56.859 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: CherryPy Checker: 2023-09-23T19:44:56.859 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: The Application mounted at '' has an empty config. 2023-09-23T19:44:56.859 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.859 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE Serving on http://:::9283 2023-09-23T19:44:56.860 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE Bus STARTED 2023-09-23T19:44:56.860 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE Bus STOPPING 2023-09-23T19:44:56.860 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:44:56.860 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE Bus STOPPED 2023-09-23T19:44:56.860 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE Bus STARTING 2023-09-23T19:44:56.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:56 smithi144 conmon[159462]: 2023-09-23T19:44:56.549+0000 7fc2a2f95700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T19:44:56.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:56 smithi144 conmon[159462]: ignoring --setuser ceph since I am not root 2023-09-23T19:44:56.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:56 smithi144 conmon[159462]: ignoring --setgroup ceph since I am not root 2023-09-23T19:44:56.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:56 smithi144 ceph-mgr[159486]: -- 172.21.15.144:0/2140665976 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561072d7b1e0 con 0x561072d53000 2023-09-23T19:44:56.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:56 smithi144 conmon[159462]: 2023-09-23T19:44:56.718+0000 7f1e3d27a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:44:56.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:56 smithi144 conmon[159462]: 2023-09-23T19:44:56.797+0000 7f1e3d27a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:44:57.170 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:56 smithi144 conmon[159462]: 2023-09-23T19:44:56.891+0000 7f1e3d27a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:44:57.170 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:56 smithi144 conmon[159462]: 2023-09-23T19:44:56.993+0000 7f1e3d27a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:44:57.170 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:57 smithi144 conmon[159462]: 2023-09-23T19:44:57.169+0000 7f1e3d27a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:44:57.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8b5ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8b5ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8b5ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8b5ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8b5ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8e5c0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8e5c0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8e5c0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8e5c0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: 2023-09-23T19:44:56.922+0000 7faf8e5c0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:44:57.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE Serving on http://:::9283 2023-09-23T19:44:57.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:44:56 smithi135 conmon[201366]: [23/Sep/2023:19:44:56] ENGINE Bus STARTED 2023-09-23T19:44:57.581 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:57 smithi144 conmon[159462]: 2023-09-23T19:44:57.580+0000 7f1e3d27a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:44:57.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T19:44:57.582 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25450 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: mgrmap e25: y(active, starting, since 0.0107114s) 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: Active manager daemon y restarted 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: Activating manager daemon y 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: mgrmap e26: y(active, starting, since 0.00653765s) 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:44:57.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:44:57.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: Manager daemon y is now available 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:57.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:57 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:57.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25450 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: mgrmap e25: y(active, starting, since 0.0107114s) 2023-09-23T19:44:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: Active manager daemon y restarted 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: Activating manager daemon y 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: mgrmap e26: y(active, starting, since 0.00653765s) 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:44:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:44:57.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: Manager daemon y is now available 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:57.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:57.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:57.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:44:57.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:44:57.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' 2023-09-23T19:44:57.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 172.21.15.144:0/423102923' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25450 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: mgrmap e25: y(active, starting, since 0.0107114s) 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: Active manager daemon y restarted 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: Activating manager daemon y 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: mgrmap e26: y(active, starting, since 0.00653765s) 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T19:44:57.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:44:57.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: Manager daemon y is now available 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:57.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:44:57.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:57.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:57 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:44:57.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:57 smithi144 conmon[159462]: 2023-09-23T19:44:57.727+0000 7f1e3d27a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:44:57.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:57 smithi144 conmon[159462]: 2023-09-23T19:44:57.822+0000 7f1e3d27a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:44:58.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:57 smithi144 conmon[159462]: 2023-09-23T19:44:57.922+0000 7f1e3d27a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:44:58.470 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 19:44:58 smithi135 conmon[199576]: ts=2023-09-23T19:44:58.105Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=5 2023-09-23T19:44:58.653 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:58 smithi144 conmon[159462]: 2023-09-23T19:44:58.381+0000 7f1e3d27a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:44:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[100251]: [23/Sep/2023:19:44:57] ENGINE Bus STARTING 2023-09-23T19:44:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[100251]: [23/Sep/2023:19:44:57] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:44:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[100251]: [23/Sep/2023:19:44:57] ENGINE Serving on http://172.21.15.135:8765 2023-09-23T19:44:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[100251]: [23/Sep/2023:19:44:57] ENGINE Bus STARTED 2023-09-23T19:44:58.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[100251]: mgrmap e27: y(active, since 1.01077s) 2023-09-23T19:44:58.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[100251]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:58.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1560203399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:58.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[118715]: [23/Sep/2023:19:44:57] ENGINE Bus STARTING 2023-09-23T19:44:58.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[118715]: [23/Sep/2023:19:44:57] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:44:58.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[118715]: [23/Sep/2023:19:44:57] ENGINE Serving on http://172.21.15.135:8765 2023-09-23T19:44:58.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[118715]: [23/Sep/2023:19:44:57] ENGINE Bus STARTED 2023-09-23T19:44:58.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[118715]: mgrmap e27: y(active, since 1.01077s) 2023-09-23T19:44:58.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[118715]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:58.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:44:58 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1560203399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:59.103 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:44:58 smithi144 conmon[172924]: ts=2023-09-23T19:44:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:44:59.103 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:58 smithi144 ceph-mon[104994]: [23/Sep/2023:19:44:57] ENGINE Bus STARTING 2023-09-23T19:44:59.103 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:58 smithi144 ceph-mon[104994]: [23/Sep/2023:19:44:57] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:44:59.103 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:58 smithi144 ceph-mon[104994]: [23/Sep/2023:19:44:57] ENGINE Serving on http://172.21.15.135:8765 2023-09-23T19:44:59.104 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:58 smithi144 ceph-mon[104994]: [23/Sep/2023:19:44:57] ENGINE Bus STARTED 2023-09-23T19:44:59.104 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:58 smithi144 ceph-mon[104994]: mgrmap e27: y(active, since 1.01077s) 2023-09-23T19:44:59.104 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:58 smithi144 ceph-mon[104994]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:44:59.104 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:44:58 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1560203399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:44:59.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:59 smithi144 conmon[159462]: 2023-09-23T19:44:59.102+0000 7f1e3d27a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:44:59.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:44:59 smithi144 conmon[159462]: 2023-09-23T19:44:59.189+0000 7f1e3d27a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:45:00.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:00 smithi144 conmon[159462]: 2023-09-23T19:45:00.079+0000 7f1e3d27a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:45:00.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:00 smithi144 conmon[159462]: 2023-09-23T19:45:00.184+0000 7f1e3d27a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:45:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:00 smithi144 ceph-mon[104994]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:00 smithi144 ceph-mon[104994]: mgrmap e28: y(active, since 3s) 2023-09-23T19:45:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:00 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:00 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:00 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3014608063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[118715]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[118715]: mgrmap e28: y(active, since 3s) 2023-09-23T19:45:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3014608063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[100251]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[100251]: mgrmap e28: y(active, since 3s) 2023-09-23T19:45:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:00 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3014608063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:00.689 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:00 smithi144 conmon[159462]: 2023-09-23T19:45:00.383+0000 7f1e3d27a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:45:00.689 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:00 smithi144 conmon[159462]: 2023-09-23T19:45:00.688+0000 7f1e3d27a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:45:01.093 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:00 smithi144 conmon[159462]: 2023-09-23T19:45:00.932+0000 7f1e3d27a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:45:01.094 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:01 smithi144 conmon[159462]: 2023-09-23T19:45:01.093+0000 7f1e3d27a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:45:01.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:01 smithi144 conmon[159462]: 2023-09-23T19:45:01.361+0000 7f1e3d27a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:45:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:01 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:01 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:01 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:01 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:01 smithi144 ceph-mon[104994]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:01 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:01 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[118715]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:01.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[100251]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:01 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:02.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:02 smithi144 conmon[159462]: 2023-09-23T19:45:02.071+0000 7f1e3d27a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:45:02.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:02 smithi144 conmon[159462]: 2023-09-23T19:45:02.206+0000 7f1e3d27a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:45:02.742 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:02 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3154838070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:02.742 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:02 smithi144 conmon[159462]: 2023-09-23T19:45:02.403+0000 7f1e3d27a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:45:02.742 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:02 smithi144 conmon[159462]: 2023-09-23T19:45:02.518+0000 7f1e3d27a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:45:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:02 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3154838070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:02.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:02 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3154838070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:03.833 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:03 smithi135 ceph-mon[118715]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:03.833 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:03 smithi135 ceph-mon[100251]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:03 smithi144 ceph-mon[104994]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:03.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:02 smithi144 conmon[159462]: 2023-09-23T19:45:02.741+0000 7f1e3d27a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:45:03.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:03 smithi144 conmon[159462]: 2023-09-23T19:45:03.608+0000 7f1e3d27a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:45:03.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:03 smithi144 conmon[159462]: [23/Sep/2023:19:45:03] ENGINE Bus STARTING 2023-09-23T19:45:03.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:03 smithi144 conmon[159462]: CherryPy Checker: 2023-09-23T19:45:03.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:03 smithi144 conmon[159462]: The Application mounted at '' has an empty config. 2023-09-23T19:45:03.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:03 smithi144 conmon[159462]: 2023-09-23T19:45:03.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:03 smithi144 conmon[159462]: [23/Sep/2023:19:45:03] ENGINE Serving on http://:::9283 2023-09-23T19:45:03.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:03 smithi144 conmon[159462]: [23/Sep/2023:19:45:03] ENGINE Bus STARTED 2023-09-23T19:45:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: Standby manager daemon x started 2023-09-23T19:45:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:45:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:04.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:04 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:04.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: Standby manager daemon x started 2023-09-23T19:45:04.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:04.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:04.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: Standby manager daemon x started 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/420405834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:45:04.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:45:04.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:04.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:04 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:05.822 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:05 smithi144 conmon[159462]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:45:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: mgrmap e29: y(active, since 7s), standbys: x 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3570100648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:05.823 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.824 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.824 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.824 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.824 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.824 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:45:05.824 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:05 smithi144 ceph-mon[104994]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:45:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: mgrmap e29: y(active, since 7s), standbys: x 2023-09-23T19:45:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:45:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3570100648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:45:05.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[100251]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:45:05.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:05.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: mgrmap e29: y(active, since 7s), standbys: x 2023-09-23T19:45:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:45:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3570100648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:05.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:05.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T19:45:05.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:05 smithi135 ceph-mon[118715]: Reconfiguring daemon prometheus.a on smithi144 2023-09-23T19:45:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:07 smithi144 ceph-mon[104994]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:07 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2272122625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:07.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:07 smithi135 ceph-mon[100251]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:07.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:07 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2272122625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:07 smithi135 ceph-mon[118715]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:07 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2272122625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:08.183 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:07 smithi144 systemd[1]: Stopping Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:45:08.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.183Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:45:08.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T19:45:08.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T19:45:08.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T19:45:08.632 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T19:45:08.633 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T19:45:08.633 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T19:45:08.633 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T19:45:08.633 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T19:45:08.633 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T19:45:08.633 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T19:45:08.633 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 conmon[172924]: ts=2023-09-23T19:45:08.184Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T19:45:09.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 bash[178345]: 400efc42515fc13d2b3cd6cf0b582a08a0bbdbe962d3f38651d83f97929174c5 2023-09-23T19:45:09.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T19:45:09.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 systemd[1]: Stopped Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:45:09.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:08 smithi144 systemd[1]: Starting Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 bash[178677]: b9776c7278f3d46328c913cbaa4ab718a335771985ff5b90acd049134c6b4a2b 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.511Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.511Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.511Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi144 (none))" 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.511Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.511Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.512Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.513Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.514Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.514Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T19:45:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 systemd[1]: Started Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.516Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.516Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.83µs 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.516Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.517Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.534Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.536Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.536Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.536Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.985µs wal_replay_duration=19.721671ms wbl_replay_duration=121ns total_replay_duration=19.776904ms 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.540Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.540Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T19:45:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.540Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T19:45:09.884 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.556Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.213715ms db_storage=989ns remote_storage=1.165µs web_handler=439ns query_engine=727ns scrape=494.222µs scrape_sd=83.421µs notify=12.711µs notify_sd=14.061µs rules=15.362205ms tracing=5.819µs 2023-09-23T19:45:09.884 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.556Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T19:45:09.884 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:09 smithi144 conmon[178718]: ts=2023-09-23T19:45:09.556Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T19:45:09.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:09 smithi135 conmon[201366]: [23/Sep/2023:19:45:09] ENGINE Bus STOPPING 2023-09-23T19:45:09.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:09 smithi135 conmon[201366]: [23/Sep/2023:19:45:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:45:09.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:09 smithi135 conmon[201366]: [23/Sep/2023:19:45:09] ENGINE Bus STOPPED 2023-09-23T19:45:09.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:09 smithi135 conmon[201366]: [23/Sep/2023:19:45:09] ENGINE Bus STARTING 2023-09-23T19:45:09.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:09 smithi135 conmon[201366]: [23/Sep/2023:19:45:09] ENGINE Serving on http://:::9283 2023-09-23T19:45:09.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:09 smithi135 conmon[201366]: [23/Sep/2023:19:45:09] ENGINE Bus STARTED 2023-09-23T19:45:10.345 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:10 smithi144 ceph-mon[104994]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:10.345 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:10 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.345 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:10 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.345 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:10 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:45:10.346 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:10 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:45:10.346 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:10 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.346 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:10 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:10.346 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:10 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1460738710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[100251]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:45:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:45:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/1460738710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[118715]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:45:10.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:45:10.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:10.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:10.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:10 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1460738710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:11 smithi144 ceph-mon[104994]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:45:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:11 smithi144 ceph-mon[104994]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:45:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:11 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:45:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:11 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:45:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:11 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:45:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:11 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:11.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[100251]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:45:11.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[100251]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:45:11.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[118715]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[118715]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:45:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:11 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[118715]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[118715]: Upgrade: Updating mgr.x 2023-09-23T19:45:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[118715]: Deploying daemon mgr.x on smithi144 2023-09-23T19:45:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2785336347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[100251]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[100251]: Upgrade: Updating mgr.x 2023-09-23T19:45:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[100251]: Deploying daemon mgr.x on smithi144 2023-09-23T19:45:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:12 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2785336347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:12.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:12 smithi144 ceph-mon[104994]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:12.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:12 smithi144 ceph-mon[104994]: Upgrade: Updating mgr.x 2023-09-23T19:45:12.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:12 smithi144 ceph-mon[104994]: Deploying daemon mgr.x on smithi144 2023-09-23T19:45:12.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:12 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:12.520 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:12 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2785336347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:12.870 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:12 smithi144 systemd[1]: Stopping Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:45:13.569 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:13 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:45:13.569 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:13 smithi144 bash[179359]: 7aa64de85678e036b0c2d0822dc31db95f0b39d5649dd94e3db25144d43aad27 2023-09-23T19:45:13.570 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:13 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T19:45:13.570 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:13 smithi144 systemd[1]: Stopped Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:45:13.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:13 smithi144 systemd[1]: Starting Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:45:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:14 smithi144 ceph-mon[104994]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:14 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:14 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:14.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:14 smithi144 ceph-mgr[179777]: -- 172.21.15.144:0/908053987 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56053e2b31e0 con 0x56053e28a800 2023-09-23T19:45:14.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:14 smithi144 bash[179712]: 81770c1349866c93fc7eb3150c18c6f921ccf61872842c4cecb7bc663c06ec3e 2023-09-23T19:45:14.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:14 smithi144 systemd[1]: Started Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:45:14.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:14 smithi144 conmon[179753]: 2023-09-23T19:45:14.231+0000 7f2edd64a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:45:14.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:14 smithi144 conmon[179753]: 2023-09-23T19:45:14.315+0000 7f2edd64a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:45:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:14 smithi135 ceph-mon[118715]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:14 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:14 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:14 smithi135 ceph-mon[100251]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:14 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:14 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:14.713 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:14 smithi144 conmon[179753]: 2023-09-23T19:45:14.408+0000 7f2edd64a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:45:14.713 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:14 smithi144 conmon[179753]: 2023-09-23T19:45:14.518+0000 7f2edd64a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:45:14.713 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:14 smithi144 conmon[179753]: 2023-09-23T19:45:14.712+0000 7f2edd64a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:45:15.361 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:15 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:15.361 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:15 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/2387832436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:15.361 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:15 smithi144 conmon[179753]: 2023-09-23T19:45:15.109+0000 7f2edd64a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:45:15.361 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:15 smithi144 conmon[179753]: 2023-09-23T19:45:15.266+0000 7f2edd64a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:45:15.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:15 smithi144 conmon[179753]: 2023-09-23T19:45:15.360+0000 7f2edd64a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:45:15.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:15 smithi144 conmon[179753]: 2023-09-23T19:45:15.461+0000 7f2edd64a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:45:15.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:15 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:15.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:15 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/2387832436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:15.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:15 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:15.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:15 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/2387832436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:16.226 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:15 smithi144 conmon[179753]: 2023-09-23T19:45:15.959+0000 7f2edd64a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:45:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:16 smithi144 ceph-mon[104994]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:16.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:16 smithi135 ceph-mon[100251]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:16 smithi135 ceph-mon[118715]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:17.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:16 smithi144 conmon[179753]: 2023-09-23T19:45:16.699+0000 7f2edd64a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:45:17.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:16 smithi144 conmon[179753]: 2023-09-23T19:45:16.791+0000 7f2edd64a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:45:17.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:17 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/4140871137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:17.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:17 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/4140871137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:17.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:17 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/4140871137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:17.930 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:17 smithi144 conmon[179753]: 2023-09-23T19:45:17.632+0000 7f2edd64a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:45:17.930 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:17 smithi144 conmon[179753]: 2023-09-23T19:45:17.734+0000 7f2edd64a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:45:17.930 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:17 smithi144 conmon[179753]: 2023-09-23T19:45:17.929+0000 7f2edd64a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:45:18.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:18 smithi144 ceph-mon[104994]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:18.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:18 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:18.234 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:18 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:18.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:18 smithi144 conmon[179753]: 2023-09-23T19:45:18.236+0000 7f2edd64a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:45:18.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:18 smithi144 conmon[179753]: 2023-09-23T19:45:18.478+0000 7f2edd64a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:45:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:18 smithi135 ceph-mon[118715]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:18 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:18 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:18 smithi135 ceph-mon[100251]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:18 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:18 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:18.921 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:18 smithi144 conmon[179753]: 2023-09-23T19:45:18.654+0000 7f2edd64a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:45:19.276 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:18 smithi144 conmon[179753]: 2023-09-23T19:45:18.943+0000 7f2edd64a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:45:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:19 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:19 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:19.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:19 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:19.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:19 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:19 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:19 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:20.043 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:19 smithi144 conmon[179753]: 2023-09-23T19:45:19.679+0000 7f2edd64a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:45:20.043 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:19 smithi144 conmon[179753]: 2023-09-23T19:45:19.826+0000 7f2edd64a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:45:20.303 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:20 smithi144 conmon[179753]: 2023-09-23T19:45:20.042+0000 7f2edd64a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:45:20.303 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:20 smithi144 conmon[179753]: 2023-09-23T19:45:20.168+0000 7f2edd64a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:45:20.631 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:20 smithi144 conmon[179753]: 2023-09-23T19:45:20.401+0000 7f2edd64a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:45:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:20 smithi144 ceph-mon[104994]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:20 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3262625601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:20 smithi135 ceph-mon[118715]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:20 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3262625601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:20.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:20 smithi135 ceph-mon[100251]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:20 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/3262625601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:21.440 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:21 smithi144 conmon[179753]: 2023-09-23T19:45:21.319+0000 7f2edd64a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:45:21.440 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:21 smithi144 conmon[179753]: [23/Sep/2023:19:45:21] ENGINE Bus STARTING 2023-09-23T19:45:21.440 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:21 smithi144 conmon[179753]: CherryPy Checker: 2023-09-23T19:45:21.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:21 smithi144 conmon[179753]: The Application mounted at '' has an empty config. 2023-09-23T19:45:21.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:21 smithi144 conmon[179753]: 2023-09-23T19:45:21.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:21 smithi144 conmon[179753]: [23/Sep/2023:19:45:21] ENGINE Serving on http://:::9283 2023-09-23T19:45:21.882 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:21 smithi144 conmon[179753]: [23/Sep/2023:19:45:21] ENGINE Bus STARTED 2023-09-23T19:45:22.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: Standby manager daemon x restarted 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: Standby manager daemon x started 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/954641465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:22.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:45:22.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: Standby manager daemon x restarted 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: Standby manager daemon x started 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/954641465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:45:22.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T19:45:22.614 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:45:22.614 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:22 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T19:45:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: Standby manager daemon x restarted 2023-09-23T19:45:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: Standby manager daemon x started 2023-09-23T19:45:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/20532852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/954641465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:45:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:22 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T19:45:23.622 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:23 smithi135 ceph-mon[100251]: Upgrade: Setting container_image for all mgr 2023-09-23T19:45:23.622 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:23 smithi135 ceph-mon[100251]: Upgrade: It appears safe to stop mon.a 2023-09-23T19:45:23.622 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:23 smithi135 ceph-mon[100251]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T19:45:23.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:23 smithi135 ceph-mon[118715]: Upgrade: Setting container_image for all mgr 2023-09-23T19:45:23.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:23 smithi135 ceph-mon[118715]: Upgrade: It appears safe to stop mon.a 2023-09-23T19:45:23.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:23 smithi135 ceph-mon[118715]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T19:45:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:23 smithi144 ceph-mon[104994]: Upgrade: Setting container_image for all mgr 2023-09-23T19:45:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:23 smithi144 ceph-mon[104994]: Upgrade: It appears safe to stop mon.a 2023-09-23T19:45:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:23 smithi144 ceph-mon[104994]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T19:45:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:45:23] "GET /metrics HTTP/1.1" 200 33784 "" "Prometheus/2.43.0" 2023-09-23T19:45:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[118715]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/707129615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[100251]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[100251]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[100251]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:24 smithi135 ceph-mon[100251]: from='client.? 172.21.15.135:0/707129615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:24 smithi144 ceph-mon[104994]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:24 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='mgr.y' 2023-09-23T19:45:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:24 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:45:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:24 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:45:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:24 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:24 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/707129615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:25.324 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:25 smithi135 systemd[1]: Stopping Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:45:25.626 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:25 smithi135 ceph-mon[118715]: Upgrade: Updating mon.a 2023-09-23T19:45:25.626 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:25 smithi135 ceph-mon[118715]: Deploying daemon mon.a on smithi135 2023-09-23T19:45:25.626 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:25 smithi135 ceph-mon[100251]: Upgrade: Updating mon.a 2023-09-23T19:45:25.626 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:25 smithi135 ceph-mon[100251]: Deploying daemon mon.a on smithi135 2023-09-23T19:45:25.626 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:25 smithi135 conmon[100227]: 2023-09-23T19:45:25.483+0000 7f0024556700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:45:25.627 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:25 smithi135 conmon[100227]: 2023-09-23T19:45:25.483+0000 7f0024556700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T19:45:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:25 smithi144 ceph-mon[104994]: Upgrade: Updating mon.a 2023-09-23T19:45:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:25 smithi144 ceph-mon[104994]: Deploying daemon mon.a on smithi135 2023-09-23T19:45:26.423 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:26 smithi135 bash[213428]: 1e28ea7cdc97a851a11b1a682dfa04021558ba887460e5e4d8f5aa740a5445e5 2023-09-23T19:45:26.423 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:26 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T19:45:26.423 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:26 smithi135 systemd[1]: Stopped Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:45:26.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:26 smithi135 systemd[1]: Starting Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:45:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:45:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 8 2023-09-23T19:45:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 bash[213802]: 7fa1f78d472573b2351f8e7ad7cd571eee8e96cc94a0c3a0a69716e892041863 2023-09-23T19:45:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: pidfile_write: ignore empty --pid-file 2023-09-23T19:45:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 systemd[1]: Started Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:45:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: load: jerasure load: lrc 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: RocksDB version: 7.9.2 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Git sha 0 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: DB SUMMARY 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: DB Session ID: J8Q8MWS0N7IRQI08529D 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: CURRENT file: CURRENT 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1170 Bytes 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 713421 ; 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.error_if_exists: 0 2023-09-23T19:45:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.create_if_missing: 0 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.paranoid_checks: 1 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.env: 0x5649336a6720 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.info_log: 0x564935e3f420 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.statistics: (nil) 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.use_fsync: 0 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_log_file_size: 0 2023-09-23T19:45:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.allow_fallocate: 1 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.use_direct_reads: 0 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.db_log_dir: 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.wal_dir: 2023-09-23T19:45:27.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.write_buffer_manager: 0x5649350cec80 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:45:27.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:45:27.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:45:27.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:45:27.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:45:27.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.unordered_write: 0 2023-09-23T19:45:27.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:45:27.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:45:27.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:45:27.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.row_cache: None 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.wal_filter: None 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.two_write_queues: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.wal_compression: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.atomic_flush: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.log_readahead_size: 0 2023-09-23T19:45:27.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_background_jobs: 2 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_background_compactions: -1 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_subcompactions: 1 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:45:27.478 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_open_files: -1 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_background_flushes: -1 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Compression algorithms supported: 2023-09-23T19:45:27.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: kZSTD supported: 0 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: kXpressCompression supported: 0 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: kZlibCompression supported: 1 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: kSnappyCompression supported: 1 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: kLZ4Compression supported: 1 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: kBZip2Compression supported: 0 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:45:27.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.merge_operator: 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_filter: None 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564935e3f520) 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout: cache_index_and_filter_blocks: 1 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout: pin_top_level_index_and_filter: 1 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout: index_type: 0 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout: data_block_index_type: 0 2023-09-23T19:45:27.481 INFO:journalctl@ceph.mon.a.smithi135.stdout: index_shortening: 1 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: checksum: 4 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: no_block_cache: 0 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_cache: 0x564935151350 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_cache_name: BinnedLRUCache 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_cache_options: 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: capacity : 536870912 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: num_shard_bits : 4 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: strict_capacity_limit : 0 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: high_pri_pool_ratio: 0.000 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_cache_compressed: (nil) 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: persistent_cache: (nil) 2023-09-23T19:45:27.482 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_size: 4096 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_size_deviation: 10 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_restart_interval: 16 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: index_block_restart_interval: 1 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: metadata_block_size: 4096 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: partition_filters: 0 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: use_delta_encoding: 1 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: filter_policy: bloomfilter 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: whole_key_filtering: 1 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: verify_compression: 0 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: read_amp_bytes_per_bit: 0 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: format_version: 5 2023-09-23T19:45:27.483 INFO:journalctl@ceph.mon.a.smithi135.stdout: enable_index_compression: 1 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout: block_align: 0 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout: max_auto_readahead_size: 262144 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout: prepopulate_block_cache: 0 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout: initial_auto_readahead_size: 8192 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression: NoCompression 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.num_levels: 7 2023-09-23T19:45:27.484 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:45:27.485 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:45:27.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:45:27.487 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T19:45:27.488 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T19:45:27.488 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:45:27.488 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:45:27.488 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:45:27.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.table_properties_collectors: 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.inplace_update_support: 0 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.bloom_locality: 0 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.max_successive_merges: 0 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:45:27.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.ttl: 2592000 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.enable_blob_files: false 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.min_blob_size: 0 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T19:45:27.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12416, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 11f3b44b-1225-4e1c-aa8f-687af70847a9 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498327023635, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498327026567, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 652261, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12403, "largest_seqno": 12858, "table_properties": {"data_size": 649322, "index_size": 1349, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 5552, "raw_average_key_size": 25, "raw_value_size": 644401, "raw_average_value_size": 2915, "num_data_blocks": 60, "num_entries": 221, "num_filter_entries": 221, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695498327, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "11f3b44b-1225-4e1c-aa8f-687af70847a9", "db_session_id": "J8Q8MWS0N7IRQI08529D", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498327026649, "job": 1, "event": "recovery_finished"} 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T19:45:27.492 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5649351ec000 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: DB pointer 0x5649351da000 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** DB Stats ** 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:45:27.493 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: L0 1/0 636.97 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 364.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: L6 1/0 6.18 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: Sum 2/0 6.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 364.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 364.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:45:27.494 INFO:journalctl@ceph.mon.a.smithi135.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 364.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:45:27.495 INFO:journalctl@ceph.mon.a.smithi135.stdout: Cumulative compaction: 0.00 GB write, 119.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout: Interval compaction: 0.00 GB write, 119.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout: Block cache BinnedLRUCache@0x564935151350#8 capacity: 512.00 MB usage: 2.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.66 KB,0.00012517%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: starting mon.a rank 0 at public addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] at bind addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: mon.a@-1(???) e3 preinit fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: mon.a@-1(???).mds e9 new map 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: mon.a@-1(???).mds e9 print_map 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout: e9 2023-09-23T19:45:27.496 INFO:journalctl@ceph.mon.a.smithi135.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: legacy client fscid: 1 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: Filesystem 'foo' (1) 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: fs_name foo 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: epoch 8 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: created 2023-09-23T19:41:39.534719+0000 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: modified 2023-09-23T19:41:55.190501+0000 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: tableserver 0 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: root 0 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: session_timeout 60 2023-09-23T19:45:27.497 INFO:journalctl@ceph.mon.a.smithi135.stdout: session_autoclose 300 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: max_file_size 1099511627776 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: required_client_features {} 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: last_failure 0 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: last_failure_osd_epoch 0 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: max_mds 1 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: in 0 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: up {0=24569} 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: failed 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: damaged 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: stopped 2023-09-23T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi135.stdout: data_pools [4] 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: metadata_pool 3 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: inline_data disabled 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: balancer 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: bal_rank_mask -1 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: standby_count_wanted 1 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: [mds.foo.smithi135.srojfl{0:24569} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: Standby daemons: 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout: [mds.foo.smithi144.plbefj{-1:24553} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: mon.a@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T19:45:27.499 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta expand map: {default=false} 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta from 'false' to 'false' 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta expanded map: {default=false} 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta expand map: {default=info} 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta from 'info' to 'info' 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta expanded map: {default=info} 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta expand map: {default=daemon} 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T19:45:27.500 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta expand map: {default=debug} 2023-09-23T19:45:27.501 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T19:45:27.501 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:27 smithi135 ceph-mon[213871]: expand_channel_meta expanded map: {default=debug} 2023-09-23T19:45:29.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:28 smithi144 conmon[178718]: ts=2023-09-23T19:45:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:45:30.381 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:30 smithi144 conmon[179753]: ignoring --setuser ceph since I am not root 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:30 smithi144 conmon[179753]: ignoring --setgroup ceph since I am not root 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mgr[179777]: -- 172.21.15.144:0/857606076 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e6889031e0 con 0x55e6888db000 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:30 smithi144 conmon[179753]: 2023-09-23T19:45:30.245+0000 7f8d08606200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:30 smithi144 conmon[179753]: 2023-09-23T19:45:30.325+0000 7f8d08606200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/3490628899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: mon.a calling monitor election 2023-09-23T19:45:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/543172015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: overall HEALTH_OK 2023-09-23T19:45:30.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:30 smithi144 ceph-mon[104994]: from='mgr.25715 ' entity='' 2023-09-23T19:45:30.426 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:30 smithi135 conmon[201366]: ignoring --setuser ceph since I am not root 2023-09-23T19:45:30.426 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:30 smithi135 conmon[201366]: ignoring --setgroup ceph since I am not root 2023-09-23T19:45:30.426 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mgr[201390]: -- 172.21.15.135:0/2692378657 <== mon.0 v2:172.21.15.135:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5637c0c671e0 con 0x5637c0c3f000 2023-09-23T19:45:30.426 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:30 smithi135 conmon[201366]: 2023-09-23T19:45:30.257+0000 7fcf9083e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T19:45:30.426 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:30 smithi135 conmon[201366]: 2023-09-23T19:45:30.336+0000 7fcf9083e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T19:45:30.426 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:30 smithi135 conmon[201366]: 2023-09-23T19:45:30.425+0000 7fcf9083e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:45:30.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/3490628899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: mon.a calling monitor election 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/543172015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T19:45:30.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: overall HEALTH_OK 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[118715]: from='mgr.25715 ' entity='' 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3490628899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: from='mgr.25715 172.21.15.135:0/286297407' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: mon.a calling monitor election 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/543172015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:45:30.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:45:30.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T19:45:30.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T19:45:30.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T19:45:30.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: overall HEALTH_OK 2023-09-23T19:45:30.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:30 smithi135 ceph-mon[213871]: from='mgr.25715 ' entity='' 2023-09-23T19:45:30.675 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:30 smithi144 conmon[179753]: 2023-09-23T19:45:30.413+0000 7f8d08606200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T19:45:30.675 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:30 smithi144 conmon[179753]: 2023-09-23T19:45:30.505+0000 7f8d08606200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:45:30.687 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:30 smithi135 conmon[201366]: 2023-09-23T19:45:30.517+0000 7fcf9083e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T19:45:30.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:30 smithi135 conmon[201366]: 2023-09-23T19:45:30.686+0000 7fcf9083e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:45:31.344 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T19:45:31.346 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:30 smithi144 conmon[179753]: 2023-09-23T19:45:30.673+0000 7f8d08606200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T19:45:31.346 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:31 smithi144 conmon[179753]: 2023-09-23T19:45:31.032+0000 7f8d08606200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:45:31.346 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:31 smithi144 conmon[179753]: 2023-09-23T19:45:31.163+0000 7f8d08606200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:45:31.346 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:31 smithi144 conmon[179753]: 2023-09-23T19:45:31.250+0000 7f8d08606200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:45:31.346 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:31 smithi144 conmon[179753]: 2023-09-23T19:45:31.345+0000 7f8d08606200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:45:31.359 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:31 smithi135 conmon[201366]: 2023-09-23T19:45:31.055+0000 7fcf9083e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T19:45:31.359 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:31 smithi135 conmon[201366]: 2023-09-23T19:45:31.191+0000 7fcf9083e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T19:45:31.359 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:31 smithi135 conmon[201366]: 2023-09-23T19:45:31.282+0000 7fcf9083e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T19:45:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:31 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/996805223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:31.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:31 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/996805223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:31.721 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:31 smithi135 conmon[201366]: 2023-09-23T19:45:31.382+0000 7fcf9083e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T19:45:31.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:31 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/996805223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:45:32.102 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:31 smithi135 conmon[201366]: 2023-09-23T19:45:31.831+0000 7fcf9083e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:45:32.124 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:45:32.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:31 smithi144 conmon[179753]: 2023-09-23T19:45:31.771+0000 7f8d08606200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T19:45:32.513 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:32.878 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:32 smithi135 conmon[201366]: 2023-09-23T19:45:32.553+0000 7fcf9083e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:45:32.878 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:32 smithi135 conmon[201366]: 2023-09-23T19:45:32.645+0000 7fcf9083e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:45:32.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:32 smithi144 conmon[179753]: 2023-09-23T19:45:32.427+0000 7f8d08606200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T19:45:32.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:32 smithi144 conmon[179753]: 2023-09-23T19:45:32.511+0000 7f8d08606200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T19:45:33.552 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:33 smithi144 conmon[179753]: 2023-09-23T19:45:33.284+0000 7f8d08606200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:45:33.552 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:33 smithi144 conmon[179753]: 2023-09-23T19:45:33.377+0000 7f8d08606200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:45:33.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:33 smithi135 conmon[201366]: 2023-09-23T19:45:33.459+0000 7fcf9083e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:45:33.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:33 smithi135 conmon[201366]: 2023-09-23T19:45:33.553+0000 7fcf9083e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T19:45:33.839 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:33 smithi144 conmon[179753]: 2023-09-23T19:45:33.551+0000 7f8d08606200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:45:34.052 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:33 smithi135 conmon[201366]: 2023-09-23T19:45:33.743+0000 7fcf9083e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T19:45:34.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:33 smithi144 conmon[179753]: 2023-09-23T19:45:33.838+0000 7f8d08606200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:45:34.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:34 smithi144 conmon[179753]: 2023-09-23T19:45:34.066+0000 7f8d08606200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:45:34.440 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:34 smithi135 conmon[201366]: 2023-09-23T19:45:34.051+0000 7fcf9083e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T19:45:34.441 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:34 smithi135 conmon[201366]: 2023-09-23T19:45:34.290+0000 7fcf9083e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T19:45:34.441 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:34 smithi135 conmon[201366]: 2023-09-23T19:45:34.440+0000 7fcf9083e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:45:34.485 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:34 smithi144 conmon[179753]: 2023-09-23T19:45:34.216+0000 7f8d08606200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T19:45:34.881 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:34 smithi144 conmon[179753]: 2023-09-23T19:45:34.484+0000 7f8d08606200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:45:34.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:34 smithi135 conmon[201366]: 2023-09-23T19:45:34.709+0000 7fcf9083e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T19:45:35.525 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:35 smithi144 conmon[179753]: 2023-09-23T19:45:35.194+0000 7f8d08606200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:45:35.525 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:35 smithi144 conmon[179753]: 2023-09-23T19:45:35.327+0000 7f8d08606200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:45:35.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:35 smithi135 conmon[201366]: 2023-09-23T19:45:35.423+0000 7fcf9083e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T19:45:35.721 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:35 smithi135 conmon[201366]: 2023-09-23T19:45:35.559+0000 7fcf9083e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T19:45:35.864 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:35 smithi144 conmon[179753]: 2023-09-23T19:45:35.524+0000 7f8d08606200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:45:35.864 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:35 smithi144 conmon[179753]: 2023-09-23T19:45:35.640+0000 7f8d08606200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:45:36.095 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:35 smithi135 conmon[201366]: 2023-09-23T19:45:35.756+0000 7fcf9083e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T19:45:36.095 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:35 smithi135 conmon[201366]: 2023-09-23T19:45:35.872+0000 7fcf9083e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T19:45:36.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:35 smithi144 conmon[179753]: 2023-09-23T19:45:35.863+0000 7f8d08606200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:45:36.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:36 smithi135 conmon[201366]: 2023-09-23T19:45:36.095+0000 7fcf9083e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T19:45:36.996 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:36 smithi144 conmon[179753]: 2023-09-23T19:45:36.733+0000 7f8d08606200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:45:36.996 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:36 smithi144 conmon[179753]: [23/Sep/2023:19:45:36] ENGINE Bus STARTING 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:36 smithi144 conmon[179753]: CherryPy Checker: 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:36 smithi144 conmon[179753]: The Application mounted at '' has an empty config. 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:36 smithi144 conmon[179753]: 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:36 smithi144 conmon[179753]: [23/Sep/2023:19:45:36] ENGINE Serving on http://:::9283 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 19:45:36 smithi144 conmon[179753]: [23/Sep/2023:19:45:36] ENGINE Bus STARTED 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:36 smithi144 ceph-mon[104994]: Standby manager daemon x restarted 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:36 smithi144 ceph-mon[104994]: Standby manager daemon x started 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:36 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:36 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:36.997 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:36 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:36.998 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:36 smithi144 ceph-mon[104994]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:37.077 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:36 smithi135 conmon[201366]: 2023-09-23T19:45:36.975+0000 7fcf9083e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T19:45:37.077 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[213871]: Standby manager daemon x restarted 2023-09-23T19:45:37.077 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[213871]: Standby manager daemon x started 2023-09-23T19:45:37.077 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[213871]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:37.077 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[213871]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:37.077 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[213871]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:37.078 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[213871]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:37.081 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[118715]: Standby manager daemon x restarted 2023-09-23T19:45:37.081 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[118715]: Standby manager daemon x started 2023-09-23T19:45:37.081 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T19:45:37.081 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T19:45:37.081 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T19:45:37.081 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:36 smithi135 ceph-mon[118715]: from='mgr.? 172.21.15.144:0/1801236750' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T19:45:37.081 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:36 smithi135 conmon[118691]: 2023-09-23T19:45:36.996+0000 7fa46921e700 -1 mon.c@2(peon).osd e90 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:45:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:36 smithi144 conmon[104970]: 2023-09-23T19:45:36.995+0000 7feb50efb700 -1 mon.b@1(peon).osd e90 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T19:45:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: [23/Sep/2023:19:45:37] ENGINE Bus STARTING 2023-09-23T19:45:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceeab6a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.470 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceeab6a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceeab6a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceeab6a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceeab6a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceedb70700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceedb70700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceedb70700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceedb70700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.149+0000 7fceedb70700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: CherryPy Checker: 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: The Application mounted at '' has an empty config. 2023-09-23T19:45:37.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: 2023-09-23T19:45:37.472 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: [23/Sep/2023:19:45:37] ENGINE Serving on http://:::9283 2023-09-23T19:45:37.472 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:37 smithi135 conmon[201366]: [23/Sep/2023:19:45:37] ENGINE Bus STARTED 2023-09-23T19:45:38.023 INFO:teuthology.orchestra.run.smithi135.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:45:38.023 INFO:teuthology.orchestra.run.smithi135.stdout:alertmanager.a smithi135 *:9093,9094 running (66s) - 4m 27.0M - 0.25.0 c8568f914cd2 22e51759a65e 2023-09-23T19:45:38.024 INFO:teuthology.orchestra.run.smithi135.stdout:grafana.a smithi144 *:3000 running (56s) - 4m 82.2M - 9.4.7 2c41d148cca3 902bff181c85 2023-09-23T19:45:38.024 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi135.srojfl smithi135 running (3m) - 3m 16.5M - 17.2.6-1123-g61431f6b 0db14aa01094 37783bce2e06 2023-09-23T19:45:38.024 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi144.plbefj smithi144 running (3m) - 3m 14.8M - 17.2.6-1123-g61431f6b 0db14aa01094 af9c7e8247c3 2023-09-23T19:45:38.024 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.x smithi144 *:8443,9283,8765 running (24s) - 8m 48.4M - 18.2.0-460-gbb0daeff e32c2a64c126 81770c134986 2023-09-23T19:45:38.024 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.y smithi135 *:8443,9283,8765 running (49s) - 10m 477M - 18.2.0-460-gbb0daeff e32c2a64c126 eb6afb9206f5 2023-09-23T19:45:38.024 INFO:teuthology.orchestra.run.smithi135.stdout:mon.a smithi135 running (10m) - 10m 103M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 1e28ea7cdc97 2023-09-23T19:45:38.025 INFO:teuthology.orchestra.run.smithi135.stdout:mon.b smithi144 running (9m) - 9m 73.0M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 40abc2a02dd4 2023-09-23T19:45:38.025 INFO:teuthology.orchestra.run.smithi135.stdout:mon.c smithi135 running (8m) - 8m 68.4M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 3ddce49c9396 2023-09-23T19:45:38.025 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.a smithi135 *:9100 running (64s) - 4m 8040k - 1.5.0 0da6a335fe13 caa9af75261e 2023-09-23T19:45:38.025 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.b smithi144 *:9100 running (54s) - 4m 8200k - 1.5.0 0da6a335fe13 a8b1ed1aeea7 2023-09-23T19:45:38.025 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 smithi135 running (8m) - 8m 79.3M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 af561f634ba3 2023-09-23T19:45:38.025 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1 smithi135 running (7m) - 7m 95.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 791e31e1318a 2023-09-23T19:45:38.025 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 smithi135 running (7m) - 7m 75.5M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 0ee5fdb5fb5e 2023-09-23T19:45:38.025 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 smithi135 running (7m) - 7m 64.2M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 0fd086d2e07f 2023-09-23T19:45:38.026 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 smithi144 running (6m) - 6m 68.3M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 076c88452d16 2023-09-23T19:45:38.026 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 smithi144 running (6m) - 6m 90.3M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 30c9f1e12bc8 2023-09-23T19:45:38.026 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 smithi144 running (6m) - 6m 70.4M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 d04b0ed25c0f 2023-09-23T19:45:38.026 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 smithi144 running (5m) - 5m 90.0M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 4b2a00379dc8 2023-09-23T19:45:38.026 INFO:teuthology.orchestra.run.smithi135.stdout:prometheus.a smithi144 *:9095 running (28s) - 5m 38.7M - 2.43.0 a07b618ecd1d b9776c7278f3 2023-09-23T19:45:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: mgrmap e32: y(active, since 40s), standbys: x 2023-09-23T19:45:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: Active manager daemon y restarted 2023-09-23T19:45:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: Activating manager daemon y 2023-09-23T19:45:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:45:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: mgrmap e33: y(active, starting, since 0.00967443s), standbys: x 2023-09-23T19:45:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:45:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:45:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:45:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:45:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:45:38.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:45:38.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:45:38.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:45:38.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:45:38.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:45:38.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:45:38.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:45:38.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:45:38.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:45:38.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:45:38.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:45:38.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:45:38.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:45:38.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: Manager daemon y is now available 2023-09-23T19:45:38.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:38.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:38.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:38.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:45:38.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:45:38.138 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.138 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:45:38.138 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:37 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:45:38.207 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:38 smithi135 conmon[201366]: 2023-09-23T19:45:38.010+0000 7fcf49f9e700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T19:45:38.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: mgrmap e32: y(active, since 40s), standbys: x 2023-09-23T19:45:38.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: Active manager daemon y restarted 2023-09-23T19:45:38.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: Activating manager daemon y 2023-09-23T19:45:38.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: mgrmap e33: y(active, starting, since 0.00967443s), standbys: x 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:45:38.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:45:38.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: Manager daemon y is now available 2023-09-23T19:45:38.210 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:38.210 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:38.210 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:38.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: mgrmap e32: y(active, since 40s), standbys: x 2023-09-23T19:45:38.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: Active manager daemon y restarted 2023-09-23T19:45:38.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: Activating manager daemon y 2023-09-23T19:45:38.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: mgrmap e33: y(active, starting, since 0.00967443s), standbys: x 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi144.plbefj"}]: dispatch 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi135.srojfl"}]: dispatch 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T19:45:38.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: Manager daemon y is now available 2023-09-23T19:45:38.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:38.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:38.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:38.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:45:38.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:45:38.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:45:38.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:45:38.214 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:45:38.214 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T19:45:38.214 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: failed to encode map e90 with expected crc 2023-09-23T19:45:38.214 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:45:38.214 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:37 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T19:45:38.795 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T19:45:39.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:39 smithi135 ceph-mon[213871]: mgrmap e34: y(active, since 1.01609s), standbys: x 2023-09-23T19:45:39.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:39 smithi135 ceph-mon[213871]: from='client.25871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:45:39.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:39 smithi135 ceph-mon[213871]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:39.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:39 smithi135 ceph-mon[118715]: mgrmap e34: y(active, since 1.01609s), standbys: x 2023-09-23T19:45:39.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:39 smithi135 ceph-mon[118715]: from='client.25871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:45:39.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:39 smithi135 ceph-mon[118715]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:39 smithi144 ceph-mon[104994]: mgrmap e34: y(active, since 1.01609s), standbys: x 2023-09-23T19:45:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:39 smithi144 ceph-mon[104994]: from='client.25871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:45:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:39 smithi144 ceph-mon[104994]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:39.430 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:40 smithi144 ceph-mon[104994]: [23/Sep/2023:19:45:38] ENGINE Bus STARTING 2023-09-23T19:45:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:40 smithi144 ceph-mon[104994]: [23/Sep/2023:19:45:38] ENGINE Serving on http://172.21.15.135:8765 2023-09-23T19:45:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:40 smithi144 ceph-mon[104994]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:40 smithi144 ceph-mon[104994]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T19:45:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:40 smithi144 ceph-mon[104994]: [23/Sep/2023:19:45:39] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:45:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:40 smithi144 ceph-mon[104994]: [23/Sep/2023:19:45:39] ENGINE Bus STARTED 2023-09-23T19:45:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[118715]: [23/Sep/2023:19:45:38] ENGINE Bus STARTING 2023-09-23T19:45:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[118715]: [23/Sep/2023:19:45:38] ENGINE Serving on http://172.21.15.135:8765 2023-09-23T19:45:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[118715]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[118715]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T19:45:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[118715]: [23/Sep/2023:19:45:39] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:45:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[118715]: [23/Sep/2023:19:45:39] ENGINE Bus STARTED 2023-09-23T19:45:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[213871]: [23/Sep/2023:19:45:38] ENGINE Bus STARTING 2023-09-23T19:45:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[213871]: [23/Sep/2023:19:45:38] ENGINE Serving on http://172.21.15.135:8765 2023-09-23T19:45:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[213871]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[213871]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T19:45:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[213871]: [23/Sep/2023:19:45:39] ENGINE Serving on https://172.21.15.135:7150 2023-09-23T19:45:40.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:40 smithi135 ceph-mon[213871]: [23/Sep/2023:19:45:39] ENGINE Bus STARTED 2023-09-23T19:45:41.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[118715]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:41.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[213871]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:41.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:41 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:41 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:41 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:41 smithi144 ceph-mon[104994]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:41.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:41 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:41 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:41.846 INFO:teuthology.orchestra.run.smithi135.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:45:42.581 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T19:45:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[118715]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:45:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[118715]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[213871]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[213871]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:42 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.806 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:42 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.807 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:42 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.807 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:42 smithi144 ceph-mon[104994]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:45:42.807 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:42 smithi144 ceph-mon[104994]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:45:42.807 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:42 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.807 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:42 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.807 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:42 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:42.925 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:45:43.773 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:43 smithi144 ceph-mon[104994]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:45:43] "GET /metrics HTTP/1.1" 200 31093 "" "Prometheus/2.43.0" 2023-09-23T19:45:43.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:43 smithi135 ceph-mon[118715]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:43.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:43 smithi135 ceph-mon[213871]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:45.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:45.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:45.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:45.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[118715]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:45.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:45.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:45.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:45:45.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:45:45.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:45.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:45.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:45.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:45.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:45 smithi135 ceph-mon[213871]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:45.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:45.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:45.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:45.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:45.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.conf 2023-09-23T19:45:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.conf 2023-09-23T19:45:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T19:45:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-09-23T19:45:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:45 smithi144 ceph-mon[104994]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:46 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: Updating smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: Updating smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:45:46.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:46 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:45:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:48 smithi144 ceph-mon[104994]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:48.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:48 smithi135 ceph-mon[213871]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:48.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:48 smithi135 ceph-mon[118715]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:49.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:48 smithi144 conmon[178718]: ts=2023-09-23T19:45:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:45:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:50 smithi144 ceph-mon[104994]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:50 smithi135 ceph-mon[118715]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:50 smithi135 ceph-mon[213871]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:52 smithi144 ceph-mon[104994]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:52 smithi135 ceph-mon[118715]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:52 smithi135 ceph-mon[213871]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:53 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:53.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:53 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:45:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:45:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:45:53] "GET /metrics HTTP/1.1" 200 33780 "" "Prometheus/2.43.0" 2023-09-23T19:45:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:54 smithi144 ceph-mon[104994]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:54 smithi135 ceph-mon[118715]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:54 smithi135 ceph-mon[213871]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:56 smithi144 ceph-mon[104994]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:56.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:56 smithi135 ceph-mon[118715]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:56 smithi135 ceph-mon[213871]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:45:58 smithi144 ceph-mon[104994]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:58.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:45:58 smithi135 ceph-mon[118715]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:58.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:45:58 smithi135 ceph-mon[213871]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:45:59.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:45:58 smithi144 conmon[178718]: ts=2023-09-23T19:45:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:46:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:00 smithi144 ceph-mon[104994]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:00 smithi135 ceph-mon[118715]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:00 smithi135 ceph-mon[213871]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:02 smithi144 ceph-mon[104994]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:02 smithi135 ceph-mon[118715]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:02.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:02 smithi135 ceph-mon[213871]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:46:03] "GET /metrics HTTP/1.1" 200 33780 "" "Prometheus/2.43.0" 2023-09-23T19:46:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:04 smithi144 ceph-mon[104994]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:04.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:04 smithi135 ceph-mon[118715]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:04.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:04 smithi135 ceph-mon[213871]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:06 smithi144 ceph-mon[104994]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:06 smithi135 ceph-mon[118715]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:06.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:06 smithi135 ceph-mon[213871]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:08 smithi144 ceph-mon[104994]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:08 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:08 smithi135 ceph-mon[118715]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:08 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:08 smithi135 ceph-mon[213871]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:09.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:46:08 smithi144 conmon[178718]: ts=2023-09-23T19:46:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:46:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:10 smithi144 ceph-mon[104994]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:10 smithi135 ceph-mon[118715]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:10 smithi135 ceph-mon[213871]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:12 smithi144 ceph-mon[104994]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:12 smithi135 ceph-mon[118715]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:12 smithi135 ceph-mon[213871]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:46:13] "GET /metrics HTTP/1.1" 200 33779 "" "Prometheus/2.43.0" 2023-09-23T19:46:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:14 smithi144 ceph-mon[104994]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:14 smithi135 ceph-mon[118715]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:14.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:14 smithi135 ceph-mon[213871]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:16 smithi144 ceph-mon[104994]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:16 smithi135 ceph-mon[118715]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:16 smithi135 ceph-mon[213871]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:18 smithi144 ceph-mon[104994]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:18.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:18 smithi135 ceph-mon[118715]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:18 smithi135 ceph-mon[213871]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:19.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:46:18 smithi144 conmon[178718]: ts=2023-09-23T19:46:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:46:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:20 smithi144 ceph-mon[104994]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:20.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:20 smithi135 ceph-mon[118715]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:20.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:20 smithi135 ceph-mon[213871]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:22 smithi144 ceph-mon[104994]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:22 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:22 smithi135 ceph-mon[118715]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:22 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:22 smithi135 ceph-mon[213871]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:46:23] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.43.0" 2023-09-23T19:46:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:24 smithi144 ceph-mon[104994]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:24 smithi135 ceph-mon[118715]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:24.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:24 smithi135 ceph-mon[213871]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:26 smithi144 ceph-mon[104994]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:26 smithi135 ceph-mon[118715]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:26 smithi135 ceph-mon[213871]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:28 smithi144 ceph-mon[104994]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:28 smithi135 ceph-mon[118715]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:28 smithi135 ceph-mon[213871]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:29.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:46:28 smithi144 conmon[178718]: ts=2023-09-23T19:46:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:46:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:30 smithi144 ceph-mon[104994]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:30 smithi135 ceph-mon[118715]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:30.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:30 smithi135 ceph-mon[213871]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:32.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:32 smithi144 ceph-mon[104994]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:32 smithi135 ceph-mon[118715]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:32.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:32 smithi135 ceph-mon[213871]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:46:33] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.43.0" 2023-09-23T19:46:34.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:34 smithi144 ceph-mon[104994]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:34 smithi135 ceph-mon[118715]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:34.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:34 smithi135 ceph-mon[213871]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:36 smithi144 ceph-mon[104994]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:36 smithi135 ceph-mon[118715]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:36 smithi135 ceph-mon[213871]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:38 smithi144 ceph-mon[104994]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:38 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:38 smithi135 ceph-mon[118715]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:38 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:38 smithi135 ceph-mon[213871]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:39.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:46:38 smithi144 conmon[178718]: ts=2023-09-23T19:46:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:46:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:40 smithi144 ceph-mon[104994]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:40 smithi135 ceph-mon[118715]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:40 smithi135 ceph-mon[213871]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:42 smithi144 ceph-mon[104994]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:42 smithi135 ceph-mon[118715]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:42.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:42 smithi135 ceph-mon[213871]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:46:43] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.43.0" 2023-09-23T19:46:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:44 smithi144 ceph-mon[104994]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:44.394 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:44 smithi135 ceph-mon[213871]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:44.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:44 smithi135 ceph-mon[118715]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:44.990 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T19:46:45.339 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:46.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:46 smithi135 ceph-mon[118715]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:46.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:46 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:46.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:46 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:46.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:46 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:46 smithi135 ceph-mon[213871]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:46 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:46 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:46 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:46 smithi144 ceph-mon[104994]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:46 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:46 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:46 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.093 INFO:teuthology.orchestra.run.smithi135.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:46:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[118715]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[118715]: from='client.25916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:46:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[118715]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:46:47.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[213871]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:47.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[213871]: from='client.25916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:46:47.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[213871]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:46:47.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:47 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:47 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:47 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:47 smithi144 ceph-mon[104994]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:47 smithi144 ceph-mon[104994]: from='client.25916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:46:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:47 smithi144 ceph-mon[104994]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:46:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:47 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:47.861 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T19:46:48.382 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:49.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:46:48 smithi144 conmon[178718]: ts=2023-09-23T19:46:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:46:50.153 INFO:teuthology.orchestra.run.smithi135.stdout:wait for majority of mons to upgrade 2023-09-23T19:46:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:50 smithi144 ceph-mon[104994]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:50 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:50 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:50.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:50 smithi135 ceph-mon[213871]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:50.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:50 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:50.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:50 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:50.397 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:50 smithi135 ceph-mon[118715]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:50.397 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:50 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:50.397 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:50 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:50.916 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T19:46:51.558 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:46:51.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T19:46:51.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: Upgrade: It appears safe to stop mon.c 2023-09-23T19:46:51.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[213871]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:51.610 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.610 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.610 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:46:51.610 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:46:51.610 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.610 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:51.610 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:46:51.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:46:51.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T19:46:51.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: Upgrade: It appears safe to stop mon.c 2023-09-23T19:46:51.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:51 smithi135 ceph-mon[118715]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:46:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:46:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:46:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:46:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T19:46:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: Upgrade: It appears safe to stop mon.c 2023-09-23T19:46:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:51 smithi144 ceph-mon[104994]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:53.306 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:53.306 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[213871]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:53.306 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:46:53.307 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:46:53.307 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:46:53.307 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:53.307 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[118715]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:53.307 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:46:53.307 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:46:53.307 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:53 smithi135 ceph-mon[118715]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:46:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:53 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:46:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:53 smithi144 ceph-mon[104994]: from='mgr.25859 ' entity='mgr.y' 2023-09-23T19:46:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:53 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:46:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:53 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:46:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:53 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:46:53.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:46:53] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.43.0" 2023-09-23T19:46:54.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:54 smithi135 ceph-mon[213871]: Upgrade: Updating mon.c 2023-09-23T19:46:54.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:54 smithi135 ceph-mon[213871]: Deploying daemon mon.c on smithi135 2023-09-23T19:46:54.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:54 smithi135 ceph-mon[213871]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:54.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1423945250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:54.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:54 smithi135 ceph-mon[118715]: Upgrade: Updating mon.c 2023-09-23T19:46:54.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:54 smithi135 ceph-mon[118715]: Deploying daemon mon.c on smithi135 2023-09-23T19:46:54.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:54 smithi135 ceph-mon[118715]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:54.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:54 smithi135 ceph-mon[118715]: from='client.? 172.21.15.135:0/1423945250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:54 smithi144 ceph-mon[104994]: Upgrade: Updating mon.c 2023-09-23T19:46:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:54 smithi144 ceph-mon[104994]: Deploying daemon mon.c on smithi135 2023-09-23T19:46:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:54 smithi144 ceph-mon[104994]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:54 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1423945250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:54.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:54 smithi135 systemd[1]: Stopping Ceph mon.c for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:46:55.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:54 smithi135 conmon[118691]: 2023-09-23T19:46:54.758+0000 7fa46ea29700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:46:55.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:54 smithi135 conmon[118691]: 2023-09-23T19:46:54.758+0000 7fa46ea29700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T19:46:55.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:54 smithi135 conmon[201366]: [23/Sep/2023:19:46:54] ENGINE Bus STOPPING 2023-09-23T19:46:55.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:54 smithi135 conmon[201366]: [23/Sep/2023:19:46:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:46:55.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:54 smithi135 conmon[201366]: [23/Sep/2023:19:46:54] ENGINE Bus STOPPED 2023-09-23T19:46:55.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:54 smithi135 conmon[201366]: [23/Sep/2023:19:46:54] ENGINE Bus STARTING 2023-09-23T19:46:55.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:54 smithi135 conmon[201366]: [23/Sep/2023:19:46:54] ENGINE Serving on http://:::9283 2023-09-23T19:46:55.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:46:54 smithi135 conmon[201366]: [23/Sep/2023:19:46:54] ENGINE Bus STARTED 2023-09-23T19:46:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:55 smithi135 bash[220658]: 3ddce49c9396205eb730b554c987a1933596df3521539a0090a6d864bf609009 2023-09-23T19:46:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:55 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T19:46:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:55 smithi135 systemd[1]: Stopped Ceph mon.c for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:46:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:55 smithi135 systemd[1]: Starting Ceph mon.c for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:46:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 bash[221025]: 6b8308cd56e0454f441271967312d5e96ceeb99846fed25d0daa93ed355ef162 2023-09-23T19:46:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:46:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T19:46:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: pidfile_write: ignore empty --pid-file 2023-09-23T19:46:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 systemd[1]: Started Ceph mon.c for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: load: jerasure load: lrc 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: RocksDB version: 7.9.2 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Git sha 0 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: DB SUMMARY 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: DB Session ID: DHQX6U0LH8LQLDSUDSRZ 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: CURRENT file: CURRENT 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:46:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1095 Bytes 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 4509720 ; 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.error_if_exists: 0 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.create_if_missing: 0 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.paranoid_checks: 1 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.env: 0x562c15b4e720 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T19:46:56.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.info_log: 0x562c18667420 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.statistics: (nil) 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.use_fsync: 0 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_log_file_size: 0 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.allow_fallocate: 1 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:46:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.use_direct_reads: 0 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.db_log_dir: 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.wal_dir: 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:46:56.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.write_buffer_manager: 0x562c178f6c80 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:46:56.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.unordered_write: 0 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.row_cache: None 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.wal_filter: None 2023-09-23T19:46:56.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.two_write_queues: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.wal_compression: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.atomic_flush: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.log_readahead_size: 0 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:46:56.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:46:56.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:46:56.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:46:56.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:46:56.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:46:56.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T19:46:56.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_background_jobs: 2 2023-09-23T19:46:56.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_background_compactions: -1 2023-09-23T19:46:56.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_subcompactions: 1 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_open_files: -1 2023-09-23T19:46:56.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_background_flushes: -1 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Compression algorithms supported: 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: kZSTD supported: 0 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: kXpressCompression supported: 0 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:46:56.732 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: kZlibCompression supported: 1 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: kSnappyCompression supported: 1 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: kLZ4Compression supported: 1 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: kBZip2Compression supported: 0 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.merge_operator: 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_filter: None 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:46:56.733 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562c18667520) 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout: cache_index_and_filter_blocks: 1 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout: pin_top_level_index_and_filter: 1 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_type: 0 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout: data_block_index_type: 0 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_shortening: 1 2023-09-23T19:46:56.734 INFO:journalctl@ceph.mon.c.smithi135.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: checksum: 4 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: no_block_cache: 0 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache: 0x562c179791f0 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_name: BinnedLRUCache 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_options: 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: capacity : 536870912 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: num_shard_bits : 4 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: strict_capacity_limit : 0 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: high_pri_pool_ratio: 0.000 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_compressed: (nil) 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: persistent_cache: (nil) 2023-09-23T19:46:56.735 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_size: 4096 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_size_deviation: 10 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_restart_interval: 16 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_block_restart_interval: 1 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: metadata_block_size: 4096 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: partition_filters: 0 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: use_delta_encoding: 1 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: filter_policy: bloomfilter 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: whole_key_filtering: 1 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: verify_compression: 0 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: read_amp_bytes_per_bit: 0 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: format_version: 5 2023-09-23T19:46:56.736 INFO:journalctl@ceph.mon.c.smithi135.stdout: enable_index_compression: 1 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_align: 0 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout: max_auto_readahead_size: 262144 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout: prepopulate_block_cache: 0 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout: initial_auto_readahead_size: 8192 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression: NoCompression 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:46:56.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.num_levels: 7 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:46:56.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:46:56.739 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:46:56.740 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:46:56.741 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.table_properties_collectors: 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.inplace_update_support: 0 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.bloom_locality: 0 2023-09-23T19:46:56.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.max_successive_merges: 0 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.ttl: 2592000 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.enable_blob_files: false 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.min_blob_size: 0 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:46:56.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12973, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5908b8e5-a4bc-450b-a85e-3b3781f3364d 2023-09-23T19:46:56.744 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498416309609, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498416326074, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2794657, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12962, "largest_seqno": 14762, "table_properties": {"data_size": 2787279, "index_size": 4631, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15346, "raw_average_key_size": 22, "raw_value_size": 2772769, "raw_average_value_size": 4150, "num_data_blocks": 211, "num_entries": 668, "num_filter_entries": 668, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695498416, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5908b8e5-a4bc-450b-a85e-3b3781f3364d", "db_session_id": "DHQX6U0LH8LQLDSUDSRZ", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498416326182, "job": 1, "event": "recovery_finished"} 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562c17a14000 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: DB pointer 0x562c17a02000 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T19:46:56.745 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** DB Stats ** 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T19:46:56.746 INFO:journalctl@ceph.mon.c.smithi135.stdout: L0 1/0 2.67 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 268.3 0.01 0.00 1 0.010 0 0 0.0 0.0 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: L6 1/0 6.18 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: Sum 2/0 8.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 268.3 0.01 0.00 1 0.010 0 0 0.0 0.0 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 268.3 0.01 0.00 1 0.010 0 0 0.0 0.0 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** Compaction Stats [default] ** 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 268.3 0.01 0.00 1 0.010 0 0 0.0 0.0 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:46:56.747 INFO:journalctl@ceph.mon.c.smithi135.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative compaction: 0.00 GB write, 139.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval compaction: 0.00 GB write, 139.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: Block cache BinnedLRUCache@0x562c179791f0#7 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: starting mon.c rank 2 at public addrs [v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0] at bind addrs [v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: mon.c@-1(???) e3 preinit fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: mon.c@-1(???).mds e9 new map 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: mon.c@-1(???).mds e9 print_map 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout: e9 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout: legacy client fscid: 1 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout: Filesystem 'foo' (1) 2023-09-23T19:46:56.749 INFO:journalctl@ceph.mon.c.smithi135.stdout: fs_name foo 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: epoch 8 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: created 2023-09-23T19:41:39.534719+0000 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: modified 2023-09-23T19:41:55.190501+0000 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: tableserver 0 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: root 0 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: session_timeout 60 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: session_autoclose 300 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: max_file_size 1099511627776 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: required_client_features {} 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: last_failure 0 2023-09-23T19:46:56.750 INFO:journalctl@ceph.mon.c.smithi135.stdout: last_failure_osd_epoch 0 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: max_mds 1 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: in 0 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: up {0=24569} 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: failed 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: damaged 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: stopped 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: data_pools [4] 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: metadata_pool 3 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: inline_data disabled 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: balancer 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: bal_rank_mask -1 2023-09-23T19:46:56.751 INFO:journalctl@ceph.mon.c.smithi135.stdout: standby_count_wanted 1 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout: [mds.foo.smithi135.srojfl{0:24569} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout: Standby daemons: 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout: [mds.foo.smithi144.plbefj{-1:24553} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: mon.c@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-09-23T19:46:56.752 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta expand map: {default=false} 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta from 'false' to 'false' 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta expanded map: {default=false} 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta expand map: {default=info} 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta from 'info' to 'info' 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta expanded map: {default=info} 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta expand map: {default=daemon} 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta expand map: {default=debug} 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T19:46:56.753 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:56 smithi135 ceph-mon[221113]: expand_channel_meta expanded map: {default=debug} 2023-09-23T19:46:57.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:57.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: mon.c calling monitor election 2023-09-23T19:46:57.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: mon.a calling monitor election 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1017594382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: mgrmap e35: y(active, since 79s), standbys: x 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: overall HEALTH_OK 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:46:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:57.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:57 smithi144 ceph-mon[104994]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:57.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: mon.c calling monitor election 2023-09-23T19:46:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: mon.a calling monitor election 2023-09-23T19:46:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1017594382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:46:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:46:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:46:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:46:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: mgrmap e35: y(active, since 79s), standbys: x 2023-09-23T19:46:57.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: overall HEALTH_OK 2023-09-23T19:46:57.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:46:57.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:46:57.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:57.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[221113]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:57.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: mon.c calling monitor election 2023-09-23T19:46:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: mon.a calling monitor election 2023-09-23T19:46:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1017594382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:46:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: monmap e3: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:46:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:46:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:46:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: mgrmap e35: y(active, since 79s), standbys: x 2023-09-23T19:46:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: overall HEALTH_OK 2023-09-23T19:46:57.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:46:57.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:46:57.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:46:57.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:57 smithi135 ceph-mon[213871]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:46:58.891 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T19:46:58.931 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:46:58 smithi144 conmon[178718]: ts=2023-09-23T19:46:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:46:59.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1713471392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:59.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1713471392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:58 smithi144 ceph-mon[104994]: from='client.? 172.21.15.135:0/1713471392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:46:59.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:47:00.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:59 smithi135 ceph-mon[221113]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:00.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:00.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:46:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:59 smithi135 ceph-mon[213871]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:46:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:00.271 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:59 smithi144 ceph-mon[104994]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:59 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:46:59 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:01.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:01 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:01.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:01 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:01.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:01 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:01.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:01 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:01 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:01 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:01.634 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:47:01 smithi135 conmon[201366]: 2023-09-23T19:47:01.335+0000 7fcf49f9e700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T19:47:02.293 INFO:teuthology.orchestra.run.smithi135.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:47:02.293 INFO:teuthology.orchestra.run.smithi135.stdout:alertmanager.a smithi135 *:9093,9094 running (2m) - 6m 29.1M - 0.25.0 c8568f914cd2 22e51759a65e 2023-09-23T19:47:02.293 INFO:teuthology.orchestra.run.smithi135.stdout:grafana.a smithi144 *:3000 running (2m) - 5m 86.7M - 9.4.7 2c41d148cca3 902bff181c85 2023-09-23T19:47:02.293 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi135.srojfl smithi135 running (5m) - 5m 17.5M - 17.2.6-1123-g61431f6b 0db14aa01094 37783bce2e06 2023-09-23T19:47:02.293 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi144.plbefj smithi144 running (5m) - 5m 15.3M - 17.2.6-1123-g61431f6b 0db14aa01094 af9c7e8247c3 2023-09-23T19:47:02.293 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.x smithi144 *:8443,9283,8765 running (108s) - 10m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 81770c134986 2023-09-23T19:47:02.293 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.y smithi135 *:8443,9283,8765 running (2m) - 11m 500M - 18.2.0-460-gbb0daeff e32c2a64c126 eb6afb9206f5 2023-09-23T19:47:02.293 INFO:teuthology.orchestra.run.smithi135.stdout:mon.a smithi135 running (95s) - 11m 48.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 7fa1f78d4725 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:mon.b smithi144 running (10m) - 10m 82.6M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 40abc2a02dd4 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:mon.c smithi135 running (6s) - 10m 25.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 6b8308cd56e0 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.a smithi135 *:9100 running (2m) - 5m 16.9M - 1.5.0 0da6a335fe13 caa9af75261e 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.b smithi144 *:9100 running (2m) - 5m 8552k - 1.5.0 0da6a335fe13 a8b1ed1aeea7 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 smithi135 running (9m) - 9m 84.9M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 af561f634ba3 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1 smithi135 running (9m) - 9m 101M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 791e31e1318a 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 smithi135 running (9m) - 9m 79.5M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 0ee5fdb5fb5e 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 smithi135 running (8m) - 8m 70.4M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 0fd086d2e07f 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 smithi144 running (8m) - 8m 73.5M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 076c88452d16 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 smithi144 running (7m) - 7m 93.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 30c9f1e12bc8 2023-09-23T19:47:02.294 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 smithi144 running (7m) - 7m 74.7M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 d04b0ed25c0f 2023-09-23T19:47:02.295 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 smithi144 running (7m) - 7m 95.2M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 4b2a00379dc8 2023-09-23T19:47:02.295 INFO:teuthology.orchestra.run.smithi135.stdout:prometheus.a smithi144 *:9095 running (113s) - 6m 48.9M - 2.43.0 a07b618ecd1d b9776c7278f3 2023-09-23T19:47:02.319 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:02 smithi135 ceph-mon[221113]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:02.319 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:02 smithi135 ceph-mon[213871]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:02 smithi144 ceph-mon[104994]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:03.294 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T19:47:03.708 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:04.497 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:04 smithi144 ceph-mon[104994]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:47:04.498 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:04 smithi144 ceph-mon[104994]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:04.498 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:47:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:47:03] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.43.0" 2023-09-23T19:47:04.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:04 smithi135 ceph-mon[221113]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:47:04.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:04 smithi135 ceph-mon[221113]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:04.499 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:04 smithi135 ceph-mon[213871]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:47:04.499 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:04 smithi135 ceph-mon[213871]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:06.078 INFO:teuthology.orchestra.run.smithi135.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:47:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:47:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:47:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:06 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T19:47:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:47:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T19:47:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T19:47:06.833 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T19:47:07.086 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:07 smithi135 ceph-mon[213871]: Upgrade: It appears safe to stop mon.b 2023-09-23T19:47:07.086 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:07 smithi135 ceph-mon[213871]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:47:07.086 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:07 smithi135 ceph-mon[213871]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:47:07.086 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:07.188 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:47:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:07 smithi144 ceph-mon[104994]: Upgrade: It appears safe to stop mon.b 2023-09-23T19:47:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:07 smithi144 ceph-mon[104994]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:47:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:07 smithi144 ceph-mon[104994]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:47:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:07 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:07.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:07 smithi135 ceph-mon[221113]: Upgrade: It appears safe to stop mon.b 2023-09-23T19:47:07.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:07 smithi135 ceph-mon[221113]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:47:07.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:07 smithi135 ceph-mon[221113]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:47:07.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 ceph-mon[104994]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 ceph-mon[104994]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[221113]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[213871]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 systemd[1]: Stopping Ceph mon.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:47:08.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:47:08 smithi144 conmon[178718]: ts=2023-09-23T19:47:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:47:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 conmon[104970]: 2023-09-23T19:47:08.915+0000 7feb56706700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:47:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:08 smithi144 conmon[104970]: 2023-09-23T19:47:08.915+0000 7feb56706700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T19:47:09.815 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:09 smithi144 bash[194410]: 40abc2a02dd4d4e43a1fd1407dd7d20476c09eaa27f89e70f31461c176e994d8 2023-09-23T19:47:09.815 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:09 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T19:47:09.815 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:09 smithi144 systemd[1]: Stopped Ceph mon.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:47:09.816 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:09 smithi144 systemd[1]: Starting Ceph mon.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:47:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 bash[194768]: 4fee8ae2a29683f3bd2944c900ac5761caacf4bb5f2a58c60fdcd2c5dc7b97bb 2023-09-23T19:47:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T19:47:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T19:47:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: pidfile_write: ignore empty --pid-file 2023-09-23T19:47:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: load: jerasure load: lrc 2023-09-23T19:47:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: RocksDB version: 7.9.2 2023-09-23T19:47:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Git sha 0 2023-09-23T19:47:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: DB SUMMARY 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: DB Session ID: WDY1HYYJ5WL33OEO27BV 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: CURRENT file: CURRENT 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1095 Bytes 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 5276172 ; 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.error_if_exists: 0 2023-09-23T19:47:10.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.create_if_missing: 0 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.paranoid_checks: 1 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.env: 0x563517f20720 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.info_log: 0x563519c89420 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T19:47:10.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.statistics: (nil) 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.use_fsync: 0 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_log_file_size: 0 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.allow_fallocate: 1 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.use_direct_reads: 0 2023-09-23T19:47:10.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T19:47:10.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T19:47:10.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.db_log_dir: 2023-09-23T19:47:10.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.wal_dir: 2023-09-23T19:47:10.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T19:47:10.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T19:47:10.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T19:47:10.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T19:47:10.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.write_buffer_manager: 0x563518f18c80 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 systemd[1]: Started Ceph mon.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T19:47:10.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T19:47:10.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T19:47:10.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T19:47:10.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.unordered_write: 0 2023-09-23T19:47:10.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T19:47:10.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T19:47:10.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T19:47:10.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T19:47:10.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.row_cache: None 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.wal_filter: None 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.two_write_queues: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.wal_compression: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.atomic_flush: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T19:47:10.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.log_readahead_size: 0 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_background_jobs: 2 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_background_compactions: -1 2023-09-23T19:47:10.640 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_subcompactions: 1 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_open_files: -1 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T19:47:10.641 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T19:47:10.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T19:47:10.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T19:47:10.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_background_flushes: -1 2023-09-23T19:47:10.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Compression algorithms supported: 2023-09-23T19:47:10.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T19:47:10.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: kZSTD supported: 0 2023-09-23T19:47:10.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: kXpressCompression supported: 0 2023-09-23T19:47:10.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: kZlibCompression supported: 1 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: kSnappyCompression supported: 1 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: kLZ4Compression supported: 1 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: kBZip2Compression supported: 0 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T19:47:10.643 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.merge_operator: 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_filter: None 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563519c89520) 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout: cache_index_and_filter_blocks: 1 2023-09-23T19:47:10.644 INFO:journalctl@ceph.mon.b.smithi144.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: pin_top_level_index_and_filter: 1 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: index_type: 0 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: data_block_index_type: 0 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: index_shortening: 1 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: checksum: 4 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: no_block_cache: 0 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_cache: 0x563518f9b350 2023-09-23T19:47:10.645 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_cache_name: BinnedLRUCache 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_cache_options: 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: capacity : 536870912 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: num_shard_bits : 4 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: strict_capacity_limit : 0 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: high_pri_pool_ratio: 0.000 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_cache_compressed: (nil) 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: persistent_cache: (nil) 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_size: 4096 2023-09-23T19:47:10.646 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_size_deviation: 10 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_restart_interval: 16 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: index_block_restart_interval: 1 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: metadata_block_size: 4096 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: partition_filters: 0 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: use_delta_encoding: 1 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: filter_policy: bloomfilter 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: whole_key_filtering: 1 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: verify_compression: 0 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: read_amp_bytes_per_bit: 0 2023-09-23T19:47:10.647 INFO:journalctl@ceph.mon.b.smithi144.stdout: format_version: 5 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout: enable_index_compression: 1 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout: block_align: 0 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout: max_auto_readahead_size: 262144 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout: prepopulate_block_cache: 0 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout: initial_auto_readahead_size: 8192 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T19:47:10.648 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression: NoCompression 2023-09-23T19:47:10.649 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T19:47:10.649 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T19:47:10.649 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T19:47:10.649 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.num_levels: 7 2023-09-23T19:47:10.649 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T19:47:10.649 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T19:47:10.649 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T19:47:10.649 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T19:47:10.650 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T19:47:10.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T19:47:10.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T19:47:10.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:47:10.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T19:47:10.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T19:47:10.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:47:10.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:47:10.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T19:47:10.652 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T19:47:10.653 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T19:47:10.654 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T19:47:10.655 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.table_properties_collectors: 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.inplace_update_support: 0 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T19:47:10.656 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.bloom_locality: 0 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.max_successive_merges: 0 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.ttl: 2592000 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T19:47:10.657 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.enable_blob_files: false 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.min_blob_size: 0 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T19:47:10.658 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13048, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e93b43bd-2451-4a01-afe5-c78e66c9762b 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498430387823, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498430399575, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3245616, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13042, "largest_seqno": 15169, "table_properties": {"data_size": 3237181, "index_size": 5431, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1989, "raw_key_size": 17750, "raw_average_key_size": 22, "raw_value_size": 3220363, "raw_average_value_size": 4149, "num_data_blocks": 248, "num_entries": 776, "num_filter_entries": 776, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695498430, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e93b43bd-2451-4a01-afe5-c78e66c9762b", "db_session_id": "WDY1HYYJ5WL33OEO27BV", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695498430399688, "job": 1, "event": "recovery_finished"} 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T19:47:10.659 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563519036000 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: DB pointer 0x563519024000 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** DB Stats ** 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T19:47:10.660 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** Compaction Stats [default] ** 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: L0 1/0 3.10 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 689.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: L6 1/0 6.18 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: Sum 2/0 9.28 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 689.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 689.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.661 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** Compaction Stats [default] ** 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 689.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T19:47:10.662 INFO:journalctl@ceph.mon.b.smithi144.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout: Cumulative compaction: 0.00 GB write, 186.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout: Interval compaction: 0.00 GB write, 186.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout: Block cache BinnedLRUCache@0x563518f9b350#7 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: starting mon.b rank 1 at public addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] at bind addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mon.b@-1(???) e3 preinit fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mon.b@-1(???).mds e9 new map 2023-09-23T19:47:10.663 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mon.b@-1(???).mds e9 print_map 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: e9 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: legacy client fscid: 1 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: Filesystem 'foo' (1) 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: fs_name foo 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: epoch 8 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: created 2023-09-23T19:41:39.534719+0000 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: modified 2023-09-23T19:41:55.190501+0000 2023-09-23T19:47:10.664 INFO:journalctl@ceph.mon.b.smithi144.stdout: tableserver 0 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: root 0 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: session_timeout 60 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: session_autoclose 300 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: max_file_size 1099511627776 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: required_client_features {} 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: last_failure 0 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: last_failure_osd_epoch 0 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: max_mds 1 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: in 0 2023-09-23T19:47:10.665 INFO:journalctl@ceph.mon.b.smithi144.stdout: up {0=24569} 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: failed 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: damaged 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: stopped 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: data_pools [4] 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: metadata_pool 3 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: inline_data disabled 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: balancer 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: bal_rank_mask -1 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: standby_count_wanted 1 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: [mds.foo.smithi135.srojfl{0:24569} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.135:6834/2786532664,v1:172.21.15.135:6835/2786532664] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.666 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout: Standby daemons: 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout: 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout: [mds.foo.smithi144.plbefj{-1:24553} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.144:6832/3628854139,v1:172.21.15.144:6833/3628854139] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mon.b@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta expand map: {default=false} 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta from 'false' to 'false' 2023-09-23T19:47:10.667 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta expanded map: {default=false} 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta expand map: {default=info} 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta from 'info' to 'info' 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta expanded map: {default=info} 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta expand map: {default=daemon} 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta expand map: {default=debug} 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: expand_channel_meta expanded map: {default=debug} 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: Upgrade: Updating mon.b 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: Deploying daemon mon.b on smithi144 2023-09-23T19:47:10.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:10 smithi144 ceph-mon[194833]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T19:47:10.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:10 smithi135 ceph-mon[213871]: Upgrade: Updating mon.b 2023-09-23T19:47:10.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:10 smithi135 ceph-mon[213871]: Deploying daemon mon.b on smithi144 2023-09-23T19:47:10.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:10 smithi135 ceph-mon[213871]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T19:47:10.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:10 smithi135 ceph-mon[221113]: Upgrade: Updating mon.b 2023-09-23T19:47:10.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:10 smithi135 ceph-mon[221113]: Deploying daemon mon.b on smithi144 2023-09-23T19:47:10.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:10 smithi135 ceph-mon[221113]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T19:47:11.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: overall HEALTH_OK 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: mon.a calling monitor election 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: mon.b calling monitor election 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: mon.c calling monitor election 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: monmap e4: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:47:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:47:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: mgrmap e36: y(active, since 93s), standbys: x 2023-09-23T19:47:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: overall HEALTH_OK 2023-09-23T19:47:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:11 smithi144 ceph-mon[194833]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:11.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: overall HEALTH_OK 2023-09-23T19:47:11.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: mon.a calling monitor election 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: mon.b calling monitor election 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: mon.c calling monitor election 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: monmap e4: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:47:11.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: mgrmap e36: y(active, since 93s), standbys: x 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: overall HEALTH_OK 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[221113]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: overall HEALTH_OK 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T19:47:11.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: mon.a calling monitor election 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: mon.b calling monitor election 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: mon.c calling monitor election 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: monmap e4: 3 mons at {a=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0],b=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],c=[v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]} removed_ranks: {} 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: fsmap foo:1 {0=foo.smithi135.srojfl=up:active} 1 up:standby 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: mgrmap e36: y(active, since 93s), standbys: x 2023-09-23T19:47:11.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: overall HEALTH_OK 2023-09-23T19:47:11.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:11.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:11.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:11.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:11 smithi135 ceph-mon[213871]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:47:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:47:13] "GET /metrics HTTP/1.1" 200 33773 "" "Prometheus/2.43.0" 2023-09-23T19:47:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:14 smithi144 ceph-mon[194833]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:14 smithi135 ceph-mon[221113]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:14 smithi135 ceph-mon[213871]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:15 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:15 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:15 smithi144 ceph-mon[194833]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:15.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:47:15 smithi135 conmon[201366]: 2023-09-23T19:47:15.409+0000 7fcf49f9e700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T19:47:15.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:15 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:15.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:15 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:15.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:15 smithi135 ceph-mon[213871]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:15.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:15 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:15.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:15 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:15.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:15 smithi135 ceph-mon[221113]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:18.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:19.096 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:47:18 smithi144 conmon[178718]: ts=2023-09-23T19:47:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:47:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:19 smithi144 ceph-mon[194833]: Reconfiguring mon.a (monmap changed)... 2023-09-23T19:47:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:19 smithi144 ceph-mon[194833]: Reconfiguring daemon mon.a on smithi135 2023-09-23T19:47:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:19 smithi135 ceph-mon[221113]: Reconfiguring mon.a (monmap changed)... 2023-09-23T19:47:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:19 smithi135 ceph-mon[221113]: Reconfiguring daemon mon.a on smithi135 2023-09-23T19:47:19.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:19 smithi135 ceph-mon[213871]: Reconfiguring mon.a (monmap changed)... 2023-09-23T19:47:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:19 smithi135 ceph-mon[213871]: Reconfiguring daemon mon.a on smithi135 2023-09-23T19:47:20.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[213871]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:20.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:20.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:20.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:47:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:47:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[221113]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:47:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:47:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:20 smithi144 ceph-mon[194833]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T19:47:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:47:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:47:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:21.365 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[221113]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T19:47:21.365 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[221113]: Reconfiguring daemon mgr.y on smithi135 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[213871]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[213871]: Reconfiguring daemon mgr.y on smithi135 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:21.366 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:21.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:21.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:21.367 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:21 smithi144 ceph-mon[194833]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T19:47:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:21 smithi144 ceph-mon[194833]: Reconfiguring daemon mgr.y on smithi135 2023-09-23T19:47:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:21.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:21.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:22 smithi144 ceph-mon[194833]: Reconfiguring mon.c (monmap changed)... 2023-09-23T19:47:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:22 smithi144 ceph-mon[194833]: Reconfiguring daemon mon.c on smithi135 2023-09-23T19:47:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:22 smithi144 ceph-mon[194833]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:22 smithi135 ceph-mon[221113]: Reconfiguring mon.c (monmap changed)... 2023-09-23T19:47:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:22 smithi135 ceph-mon[221113]: Reconfiguring daemon mon.c on smithi135 2023-09-23T19:47:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:22 smithi135 ceph-mon[221113]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:22 smithi135 ceph-mon[213871]: Reconfiguring mon.c (monmap changed)... 2023-09-23T19:47:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:22 smithi135 ceph-mon[213871]: Reconfiguring daemon mon.c on smithi135 2023-09-23T19:47:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:22 smithi135 ceph-mon[213871]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[221113]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T19:47:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:47:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[221113]: Reconfiguring daemon osd.0 on smithi135 2023-09-23T19:47:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[221113]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[213871]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T19:47:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:47:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[213871]: Reconfiguring daemon osd.0 on smithi135 2023-09-23T19:47:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:23 smithi135 ceph-mon[213871]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:23 smithi144 ceph-mon[194833]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T19:47:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:47:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:23 smithi144 ceph-mon[194833]: Reconfiguring daemon osd.0 on smithi135 2023-09-23T19:47:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:23 smithi144 ceph-mon[194833]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:47:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:47:23] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.43.0" 2023-09-23T19:47:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[221113]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T19:47:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:47:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[221113]: Reconfiguring daemon osd.1 on smithi135 2023-09-23T19:47:24.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:24.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:24.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[213871]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T19:47:24.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:47:24.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:24.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:24 smithi135 ceph-mon[213871]: Reconfiguring daemon osd.1 on smithi135 2023-09-23T19:47:25.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:25.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:25.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:24 smithi144 ceph-mon[194833]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T19:47:25.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:47:25.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:25.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:24 smithi144 ceph-mon[194833]: Reconfiguring daemon osd.1 on smithi135 2023-09-23T19:47:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[221113]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[221113]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T19:47:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:47:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[213871]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[213871]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T19:47:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:47:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:26 smithi144 ceph-mon[194833]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:26 smithi144 ceph-mon[194833]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T19:47:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:47:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:27.382 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[221113]: Reconfiguring daemon osd.2 on smithi135 2023-09-23T19:47:27.382 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:27.382 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:27.382 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:47:27.382 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:27.382 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[213871]: Reconfiguring daemon osd.2 on smithi135 2023-09-23T19:47:27.383 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:27.383 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:27.383 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:47:27.383 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:27 smithi144 ceph-mon[194833]: Reconfiguring daemon osd.2 on smithi135 2023-09-23T19:47:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:47:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[221113]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T19:47:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[221113]: Reconfiguring daemon osd.3 on smithi135 2023-09-23T19:47:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[221113]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:47:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[213871]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T19:47:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[213871]: Reconfiguring daemon osd.3 on smithi135 2023-09-23T19:47:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[213871]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:28.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:28.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:47:28.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:28 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:28 smithi144 ceph-mon[194833]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T19:47:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:28 smithi144 ceph-mon[194833]: Reconfiguring daemon osd.3 on smithi135 2023-09-23T19:47:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:28 smithi144 ceph-mon[194833]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:28 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:28 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:28 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi135.srojfl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:47:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:28 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:29.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:47:28 smithi144 conmon[178718]: ts=2023-09-23T19:47:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:47:29.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:29 smithi135 ceph-mon[213871]: Reconfiguring mds.foo.smithi135.srojfl (monmap changed)... 2023-09-23T19:47:29.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:29 smithi135 ceph-mon[213871]: Reconfiguring daemon mds.foo.smithi135.srojfl on smithi135 2023-09-23T19:47:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:29 smithi135 ceph-mon[221113]: Reconfiguring mds.foo.smithi135.srojfl (monmap changed)... 2023-09-23T19:47:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:29 smithi135 ceph-mon[221113]: Reconfiguring daemon mds.foo.smithi135.srojfl on smithi135 2023-09-23T19:47:29.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:29 smithi144 ceph-mon[194833]: Reconfiguring mds.foo.smithi135.srojfl (monmap changed)... 2023-09-23T19:47:29.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:29 smithi144 ceph-mon[194833]: Reconfiguring daemon mds.foo.smithi135.srojfl on smithi135 2023-09-23T19:47:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[221113]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[213871]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:30.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:30 smithi144 ceph-mon[194833]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:30.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:30.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:30.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T19:47:30.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T19:47:30.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:31.461 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:31 smithi144 ceph-mon[194833]: Reconfiguring mon.b (monmap changed)... 2023-09-23T19:47:31.461 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:31 smithi144 ceph-mon[194833]: Reconfiguring daemon mon.b on smithi144 2023-09-23T19:47:31.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:31 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:31.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:31 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:31.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:31 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:47:31.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:31 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:47:31.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:31 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[221113]: Reconfiguring mon.b (monmap changed)... 2023-09-23T19:47:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[221113]: Reconfiguring daemon mon.b on smithi144 2023-09-23T19:47:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:47:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:47:31.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[213871]: Reconfiguring mon.b (monmap changed)... 2023-09-23T19:47:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[213871]: Reconfiguring daemon mon.b on smithi144 2023-09-23T19:47:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T19:47:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T19:47:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:31 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:32 smithi135 ceph-mon[221113]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T19:47:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:32 smithi135 ceph-mon[221113]: Reconfiguring daemon mgr.x on smithi144 2023-09-23T19:47:32.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:32 smithi135 ceph-mon[221113]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:32 smithi135 ceph-mon[213871]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T19:47:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:32 smithi135 ceph-mon[213871]: Reconfiguring daemon mgr.x on smithi144 2023-09-23T19:47:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:32 smithi135 ceph-mon[213871]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:32 smithi144 ceph-mon[194833]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T19:47:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:32 smithi144 ceph-mon[194833]: Reconfiguring daemon mgr.x on smithi144 2023-09-23T19:47:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:32 smithi144 ceph-mon[194833]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:33 smithi144 ceph-mon[194833]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T19:47:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:47:33.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:33.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:33 smithi144 ceph-mon[194833]: Reconfiguring daemon osd.4 on smithi144 2023-09-23T19:47:33.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:33 smithi144 ceph-mon[194833]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:33.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:33.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:33.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[213871]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[213871]: Reconfiguring daemon osd.4 on smithi144 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[213871]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[221113]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T19:47:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[221113]: Reconfiguring daemon osd.4 on smithi144 2023-09-23T19:47:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:33 smithi135 ceph-mon[221113]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:47:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:47:33] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.43.0" 2023-09-23T19:47:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:34 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:34 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:34 smithi144 ceph-mon[194833]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T19:47:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:34 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:47:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:34 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:34 smithi144 ceph-mon[194833]: Reconfiguring daemon osd.5 on smithi144 2023-09-23T19:47:34.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:34.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:34.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[221113]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T19:47:34.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:47:34.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:34.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[221113]: Reconfiguring daemon osd.5 on smithi144 2023-09-23T19:47:34.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:34.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:34.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[213871]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T19:47:34.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T19:47:34.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:34.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:34 smithi135 ceph-mon[213871]: Reconfiguring daemon osd.5 on smithi144 2023-09-23T19:47:36.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:36.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:36.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:35 smithi144 ceph-mon[194833]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T19:47:36.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:47:36.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:35 smithi144 ceph-mon[194833]: Reconfiguring daemon osd.6 on smithi144 2023-09-23T19:47:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:35 smithi144 ceph-mon[194833]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:36.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:36.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:36.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[221113]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T19:47:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:47:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[221113]: Reconfiguring daemon osd.6 on smithi144 2023-09-23T19:47:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[221113]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[213871]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T19:47:36.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T19:47:36.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:36.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[213871]: Reconfiguring daemon osd.6 on smithi144 2023-09-23T19:47:36.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:35 smithi135 ceph-mon[213871]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:37 smithi144 ceph-mon[194833]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T19:47:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:47:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:37 smithi144 ceph-mon[194833]: Reconfiguring daemon osd.7 on smithi144 2023-09-23T19:47:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:37 smithi144 ceph-mon[194833]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[221113]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T19:47:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[221113]: Reconfiguring daemon osd.7 on smithi144 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[221113]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[213871]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T19:47:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[213871]: Reconfiguring daemon osd.7 on smithi144 2023-09-23T19:47:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[213871]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:38.881 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:47:38 smithi144 conmon[178718]: ts=2023-09-23T19:47:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:47:38.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:38.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:38 smithi144 ceph-mon[194833]: Reconfiguring mds.foo.smithi144.plbefj (monmap changed)... 2023-09-23T19:47:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:47:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:38 smithi144 ceph-mon[194833]: Reconfiguring daemon mds.foo.smithi144.plbefj on smithi144 2023-09-23T19:47:38.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:38.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:38.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[221113]: Reconfiguring mds.foo.smithi144.plbefj (monmap changed)... 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[221113]: Reconfiguring daemon mds.foo.smithi144.plbefj on smithi144 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[213871]: Reconfiguring mds.foo.smithi144.plbefj (monmap changed)... 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi144.plbefj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:38.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:38 smithi135 ceph-mon[213871]: Reconfiguring daemon mds.foo.smithi144.plbefj on smithi144 2023-09-23T19:47:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:39 smithi144 ceph-mon[194833]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[221113]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[213871]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:47:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:47:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:47:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:42 smithi144 ceph-mon[194833]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:42 smithi135 ceph-mon[221113]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:42 smithi135 ceph-mon[213871]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:47:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:47:43] "GET /metrics HTTP/1.1" 200 33769 "" "Prometheus/2.43.0" 2023-09-23T19:47:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:44 smithi144 ceph-mon[194833]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:44 smithi135 ceph-mon[213871]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:44.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:44 smithi135 ceph-mon[221113]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:46 smithi144 ceph-mon[194833]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:46 smithi135 ceph-mon[221113]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:46.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:46 smithi135 ceph-mon[213871]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:48 smithi144 ceph-mon[194833]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:48.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:48 smithi135 ceph-mon[221113]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:48.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:48 smithi135 ceph-mon[213871]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:49.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:47:48 smithi144 conmon[178718]: ts=2023-09-23T19:47:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:47:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:50 smithi144 ceph-mon[194833]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:50 smithi135 ceph-mon[213871]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:50 smithi135 ceph-mon[221113]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:52 smithi144 ceph-mon[194833]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:52 smithi135 ceph-mon[213871]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:52 smithi135 ceph-mon[221113]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:47:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:47:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:47:53] "GET /metrics HTTP/1.1" 200 33781 "" "Prometheus/2.43.0" 2023-09-23T19:47:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:54 smithi144 ceph-mon[194833]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:54 smithi135 ceph-mon[221113]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:54 smithi135 ceph-mon[213871]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:56 smithi144 ceph-mon[194833]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:56.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:56 smithi135 ceph-mon[221113]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:56 smithi135 ceph-mon[213871]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:47:58 smithi144 ceph-mon[194833]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:58.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:47:58 smithi135 ceph-mon[213871]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:47:58 smithi135 ceph-mon[221113]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:47:59.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:47:58 smithi144 conmon[178718]: ts=2023-09-23T19:47:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:48:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:00 smithi144 ceph-mon[194833]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:00 smithi135 ceph-mon[213871]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:00 smithi135 ceph-mon[221113]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:02 smithi144 ceph-mon[194833]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:02 smithi135 ceph-mon[221113]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:02.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:02 smithi135 ceph-mon[213871]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:48:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:48:03] "GET /metrics HTTP/1.1" 200 33781 "" "Prometheus/2.43.0" 2023-09-23T19:48:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:04 smithi144 ceph-mon[194833]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:04.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:04 smithi135 ceph-mon[221113]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:04.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:04 smithi135 ceph-mon[213871]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:06 smithi135 ceph-mon[221113]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:06.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:06 smithi135 ceph-mon[213871]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:06 smithi144 ceph-mon[194833]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:08 smithi144 ceph-mon[194833]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:08 smithi135 ceph-mon[221113]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:08.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:08 smithi135 ceph-mon[213871]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:09.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:48:08 smithi144 conmon[178718]: ts=2023-09-23T19:48:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:48:09.204 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T19:48:09.552 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:10.073 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:10 smithi135 ceph-mon[213871]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:10 smithi144 ceph-mon[194833]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:10 smithi135 ceph-mon[221113]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:11.354 INFO:teuthology.orchestra.run.smithi135.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:48:12.164 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T19:48:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:12 smithi144 ceph-mon[194833]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:12 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:12 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:12 smithi135 ceph-mon[221113]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:12 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:12 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:12 smithi135 ceph-mon[213871]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:12 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:12 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:12.822 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:13.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:13 smithi144 ceph-mon[194833]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:13.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:13 smithi144 ceph-mon[194833]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:48:13.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:13 smithi135 ceph-mon[221113]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:13.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:13 smithi135 ceph-mon[221113]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:48:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:13 smithi135 ceph-mon[213871]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:13 smithi135 ceph-mon[213871]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:48:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:48:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:48:13] "GET /metrics HTTP/1.1" 200 33780 "" "Prometheus/2.43.0" 2023-09-23T19:48:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:14 smithi144 ceph-mon[194833]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:14 smithi135 ceph-mon[221113]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:14.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:14 smithi135 ceph-mon[213871]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:14.682 INFO:teuthology.orchestra.run.smithi135.stdout:wait for all mons to upgrade 2023-09-23T19:48:15.423 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T19:48:15.877 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:16 smithi144 ceph-mon[194833]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:16 smithi135 ceph-mon[221113]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:16 smithi135 ceph-mon[213871]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:17.890 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T19:48:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T19:48:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T19:48:18.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:48:18.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/198469642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:48:18.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:18.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T19:48:18.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T19:48:18.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:48:18.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/198469642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:48:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T19:48:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T19:48:18.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T19:48:18.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T19:48:18.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:48:18.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:18.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:48:18.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/198469642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T19:48:18.605 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:48:19.059 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:19.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:48:18 smithi144 conmon[178718]: ts=2023-09-23T19:48:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:48:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:19 smithi135 ceph-mon[221113]: Upgrade: Setting container_image for all mon 2023-09-23T19:48:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:19 smithi135 ceph-mon[221113]: Upgrade: Setting container_image for all crash 2023-09-23T19:48:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:19 smithi135 ceph-mon[221113]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:48:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:19 smithi135 ceph-mon[221113]: Upgrade: osd.0 is safe to restart 2023-09-23T19:48:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:19 smithi135 ceph-mon[213871]: Upgrade: Setting container_image for all mon 2023-09-23T19:48:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:19 smithi135 ceph-mon[213871]: Upgrade: Setting container_image for all crash 2023-09-23T19:48:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:19 smithi135 ceph-mon[213871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:48:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:19 smithi135 ceph-mon[213871]: Upgrade: osd.0 is safe to restart 2023-09-23T19:48:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:19 smithi144 ceph-mon[194833]: Upgrade: Setting container_image for all mon 2023-09-23T19:48:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:19 smithi144 ceph-mon[194833]: Upgrade: Setting container_image for all crash 2023-09-23T19:48:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:19 smithi144 ceph-mon[194833]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T19:48:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:19 smithi144 ceph-mon[194833]: Upgrade: osd.0 is safe to restart 2023-09-23T19:48:20.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:20 smithi135 ceph-mon[213871]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:20.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:20.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:48:20.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:20.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:20 smithi135 ceph-mon[221113]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:20.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:20.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:48:20.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:20 smithi144 ceph-mon[194833]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T19:48:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:21.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:21 smithi135 ceph-mon[221113]: Upgrade: Updating osd.0 2023-09-23T19:48:21.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:21 smithi135 ceph-mon[221113]: Deploying daemon osd.0 on smithi135 2023-09-23T19:48:21.366 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:21 smithi135 ceph-mon[213871]: Upgrade: Updating osd.0 2023-09-23T19:48:21.366 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:21 smithi135 ceph-mon[213871]: Deploying daemon osd.0 on smithi135 2023-09-23T19:48:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:21 smithi144 ceph-mon[194833]: Upgrade: Updating osd.0 2023-09-23T19:48:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:21 smithi144 ceph-mon[194833]: Deploying daemon osd.0 on smithi135 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:alertmanager.a smithi135 *:9093,9094 running (3m) - 7m 29.1M - 0.25.0 c8568f914cd2 22e51759a65e 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:grafana.a smithi144 *:3000 running (3m) - 7m 90.7M - 9.4.7 2c41d148cca3 902bff181c85 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi135.srojfl smithi135 running (6m) - 6m 18.1M - 17.2.6-1123-g61431f6b 0db14aa01094 37783bce2e06 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi144.plbefj smithi144 running (6m) - 6m 15.4M - 17.2.6-1123-g61431f6b 0db14aa01094 af9c7e8247c3 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.x smithi144 *:8443,9283,8765 running (3m) - 11m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 81770c134986 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.y smithi135 *:8443,9283,8765 running (3m) - 13m 504M - 18.2.0-460-gbb0daeff e32c2a64c126 eb6afb9206f5 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:mon.a smithi135 running (2m) - 13m 58.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 7fa1f78d4725 2023-09-23T19:48:22.054 INFO:teuthology.orchestra.run.smithi135.stdout:mon.b smithi144 running (71s) - 11m 27.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 4fee8ae2a296 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:mon.c smithi135 running (86s) - 11m 43.8M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 6b8308cd56e0 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.a smithi135 *:9100 running (3m) - 7m 16.7M - 1.5.0 0da6a335fe13 caa9af75261e 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.b smithi144 *:9100 running (3m) - 6m 11.8M - 1.5.0 0da6a335fe13 a8b1ed1aeea7 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 smithi135 running (11m) - 11m 88.0M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 af561f634ba3 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1 smithi135 running (10m) - 10m 105M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 791e31e1318a 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 smithi135 running (10m) - 10m 82.1M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 0ee5fdb5fb5e 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 smithi135 running (9m) - 9m 73.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 0fd086d2e07f 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 smithi144 running (9m) - 9m 74.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 076c88452d16 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 smithi144 running (9m) - 9m 94.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 30c9f1e12bc8 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 smithi144 running (8m) - 8m 75.9M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 d04b0ed25c0f 2023-09-23T19:48:22.055 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 smithi144 running (8m) - 8m 96.3M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 4b2a00379dc8 2023-09-23T19:48:22.056 INFO:teuthology.orchestra.run.smithi135.stdout:prometheus.a smithi144 *:9095 running (3m) - 7m 49.2M - 2.43.0 a07b618ecd1d b9776c7278f3 2023-09-23T19:48:22.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:22 smithi135 ceph-mon[221113]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:22.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:22.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:22 smithi135 ceph-mon[213871]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:22.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:22 smithi144 ceph-mon[194833]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:22.931 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T19:48:23.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:23 smithi135 ceph-mon[221113]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:23.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:23 smithi135 ceph-mon[213871]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:23 smithi144 ceph-mon[194833]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:23.804 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:48:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:48:23] "GET /metrics HTTP/1.1" 200 33777 "" "Prometheus/2.43.0" 2023-09-23T19:48:24.289 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:24 smithi135 ceph-mon[221113]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:24.289 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:24 smithi135 systemd[1]: Stopping Ceph osd.0 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:48:24.290 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:24 smithi135 ceph-mon[213871]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:24 smithi144 ceph-mon[194833]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:25.292 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:25 smithi135 ceph-mon[221113]: osd.0 marked itself down and dead 2023-09-23T19:48:25.293 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:25 smithi135 conmon[124960]: 2023-09-23T19:48:25.036+0000 7f3761086700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:48:25.293 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:25 smithi135 conmon[124960]: 2023-09-23T19:48:25.036+0000 7f3761086700 -1 osd.0 90 *** Got signal Terminated *** 2023-09-23T19:48:25.293 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:25 smithi135 conmon[124960]: 2023-09-23T19:48:25.036+0000 7f3761086700 -1 osd.0 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:48:25.293 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:25 smithi135 ceph-mon[213871]: osd.0 marked itself down and dead 2023-09-23T19:48:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:25 smithi144 ceph-mon[194833]: osd.0 marked itself down and dead 2023-09-23T19:48:25.713 INFO:teuthology.orchestra.run.smithi135.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:48:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[213871]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[213871]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:48:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[213871]: failed to encode map e91 with expected crc 2023-09-23T19:48:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[213871]: failed to encode map e91 with expected crc 2023-09-23T19:48:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[213871]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T19:48:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[221113]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[221113]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:48:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[221113]: failed to encode map e91 with expected crc 2023-09-23T19:48:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[221113]: failed to encode map e91 with expected crc 2023-09-23T19:48:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[221113]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T19:48:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:26 smithi144 ceph-mon[194833]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:26 smithi144 ceph-mon[194833]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:48:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:26 smithi144 ceph-mon[194833]: failed to encode map e91 with expected crc 2023-09-23T19:48:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:26 smithi144 ceph-mon[194833]: failed to encode map e91 with expected crc 2023-09-23T19:48:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:26 smithi144 ceph-mon[194833]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T19:48:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:26.760 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T19:48:26.970 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:26 smithi135 bash[233714]: af561f634ba390d4a0c9beb5fbc7270a7b385c2d72251ca0add641e4a4eabb13 2023-09-23T19:48:27.186 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:48:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[221113]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:27 smithi135 ceph-mon[213871]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:48:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:48:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T19:48:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:27 smithi144 ceph-mon[194833]: failed to encode map e92 with expected crc 2023-09-23T19:48:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:28 smithi135 ceph-mon[221113]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:28 smithi135 ceph-mon[213871]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:28 smithi144 ceph-mon[194833]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:29.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:48:28 smithi144 conmon[178718]: ts=2023-09-23T19:48:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:48:29.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:29 smithi135 ceph-mon[221113]: pgmap v91: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:29.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:29 smithi135 ceph-mon[213871]: pgmap v91: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:29.221 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:28 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T19:48:29.221 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:28 smithi135 systemd[1]: Stopped Ceph osd.0 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:48:29.221 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:29 smithi135 systemd[1]: Starting Ceph osd.0 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:48:29.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:29 smithi144 ceph-mon[194833]: pgmap v91: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:30 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:48:30.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:30 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:48:30.471 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:30 smithi135 bash[234836]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:48:30.471 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:30 smithi135 bash[234836]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T19:48:30.471 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:30 smithi135 bash[234836]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T19:48:30.472 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:30 smithi135 bash[234836]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T19:48:30.472 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:30 smithi135 bash[234836]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T19:48:30.472 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:30 smithi135 bash[234836]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T19:48:30.472 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:30 smithi135 bash[234836]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T19:48:30.472 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:30 smithi135 bash[234836]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T19:48:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:30 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T19:48:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:31 smithi135 ceph-mon[221113]: pgmap v92: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:31.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:31 smithi135 ceph-mon[213871]: pgmap v92: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:31.471 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:31 smithi135 ceph-osd[235328]: -- 172.21.15.135:0/3965557034 <== mon.0 v2:172.21.15.135:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5573b283c4e0 con 0x5573b280f400 2023-09-23T19:48:31.471 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:31 smithi135 bash[235263]: 28ca011335d607c5ae8af5254f8b5921b8fb53099a87b43ae6c625791925e59e 2023-09-23T19:48:31.471 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:31 smithi135 systemd[1]: Started Ceph osd.0 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:48:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:31 smithi144 ceph-mon[194833]: pgmap v92: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:32.220 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:31 smithi135 conmon[235304]: 2023-09-23T19:48:31.831+0000 7fe357732640 -1 Falling back to public interface 2023-09-23T19:48:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:32 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:32 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:32.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:32 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:32 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:32 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:32 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:32.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:32 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:32 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:32 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:33 smithi144 ceph-mon[194833]: pgmap v93: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:33.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:33 smithi135 ceph-mon[221113]: pgmap v93: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:33.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:33 smithi135 ceph-mon[213871]: pgmap v93: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:48:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:48:33] "GET /metrics HTTP/1.1" 200 33777 "" "Prometheus/2.43.0" 2023-09-23T19:48:34.970 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:34 smithi135 conmon[235304]: 2023-09-23T19:48:34.582+0000 7fe357732640 -1 osd.0 90 log_to_monitors true 2023-09-23T19:48:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:35 smithi144 ceph-mon[194833]: from='osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:48:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:35 smithi144 ceph-mon[194833]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:48:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[221113]: from='osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[221113]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[213871]: from='osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:48:35.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[213871]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T19:48:35.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:35.970 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 19:48:35 smithi135 conmon[235304]: 2023-09-23T19:48:35.572+0000 7fe346b9b700 -1 osd.0 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:48:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:36 smithi144 ceph-mon[194833]: pgmap v94: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:36 smithi144 ceph-mon[194833]: failed to encode map e93 with expected crc 2023-09-23T19:48:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:36 smithi144 ceph-mon[194833]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:48:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:36 smithi144 ceph-mon[194833]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T19:48:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:36 smithi144 ceph-mon[194833]: from='osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:48:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:36 smithi144 ceph-mon[194833]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:48:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[221113]: pgmap v94: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[221113]: failed to encode map e93 with expected crc 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[221113]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[221113]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[221113]: from='osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[221113]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[213871]: pgmap v94: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[213871]: failed to encode map e93 with expected crc 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[213871]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[213871]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T19:48:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[213871]: from='osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:48:36.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:36 smithi135 ceph-mon[213871]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:48:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:37 smithi144 ceph-mon[194833]: failed to encode map e93 with expected crc 2023-09-23T19:48:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:37 smithi144 ceph-mon[194833]: failed to encode map e93 with expected crc 2023-09-23T19:48:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:37 smithi144 ceph-mon[194833]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:48:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:37 smithi144 ceph-mon[194833]: failed to encode map e94 with expected crc 2023-09-23T19:48:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:37 smithi144 ceph-mon[194833]: osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857] boot 2023-09-23T19:48:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:37 smithi144 ceph-mon[194833]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T19:48:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[213871]: failed to encode map e93 with expected crc 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[213871]: failed to encode map e93 with expected crc 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[213871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[213871]: failed to encode map e94 with expected crc 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[213871]: osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857] boot 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[213871]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[221113]: failed to encode map e93 with expected crc 2023-09-23T19:48:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[221113]: failed to encode map e93 with expected crc 2023-09-23T19:48:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[221113]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:48:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[221113]: failed to encode map e94 with expected crc 2023-09-23T19:48:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[221113]: osd.0 [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857] boot 2023-09-23T19:48:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[221113]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T19:48:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T19:48:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:38 smithi144 ceph-mon[194833]: failed to encode map e94 with expected crc 2023-09-23T19:48:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:38 smithi144 ceph-mon[194833]: pgmap v97: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:38 smithi144 ceph-mon[194833]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T19:48:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:38 smithi144 ceph-mon[194833]: failed to encode map e95 with expected crc 2023-09-23T19:48:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:38 smithi144 ceph-mon[194833]: failed to encode map e95 with expected crc 2023-09-23T19:48:38.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[221113]: failed to encode map e94 with expected crc 2023-09-23T19:48:38.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[221113]: pgmap v97: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:38.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:38.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:38.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[221113]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T19:48:38.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[221113]: failed to encode map e95 with expected crc 2023-09-23T19:48:38.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[221113]: failed to encode map e95 with expected crc 2023-09-23T19:48:38.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[213871]: failed to encode map e94 with expected crc 2023-09-23T19:48:38.386 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[213871]: pgmap v97: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T19:48:38.386 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:38.386 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:38.386 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[213871]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T19:48:38.386 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[213871]: failed to encode map e95 with expected crc 2023-09-23T19:48:38.386 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:38 smithi135 ceph-mon[213871]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.096 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:48:38 smithi144 conmon[178718]: ts=2023-09-23T19:48:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:48:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: failed to encode map e95 with expected crc 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:48:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:40 smithi144 ceph-mon[194833]: pgmap v99: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:48:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:40 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:48:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:40 smithi135 ceph-mon[221113]: pgmap v99: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:48:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:40 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:48:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:40 smithi135 ceph-mon[213871]: pgmap v99: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:48:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:40 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:48:41.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:41 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T19:48:41.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:41 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T19:48:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:41 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T19:48:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:41 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T19:48:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:41 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T19:48:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:41 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T19:48:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:42 smithi135 ceph-mon[221113]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:42.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:42 smithi135 ceph-mon[213871]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:42 smithi144 ceph-mon[194833]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:48:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:48:43] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2023-09-23T19:48:44.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:44 smithi135 ceph-mon[221113]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:44 smithi135 ceph-mon[213871]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:44 smithi144 ceph-mon[194833]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:46.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:46 smithi135 ceph-mon[213871]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:46 smithi135 ceph-mon[221113]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:46 smithi144 ceph-mon[194833]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:48 smithi144 ceph-mon[194833]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:48 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:48.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:48 smithi135 ceph-mon[221113]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:48.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:48 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:48 smithi135 ceph-mon[213871]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:48 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:49.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:48:48 smithi144 conmon[178718]: ts=2023-09-23T19:48:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:48:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:50 smithi144 ceph-mon[194833]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:50 smithi135 ceph-mon[213871]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:50 smithi135 ceph-mon[221113]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:52 smithi144 ceph-mon[194833]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:52 smithi135 ceph-mon[221113]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:52 smithi135 ceph-mon[213871]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:48:52.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:48:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:48:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:48:53] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2023-09-23T19:48:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:54 smithi144 ceph-mon[194833]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:54 smithi135 ceph-mon[221113]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:54 smithi135 ceph-mon[213871]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:56 smithi144 ceph-mon[194833]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:56.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:56 smithi135 ceph-mon[221113]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:56 smithi135 ceph-mon[213871]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:48:58 smithi144 ceph-mon[194833]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:58.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:48:58 smithi135 ceph-mon[213871]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:58.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:48:58 smithi135 ceph-mon[221113]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:48:59.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:48:58 smithi144 conmon[178718]: ts=2023-09-23T19:48:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:49:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:00 smithi144 ceph-mon[194833]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:00 smithi135 ceph-mon[221113]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:00 smithi135 ceph-mon[213871]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:02 smithi144 ceph-mon[194833]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:02.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:02 smithi135 ceph-mon[213871]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:02.504 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:02 smithi135 ceph-mon[221113]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:49:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:49:03] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2023-09-23T19:49:04.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:04 smithi135 ceph-mon[213871]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:04.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:04 smithi135 ceph-mon[221113]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:04 smithi144 ceph-mon[194833]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:06 smithi135 ceph-mon[221113]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:06.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:06 smithi135 ceph-mon[213871]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:06 smithi144 ceph-mon[194833]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:08 smithi144 ceph-mon[194833]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:08.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:08 smithi135 ceph-mon[213871]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:08.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:08 smithi135 ceph-mon[221113]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:09.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:49:08 smithi144 conmon[178718]: ts=2023-09-23T19:49:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:49:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:10 smithi144 ceph-mon[194833]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:10 smithi135 ceph-mon[221113]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:10 smithi135 ceph-mon[213871]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:12 smithi144 ceph-mon[194833]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:12 smithi135 ceph-mon[221113]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:12 smithi135 ceph-mon[213871]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:49:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:49:13] "GET /metrics HTTP/1.1" 200 33842 "" "Prometheus/2.43.0" 2023-09-23T19:49:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:14 smithi144 ceph-mon[194833]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:14 smithi135 ceph-mon[221113]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:14.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:14 smithi135 ceph-mon[213871]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:16 smithi144 ceph-mon[194833]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:16 smithi135 ceph-mon[221113]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:16 smithi135 ceph-mon[213871]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:18 smithi144 ceph-mon[194833]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:18.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:18 smithi135 ceph-mon[221113]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:18.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:18 smithi135 ceph-mon[213871]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:19.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:49:18 smithi144 conmon[178718]: ts=2023-09-23T19:49:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:49:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:20 smithi144 ceph-mon[194833]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:20.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:20 smithi135 ceph-mon[221113]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:20.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:20 smithi135 ceph-mon[213871]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:22 smithi144 ceph-mon[194833]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:22 smithi135 ceph-mon[221113]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:22 smithi135 ceph-mon[213871]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:49:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:49:23] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2023-09-23T19:49:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:24 smithi144 ceph-mon[194833]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:24 smithi135 ceph-mon[221113]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:24.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:24 smithi135 ceph-mon[213871]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:26 smithi144 ceph-mon[194833]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:26 smithi135 ceph-mon[221113]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:26 smithi135 ceph-mon[213871]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:28 smithi144 ceph-mon[194833]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:28 smithi135 ceph-mon[221113]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:28 smithi135 ceph-mon[213871]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:29.131 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 19:49:28 smithi144 conmon[178718]: ts=2023-09-23T19:49:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi135:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.135\", device_class=\"ssd\", hostname=\"smithi135\", instance=\"smithi144.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.135\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T19:49:30.109 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T19:49:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:30 smithi144 ceph-mon[194833]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:30.460 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:30 smithi135 ceph-mon[221113]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:30.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:30 smithi135 ceph-mon[213871]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:32.211 INFO:teuthology.orchestra.run.smithi135.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:49:32.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:32 smithi144 ceph-mon[194833]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:32.455 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:32 smithi135 ceph-mon[221113]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:32.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:32 smithi135 ceph-mon[213871]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:32.995 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T19:49:33.297 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:49:33.297 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:49:33.297 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:33.297 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: Upgrade: osd.1 is safe to restart 2023-09-23T19:49:33.298 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[221113]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:33.299 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.300 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.300 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.300 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:49:33.300 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:49:33.300 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: Upgrade: osd.1 is safe to restart 2023-09-23T19:49:33.300 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:33 smithi135 ceph-mon[213871]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:33.551 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:49:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:49:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:33.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:33.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:49:33.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T19:49:33.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: Upgrade: osd.1 is safe to restart 2023-09-23T19:49:33.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:33 smithi144 ceph-mon[194833]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:33.879 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:49:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:49:33] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2023-09-23T19:49:35.608 INFO:teuthology.orchestra.run.smithi135.stdout:wait for half of osds to upgrade 2023-09-23T19:49:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:36 smithi144 ceph-mon[194833]: Upgrade: Updating osd.1 2023-09-23T19:49:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:49:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:36 smithi144 ceph-mon[194833]: Deploying daemon osd.1 on smithi135 2023-09-23T19:49:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:36 smithi144 ceph-mon[194833]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[221113]: Upgrade: Updating osd.1 2023-09-23T19:49:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:49:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[221113]: Deploying daemon osd.1 on smithi135 2023-09-23T19:49:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[221113]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[213871]: Upgrade: Updating osd.1 2023-09-23T19:49:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T19:49:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[213871]: Deploying daemon osd.1 on smithi135 2023-09-23T19:49:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:36 smithi135 ceph-mon[213871]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:36.493 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T19:49:37.283 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:49:37.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:38.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:38 smithi135 ceph-mon[213871]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:38 smithi135 ceph-mon[221113]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:38.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:38 smithi144 ceph-mon[194833]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:39.720 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:39 smithi135 systemd[1]: Stopping Ceph osd.1 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:49:40.141 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:39 smithi135 conmon[134619]: 2023-09-23T19:49:39.775+0000 7f89d149b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:49:40.142 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:39 smithi135 conmon[134619]: 2023-09-23T19:49:39.775+0000 7f89d149b700 -1 osd.1 95 *** Got signal Terminated *** 2023-09-23T19:49:40.142 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:39 smithi135 conmon[134619]: 2023-09-23T19:49:39.775+0000 7f89d149b700 -1 osd.1 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:49:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:40 smithi135 ceph-mon[221113]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:40 smithi135 ceph-mon[221113]: osd.1 marked itself down and dead 2023-09-23T19:49:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:40 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2583973458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:40 smithi135 ceph-mon[213871]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:40 smithi135 ceph-mon[213871]: osd.1 marked itself down and dead 2023-09-23T19:49:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:40 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2583973458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:40 smithi144 ceph-mon[194833]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:40 smithi144 ceph-mon[194833]: osd.1 marked itself down and dead 2023-09-23T19:49:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:40 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2583973458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:41.440 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:41 smithi135 ceph-mon[213871]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:49:41.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:41 smithi135 ceph-mon[213871]: failed to encode map e96 with expected crc 2023-09-23T19:49:41.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:41 smithi135 ceph-mon[213871]: failed to encode map e96 with expected crc 2023-09-23T19:49:41.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:41 smithi135 ceph-mon[213871]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T19:49:41.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:41 smithi135 ceph-mon[221113]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:49:41.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:41 smithi135 ceph-mon[221113]: failed to encode map e96 with expected crc 2023-09-23T19:49:41.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:41 smithi135 ceph-mon[221113]: failed to encode map e96 with expected crc 2023-09-23T19:49:41.442 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:41 smithi135 ceph-mon[221113]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T19:49:41.442 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:41 smithi135 bash[242735]: 791e31e1318a1e67eed5481206f0ba70b16ba213522ca1c057a5e0f0d97732fa 2023-09-23T19:49:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:41 smithi144 ceph-mon[194833]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:49:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:41 smithi144 ceph-mon[194833]: failed to encode map e96 with expected crc 2023-09-23T19:49:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:41 smithi144 ceph-mon[194833]: failed to encode map e96 with expected crc 2023-09-23T19:49:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:41 smithi144 ceph-mon[194833]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T19:49:42.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: pgmap v131: 73 pgs: 3 peering, 9 stale+active+clean, 61 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:42.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:49:42.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[221113]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: pgmap v131: 73 pgs: 3 peering, 9 stale+active+clean, 61 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:42.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.442 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.443 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:42 smithi135 ceph-mon[213871]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: pgmap v131: 73 pgs: 3 peering, 9 stale+active+clean, 61 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:42 smithi144 ceph-mon[194833]: failed to encode map e97 with expected crc 2023-09-23T19:49:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3068850267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3068850267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:43.471 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:43 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T19:49:43.471 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:43 smithi135 systemd[1]: Stopped Ceph osd.1 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:49:43.471 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:43 smithi135 systemd[1]: Starting Ceph osd.1 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:49:43.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3068850267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:49:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:49:43] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2023-09-23T19:49:44.437 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:44 smithi135 ceph-mon[213871]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 3 peering, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:49:44.437 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:44 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:49:44.438 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:44 smithi135 ceph-mon[221113]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 3 peering, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:49:44.438 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:44 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:49:44.438 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:44 smithi135 bash[243400]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:49:44.438 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:44 smithi135 bash[243400]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T19:49:44.438 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:44 smithi135 bash[243400]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T19:49:44.438 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:44 smithi135 bash[243400]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T19:49:44.438 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:44 smithi135 bash[243400]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T19:49:44.438 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:44 smithi135 bash[243400]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T19:49:44.439 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:44 smithi135 bash[243400]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T19:49:44.439 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:44 smithi135 bash[243400]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T19:49:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:44 smithi144 ceph-mon[194833]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 3 peering, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:49:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:44 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:49:45.439 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2934272454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:45.440 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2934272454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2934272454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:45.720 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:45 smithi135 ceph-osd[243924]: -- 172.21.15.135:0/555158728 <== mon.2 v2:172.21.15.135:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55677a2764e0 con 0x55677a248c00 2023-09-23T19:49:45.720 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:45 smithi135 bash[243859]: aa8c0ac7e9ea7c1f63854f6cc83a86ec2c95b5e3f4330a1c615187ce3c39290b 2023-09-23T19:49:45.720 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:45 smithi135 systemd[1]: Started Ceph osd.1 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:49:46.220 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:45 smithi135 conmon[243900]: 2023-09-23T19:49:45.960+0000 7eff0d3b9640 -1 Falling back to public interface 2023-09-23T19:49:46.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:46 smithi135 ceph-mon[221113]: pgmap v134: 73 pgs: 3 active+undersized+degraded, 3 peering, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:49:46.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:46 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:46.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:46 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:46.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:46 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:46 smithi135 ceph-mon[213871]: pgmap v134: 73 pgs: 3 active+undersized+degraded, 3 peering, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:49:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:46 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:46 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:46 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:46 smithi144 ceph-mon[194833]: pgmap v134: 73 pgs: 3 active+undersized+degraded, 3 peering, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:49:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:46 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:46 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:46 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3381015387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3381015387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3381015387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:48.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:48 smithi135 ceph-mon[221113]: pgmap v135: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T19:49:48.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:48 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2023-09-23T19:49:48.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:48 smithi135 ceph-mon[213871]: pgmap v135: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T19:49:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:48 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2023-09-23T19:49:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:48 smithi144 ceph-mon[194833]: pgmap v135: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T19:49:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:48 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2023-09-23T19:49:49.220 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:48 smithi135 conmon[243900]: 2023-09-23T19:49:48.725+0000 7eff0d3b9640 -1 osd.1 95 log_to_monitors true 2023-09-23T19:49:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:49:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:49:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: pgmap v136: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T19:49:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:49 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2790791689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:49.690 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 19:49:49 smithi135 conmon[243900]: 2023-09-23T19:49:49.374+0000 7efefb820700 -1 osd.1 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:49:49.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:49:49.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:49:49.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: pgmap v136: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2790791689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:49.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: pgmap v136: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T19:49:49.692 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:49 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2790791689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:50 smithi144 ceph-mon[194833]: failed to encode map e98 with expected crc 2023-09-23T19:49:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:50 smithi144 ceph-mon[194833]: failed to encode map e98 with expected crc 2023-09-23T19:49:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:50 smithi144 ceph-mon[194833]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:49:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:50 smithi144 ceph-mon[194833]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T19:49:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:50 smithi144 ceph-mon[194833]: from='osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:49:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:50 smithi144 ceph-mon[194833]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:49:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:50 smithi144 ceph-mon[194833]: failed to encode map e98 with expected crc 2023-09-23T19:49:50.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[221113]: failed to encode map e98 with expected crc 2023-09-23T19:49:50.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[221113]: failed to encode map e98 with expected crc 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[221113]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[221113]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[221113]: from='osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[221113]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[221113]: failed to encode map e98 with expected crc 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[213871]: failed to encode map e98 with expected crc 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[213871]: failed to encode map e98 with expected crc 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[213871]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T19:49:50.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[213871]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T19:49:50.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[213871]: from='osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:49:50.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[213871]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:49:50.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:50 smithi135 ceph-mon[213871]: failed to encode map e98 with expected crc 2023-09-23T19:49:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: failed to encode map e98 with expected crc 2023-09-23T19:49:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: failed to encode map e98 with expected crc 2023-09-23T19:49:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:49:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: failed to encode map e99 with expected crc 2023-09-23T19:49:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760] boot 2023-09-23T19:49:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T19:49:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:49:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: pgmap v139: 73 pgs: 2 peering, 4 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:49:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:51 smithi144 ceph-mon[194833]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T19:49:51.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: failed to encode map e98 with expected crc 2023-09-23T19:49:51.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: failed to encode map e98 with expected crc 2023-09-23T19:49:51.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:49:51.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: failed to encode map e99 with expected crc 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760] boot 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: pgmap v139: 73 pgs: 2 peering, 4 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[221113]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: failed to encode map e98 with expected crc 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: failed to encode map e98 with expected crc 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:49:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: failed to encode map e99 with expected crc 2023-09-23T19:49:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: osd.1 [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760] boot 2023-09-23T19:49:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T19:49:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T19:49:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: pgmap v139: 73 pgs: 2 peering, 4 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:49:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:51 smithi135 ceph-mon[213871]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T19:49:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3780026922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:52.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3780026922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: failed to encode map e100 with expected crc 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3780026922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:49:52.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.220 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:49:52 smithi135 conmon[201366]: 2023-09-23T19:49:52.770+0000 7fcf49f9e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T19:49:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:49:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:49:53.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T19:49:53.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:53 smithi144 ceph-mon[194833]: pgmap v141: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:49:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:49:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:49:53] "GET /metrics HTTP/1.1" 200 33700 "" "Prometheus/2.43.0" 2023-09-23T19:49:53.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:49:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[213871]: pgmap v141: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:49:53.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:49:53.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:49:53.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T19:49:53.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:53 smithi135 ceph-mon[221113]: pgmap v141: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:49:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2058976608' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:54.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2058976608' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:54.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2058976608' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:55.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:55 smithi135 ceph-mon[221113]: pgmap v142: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:55.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:55 smithi135 ceph-mon[213871]: pgmap v142: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:56.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:55 smithi144 ceph-mon[194833]: pgmap v142: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:56.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:56 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T19:49:56.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:56 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T19:49:56.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2824191675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:56.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:56 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T19:49:56.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:56 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T19:49:56.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2824191675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:57.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:56 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T19:49:57.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:56 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T19:49:57.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2824191675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:58 smithi144 ceph-mon[194833]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:58.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:58 smithi135 ceph-mon[221113]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:58.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:58 smithi135 ceph-mon[213871]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:49:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:49:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1537423311' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:59.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:49:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1537423311' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:49:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:49:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1537423311' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:00 smithi144 ceph-mon[194833]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:00 smithi144 ceph-mon[194833]: overall HEALTH_OK 2023-09-23T19:50:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:00 smithi135 ceph-mon[221113]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:00 smithi135 ceph-mon[221113]: overall HEALTH_OK 2023-09-23T19:50:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:00 smithi135 ceph-mon[213871]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:00 smithi135 ceph-mon[213871]: overall HEALTH_OK 2023-09-23T19:50:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:01 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3884875807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:01.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:01 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3884875807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:01 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3884875807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:02 smithi144 ceph-mon[194833]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:02 smithi135 ceph-mon[221113]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:02.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:02 smithi135 ceph-mon[213871]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:03.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:03 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:03.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:03 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1211036558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:03.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:03 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:03 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1211036558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:03 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:03 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1211036558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:50:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:50:03] "GET /metrics HTTP/1.1" 200 33700 "" "Prometheus/2.43.0" 2023-09-23T19:50:04.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:04 smithi135 ceph-mon[213871]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:04 smithi135 ceph-mon[221113]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:04 smithi144 ceph-mon[194833]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:06.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:06 smithi135 ceph-mon[213871]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:06.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4248351421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:06 smithi135 ceph-mon[221113]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4248351421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:06 smithi144 ceph-mon[194833]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4248351421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:08.282 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[221113]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:08.282 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:08.282 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:08.283 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4158863474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:08.283 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:50:08.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[213871]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:08.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:08.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:08.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4158863474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:08.283 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:50:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:08 smithi144 ceph-mon[194833]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:08 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4158863474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:50:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:09 smithi144 ceph-mon[194833]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:50:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:09 smithi144 ceph-mon[194833]: Upgrade: osd.2 is safe to restart 2023-09-23T19:50:09.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:09 smithi135 ceph-mon[221113]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:50:09.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:09 smithi135 ceph-mon[221113]: Upgrade: osd.2 is safe to restart 2023-09-23T19:50:09.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:09 smithi135 ceph-mon[213871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T19:50:09.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:09 smithi135 ceph-mon[213871]: Upgrade: osd.2 is safe to restart 2023-09-23T19:50:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:10 smithi144 ceph-mon[194833]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:10 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:10 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:50:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:10 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:10.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/199629781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[213871]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:50:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/199629781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[221113]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T19:50:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/199629781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:11 smithi144 ceph-mon[194833]: Upgrade: Updating osd.2 2023-09-23T19:50:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:11 smithi144 ceph-mon[194833]: Deploying daemon osd.2 on smithi135 2023-09-23T19:50:11.383 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:11 smithi135 ceph-mon[221113]: Upgrade: Updating osd.2 2023-09-23T19:50:11.384 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:11 smithi135 ceph-mon[221113]: Deploying daemon osd.2 on smithi135 2023-09-23T19:50:11.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:11 smithi135 ceph-mon[213871]: Upgrade: Updating osd.2 2023-09-23T19:50:11.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:11 smithi135 ceph-mon[213871]: Deploying daemon osd.2 on smithi135 2023-09-23T19:50:12.359 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:12 smithi135 ceph-mon[221113]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:12.359 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:12 smithi135 ceph-mon[213871]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:12 smithi144 ceph-mon[194833]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:12.720 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:12 smithi135 systemd[1]: Stopping Ceph osd.2 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:50:12.720 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:12 smithi135 conmon[143550]: 2023-09-23T19:50:12.587+0000 7f019559a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:50:12.720 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:12 smithi135 conmon[143550]: 2023-09-23T19:50:12.587+0000 7f019559a700 -1 osd.2 100 *** Got signal Terminated *** 2023-09-23T19:50:12.720 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:12 smithi135 conmon[143550]: 2023-09-23T19:50:12.587+0000 7f019559a700 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:50:13.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1546214990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:13.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:13 smithi144 ceph-mon[194833]: osd.2 marked itself down and dead 2023-09-23T19:50:13.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1546214990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:13.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:13 smithi135 ceph-mon[221113]: osd.2 marked itself down and dead 2023-09-23T19:50:13.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1546214990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:13 smithi135 ceph-mon[213871]: osd.2 marked itself down and dead 2023-09-23T19:50:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:50:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:50:13] "GET /metrics HTTP/1.1" 200 33842 "" "Prometheus/2.43.0" 2023-09-23T19:50:13.971 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:13 smithi135 bash[250999]: 0ee5fdb5fb5eb8bd7d7d68b6531e86e7af59270136f3687332babcecb357ef82 2023-09-23T19:50:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:14 smithi144 ceph-mon[194833]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:14 smithi144 ceph-mon[194833]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:50:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:14 smithi144 ceph-mon[194833]: failed to encode map e101 with expected crc 2023-09-23T19:50:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:14 smithi144 ceph-mon[194833]: failed to encode map e101 with expected crc 2023-09-23T19:50:14.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:14 smithi144 ceph-mon[194833]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T19:50:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[213871]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[213871]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:50:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[213871]: failed to encode map e101 with expected crc 2023-09-23T19:50:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[213871]: failed to encode map e101 with expected crc 2023-09-23T19:50:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[213871]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T19:50:14.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[221113]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:14.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[221113]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:50:14.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[221113]: failed to encode map e101 with expected crc 2023-09-23T19:50:14.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[221113]: failed to encode map e101 with expected crc 2023-09-23T19:50:14.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:14 smithi135 ceph-mon[221113]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T19:50:15.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:15 smithi144 ceph-mon[194833]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:15 smithi144 ceph-mon[194833]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:15 smithi144 ceph-mon[194833]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T19:50:15.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:15 smithi144 ceph-mon[194833]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:15 smithi144 ceph-mon[194833]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:15 smithi144 ceph-mon[194833]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:15 smithi144 ceph-mon[194833]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3733071837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[221113]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[221113]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[221113]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[221113]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[221113]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[221113]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[221113]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3733071837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[213871]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[213871]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[213871]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[213871]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[213871]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[213871]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[213871]: failed to encode map e102 with expected crc 2023-09-23T19:50:15.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3733071837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:16.119 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:15 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T19:50:16.119 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:15 smithi135 systemd[1]: Stopped Ceph osd.2 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:50:16.119 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:15 smithi135 systemd[1]: Starting Ceph osd.2 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:50:16.375 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:16 smithi135 ceph-mon[221113]: pgmap v154: 73 pgs: 1 stale+active+clean, 5 active+undersized, 67 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:16.375 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:16 smithi135 ceph-mon[213871]: pgmap v154: 73 pgs: 1 stale+active+clean, 5 active+undersized, 67 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:16 smithi144 ceph-mon[194833]: pgmap v154: 73 pgs: 1 stale+active+clean, 5 active+undersized, 67 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:17.123 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2070096802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:17.124 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:16 smithi135 bash[251671]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:50:17.124 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:16 smithi135 bash[251671]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T19:50:17.124 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:16 smithi135 bash[251671]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T19:50:17.124 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:16 smithi135 bash[251671]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T19:50:17.124 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:16 smithi135 bash[251671]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T19:50:17.124 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:16 smithi135 bash[251671]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T19:50:17.124 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:16 smithi135 bash[251671]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T19:50:17.124 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:16 smithi135 bash[251671]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T19:50:17.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2070096802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:17.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2070096802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:18.126 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:18 smithi144 ceph-mon[194833]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:18.126 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:18 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:50:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:18 smithi135 ceph-mon[213871]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:18.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:18 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:50:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:18 smithi135 ceph-mon[221113]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:18.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:18 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:50:18.471 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:18 smithi135 ceph-osd[252183]: -- 172.21.15.135:0/1620717305 <== mon.2 v2:172.21.15.135:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5645c34d44e0 con 0x5645c34a6c00 2023-09-23T19:50:18.471 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:18 smithi135 bash[252118]: 4d5310d5a38b331f4ab04398bb045b512e2602d2cc1ffc3f687f9a9a07621b62 2023-09-23T19:50:18.471 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:18 smithi135 systemd[1]: Started Ceph osd.2 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:50:18.970 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:18 smithi135 conmon[252159]: 2023-09-23T19:50:18.631+0000 7f97ba364640 -1 Falling back to public interface 2023-09-23T19:50:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:19.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:20.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:20 smithi135 ceph-mon[221113]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:50:20.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2315634562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:20.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:20 smithi135 ceph-mon[213871]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:50:20.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2315634562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:20 smithi144 ceph-mon[194833]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:50:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2315634562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:21.720 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:21 smithi135 conmon[252159]: 2023-09-23T19:50:21.370+0000 7f97ba364640 -1 osd.2 100 log_to_monitors true 2023-09-23T19:50:22.220 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 19:50:22 smithi135 conmon[252159]: 2023-09-23T19:50:22.002+0000 7f97aafd0700 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:50:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[213871]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:50:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[213871]: from='osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:50:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[213871]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:50:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/580207773' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[221113]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:50:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[221113]: from='osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:50:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[221113]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:50:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/580207773' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:21 smithi144 ceph-mon[194833]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:50:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:21 smithi144 ceph-mon[194833]: from='osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:50:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:21 smithi144 ceph-mon[194833]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T19:50:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/580207773' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:22 smithi144 ceph-mon[194833]: failed to encode map e103 with expected crc 2023-09-23T19:50:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:22 smithi144 ceph-mon[194833]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:50:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:22 smithi144 ceph-mon[194833]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T19:50:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:22 smithi144 ceph-mon[194833]: from='osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:22 smithi144 ceph-mon[194833]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:22 smithi144 ceph-mon[194833]: failed to encode map e103 with expected crc 2023-09-23T19:50:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[221113]: failed to encode map e103 with expected crc 2023-09-23T19:50:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[221113]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[221113]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[221113]: from='osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[221113]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[221113]: failed to encode map e103 with expected crc 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[213871]: failed to encode map e103 with expected crc 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[213871]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[213871]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T19:50:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[213871]: from='osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[213871]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:22 smithi135 ceph-mon[213871]: failed to encode map e103 with expected crc 2023-09-23T19:50:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:50:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:50:23] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-23T19:50:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: failed to encode map e103 with expected crc 2023-09-23T19:50:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:50:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: failed to encode map e104 with expected crc 2023-09-23T19:50:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: failed to encode map e104 with expected crc 2023-09-23T19:50:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510] boot 2023-09-23T19:50:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T19:50:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:50:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: pgmap v160: 73 pgs: 11 peering, 1 active+undersized+degraded, 7 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:24 smithi144 ceph-mon[194833]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T19:50:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: failed to encode map e103 with expected crc 2023-09-23T19:50:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:50:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: failed to encode map e104 with expected crc 2023-09-23T19:50:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: failed to encode map e104 with expected crc 2023-09-23T19:50:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510] boot 2023-09-23T19:50:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T19:50:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:50:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: pgmap v160: 73 pgs: 11 peering, 1 active+undersized+degraded, 7 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:24.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[221113]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T19:50:24.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: failed to encode map e103 with expected crc 2023-09-23T19:50:24.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:50:24.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: failed to encode map e104 with expected crc 2023-09-23T19:50:24.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: failed to encode map e104 with expected crc 2023-09-23T19:50:24.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: osd.2 [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510] boot 2023-09-23T19:50:24.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T19:50:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T19:50:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: pgmap v160: 73 pgs: 11 peering, 1 active+undersized+degraded, 7 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:24 smithi135 ceph-mon[213871]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T19:50:25.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1232602027' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:25.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1232602027' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.387 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.388 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.388 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T19:50:25.388 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: failed to encode map e105 with expected crc 2023-09-23T19:50:25.388 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1232602027' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:25.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:50:25 smithi135 conmon[201366]: 2023-09-23T19:50:25.592+0000 7fcf49f9e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T19:50:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: pgmap v162: 73 pgs: 11 peering, 1 active+undersized+degraded, 3 active+undersized, 58 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: pgmap v162: 73 pgs: 11 peering, 1 active+undersized+degraded, 3 active+undersized, 58 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: pgmap v162: 73 pgs: 11 peering, 1 active+undersized+degraded, 3 active+undersized, 58 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:27.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:27 smithi144 ceph-mon[194833]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:27.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:27 smithi144 ceph-mon[194833]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T19:50:27.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:27 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3488730798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:27 smithi135 ceph-mon[221113]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:27 smithi135 ceph-mon[221113]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T19:50:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:27 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3488730798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:27 smithi135 ceph-mon[213871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:27 smithi135 ceph-mon[213871]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T19:50:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:27 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3488730798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:28 smithi135 ceph-mon[221113]: pgmap v163: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:28 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:28 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T19:50:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:28 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T19:50:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:28 smithi135 ceph-mon[213871]: pgmap v163: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:28 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:28 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T19:50:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:28 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T19:50:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:28 smithi144 ceph-mon[194833]: pgmap v163: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:28 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:28 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T19:50:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:28 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T19:50:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2113194187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:29.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2113194187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:29.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2113194187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:30 smithi135 ceph-mon[221113]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:30 smithi135 ceph-mon[213871]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:30 smithi144 ceph-mon[194833]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:31 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/715960431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:31.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:31 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/715960431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:31 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/715960431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:32 smithi135 ceph-mon[221113]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:32.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:32 smithi135 ceph-mon[213871]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:32 smithi144 ceph-mon[194833]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:50:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:50:33] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-23T19:50:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:34 smithi135 ceph-mon[221113]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:34 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1978972113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:34.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:34 smithi135 ceph-mon[213871]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:34.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:34 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1978972113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:34 smithi144 ceph-mon[194833]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:34.642 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:34 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1978972113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:36 smithi135 ceph-mon[221113]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/710739810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:36 smithi135 ceph-mon[213871]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/710739810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:36.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:36 smithi144 ceph-mon[194833]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:36.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/710739810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:38 smithi144 ceph-mon[194833]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1335201751' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[221113]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1335201751' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[213871]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1335201751' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:40 smithi144 ceph-mon[194833]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:40 smithi135 ceph-mon[221113]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:40 smithi135 ceph-mon[213871]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:41.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2700430918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:41.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:41 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:41.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2700430918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:41.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:41 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:41.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2700430918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:41.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:41 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:42 smithi144 ceph-mon[194833]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:42 smithi144 ceph-mon[194833]: Upgrade: osd.3 is safe to restart 2023-09-23T19:50:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:42 smithi144 ceph-mon[194833]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:42 smithi135 ceph-mon[221113]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:42 smithi135 ceph-mon[221113]: Upgrade: osd.3 is safe to restart 2023-09-23T19:50:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:42 smithi135 ceph-mon[221113]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:42 smithi135 ceph-mon[213871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T19:50:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:42 smithi135 ceph-mon[213871]: Upgrade: osd.3 is safe to restart 2023-09-23T19:50:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:42 smithi135 ceph-mon[213871]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[221113]: Upgrade: Updating osd.3 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[221113]: Deploying daemon osd.3 on smithi135 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/676757640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[221113]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[213871]: Upgrade: Updating osd.3 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:50:43.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:43.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[213871]: Deploying daemon osd.3 on smithi135 2023-09-23T19:50:43.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/676757640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:43.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:43 smithi135 ceph-mon[213871]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:43.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:43 smithi144 ceph-mon[194833]: Upgrade: Updating osd.3 2023-09-23T19:50:43.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:43 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:43.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:43 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T19:50:43.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:43 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:43 smithi144 ceph-mon[194833]: Deploying daemon osd.3 on smithi135 2023-09-23T19:50:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/676757640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:43 smithi144 ceph-mon[194833]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:43.921 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:50:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:50:43] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.43.0" 2023-09-23T19:50:45.343 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:44 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2840177130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:45.343 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:44 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2840177130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:44 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2840177130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:45.720 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:45 smithi135 systemd[1]: Stopping Ceph osd.3 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:50:45.720 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:45 smithi135 conmon[152525]: 2023-09-23T19:50:45.574+0000 7f65c53a6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:50:45.720 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:45 smithi135 conmon[152525]: 2023-09-23T19:50:45.574+0000 7f65c53a6700 -1 osd.3 105 *** Got signal Terminated *** 2023-09-23T19:50:45.720 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:45 smithi135 conmon[152525]: 2023-09-23T19:50:45.574+0000 7f65c53a6700 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:50:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:46 smithi144 ceph-mon[194833]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:46 smithi144 ceph-mon[194833]: osd.3 marked itself down and dead 2023-09-23T19:50:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:46 smithi135 ceph-mon[221113]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:46 smithi135 ceph-mon[221113]: osd.3 marked itself down and dead 2023-09-23T19:50:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:46 smithi135 ceph-mon[213871]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:46 smithi135 ceph-mon[213871]: osd.3 marked itself down and dead 2023-09-23T19:50:46.970 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:46 smithi135 bash[259258]: 0fd086d2e07fc174402d19457b0d371c401fe534ed895854745c2be2d8b67c82 2023-09-23T19:50:47.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:47 smithi144 ceph-mon[194833]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:50:47.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:47 smithi144 ceph-mon[194833]: failed to encode map e106 with expected crc 2023-09-23T19:50:47.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:47 smithi144 ceph-mon[194833]: failed to encode map e106 with expected crc 2023-09-23T19:50:47.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:47 smithi144 ceph-mon[194833]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T19:50:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:47 smithi135 ceph-mon[221113]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:50:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:47 smithi135 ceph-mon[221113]: failed to encode map e106 with expected crc 2023-09-23T19:50:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:47 smithi135 ceph-mon[221113]: failed to encode map e106 with expected crc 2023-09-23T19:50:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:47 smithi135 ceph-mon[221113]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T19:50:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:47 smithi135 ceph-mon[213871]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:50:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:47 smithi135 ceph-mon[213871]: failed to encode map e106 with expected crc 2023-09-23T19:50:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:47 smithi135 ceph-mon[213871]: failed to encode map e106 with expected crc 2023-09-23T19:50:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:47 smithi135 ceph-mon[213871]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T19:50:48.285 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: pgmap v175: 73 pgs: 4 peering, 11 stale+active+clean, 58 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:48.286 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3232289971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: pgmap v175: 73 pgs: 4 peering, 11 stale+active+clean, 58 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:48.288 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:48 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3232289971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: failed to encode map e107 with expected crc 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: pgmap v175: 73 pgs: 4 peering, 11 stale+active+clean, 58 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:50:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:48 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3232289971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:49.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:49 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:50:49.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:49 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:50:49.221 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:48 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T19:50:49.221 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:48 smithi135 systemd[1]: Stopped Ceph osd.3 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:50:49.221 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:49 smithi135 systemd[1]: Starting Ceph osd.3 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:50:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:49 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T19:50:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:50 smithi135 ceph-mon[221113]: pgmap v176: 73 pgs: 2 active+undersized+degraded, 4 peering, 5 stale+active+clean, 12 active+undersized, 50 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:50:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1302704758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:50 smithi135 ceph-mon[213871]: pgmap v176: 73 pgs: 2 active+undersized+degraded, 4 peering, 5 stale+active+clean, 12 active+undersized, 50 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:50:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1302704758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:50.471 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:50 smithi135 bash[259926]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T19:50:50.471 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:50 smithi135 bash[259926]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T19:50:50.471 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:50 smithi135 bash[259926]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T19:50:50.471 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:50 smithi135 bash[259926]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T19:50:50.471 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:50 smithi135 bash[259926]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T19:50:50.472 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:50 smithi135 bash[259926]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T19:50:50.472 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:50 smithi135 bash[259926]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T19:50:50.472 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:50 smithi135 bash[259926]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T19:50:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:50 smithi144 ceph-mon[194833]: pgmap v176: 73 pgs: 2 active+undersized+degraded, 4 peering, 5 stale+active+clean, 12 active+undersized, 50 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T19:50:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1302704758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:51.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:51 smithi135 ceph-mon[221113]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 4 peering, 24 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:51.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:51 smithi135 ceph-mon[213871]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 4 peering, 24 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:51 smithi144 ceph-mon[194833]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 4 peering, 24 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:51.970 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:51 smithi135 ceph-osd[260438]: -- 172.21.15.135:0/2533931868 <== mon.2 v2:172.21.15.135:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fc2f9224e0 con 0x55fc2f8f4c00 2023-09-23T19:50:51.970 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:51 smithi135 bash[260372]: 8bbface9be59e32010cc1d2a386581ccdf17d6b94e329c7562be3448353e8164 2023-09-23T19:50:51.970 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:51 smithi135 systemd[1]: Started Ceph osd.3 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:50:52.470 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:52 smithi135 conmon[260414]: 2023-09-23T19:50:52.007+0000 7fd860f8a640 -1 Falling back to public interface 2023-09-23T19:50:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3535429664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:52.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3535429664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3535429664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:50:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:53 smithi144 ceph-mon[194833]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:53.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:53 smithi135 ceph-mon[221113]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:53 smithi135 ceph-mon[213871]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:53.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:50:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:50:53] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.43.0" 2023-09-23T19:50:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1386288841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:54 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:54 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:54.970 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:54 smithi135 conmon[260414]: 2023-09-23T19:50:54.786+0000 7fd860f8a640 -1 osd.3 105 log_to_monitors true 2023-09-23T19:50:54.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1386288841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:54.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:54 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:54.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:54 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1386288841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:54 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:54 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.970 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 19:50:55 smithi135 conmon[260414]: 2023-09-23T19:50:55.782+0000 7fd8503f3700 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:50:55.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[213871]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[213871]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[221113]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:50:55.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:55.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:55 smithi135 ceph-mon[221113]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:56.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:55 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:56.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:55 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:56.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:55 smithi144 ceph-mon[194833]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T19:50:56.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:55 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:55 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:55 smithi144 ceph-mon[194833]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:56.895 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[221113]: failed to encode map e108 with expected crc 2023-09-23T19:50:56.895 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[221113]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:50:56.895 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[221113]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T19:50:56.895 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[221113]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:56.895 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[221113]: failed to encode map e108 with expected crc 2023-09-23T19:50:56.895 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/179238586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:56.895 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[213871]: failed to encode map e108 with expected crc 2023-09-23T19:50:56.895 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[213871]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:50:56.896 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[213871]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T19:50:56.896 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[213871]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:56.896 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[213871]: failed to encode map e108 with expected crc 2023-09-23T19:50:56.896 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/179238586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:57.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:56 smithi144 ceph-mon[194833]: failed to encode map e108 with expected crc 2023-09-23T19:50:57.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:56 smithi144 ceph-mon[194833]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T19:50:57.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:56 smithi144 ceph-mon[194833]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T19:50:57.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:56 smithi144 ceph-mon[194833]: from='osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T19:50:57.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:56 smithi144 ceph-mon[194833]: failed to encode map e108 with expected crc 2023-09-23T19:50:57.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/179238586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:58.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: failed to encode map e108 with expected crc 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: failed to encode map e108 with expected crc 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: failed to encode map e109 with expected crc 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: failed to encode map e109 with expected crc 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678] boot 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: pgmap v182: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:50:58.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:58.133 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: failed to encode map e108 with expected crc 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: failed to encode map e108 with expected crc 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: failed to encode map e109 with expected crc 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: failed to encode map e109 with expected crc 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678] boot 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: pgmap v182: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:50:58.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: failed to encode map e108 with expected crc 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: failed to encode map e108 with expected crc 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: failed to encode map e109 with expected crc 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: failed to encode map e109 with expected crc 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: osd.3 [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678] boot 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: pgmap v182: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T19:50:58.135 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:50:58.136 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:58.923 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T19:50:58.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:50:58 smithi135 conmon[201366]: 2023-09-23T19:50:58.900+0000 7fcf49f9e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T19:50:58.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T19:50:58.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: failed to encode map e110 with expected crc 2023-09-23T19:50:58.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:59.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: failed to encode map e110 with expected crc 2023-09-23T19:50:59.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T19:50:59.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: failed to encode map e110 with expected crc 2023-09-23T19:50:59.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: failed to encode map e110 with expected crc 2023-09-23T19:50:59.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: failed to encode map e110 with expected crc 2023-09-23T19:50:59.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: failed to encode map e110 with expected crc 2023-09-23T19:50:59.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: failed to encode map e110 with expected crc 2023-09-23T19:50:59.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: failed to encode map e110 with expected crc 2023-09-23T19:50:59.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:59.724 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T19:50:59.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:50:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1111505250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[221113]: pgmap v184: 73 pgs: 2 peering, 2 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:50:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:50:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:50:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T19:50:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1111505250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:50:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:50:59 smithi135 ceph-mon[213871]: pgmap v184: 73 pgs: 2 peering, 2 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:51:00.072 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:00.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:00.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:00.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1111505250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T19:51:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:50:59 smithi144 ceph-mon[194833]: pgmap v184: 73 pgs: 2 peering, 2 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T19:51:01.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:01 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T19:51:01.394 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:01 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T19:51:01.394 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:01 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T19:51:01.394 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:01 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T19:51:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:01 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T19:51:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:01 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T19:51:01.804 INFO:teuthology.orchestra.run.smithi135.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:51:02.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:02 smithi135 ceph-mon[213871]: pgmap v185: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:02.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:02 smithi135 ceph-mon[213871]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:02 smithi135 ceph-mon[213871]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:51:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:02 smithi135 ceph-mon[221113]: pgmap v185: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:02 smithi135 ceph-mon[221113]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:02 smithi135 ceph-mon[221113]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:51:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:02 smithi144 ceph-mon[194833]: pgmap v185: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:02 smithi144 ceph-mon[194833]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:02 smithi144 ceph-mon[194833]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T19:51:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:02.641 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:51:02.992 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:03.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:03 smithi135 ceph-mon[221113]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:03.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:03 smithi135 ceph-mon[213871]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:03 smithi144 ceph-mon[194833]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:51:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:51:03] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.43.0" 2023-09-23T19:51:04.722 INFO:teuthology.orchestra.run.smithi135.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:alertmanager.a smithi135 *:9093,9094 running (6m) - 10m 29.1M - 0.25.0 c8568f914cd2 22e51759a65e 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:grafana.a smithi144 *:3000 running (6m) - 9m 92.6M - 9.4.7 2c41d148cca3 902bff181c85 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi135.srojfl smithi135 running (9m) - 9m 18.9M - 17.2.6-1123-g61431f6b 0db14aa01094 37783bce2e06 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi144.plbefj smithi144 running (9m) - 9m 16.1M - 17.2.6-1123-g61431f6b 0db14aa01094 af9c7e8247c3 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.x smithi144 *:8443,9283,8765 running (5m) - 14m 449M - 18.2.0-460-gbb0daeff e32c2a64c126 81770c134986 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.y smithi135 *:8443,9283,8765 running (6m) - 15m 515M - 18.2.0-460-gbb0daeff e32c2a64c126 eb6afb9206f5 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:mon.a smithi135 running (5m) - 15m 80.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 7fa1f78d4725 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:mon.b smithi144 running (3m) - 14m 54.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 4fee8ae2a296 2023-09-23T19:51:04.723 INFO:teuthology.orchestra.run.smithi135.stdout:mon.c smithi135 running (4m) - 14m 59.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 6b8308cd56e0 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.a smithi135 *:9100 running (6m) - 9m 17.1M - 1.5.0 0da6a335fe13 caa9af75261e 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.b smithi144 *:9100 running (6m) - 9m 17.9M - 1.5.0 0da6a335fe13 a8b1ed1aeea7 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 smithi135 running (2m) - 13m 60.4M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 28ca011335d6 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1 smithi135 running (79s) - 13m 58.8M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 aa8c0ac7e9ea 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 smithi135 running (46s) - 13m 50.5M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 4d5310d5a38b 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 smithi135 running (13s) - 12m 11.5M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 8bbface9be59 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 smithi144 running (12m) - 12m 97.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 076c88452d16 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 smithi144 running (11m) - 11m 111M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 30c9f1e12bc8 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 smithi144 running (11m) - 11m 91.9M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 d04b0ed25c0f 2023-09-23T19:51:04.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 smithi144 running (11m) - 11m 116M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 4b2a00379dc8 2023-09-23T19:51:04.725 INFO:teuthology.orchestra.run.smithi135.stdout:prometheus.a smithi144 *:9095 running (5m) - 10m 62.4M - 2.43.0 a07b618ecd1d b9776c7278f3 2023-09-23T19:51:05.540 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T19:51:05.887 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:06 smithi135 ceph-mon[221113]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:06 smithi135 ceph-mon[221113]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:06.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:06 smithi135 ceph-mon[213871]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:06.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:06 smithi135 ceph-mon[213871]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:06 smithi144 ceph-mon[194833]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:06 smithi144 ceph-mon[194833]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T19:51:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:alertmanager.a smithi135 *:9093,9094 running (6m) - 10m 29.1M - 0.25.0 c8568f914cd2 22e51759a65e 2023-09-23T19:51:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:grafana.a smithi144 *:3000 running (6m) - 9m 92.6M - 9.4.7 2c41d148cca3 902bff181c85 2023-09-23T19:51:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi135.srojfl smithi135 running (9m) - 9m 18.9M - 17.2.6-1123-g61431f6b 0db14aa01094 37783bce2e06 2023-09-23T19:51:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:mds.foo.smithi144.plbefj smithi144 running (9m) - 9m 16.1M - 17.2.6-1123-g61431f6b 0db14aa01094 af9c7e8247c3 2023-09-23T19:51:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.x smithi144 *:8443,9283,8765 running (5m) - 14m 449M - 18.2.0-460-gbb0daeff e32c2a64c126 81770c134986 2023-09-23T19:51:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:mgr.y smithi135 *:8443,9283,8765 running (6m) - 15m 515M - 18.2.0-460-gbb0daeff e32c2a64c126 eb6afb9206f5 2023-09-23T19:51:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:mon.a smithi135 running (5m) - 15m 80.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 7fa1f78d4725 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:mon.b smithi144 running (3m) - 14m 54.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 4fee8ae2a296 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:mon.c smithi135 running (4m) - 14m 59.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 6b8308cd56e0 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.a smithi135 *:9100 running (6m) - 9m 17.1M - 1.5.0 0da6a335fe13 caa9af75261e 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:node-exporter.b smithi144 *:9100 running (6m) - 9m 17.9M - 1.5.0 0da6a335fe13 a8b1ed1aeea7 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 smithi135 running (2m) - 13m 60.4M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 28ca011335d6 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1 smithi135 running (82s) - 13m 58.8M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 aa8c0ac7e9ea 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 smithi135 running (49s) - 13m 50.5M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 4d5310d5a38b 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 smithi135 running (16s) - 12m 11.5M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 8bbface9be59 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 smithi144 running (12m) - 12m 97.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 076c88452d16 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 smithi144 running (11m) - 11m 111M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 30c9f1e12bc8 2023-09-23T19:51:07.761 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 smithi144 running (11m) - 11m 91.9M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 d04b0ed25c0f 2023-09-23T19:51:07.762 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 smithi144 running (11m) - 11m 116M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 4b2a00379dc8 2023-09-23T19:51:07.762 INFO:teuthology.orchestra.run.smithi135.stdout:prometheus.a smithi144 *:9095 running (5m) - 10m 62.4M - 2.43.0 a07b618ecd1d b9776c7278f3 2023-09-23T19:51:08.096 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:08.096 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:08.097 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:08 smithi135 ceph-mon[213871]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:08 smithi144 ceph-mon[194833]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:08 smithi135 ceph-mon[221113]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:08.690 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T19:51:09.036 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:09 smithi144 ceph-mon[194833]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:09.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:09 smithi135 ceph-mon[221113]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:09.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:09 smithi135 ceph-mon[213871]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:51:10.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:10 smithi135 ceph-mon[221113]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:10.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:10 smithi135 ceph-mon[213871]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:10 smithi144 ceph-mon[194833]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:10.936 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:51:10.936 INFO:teuthology.orchestra.run.smithi135.stdout: "mon": { 2023-09-23T19:51:10.936 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T19:51:10.936 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:51:10.936 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr": { 2023-09-23T19:51:10.936 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T19:51:10.936 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: "osd": { 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 4, 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: "mds": { 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 2 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: "overall": { 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 6, 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:51:10.937 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:51:11.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:11 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1136969605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:51:11.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:11 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1136969605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:51:11.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:11 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1136969605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T19:51:11.748 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:11.748 INFO:teuthology.run_tasks:Running task print... 2023-09-23T19:51:11.757 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T19:51:11.757 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T19:51:11.776 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T19:51:11.776 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T19:51:11.777 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T19:51:12.126 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:12.158 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:12 smithi135 ceph-mon[213871]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:12.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:12 smithi135 ceph-mon[221113]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:12 smithi144 ceph-mon[194833]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:13.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:13 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:13.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:13 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:13 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:51:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:51:13] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2023-09-23T19:51:14.064 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:51:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1581991007' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:14.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1581991007' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:51:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:51:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:51:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1581991007' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T19:51:14.855 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":36,"active_gid":25859,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":4237859707},{"type":"v1","addr":"172.21.15.135:6801","nonce":4237859707}]},"active_addr":"172.21.15.135:6801/4237859707","active_change":"2023-09-23T19:45:36.991689+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25729,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":90,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":675532673}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":1533244044}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":69241247}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":2229160446}]}]} 2023-09-23T19:51:14.859 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T19:51:14.860 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T19:51:14.860 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:51:15.210 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:16 smithi135 ceph-mon[221113]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:16 smithi135 ceph-mon[213871]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:16 smithi144 ceph-mon[194833]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:17.065 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:51:17.065 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":110,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T19:50:57.767213+0000","last_up_change":"2023-09-23T19:50:56.756515+0000","last_in_change":"2023-09-23T19:39:46.254783+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:40:39.075611+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:41:38.466408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:41:38.762420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6803","nonce":100269857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6805","nonce":100269857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6809","nonce":100269857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6807","nonce":100269857}]},"public_addr":"172.21.15.135:6803/100269857","cluster_addr":"172.21.15.135:6805/100269857","heartbeat_back_addr":"172.21.15.135:6809/100269857","heartbeat_front_addr":"172.21.15.135:6807/100269857","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6811","nonce":2160782760}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6813","nonce":2160782760}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6817","nonce":2160782760}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6815","nonce":2160782760}]},"public_addr":"172.21.15.135:6811/2160782760","cluster_addr":"172.21.15.135:6813/2160782760","heartbeat_back_addr":"172.21.15.135:6817/2160782760","heartbeat_front_addr":"172.21.15.135:6815/2160782760","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":209403510},{"type":"v1","addr":"172.21.15.135:6819","nonce":209403510}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":209403510},{"type":"v1","addr":"172.21.15.135:6821","nonce":209403510}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":209403510},{"type":"v1","addr":"172.21.15.135:6825","nonce":209403510}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":209403510},{"type":"v1","addr":"172.21.15.135:6823","nonce":209403510}]},"public_addr":"172.21.15.135:6819/209403510","cluster_addr":"172.21.15.135:6821/209403510","heartbeat_back_addr":"172.21.15.135:6825/209403510","heartbeat_front_addr":"172.21.15.135:6823/209403510","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6827","nonce":2232272678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6829","nonce":2232272678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6833","nonce":2232272678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6831","nonce":2232272678}]},"public_addr":"172.21.15.135:6827/2232272678","cluster_addr":"172.21.15.135:6829/2232272678","heartbeat_back_addr":"172.21.15.135:6833/2232272678","heartbeat_front_addr":"172.21.15.135:6831/2232272678","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6817","nonce":1530810554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6819","nonce":1530810554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6823","nonce":1530810554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6821","nonce":1530810554}]},"public_addr":"172.21.15.144:6817/1530810554","cluster_addr":"172.21.15.144:6819/1530810554","heartbeat_back_addr":"172.21.15.144:6823/1530810554","heartbeat_front_addr":"172.21.15.144:6821/1530810554","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/1867076080":"2023-09-24T19:45:36.991484+0000","172.21.15.144:6834/1742442049":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.144:0/3940572722":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/456433686":"2023-09-24T19:45:36.991484+0000","172.21.15.144:0/2226038972":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1005337745":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/1867076080":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.144:6835/1742442049":"2023-09-24T19:44:56.542176+0000","172.21.15.144:0/3293023812":"2023-09-24T19:44:56.542176+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/173145882":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/4159692573":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/775791172":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/3361160768":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/2123602186":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6801/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/2749491827":"2023-09-24T19:45:36.991484+0000","172.21.15.144:0/3756303050":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/3668294279":"2023-09-24T19:44:16.725201+0000","172.21.15.135:6801/1642850725":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/4238089144":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/1017551752":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/1625057187":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/1642850725":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/3693941440":"2023-09-24T19:44:16.725201+0000","172.21.15.144:0/1990744051":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.144:0/2993818522":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/3516425720":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:51:17.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/488913333' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:51:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/488913333' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:51:17.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/488913333' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:51:17.796 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T19:51:17.796 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:51:18.140 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:18.181 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:18 smithi135 ceph-mon[213871]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:18.436 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:18 smithi135 ceph-mon[221113]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:18.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:18 smithi144 ceph-mon[194833]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:51:19.939 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":110,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T19:50:57.767213+0000","last_up_change":"2023-09-23T19:50:56.756515+0000","last_in_change":"2023-09-23T19:39:46.254783+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:40:39.075611+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:41:38.466408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:41:38.762420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6803","nonce":100269857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6805","nonce":100269857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6809","nonce":100269857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6807","nonce":100269857}]},"public_addr":"172.21.15.135:6803/100269857","cluster_addr":"172.21.15.135:6805/100269857","heartbeat_back_addr":"172.21.15.135:6809/100269857","heartbeat_front_addr":"172.21.15.135:6807/100269857","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6811","nonce":2160782760}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6813","nonce":2160782760}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6817","nonce":2160782760}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6815","nonce":2160782760}]},"public_addr":"172.21.15.135:6811/2160782760","cluster_addr":"172.21.15.135:6813/2160782760","heartbeat_back_addr":"172.21.15.135:6817/2160782760","heartbeat_front_addr":"172.21.15.135:6815/2160782760","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":209403510},{"type":"v1","addr":"172.21.15.135:6819","nonce":209403510}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":209403510},{"type":"v1","addr":"172.21.15.135:6821","nonce":209403510}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":209403510},{"type":"v1","addr":"172.21.15.135:6825","nonce":209403510}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":209403510},{"type":"v1","addr":"172.21.15.135:6823","nonce":209403510}]},"public_addr":"172.21.15.135:6819/209403510","cluster_addr":"172.21.15.135:6821/209403510","heartbeat_back_addr":"172.21.15.135:6825/209403510","heartbeat_front_addr":"172.21.15.135:6823/209403510","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6827","nonce":2232272678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6829","nonce":2232272678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6833","nonce":2232272678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6831","nonce":2232272678}]},"public_addr":"172.21.15.135:6827/2232272678","cluster_addr":"172.21.15.135:6829/2232272678","heartbeat_back_addr":"172.21.15.135:6833/2232272678","heartbeat_front_addr":"172.21.15.135:6831/2232272678","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6817","nonce":1530810554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6819","nonce":1530810554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6823","nonce":1530810554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1530810554},{"type":"v1","addr":"172.21.15.144:6821","nonce":1530810554}]},"public_addr":"172.21.15.144:6817/1530810554","cluster_addr":"172.21.15.144:6819/1530810554","heartbeat_back_addr":"172.21.15.144:6823/1530810554","heartbeat_front_addr":"172.21.15.144:6821/1530810554","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:6801/1867076080":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/1867076080":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/3668294279":"2023-09-24T19:44:16.725201+0000","172.21.15.135:6801/1642850725":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/4238089144":"2023-09-24T19:44:16.725201+0000","172.21.15.135:6801/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1005337745":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.144:6834/1742442049":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/775791172":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/3361160768":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/173145882":"2023-09-24T19:40:37.220934+0000","172.21.15.144:0/3293023812":"2023-09-24T19:44:56.542176+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.144:6835/1742442049":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.144:0/3940572722":"2023-09-24T19:44:56.542176+0000","172.21.15.144:0/2226038972":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/456433686":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/1642850725":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/3693941440":"2023-09-24T19:44:16.725201+0000","172.21.15.144:0/3756303050":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/2749491827":"2023-09-24T19:45:36.991484+0000","172.21.15.144:0/1990744051":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.144:0/2993818522":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/3516425720":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/2123602186":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/4159692573":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/1017551752":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/1625057187":"2023-09-24T19:45:36.991484+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:51:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:20 smithi135 ceph-mon[221113]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2252222521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:51:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:20 smithi135 ceph-mon[213871]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2252222521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:51:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:20 smithi144 ceph-mon[194833]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2252222521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:51:20.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T19:51:20.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T19:51:20.648 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T19:51:20.648 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T19:51:20.649 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T19:51:20.649 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T19:51:20.649 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T19:51:20.650 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T19:51:22.224 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:22.224 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:22.271 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:22.323 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:22.394 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:22.427 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:22 smithi135 ceph-mon[221113]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:22 smithi135 ceph-mon[213871]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:22.538 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:22 smithi144 ceph-mon[194833]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:22.721 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:51:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:51:23] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2023-09-23T19:51:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:24 smithi135 ceph-mon[221113]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:24.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:24 smithi135 ceph-mon[213871]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:24 smithi144 ceph-mon[194833]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:26.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:26 smithi135 ceph-mon[213871]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:26.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:26 smithi135 ceph-mon[221113]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:26 smithi144 ceph-mon[194833]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:28.416 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:28 smithi135 ceph-mon[213871]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:28.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:28 smithi135 ceph-mon[221113]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:28.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:28 smithi144 ceph-mon[194833]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:29.931 INFO:teuthology.orchestra.run.smithi135.stdout:197568495760 2023-09-23T19:51:29.932 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T19:51:30.448 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:30 smithi135 ceph-mon[221113]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:30.449 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:30 smithi135 ceph-mon[213871]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:30.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:30 smithi144 ceph-mon[194833]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:32.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:32 smithi135 ceph-mon[213871]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:32.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:32 smithi135 ceph-mon[221113]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:32.396 INFO:teuthology.orchestra.run.smithi135.stdout:468151435272 2023-09-23T19:51:32.397 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T19:51:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:32 smithi144 ceph-mon[194833]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:33.839 INFO:teuthology.orchestra.run.smithi135.stdout:446676598799 2023-09-23T19:51:33.839 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T19:51:33.869 INFO:teuthology.orchestra.run.smithi135.stdout:128849019034 2023-09-23T19:51:33.869 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T19:51:33.910 INFO:teuthology.orchestra.run.smithi135.stdout:403726925861 2023-09-23T19:51:33.910 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T19:51:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:51:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:51:33] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2023-09-23T19:51:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:34 smithi135 ceph-mon[221113]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:34.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:34 smithi135 ceph-mon[213871]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:34 smithi144 ceph-mon[194833]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:34.789 INFO:teuthology.orchestra.run.smithi135.stdout:425201762326 2023-09-23T19:51:34.789 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T19:51:35.509 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:35 smithi135 ceph-mon[221113]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:35.509 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:35 smithi135 ceph-mon[213871]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:35.590 INFO:teuthology.orchestra.run.smithi135.stdout:231928234125 2023-09-23T19:51:35.591 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T19:51:35.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:35 smithi144 ceph-mon[194833]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:35.826 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:35.835 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:36.309 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:36.485 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:36.636 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:36.657 INFO:teuthology.orchestra.run.smithi135.stdout:163208757399 2023-09-23T19:51:36.658 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T19:51:37.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:37.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:37.519 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:38.320 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:38 smithi135 ceph-mon[221113]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:38.320 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:38 smithi135 ceph-mon[213871]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:38.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:38 smithi144 ceph-mon[194833]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:38.778 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:40 smithi135 ceph-mon[221113]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:40 smithi135 ceph-mon[213871]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:40 smithi144 ceph-mon[194833]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:42.422 INFO:teuthology.orchestra.run.smithi135.stdout:403726925863 2023-09-23T19:51:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:42 smithi135 ceph-mon[221113]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4213431689' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:51:42.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:42 smithi135 ceph-mon[213871]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:42.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4213431689' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:51:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:42 smithi144 ceph-mon[194833]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4213431689' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:51:42.884 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:43.191 INFO:teuthology.orchestra.run.smithi135.stdout:425201762328 2023-09-23T19:51:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:51:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:51:43] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2023-09-23T19:51:44.105 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925861 got 403726925863 for osd.0 2023-09-23T19:51:44.106 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:44.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:44 smithi135 ceph-mon[221113]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:44.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:44 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4214231164' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:51:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:44 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/279263102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:51:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:44 smithi135 ceph-mon[213871]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:44 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4214231164' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:51:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:44 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/279263102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:51:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:44 smithi144 ceph-mon[194833]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:44 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4214231164' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:51:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:44 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/279263102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:51:44.714 INFO:teuthology.orchestra.run.smithi135.stdout:446676598802 2023-09-23T19:51:44.775 INFO:teuthology.orchestra.run.smithi135.stdout:197568495764 2023-09-23T19:51:44.823 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762326 got 425201762328 for osd.1 2023-09-23T19:51:44.823 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:45.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2724317473' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:51:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2724317473' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:51:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2724317473' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:51:45.800 INFO:teuthology.orchestra.run.smithi135.stdout:468151435275 2023-09-23T19:51:46.423 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:46 smithi135 ceph-mon[221113]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:46.424 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1799736568' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:51:46.424 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:46 smithi135 ceph-mon[213871]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:46.424 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1799736568' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:51:46.440 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495760 got 197568495764 for osd.6 2023-09-23T19:51:46.440 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:46 smithi144 ceph-mon[194833]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:46.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1799736568' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:51:47.108 INFO:teuthology.orchestra.run.smithi135.stdout:128849019037 2023-09-23T19:51:47.142 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435272 got 468151435275 for osd.3 2023-09-23T19:51:47.142 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:47.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3269074482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:51:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3269074482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:51:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3269074482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:51:48.176 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598799 got 446676598802 for osd.2 2023-09-23T19:51:48.176 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:48.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:48 smithi135 ceph-mon[221113]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:48 smithi135 ceph-mon[213871]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:48 smithi144 ceph-mon[194833]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:48.644 INFO:teuthology.orchestra.run.smithi135.stdout:231928234127 2023-09-23T19:51:49.364 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019034 got 128849019037 for osd.4 2023-09-23T19:51:49.365 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:49.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:49 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3092862892' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:51:49.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:49 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3092862892' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:51:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:49 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3092862892' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:51:49.695 INFO:teuthology.orchestra.run.smithi135.stdout:163208757402 2023-09-23T19:51:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:50 smithi135 ceph-mon[213871]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/89449866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:51:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:50 smithi135 ceph-mon[221113]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/89449866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:51:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:50 smithi144 ceph-mon[194833]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/89449866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:51:50.661 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234125 got 231928234127 for osd.7 2023-09-23T19:51:50.661 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:50.708 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757399 got 163208757402 for osd.5 2023-09-23T19:51:50.708 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:50.709 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T19:51:50.709 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:51:51.061 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:52 smithi135 ceph-mon[213871]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:52.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:52 smithi135 ceph-mon[221113]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:52 smithi144 ceph-mon[194833]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:51:52.913 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:51:52.915 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:51:53.675 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":210,"stamp":"2023-09-23T19:51:51.107204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164336,"kb_used_data":23024,"kb_used_omap":32,"kb_used_meta":141215,"kb_avail":749796880,"statfs":{"total":767960285184,"available":767792005120,"internally_reserved":0,"allocated":23576576,"data_stored":19324717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":144604678},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002784"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764512+0000","last_change":"2023-09-23T19:49:51.378856+0000","last_active":"2023-09-23T19:50:56.764512+0000","last_peered":"2023-09-23T19:50:56.764512+0000","last_clean":"2023-09-23T19:50:56.764512+0000","last_became_active":"2023-09-23T19:49:51.378451+0000","last_became_peered":"2023-09-23T19:49:51.378451+0000","last_unstale":"2023-09-23T19:50:56.764512+0000","last_undegraded":"2023-09-23T19:50:56.764512+0000","last_fullsized":"2023-09-23T19:50:56.764512+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:03.276740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770013+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:50:56.770013+0000","last_peered":"2023-09-23T19:50:56.770013+0000","last_clean":"2023-09-23T19:50:56.770013+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:50:56.770013+0000","last_undegraded":"2023-09-23T19:50:56.770013+0000","last_fullsized":"2023-09-23T19:50:56.770013+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.785311+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:50:57.785311+0000","last_peered":"2023-09-23T19:50:57.785311+0000","last_clean":"2023-09-23T19:50:57.785311+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:50:57.785311+0000","last_undegraded":"2023-09-23T19:50:57.785311+0000","last_fullsized":"2023-09-23T19:50:57.785311+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786561+0000","last_change":"2023-09-23T19:50:57.786561+0000","last_active":"2023-09-23T19:50:57.786561+0000","last_peered":"2023-09-23T19:50:57.786561+0000","last_clean":"2023-09-23T19:50:57.786561+0000","last_became_active":"2023-09-23T19:50:57.786231+0000","last_became_peered":"2023-09-23T19:50:57.786231+0000","last_unstale":"2023-09-23T19:50:57.786561+0000","last_undegraded":"2023-09-23T19:50:57.786561+0000","last_fullsized":"2023-09-23T19:50:57.786561+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:27.327075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790948+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:50:57.790948+0000","last_peered":"2023-09-23T19:50:57.790948+0000","last_clean":"2023-09-23T19:50:57.790948+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:50:57.790948+0000","last_undegraded":"2023-09-23T19:50:57.790948+0000","last_fullsized":"2023-09-23T19:50:57.790948+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768019+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:50:56.768019+0000","last_peered":"2023-09-23T19:50:56.768019+0000","last_clean":"2023-09-23T19:50:56.768019+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:50:56.768019+0000","last_undegraded":"2023-09-23T19:50:56.768019+0000","last_fullsized":"2023-09-23T19:50:56.768019+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789973+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:50:57.789973+0000","last_peered":"2023-09-23T19:50:57.789973+0000","last_clean":"2023-09-23T19:50:57.789973+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:50:57.789973+0000","last_undegraded":"2023-09-23T19:50:57.789973+0000","last_fullsized":"2023-09-23T19:50:57.789973+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768916+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:50:56.768916+0000","last_peered":"2023-09-23T19:50:56.768916+0000","last_clean":"2023-09-23T19:50:56.768916+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:50:56.768916+0000","last_undegraded":"2023-09-23T19:50:56.768916+0000","last_fullsized":"2023-09-23T19:50:56.768916+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790667+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:50:57.790667+0000","last_peered":"2023-09-23T19:50:57.790667+0000","last_clean":"2023-09-23T19:50:57.790667+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:50:57.790667+0000","last_undegraded":"2023-09-23T19:50:57.790667+0000","last_fullsized":"2023-09-23T19:50:57.790667+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.777637+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:50:57.777637+0000","last_peered":"2023-09-23T19:50:57.777637+0000","last_clean":"2023-09-23T19:50:57.777637+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:50:57.777637+0000","last_undegraded":"2023-09-23T19:50:57.777637+0000","last_fullsized":"2023-09-23T19:50:57.777637+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765709+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:50:56.765709+0000","last_peered":"2023-09-23T19:50:56.765709+0000","last_clean":"2023-09-23T19:50:56.765709+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:50:56.765709+0000","last_undegraded":"2023-09-23T19:50:56.765709+0000","last_fullsized":"2023-09-23T19:50:56.765709+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.777187+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:50:56.777187+0000","last_peered":"2023-09-23T19:50:56.777187+0000","last_clean":"2023-09-23T19:50:56.777187+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:50:56.777187+0000","last_undegraded":"2023-09-23T19:50:56.777187+0000","last_fullsized":"2023-09-23T19:50:56.777187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":342,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770704+0000","last_change":"2023-09-23T19:50:23.999651+0000","last_active":"2023-09-23T19:50:56.770704+0000","last_peered":"2023-09-23T19:50:56.770704+0000","last_clean":"2023-09-23T19:50:56.770704+0000","last_became_active":"2023-09-23T19:50:23.999184+0000","last_became_peered":"2023-09-23T19:50:23.999184+0000","last_unstale":"2023-09-23T19:50:56.770704+0000","last_undegraded":"2023-09-23T19:50:56.770704+0000","last_fullsized":"2023-09-23T19:50:56.770704+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:41.010259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793288+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:50:57.793288+0000","last_peered":"2023-09-23T19:50:57.793288+0000","last_clean":"2023-09-23T19:50:57.793288+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:50:57.793288+0000","last_undegraded":"2023-09-23T19:50:57.793288+0000","last_fullsized":"2023-09-23T19:50:57.793288+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":370,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.767547+0000","last_change":"2023-09-23T19:49:51.380375+0000","last_active":"2023-09-23T19:50:56.767547+0000","last_peered":"2023-09-23T19:50:56.767547+0000","last_clean":"2023-09-23T19:50:56.767547+0000","last_became_active":"2023-09-23T19:49:51.379793+0000","last_became_peered":"2023-09-23T19:49:51.379793+0000","last_unstale":"2023-09-23T19:50:56.767547+0000","last_undegraded":"2023-09-23T19:50:56.767547+0000","last_fullsized":"2023-09-23T19:50:56.767547+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:26.355152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789467+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:50:57.789467+0000","last_peered":"2023-09-23T19:50:57.789467+0000","last_clean":"2023-09-23T19:50:57.789467+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:50:57.789467+0000","last_undegraded":"2023-09-23T19:50:57.789467+0000","last_fullsized":"2023-09-23T19:50:57.789467+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.767556+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:50:56.767556+0000","last_peered":"2023-09-23T19:50:56.767556+0000","last_clean":"2023-09-23T19:50:56.767556+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:50:56.767556+0000","last_undegraded":"2023-09-23T19:50:56.767556+0000","last_fullsized":"2023-09-23T19:50:56.767556+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768204+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:50:56.768204+0000","last_peered":"2023-09-23T19:50:56.768204+0000","last_clean":"2023-09-23T19:50:56.768204+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:50:56.768204+0000","last_undegraded":"2023-09-23T19:50:56.768204+0000","last_fullsized":"2023-09-23T19:50:56.768204+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":188,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774652+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:50:56.774652+0000","last_peered":"2023-09-23T19:50:56.774652+0000","last_clean":"2023-09-23T19:50:56.774652+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:50:56.774652+0000","last_undegraded":"2023-09-23T19:50:56.774652+0000","last_fullsized":"2023-09-23T19:50:56.774652+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.777263+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:50:56.777263+0000","last_peered":"2023-09-23T19:50:56.777263+0000","last_clean":"2023-09-23T19:50:56.777263+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:50:56.777263+0000","last_undegraded":"2023-09-23T19:50:56.777263+0000","last_fullsized":"2023-09-23T19:50:56.777263+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794256+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:50:57.794256+0000","last_peered":"2023-09-23T19:50:57.794256+0000","last_clean":"2023-09-23T19:50:57.794256+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:50:57.794256+0000","last_undegraded":"2023-09-23T19:50:57.794256+0000","last_fullsized":"2023-09-23T19:50:57.794256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":828,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.772649+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:50:56.772649+0000","last_peered":"2023-09-23T19:50:56.772649+0000","last_clean":"2023-09-23T19:50:56.772649+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:50:56.772649+0000","last_undegraded":"2023-09-23T19:50:56.772649+0000","last_fullsized":"2023-09-23T19:50:56.772649+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770724+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:50:56.770724+0000","last_peered":"2023-09-23T19:50:56.770724+0000","last_clean":"2023-09-23T19:50:56.770724+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:50:56.770724+0000","last_undegraded":"2023-09-23T19:50:56.770724+0000","last_fullsized":"2023-09-23T19:50:56.770724+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787513+0000","last_change":"2023-09-23T19:50:57.787513+0000","last_active":"2023-09-23T19:50:57.787513+0000","last_peered":"2023-09-23T19:50:57.787513+0000","last_clean":"2023-09-23T19:50:57.787513+0000","last_became_active":"2023-09-23T19:50:57.787175+0000","last_became_peered":"2023-09-23T19:50:57.787175+0000","last_unstale":"2023-09-23T19:50:57.787513+0000","last_undegraded":"2023-09-23T19:50:57.787513+0000","last_fullsized":"2023-09-23T19:50:57.787513+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:19.081986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":450,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.772552+0000","last_change":"2023-09-23T19:48:37.575593+0000","last_active":"2023-09-23T19:50:56.772552+0000","last_peered":"2023-09-23T19:50:56.772552+0000","last_clean":"2023-09-23T19:50:56.772552+0000","last_became_active":"2023-09-23T19:48:37.575214+0000","last_became_peered":"2023-09-23T19:48:37.575214+0000","last_unstale":"2023-09-23T19:50:56.772552+0000","last_undegraded":"2023-09-23T19:50:56.772552+0000","last_fullsized":"2023-09-23T19:50:56.772552+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:13.335201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":919,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766899+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:50:56.766899+0000","last_peered":"2023-09-23T19:50:56.766899+0000","last_clean":"2023-09-23T19:50:56.766899+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:50:56.766899+0000","last_undegraded":"2023-09-23T19:50:56.766899+0000","last_fullsized":"2023-09-23T19:50:56.766899+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.780129+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:50:56.780129+0000","last_peered":"2023-09-23T19:50:56.780129+0000","last_clean":"2023-09-23T19:50:56.780129+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:50:56.780129+0000","last_undegraded":"2023-09-23T19:50:56.780129+0000","last_fullsized":"2023-09-23T19:50:56.780129+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786740+0000","last_change":"2023-09-23T19:50:23.997623+0000","last_active":"2023-09-23T19:50:57.786740+0000","last_peered":"2023-09-23T19:50:57.786740+0000","last_clean":"2023-09-23T19:50:57.786740+0000","last_became_active":"2023-09-23T19:50:23.997428+0000","last_became_peered":"2023-09-23T19:50:23.997428+0000","last_unstale":"2023-09-23T19:50:57.786740+0000","last_undegraded":"2023-09-23T19:50:57.786740+0000","last_fullsized":"2023-09-23T19:50:57.786740+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:50.529980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":594,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786571+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:50:57.786571+0000","last_peered":"2023-09-23T19:50:57.786571+0000","last_clean":"2023-09-23T19:50:57.786571+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:50:57.786571+0000","last_undegraded":"2023-09-23T19:50:57.786571+0000","last_fullsized":"2023-09-23T19:50:57.786571+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794107+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:50:57.794107+0000","last_peered":"2023-09-23T19:50:57.794107+0000","last_clean":"2023-09-23T19:50:57.794107+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:50:57.794107+0000","last_undegraded":"2023-09-23T19:50:57.794107+0000","last_fullsized":"2023-09-23T19:50:57.794107+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765277+0000","last_change":"2023-09-23T19:48:37.573541+0000","last_active":"2023-09-23T19:50:56.765277+0000","last_peered":"2023-09-23T19:50:56.765277+0000","last_clean":"2023-09-23T19:50:56.765277+0000","last_became_active":"2023-09-23T19:48:37.573213+0000","last_became_peered":"2023-09-23T19:48:37.573213+0000","last_unstale":"2023-09-23T19:50:56.765277+0000","last_undegraded":"2023-09-23T19:50:56.765277+0000","last_fullsized":"2023-09-23T19:50:56.765277+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:16.981072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":634,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766698+0000","last_change":"2023-09-23T19:49:51.382830+0000","last_active":"2023-09-23T19:50:56.766698+0000","last_peered":"2023-09-23T19:50:56.766698+0000","last_clean":"2023-09-23T19:50:56.766698+0000","last_became_active":"2023-09-23T19:49:51.382367+0000","last_became_peered":"2023-09-23T19:49:51.382367+0000","last_unstale":"2023-09-23T19:50:56.766698+0000","last_undegraded":"2023-09-23T19:50:56.766698+0000","last_fullsized":"2023-09-23T19:50:56.766698+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:48.082643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770832+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:50:56.770832+0000","last_peered":"2023-09-23T19:50:56.770832+0000","last_clean":"2023-09-23T19:50:56.770832+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:50:56.770832+0000","last_undegraded":"2023-09-23T19:50:56.770832+0000","last_fullsized":"2023-09-23T19:50:56.770832+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787126+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:50:57.787126+0000","last_peered":"2023-09-23T19:50:57.787126+0000","last_clean":"2023-09-23T19:50:57.787126+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:50:57.787126+0000","last_undegraded":"2023-09-23T19:50:57.787126+0000","last_fullsized":"2023-09-23T19:50:57.787126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":444,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768038+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:50:56.768038+0000","last_peered":"2023-09-23T19:50:56.768038+0000","last_clean":"2023-09-23T19:50:56.768038+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:50:56.768038+0000","last_undegraded":"2023-09-23T19:50:56.768038+0000","last_fullsized":"2023-09-23T19:50:56.768038+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790403+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:50:57.790403+0000","last_peered":"2023-09-23T19:50:57.790403+0000","last_clean":"2023-09-23T19:50:57.790403+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:50:57.790403+0000","last_undegraded":"2023-09-23T19:50:57.790403+0000","last_fullsized":"2023-09-23T19:50:57.790403+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793102+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:50:57.793102+0000","last_peered":"2023-09-23T19:50:57.793102+0000","last_clean":"2023-09-23T19:50:57.793102+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:50:57.793102+0000","last_undegraded":"2023-09-23T19:50:57.793102+0000","last_fullsized":"2023-09-23T19:50:57.793102+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765182+0000","last_change":"2023-09-23T19:50:23.998753+0000","last_active":"2023-09-23T19:50:56.765182+0000","last_peered":"2023-09-23T19:50:56.765182+0000","last_clean":"2023-09-23T19:50:56.765182+0000","last_became_active":"2023-09-23T19:50:23.998417+0000","last_became_peered":"2023-09-23T19:50:23.998417+0000","last_unstale":"2023-09-23T19:50:56.765182+0000","last_undegraded":"2023-09-23T19:50:56.765182+0000","last_fullsized":"2023-09-23T19:50:56.765182+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:11.002531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765806+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:50:56.765806+0000","last_peered":"2023-09-23T19:50:56.765806+0000","last_clean":"2023-09-23T19:50:56.765806+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:50:56.765806+0000","last_undegraded":"2023-09-23T19:50:56.765806+0000","last_fullsized":"2023-09-23T19:50:56.765806+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766899+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:50:56.766899+0000","last_peered":"2023-09-23T19:50:56.766899+0000","last_clean":"2023-09-23T19:50:56.766899+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:50:56.766899+0000","last_undegraded":"2023-09-23T19:50:56.766899+0000","last_fullsized":"2023-09-23T19:50:56.766899+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770764+0000","last_change":"2023-09-23T19:49:51.380925+0000","last_active":"2023-09-23T19:50:56.770764+0000","last_peered":"2023-09-23T19:50:56.770764+0000","last_clean":"2023-09-23T19:50:56.770764+0000","last_became_active":"2023-09-23T19:49:51.380427+0000","last_became_peered":"2023-09-23T19:49:51.380427+0000","last_unstale":"2023-09-23T19:50:56.770764+0000","last_undegraded":"2023-09-23T19:50:56.770764+0000","last_fullsized":"2023-09-23T19:50:56.770764+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:49.879887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790231+0000","last_change":"2023-09-23T19:50:57.790231+0000","last_active":"2023-09-23T19:50:57.790231+0000","last_peered":"2023-09-23T19:50:57.790231+0000","last_clean":"2023-09-23T19:50:57.790231+0000","last_became_active":"2023-09-23T19:50:57.789446+0000","last_became_peered":"2023-09-23T19:50:57.789446+0000","last_unstale":"2023-09-23T19:50:57.790231+0000","last_undegraded":"2023-09-23T19:50:57.790231+0000","last_fullsized":"2023-09-23T19:50:57.790231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:41.313276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793011+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:50:57.793011+0000","last_peered":"2023-09-23T19:50:57.793011+0000","last_clean":"2023-09-23T19:50:57.793011+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:50:57.793011+0000","last_undegraded":"2023-09-23T19:50:57.793011+0000","last_fullsized":"2023-09-23T19:50:57.793011+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768198+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:50:56.768198+0000","last_peered":"2023-09-23T19:50:56.768198+0000","last_clean":"2023-09-23T19:50:56.768198+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:50:56.768198+0000","last_undegraded":"2023-09-23T19:50:56.768198+0000","last_fullsized":"2023-09-23T19:50:56.768198+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.788389+0000","last_change":"2023-09-23T19:50:57.788389+0000","last_active":"2023-09-23T19:50:57.788389+0000","last_peered":"2023-09-23T19:50:57.788389+0000","last_clean":"2023-09-23T19:50:57.788389+0000","last_became_active":"2023-09-23T19:50:57.788075+0000","last_became_peered":"2023-09-23T19:50:57.788075+0000","last_unstale":"2023-09-23T19:50:57.788389+0000","last_undegraded":"2023-09-23T19:50:57.788389+0000","last_fullsized":"2023-09-23T19:50:57.788389+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:06.180629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770475+0000","last_change":"2023-09-23T19:49:51.380726+0000","last_active":"2023-09-23T19:50:56.770475+0000","last_peered":"2023-09-23T19:50:56.770475+0000","last_clean":"2023-09-23T19:50:56.770475+0000","last_became_active":"2023-09-23T19:49:51.380021+0000","last_became_peered":"2023-09-23T19:49:51.380021+0000","last_unstale":"2023-09-23T19:50:56.770475+0000","last_undegraded":"2023-09-23T19:50:56.770475+0000","last_fullsized":"2023-09-23T19:50:56.770475+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:16.892029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774672+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:50:56.774672+0000","last_peered":"2023-09-23T19:50:56.774672+0000","last_clean":"2023-09-23T19:50:56.774672+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:50:56.774672+0000","last_undegraded":"2023-09-23T19:50:56.774672+0000","last_fullsized":"2023-09-23T19:50:56.774672+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.795290+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:50:57.795290+0000","last_peered":"2023-09-23T19:50:57.795290+0000","last_clean":"2023-09-23T19:50:57.795290+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:50:57.795290+0000","last_undegraded":"2023-09-23T19:50:57.795290+0000","last_fullsized":"2023-09-23T19:50:57.795290+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.773415+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:50:56.773415+0000","last_peered":"2023-09-23T19:50:56.773415+0000","last_clean":"2023-09-23T19:50:56.773415+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:50:56.773415+0000","last_undegraded":"2023-09-23T19:50:56.773415+0000","last_fullsized":"2023-09-23T19:50:56.773415+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765786+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:50:56.765786+0000","last_peered":"2023-09-23T19:50:56.765786+0000","last_clean":"2023-09-23T19:50:56.765786+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:50:56.765786+0000","last_undegraded":"2023-09-23T19:50:56.765786+0000","last_fullsized":"2023-09-23T19:50:56.765786+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774219+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:50:56.774219+0000","last_peered":"2023-09-23T19:50:56.774219+0000","last_clean":"2023-09-23T19:50:56.774219+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:50:56.774219+0000","last_undegraded":"2023-09-23T19:50:56.774219+0000","last_fullsized":"2023-09-23T19:50:56.774219+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789346+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:50:57.789346+0000","last_peered":"2023-09-23T19:50:57.789346+0000","last_clean":"2023-09-23T19:50:57.789346+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:50:57.789346+0000","last_undegraded":"2023-09-23T19:50:57.789346+0000","last_fullsized":"2023-09-23T19:50:57.789346+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770341+0000","last_change":"2023-09-23T19:48:37.567607+0000","last_active":"2023-09-23T19:50:56.770341+0000","last_peered":"2023-09-23T19:50:56.770341+0000","last_clean":"2023-09-23T19:50:56.770341+0000","last_became_active":"2023-09-23T19:48:37.567300+0000","last_became_peered":"2023-09-23T19:48:37.567300+0000","last_unstale":"2023-09-23T19:50:56.770341+0000","last_undegraded":"2023-09-23T19:50:56.770341+0000","last_fullsized":"2023-09-23T19:50:56.770341+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.088452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793018+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:50:57.793018+0000","last_peered":"2023-09-23T19:50:57.793018+0000","last_clean":"2023-09-23T19:50:57.793018+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:50:57.793018+0000","last_undegraded":"2023-09-23T19:50:57.793018+0000","last_fullsized":"2023-09-23T19:50:57.793018+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.771560+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:50:56.771560+0000","last_peered":"2023-09-23T19:50:56.771560+0000","last_clean":"2023-09-23T19:50:56.771560+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:50:56.771560+0000","last_undegraded":"2023-09-23T19:50:56.771560+0000","last_fullsized":"2023-09-23T19:50:56.771560+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790293+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:50:57.790293+0000","last_peered":"2023-09-23T19:50:57.790293+0000","last_clean":"2023-09-23T19:50:57.790293+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:50:57.790293+0000","last_undegraded":"2023-09-23T19:50:57.790293+0000","last_fullsized":"2023-09-23T19:50:57.790293+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.788133+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:50:57.788133+0000","last_peered":"2023-09-23T19:50:57.788133+0000","last_clean":"2023-09-23T19:50:57.788133+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:50:57.788133+0000","last_undegraded":"2023-09-23T19:50:57.788133+0000","last_fullsized":"2023-09-23T19:50:57.788133+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790499+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:50:57.790499+0000","last_peered":"2023-09-23T19:50:57.790499+0000","last_clean":"2023-09-23T19:50:57.790499+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:50:57.790499+0000","last_undegraded":"2023-09-23T19:50:57.790499+0000","last_fullsized":"2023-09-23T19:50:57.790499+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.773321+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:50:56.773321+0000","last_peered":"2023-09-23T19:50:56.773321+0000","last_clean":"2023-09-23T19:50:56.773321+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:50:56.773321+0000","last_undegraded":"2023-09-23T19:50:56.773321+0000","last_fullsized":"2023-09-23T19:50:56.773321+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764907+0000","last_change":"2023-09-23T19:49:51.379282+0000","last_active":"2023-09-23T19:50:56.764907+0000","last_peered":"2023-09-23T19:50:56.764907+0000","last_clean":"2023-09-23T19:50:56.764907+0000","last_became_active":"2023-09-23T19:49:51.378849+0000","last_became_peered":"2023-09-23T19:49:51.378849+0000","last_unstale":"2023-09-23T19:50:56.764907+0000","last_undegraded":"2023-09-23T19:50:56.764907+0000","last_fullsized":"2023-09-23T19:50:56.764907+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:00.093034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766139+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:50:56.766139+0000","last_peered":"2023-09-23T19:50:56.766139+0000","last_clean":"2023-09-23T19:50:56.766139+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:50:56.766139+0000","last_undegraded":"2023-09-23T19:50:56.766139+0000","last_fullsized":"2023-09-23T19:50:56.766139+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770911+0000","last_change":"2023-09-23T19:50:23.998186+0000","last_active":"2023-09-23T19:50:56.770911+0000","last_peered":"2023-09-23T19:50:56.770911+0000","last_clean":"2023-09-23T19:50:56.770911+0000","last_became_active":"2023-09-23T19:50:23.997897+0000","last_became_peered":"2023-09-23T19:50:23.997897+0000","last_unstale":"2023-09-23T19:50:56.770911+0000","last_undegraded":"2023-09-23T19:50:56.770911+0000","last_fullsized":"2023-09-23T19:50:56.770911+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:57.792861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793984+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:50:57.793984+0000","last_peered":"2023-09-23T19:50:57.793984+0000","last_clean":"2023-09-23T19:50:57.793984+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:50:57.793984+0000","last_undegraded":"2023-09-23T19:50:57.793984+0000","last_fullsized":"2023-09-23T19:50:57.793984+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789044+0000","last_change":"2023-09-23T19:50:57.789044+0000","last_active":"2023-09-23T19:50:57.789044+0000","last_peered":"2023-09-23T19:50:57.789044+0000","last_clean":"2023-09-23T19:50:57.789044+0000","last_became_active":"2023-09-23T19:50:57.788730+0000","last_became_peered":"2023-09-23T19:50:57.788730+0000","last_unstale":"2023-09-23T19:50:57.789044+0000","last_undegraded":"2023-09-23T19:50:57.789044+0000","last_fullsized":"2023-09-23T19:50:57.789044+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:23.585694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765794+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:50:56.765794+0000","last_peered":"2023-09-23T19:50:56.765794+0000","last_clean":"2023-09-23T19:50:56.765794+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:50:56.765794+0000","last_undegraded":"2023-09-23T19:50:56.765794+0000","last_fullsized":"2023-09-23T19:50:56.765794+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789294+0000","last_change":"2023-09-23T19:50:57.789294+0000","last_active":"2023-09-23T19:50:57.789294+0000","last_peered":"2023-09-23T19:50:57.789294+0000","last_clean":"2023-09-23T19:50:57.789294+0000","last_became_active":"2023-09-23T19:50:57.788922+0000","last_became_peered":"2023-09-23T19:50:57.788922+0000","last_unstale":"2023-09-23T19:50:57.789294+0000","last_undegraded":"2023-09-23T19:50:57.789294+0000","last_fullsized":"2023-09-23T19:50:57.789294+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:11.829374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789127+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:50:57.789127+0000","last_peered":"2023-09-23T19:50:57.789127+0000","last_clean":"2023-09-23T19:50:57.789127+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:50:57.789127+0000","last_undegraded":"2023-09-23T19:50:57.789127+0000","last_fullsized":"2023-09-23T19:50:57.789127+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770478+0000","last_change":"2023-09-23T19:49:51.374353+0000","last_active":"2023-09-23T19:50:56.770478+0000","last_peered":"2023-09-23T19:50:56.770478+0000","last_clean":"2023-09-23T19:50:56.770478+0000","last_became_active":"2023-09-23T19:49:51.373992+0000","last_became_peered":"2023-09-23T19:49:51.373992+0000","last_unstale":"2023-09-23T19:50:56.770478+0000","last_undegraded":"2023-09-23T19:50:56.770478+0000","last_fullsized":"2023-09-23T19:50:56.770478+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.344350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789164+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:50:57.789164+0000","last_peered":"2023-09-23T19:50:57.789164+0000","last_clean":"2023-09-23T19:50:57.789164+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:50:57.789164+0000","last_undegraded":"2023-09-23T19:50:57.789164+0000","last_fullsized":"2023-09-23T19:50:57.789164+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.791240+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:50:57.791240+0000","last_peered":"2023-09-23T19:50:57.791240+0000","last_clean":"2023-09-23T19:50:57.791240+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:50:57.791240+0000","last_undegraded":"2023-09-23T19:50:57.791240+0000","last_fullsized":"2023-09-23T19:50:57.791240+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794931+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:50:57.794931+0000","last_peered":"2023-09-23T19:50:57.794931+0000","last_clean":"2023-09-23T19:50:57.794931+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:50:57.794931+0000","last_undegraded":"2023-09-23T19:50:57.794931+0000","last_fullsized":"2023-09-23T19:50:57.794931+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.792804+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:50:57.792804+0000","last_peered":"2023-09-23T19:50:57.792804+0000","last_clean":"2023-09-23T19:50:57.792804+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:50:57.792804+0000","last_undegraded":"2023-09-23T19:50:57.792804+0000","last_fullsized":"2023-09-23T19:50:57.792804+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764710+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:50:56.764710+0000","last_peered":"2023-09-23T19:50:56.764710+0000","last_clean":"2023-09-23T19:50:56.764710+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:50:56.764710+0000","last_undegraded":"2023-09-23T19:50:56.764710+0000","last_fullsized":"2023-09-23T19:50:56.764710+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234128,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17132,"kb_used_data":5796,"kb_used_omap":0,"kb_used_meta":11328,"kb_avail":93728020,"statfs":{"total":95995035648,"available":95977492480,"internally_reserved":0,"allocated":5935104,"data_stored":5405311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11599872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.61199999999999999,"15min":0.59299999999999997},"min":{"1min":0.36899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.9850000000000003,"5min":4.9850000000000003,"15min":4.9850000000000003},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":0.57399999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.628}]},{"osd":2,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":5.1600000000000001,"5min":5.1600000000000001,"15min":5.1600000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.66600000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59099999999999997,"15min":0.53300000000000003},"min":{"1min":0.45300000000000001,"5min":0.39800000000000002,"15min":0.36299999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":1.546},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.54500000000000004},"min":{"1min":0.379,"5min":0.35899999999999999,"15min":0.317},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.72599999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.504,"15min":0.47199999999999998},"min":{"1min":0.36499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73999999999999999,"5min":0.85299999999999998,"15min":0.878},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.56599999999999995,"15min":0.51900000000000002},"min":{"1min":0.39800000000000002,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":0.79200000000000004,"5min":0.93300000000000005,"15min":1.47},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.51800000000000002},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":0.77000000000000002,"5min":0.87,"15min":0.89100000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58199999999999996,"15min":0.53600000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69699999999999995,"5min":0.80400000000000005,"15min":1.3500000000000001},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.61099999999999999,"15min":0.56200000000000006},"min":{"1min":0.41499999999999998,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":0.77600000000000002,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.64700000000000002,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.82599999999999996,"5min":0.86199999999999999,"15min":1.52},"last":0.67900000000000005}]}]},{"osd":6,"up_from":46,"seq":197568495765,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8352,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93736800,"statfs":{"total":95995035648,"available":95986483200,"internally_reserved":0,"allocated":1728512,"data_stored":1210673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.57299999999999995,"15min":0.59299999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.60599999999999998,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.63900000000000001,"15min":0.65500000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.71999999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.55100000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.441}]},{"osd":2,"last update":"Sat Sep 23 19:51:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.501}]},{"osd":3,"last update":"Sat Sep 23 19:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58199999999999996,"15min":0.55400000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.32400000000000001},"max":{"1min":0.76800000000000002,"5min":0.83399999999999996,"15min":0.996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.39600000000000002,"5min":0.34999999999999998,"15min":0.32100000000000001},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":2.0579999999999998},"last":0.57799999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.503,"15min":0.47799999999999998},"min":{"1min":0.373,"5min":0.33800000000000002,"15min":0.28999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.84699999999999998},"last":0.623},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.52200000000000002,"15min":0.5},"min":{"1min":0.38500000000000001,"5min":0.34100000000000003,"15min":0.26500000000000001},"max":{"1min":0.73199999999999998,"5min":0.75700000000000001,"15min":0.93799999999999994},"last":0.68899999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52300000000000002,"15min":0.48699999999999999},"min":{"1min":0.38200000000000001,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":0.69899999999999995,"5min":0.70799999999999996,"15min":0.76500000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.56399999999999995,"15min":0.54100000000000004},"min":{"1min":0.373,"5min":0.32200000000000001,"15min":0.30499999999999999},"max":{"1min":0.72499999999999998,"5min":0.79200000000000004,"15min":0.92400000000000004},"last":0.72299999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:51:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.57999999999999996,"15min":0.56899999999999995},"min":{"1min":0.42399999999999999,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":0.877,"5min":0.877,"15min":0.89700000000000002},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.63500000000000001,"15min":0.60699999999999998},"min":{"1min":0.45200000000000001,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":0.82999999999999996,"5min":0.86499999999999999,"15min":1.0029999999999999},"last":0.64400000000000002}]}]},{"osd":1,"up_from":99,"seq":425201762330,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32832,"kb_used_data":5240,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93712320,"statfs":{"total":95995035648,"available":95961415680,"internally_reserved":0,"allocated":5365760,"data_stored":4818922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":28238650},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.35799999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.107,"5min":1.107,"15min":1.107},"last":0.54400000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":0.627}]},{"osd":6,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.57899999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.246,"5min":1.246,"15min":1.246},"last":0.60799999999999998}]}]},{"osd":0,"up_from":94,"seq":403726925865,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33416,"kb_used_data":5824,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93711736,"statfs":{"total":95995035648,"available":95960817664,"internally_reserved":0,"allocated":5963776,"data_stored":5409717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":28237409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8029999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.311}]},{"osd":3,"last update":"Sat Sep 23 19:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.47999999999999998,"15min":0.498},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.55400000000000005,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.47599999999999998,"15min":0.48799999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.64400000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.623,"15min":0.64300000000000002},"min":{"1min":0.41699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.496},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.64500000000000002,"15min":0.67700000000000005},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79000000000000004,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67800000000000005,"15min":0.70199999999999996},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.66900000000000004,"15min":0.70199999999999996},"min":{"1min":0.433,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.66700000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.68000000000000005,"15min":0.70999999999999996},"min":{"1min":0.46200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67600000000000005,"15min":0.70399999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.63300000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.68500000000000005,"15min":0.71899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.69699999999999995}]}]},{"osd":2,"up_from":104,"seq":446676598803,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28664,"kb_used_data":1136,"kb_used_omap":12,"kb_used_meta":27507,"kb_avail":93716488,"statfs":{"total":95995035648,"available":95965683712,"internally_reserved":0,"allocated":1163264,"data_stored":622590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":28168190},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.376},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.50600000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.1950000000000001,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.64600000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.622},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.71599999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1120,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1146880,"data_stored":620545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":28175469},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52700000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58999999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69899999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019038,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7844,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":6720,"kb_avail":93737308,"statfs":{"total":95995035648,"available":95987003392,"internally_reserved":0,"allocated":1142784,"data_stored":620627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6881280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58599999999999997,"15min":0.59999999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78000000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.626,"15min":0.63800000000000001},"min":{"1min":0.47399999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82999999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.44800000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:50:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56200000000000006,"15min":0.52900000000000003},"min":{"1min":0.39600000000000002,"5min":0.315,"15min":0.315},"max":{"1min":1.012,"5min":1.012,"15min":1.2430000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.624,"5min":0.58399999999999996,"15min":0.54500000000000004},"min":{"1min":0.42699999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":1.2090000000000001},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.502,"15min":0.45200000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":0.68500000000000005,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.47899999999999998},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":0.66100000000000003,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:50:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.53500000000000003,"15min":0.48299999999999998},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.27100000000000002},"max":{"1min":0.85499999999999998,"5min":0.86399999999999999,"15min":1.0589999999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.56499999999999995,"15min":0.51400000000000001},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.309},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.073},"last":0.69599999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59199999999999997,"15min":0.56699999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.29599999999999999},"max":{"1min":0.79400000000000004,"5min":1.117,"15min":1.117},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.63,"15min":0.58599999999999997},"min":{"1min":0.51500000000000001,"5min":0.376,"15min":0.35799999999999998},"max":{"1min":0.79200000000000004,"5min":1.016,"15min":1.016},"last":0.621}]}]},{"osd":5,"up_from":38,"seq":163208757402,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7448,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":6336,"kb_avail":93737704,"statfs":{"total":95995035648,"available":95987408896,"internally_reserved":0,"allocated":1130496,"data_stored":616332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6488064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.54200000000000004,"15min":0.56699999999999995},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.70499999999999996,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73599999999999999,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.54000000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.47099999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:51:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.501},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.66500000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.59299999999999997,"15min":0.55100000000000005},"min":{"1min":0.35399999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78900000000000003,"5min":1.6599999999999999,"15min":2.3519999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58599999999999997,"15min":0.58799999999999997},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.86899999999999999,"5min":0.88200000000000001,"15min":4.008},"last":0.626}]},{"osd":4,"last update":"Sat Sep 23 19:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.46700000000000003,"15min":0.44400000000000001},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.23699999999999999},"max":{"1min":0.78900000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.50700000000000001,"15min":0.47599999999999998},"min":{"1min":0.39400000000000002,"5min":0.28599999999999998,"15min":0.27500000000000002},"max":{"1min":0.753,"5min":0.76800000000000002,"15min":0.86399999999999999},"last":0.68300000000000005}]},{"osd":6,"last update":"Sat Sep 23 19:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52100000000000002,"15min":0.48599999999999999},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":0.72299999999999998,"5min":0.84199999999999997,"15min":0.86599999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.54100000000000004},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.307},"max":{"1min":0.75700000000000001,"5min":0.80400000000000005,"15min":0.82799999999999996},"last":0.751}]},{"osd":7,"last update":"Sat Sep 23 19:51:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.55200000000000005,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":1.0309999999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.55800000000000005,"15min":0.55500000000000005},"min":{"1min":0.46000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.86499999999999999},"last":0.65400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:51:53.680 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:51:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:51:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:51:53] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.43.0" 2023-09-23T19:51:54.028 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:54 smithi135 ceph-mon[221113]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:51:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:54 smithi135 ceph-mon[221113]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:54 smithi135 ceph-mon[213871]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:51:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:54 smithi135 ceph-mon[213871]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:54 smithi144 ceph-mon[194833]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:51:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:54 smithi144 ceph-mon[194833]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:55.768 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:51:55.770 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:51:56.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:56 smithi135 ceph-mon[221113]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:56 smithi135 ceph-mon[213871]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:56.591 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-09-23T19:51:55.108040+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164336,"kb_used_data":23024,"kb_used_omap":32,"kb_used_meta":141215,"kb_avail":749796880,"statfs":{"total":767960285184,"available":767792005120,"internally_reserved":0,"allocated":23576576,"data_stored":19324717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":144604678},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002528"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764512+0000","last_change":"2023-09-23T19:49:51.378856+0000","last_active":"2023-09-23T19:50:56.764512+0000","last_peered":"2023-09-23T19:50:56.764512+0000","last_clean":"2023-09-23T19:50:56.764512+0000","last_became_active":"2023-09-23T19:49:51.378451+0000","last_became_peered":"2023-09-23T19:49:51.378451+0000","last_unstale":"2023-09-23T19:50:56.764512+0000","last_undegraded":"2023-09-23T19:50:56.764512+0000","last_fullsized":"2023-09-23T19:50:56.764512+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:03.276740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770013+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:50:56.770013+0000","last_peered":"2023-09-23T19:50:56.770013+0000","last_clean":"2023-09-23T19:50:56.770013+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:50:56.770013+0000","last_undegraded":"2023-09-23T19:50:56.770013+0000","last_fullsized":"2023-09-23T19:50:56.770013+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.785311+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:50:57.785311+0000","last_peered":"2023-09-23T19:50:57.785311+0000","last_clean":"2023-09-23T19:50:57.785311+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:50:57.785311+0000","last_undegraded":"2023-09-23T19:50:57.785311+0000","last_fullsized":"2023-09-23T19:50:57.785311+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786561+0000","last_change":"2023-09-23T19:50:57.786561+0000","last_active":"2023-09-23T19:50:57.786561+0000","last_peered":"2023-09-23T19:50:57.786561+0000","last_clean":"2023-09-23T19:50:57.786561+0000","last_became_active":"2023-09-23T19:50:57.786231+0000","last_became_peered":"2023-09-23T19:50:57.786231+0000","last_unstale":"2023-09-23T19:50:57.786561+0000","last_undegraded":"2023-09-23T19:50:57.786561+0000","last_fullsized":"2023-09-23T19:50:57.786561+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:27.327075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790948+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:50:57.790948+0000","last_peered":"2023-09-23T19:50:57.790948+0000","last_clean":"2023-09-23T19:50:57.790948+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:50:57.790948+0000","last_undegraded":"2023-09-23T19:50:57.790948+0000","last_fullsized":"2023-09-23T19:50:57.790948+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768019+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:50:56.768019+0000","last_peered":"2023-09-23T19:50:56.768019+0000","last_clean":"2023-09-23T19:50:56.768019+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:50:56.768019+0000","last_undegraded":"2023-09-23T19:50:56.768019+0000","last_fullsized":"2023-09-23T19:50:56.768019+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789973+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:50:57.789973+0000","last_peered":"2023-09-23T19:50:57.789973+0000","last_clean":"2023-09-23T19:50:57.789973+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:50:57.789973+0000","last_undegraded":"2023-09-23T19:50:57.789973+0000","last_fullsized":"2023-09-23T19:50:57.789973+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768916+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:50:56.768916+0000","last_peered":"2023-09-23T19:50:56.768916+0000","last_clean":"2023-09-23T19:50:56.768916+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:50:56.768916+0000","last_undegraded":"2023-09-23T19:50:56.768916+0000","last_fullsized":"2023-09-23T19:50:56.768916+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790667+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:50:57.790667+0000","last_peered":"2023-09-23T19:50:57.790667+0000","last_clean":"2023-09-23T19:50:57.790667+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:50:57.790667+0000","last_undegraded":"2023-09-23T19:50:57.790667+0000","last_fullsized":"2023-09-23T19:50:57.790667+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.777637+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:50:57.777637+0000","last_peered":"2023-09-23T19:50:57.777637+0000","last_clean":"2023-09-23T19:50:57.777637+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:50:57.777637+0000","last_undegraded":"2023-09-23T19:50:57.777637+0000","last_fullsized":"2023-09-23T19:50:57.777637+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765709+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:50:56.765709+0000","last_peered":"2023-09-23T19:50:56.765709+0000","last_clean":"2023-09-23T19:50:56.765709+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:50:56.765709+0000","last_undegraded":"2023-09-23T19:50:56.765709+0000","last_fullsized":"2023-09-23T19:50:56.765709+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.777187+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:50:56.777187+0000","last_peered":"2023-09-23T19:50:56.777187+0000","last_clean":"2023-09-23T19:50:56.777187+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:50:56.777187+0000","last_undegraded":"2023-09-23T19:50:56.777187+0000","last_fullsized":"2023-09-23T19:50:56.777187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":342,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770704+0000","last_change":"2023-09-23T19:50:23.999651+0000","last_active":"2023-09-23T19:50:56.770704+0000","last_peered":"2023-09-23T19:50:56.770704+0000","last_clean":"2023-09-23T19:50:56.770704+0000","last_became_active":"2023-09-23T19:50:23.999184+0000","last_became_peered":"2023-09-23T19:50:23.999184+0000","last_unstale":"2023-09-23T19:50:56.770704+0000","last_undegraded":"2023-09-23T19:50:56.770704+0000","last_fullsized":"2023-09-23T19:50:56.770704+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:41.010259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793288+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:50:57.793288+0000","last_peered":"2023-09-23T19:50:57.793288+0000","last_clean":"2023-09-23T19:50:57.793288+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:50:57.793288+0000","last_undegraded":"2023-09-23T19:50:57.793288+0000","last_fullsized":"2023-09-23T19:50:57.793288+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":370,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.767547+0000","last_change":"2023-09-23T19:49:51.380375+0000","last_active":"2023-09-23T19:50:56.767547+0000","last_peered":"2023-09-23T19:50:56.767547+0000","last_clean":"2023-09-23T19:50:56.767547+0000","last_became_active":"2023-09-23T19:49:51.379793+0000","last_became_peered":"2023-09-23T19:49:51.379793+0000","last_unstale":"2023-09-23T19:50:56.767547+0000","last_undegraded":"2023-09-23T19:50:56.767547+0000","last_fullsized":"2023-09-23T19:50:56.767547+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:26.355152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789467+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:50:57.789467+0000","last_peered":"2023-09-23T19:50:57.789467+0000","last_clean":"2023-09-23T19:50:57.789467+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:50:57.789467+0000","last_undegraded":"2023-09-23T19:50:57.789467+0000","last_fullsized":"2023-09-23T19:50:57.789467+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.767556+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:50:56.767556+0000","last_peered":"2023-09-23T19:50:56.767556+0000","last_clean":"2023-09-23T19:50:56.767556+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:50:56.767556+0000","last_undegraded":"2023-09-23T19:50:56.767556+0000","last_fullsized":"2023-09-23T19:50:56.767556+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787777+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:50:57.787777+0000","last_peered":"2023-09-23T19:50:57.787777+0000","last_clean":"2023-09-23T19:50:57.787777+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:50:57.787777+0000","last_undegraded":"2023-09-23T19:50:57.787777+0000","last_fullsized":"2023-09-23T19:50:57.787777+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":188,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774652+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:50:56.774652+0000","last_peered":"2023-09-23T19:50:56.774652+0000","last_clean":"2023-09-23T19:50:56.774652+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:50:56.774652+0000","last_undegraded":"2023-09-23T19:50:56.774652+0000","last_fullsized":"2023-09-23T19:50:56.774652+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.777263+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:50:56.777263+0000","last_peered":"2023-09-23T19:50:56.777263+0000","last_clean":"2023-09-23T19:50:56.777263+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:50:56.777263+0000","last_undegraded":"2023-09-23T19:50:56.777263+0000","last_fullsized":"2023-09-23T19:50:56.777263+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794256+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:50:57.794256+0000","last_peered":"2023-09-23T19:50:57.794256+0000","last_clean":"2023-09-23T19:50:57.794256+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:50:57.794256+0000","last_undegraded":"2023-09-23T19:50:57.794256+0000","last_fullsized":"2023-09-23T19:50:57.794256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":828,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.772649+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:50:56.772649+0000","last_peered":"2023-09-23T19:50:56.772649+0000","last_clean":"2023-09-23T19:50:56.772649+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:50:56.772649+0000","last_undegraded":"2023-09-23T19:50:56.772649+0000","last_fullsized":"2023-09-23T19:50:56.772649+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770724+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:50:56.770724+0000","last_peered":"2023-09-23T19:50:56.770724+0000","last_clean":"2023-09-23T19:50:56.770724+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:50:56.770724+0000","last_undegraded":"2023-09-23T19:50:56.770724+0000","last_fullsized":"2023-09-23T19:50:56.770724+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787513+0000","last_change":"2023-09-23T19:50:57.787513+0000","last_active":"2023-09-23T19:50:57.787513+0000","last_peered":"2023-09-23T19:50:57.787513+0000","last_clean":"2023-09-23T19:50:57.787513+0000","last_became_active":"2023-09-23T19:50:57.787175+0000","last_became_peered":"2023-09-23T19:50:57.787175+0000","last_unstale":"2023-09-23T19:50:57.787513+0000","last_undegraded":"2023-09-23T19:50:57.787513+0000","last_fullsized":"2023-09-23T19:50:57.787513+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:19.081986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":450,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.772552+0000","last_change":"2023-09-23T19:48:37.575593+0000","last_active":"2023-09-23T19:50:56.772552+0000","last_peered":"2023-09-23T19:50:56.772552+0000","last_clean":"2023-09-23T19:50:56.772552+0000","last_became_active":"2023-09-23T19:48:37.575214+0000","last_became_peered":"2023-09-23T19:48:37.575214+0000","last_unstale":"2023-09-23T19:50:56.772552+0000","last_undegraded":"2023-09-23T19:50:56.772552+0000","last_fullsized":"2023-09-23T19:50:56.772552+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:13.335201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":919,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766899+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:50:56.766899+0000","last_peered":"2023-09-23T19:50:56.766899+0000","last_clean":"2023-09-23T19:50:56.766899+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:50:56.766899+0000","last_undegraded":"2023-09-23T19:50:56.766899+0000","last_fullsized":"2023-09-23T19:50:56.766899+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.780129+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:50:56.780129+0000","last_peered":"2023-09-23T19:50:56.780129+0000","last_clean":"2023-09-23T19:50:56.780129+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:50:56.780129+0000","last_undegraded":"2023-09-23T19:50:56.780129+0000","last_fullsized":"2023-09-23T19:50:56.780129+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786740+0000","last_change":"2023-09-23T19:50:23.997623+0000","last_active":"2023-09-23T19:50:57.786740+0000","last_peered":"2023-09-23T19:50:57.786740+0000","last_clean":"2023-09-23T19:50:57.786740+0000","last_became_active":"2023-09-23T19:50:23.997428+0000","last_became_peered":"2023-09-23T19:50:23.997428+0000","last_unstale":"2023-09-23T19:50:57.786740+0000","last_undegraded":"2023-09-23T19:50:57.786740+0000","last_fullsized":"2023-09-23T19:50:57.786740+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:50.529980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":594,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786571+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:50:57.786571+0000","last_peered":"2023-09-23T19:50:57.786571+0000","last_clean":"2023-09-23T19:50:57.786571+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:50:57.786571+0000","last_undegraded":"2023-09-23T19:50:57.786571+0000","last_fullsized":"2023-09-23T19:50:57.786571+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794391+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:50:57.794391+0000","last_peered":"2023-09-23T19:50:57.794391+0000","last_clean":"2023-09-23T19:50:57.794391+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:50:57.794391+0000","last_undegraded":"2023-09-23T19:50:57.794391+0000","last_fullsized":"2023-09-23T19:50:57.794391+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765277+0000","last_change":"2023-09-23T19:48:37.573541+0000","last_active":"2023-09-23T19:50:56.765277+0000","last_peered":"2023-09-23T19:50:56.765277+0000","last_clean":"2023-09-23T19:50:56.765277+0000","last_became_active":"2023-09-23T19:48:37.573213+0000","last_became_peered":"2023-09-23T19:48:37.573213+0000","last_unstale":"2023-09-23T19:50:56.765277+0000","last_undegraded":"2023-09-23T19:50:56.765277+0000","last_fullsized":"2023-09-23T19:50:56.765277+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:16.981072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":634,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766698+0000","last_change":"2023-09-23T19:49:51.382830+0000","last_active":"2023-09-23T19:50:56.766698+0000","last_peered":"2023-09-23T19:50:56.766698+0000","last_clean":"2023-09-23T19:50:56.766698+0000","last_became_active":"2023-09-23T19:49:51.382367+0000","last_became_peered":"2023-09-23T19:49:51.382367+0000","last_unstale":"2023-09-23T19:50:56.766698+0000","last_undegraded":"2023-09-23T19:50:56.766698+0000","last_fullsized":"2023-09-23T19:50:56.766698+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:48.082643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.781187+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:50:57.781187+0000","last_peered":"2023-09-23T19:50:57.781187+0000","last_clean":"2023-09-23T19:50:57.781187+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:50:57.781187+0000","last_undegraded":"2023-09-23T19:50:57.781187+0000","last_fullsized":"2023-09-23T19:50:57.781187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787126+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:50:57.787126+0000","last_peered":"2023-09-23T19:50:57.787126+0000","last_clean":"2023-09-23T19:50:57.787126+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:50:57.787126+0000","last_undegraded":"2023-09-23T19:50:57.787126+0000","last_fullsized":"2023-09-23T19:50:57.787126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":445,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787693+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:50:57.787693+0000","last_peered":"2023-09-23T19:50:57.787693+0000","last_clean":"2023-09-23T19:50:57.787693+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:50:57.787693+0000","last_undegraded":"2023-09-23T19:50:57.787693+0000","last_fullsized":"2023-09-23T19:50:57.787693+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790403+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:50:57.790403+0000","last_peered":"2023-09-23T19:50:57.790403+0000","last_clean":"2023-09-23T19:50:57.790403+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:50:57.790403+0000","last_undegraded":"2023-09-23T19:50:57.790403+0000","last_fullsized":"2023-09-23T19:50:57.790403+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793102+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:50:57.793102+0000","last_peered":"2023-09-23T19:50:57.793102+0000","last_clean":"2023-09-23T19:50:57.793102+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:50:57.793102+0000","last_undegraded":"2023-09-23T19:50:57.793102+0000","last_fullsized":"2023-09-23T19:50:57.793102+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765182+0000","last_change":"2023-09-23T19:50:23.998753+0000","last_active":"2023-09-23T19:50:56.765182+0000","last_peered":"2023-09-23T19:50:56.765182+0000","last_clean":"2023-09-23T19:50:56.765182+0000","last_became_active":"2023-09-23T19:50:23.998417+0000","last_became_peered":"2023-09-23T19:50:23.998417+0000","last_unstale":"2023-09-23T19:50:56.765182+0000","last_undegraded":"2023-09-23T19:50:56.765182+0000","last_fullsized":"2023-09-23T19:50:56.765182+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:11.002531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765806+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:50:56.765806+0000","last_peered":"2023-09-23T19:50:56.765806+0000","last_clean":"2023-09-23T19:50:56.765806+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:50:56.765806+0000","last_undegraded":"2023-09-23T19:50:56.765806+0000","last_fullsized":"2023-09-23T19:50:56.765806+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766899+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:50:56.766899+0000","last_peered":"2023-09-23T19:50:56.766899+0000","last_clean":"2023-09-23T19:50:56.766899+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:50:56.766899+0000","last_undegraded":"2023-09-23T19:50:56.766899+0000","last_fullsized":"2023-09-23T19:50:56.766899+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770764+0000","last_change":"2023-09-23T19:49:51.380925+0000","last_active":"2023-09-23T19:50:56.770764+0000","last_peered":"2023-09-23T19:50:56.770764+0000","last_clean":"2023-09-23T19:50:56.770764+0000","last_became_active":"2023-09-23T19:49:51.380427+0000","last_became_peered":"2023-09-23T19:49:51.380427+0000","last_unstale":"2023-09-23T19:50:56.770764+0000","last_undegraded":"2023-09-23T19:50:56.770764+0000","last_fullsized":"2023-09-23T19:50:56.770764+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:49.879887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790231+0000","last_change":"2023-09-23T19:50:57.790231+0000","last_active":"2023-09-23T19:50:57.790231+0000","last_peered":"2023-09-23T19:50:57.790231+0000","last_clean":"2023-09-23T19:50:57.790231+0000","last_became_active":"2023-09-23T19:50:57.789446+0000","last_became_peered":"2023-09-23T19:50:57.789446+0000","last_unstale":"2023-09-23T19:50:57.790231+0000","last_undegraded":"2023-09-23T19:50:57.790231+0000","last_fullsized":"2023-09-23T19:50:57.790231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:41.313276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793011+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:50:57.793011+0000","last_peered":"2023-09-23T19:50:57.793011+0000","last_clean":"2023-09-23T19:50:57.793011+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:50:57.793011+0000","last_undegraded":"2023-09-23T19:50:57.793011+0000","last_fullsized":"2023-09-23T19:50:57.793011+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768198+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:50:56.768198+0000","last_peered":"2023-09-23T19:50:56.768198+0000","last_clean":"2023-09-23T19:50:56.768198+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:50:56.768198+0000","last_undegraded":"2023-09-23T19:50:56.768198+0000","last_fullsized":"2023-09-23T19:50:56.768198+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.788389+0000","last_change":"2023-09-23T19:50:57.788389+0000","last_active":"2023-09-23T19:50:57.788389+0000","last_peered":"2023-09-23T19:50:57.788389+0000","last_clean":"2023-09-23T19:50:57.788389+0000","last_became_active":"2023-09-23T19:50:57.788075+0000","last_became_peered":"2023-09-23T19:50:57.788075+0000","last_unstale":"2023-09-23T19:50:57.788389+0000","last_undegraded":"2023-09-23T19:50:57.788389+0000","last_fullsized":"2023-09-23T19:50:57.788389+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:06.180629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770475+0000","last_change":"2023-09-23T19:49:51.380726+0000","last_active":"2023-09-23T19:50:56.770475+0000","last_peered":"2023-09-23T19:50:56.770475+0000","last_clean":"2023-09-23T19:50:56.770475+0000","last_became_active":"2023-09-23T19:49:51.380021+0000","last_became_peered":"2023-09-23T19:49:51.380021+0000","last_unstale":"2023-09-23T19:50:56.770475+0000","last_undegraded":"2023-09-23T19:50:56.770475+0000","last_fullsized":"2023-09-23T19:50:56.770475+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:16.892029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774672+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:50:56.774672+0000","last_peered":"2023-09-23T19:50:56.774672+0000","last_clean":"2023-09-23T19:50:56.774672+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:50:56.774672+0000","last_undegraded":"2023-09-23T19:50:56.774672+0000","last_fullsized":"2023-09-23T19:50:56.774672+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.795290+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:50:57.795290+0000","last_peered":"2023-09-23T19:50:57.795290+0000","last_clean":"2023-09-23T19:50:57.795290+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:50:57.795290+0000","last_undegraded":"2023-09-23T19:50:57.795290+0000","last_fullsized":"2023-09-23T19:50:57.795290+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.773415+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:50:56.773415+0000","last_peered":"2023-09-23T19:50:56.773415+0000","last_clean":"2023-09-23T19:50:56.773415+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:50:56.773415+0000","last_undegraded":"2023-09-23T19:50:56.773415+0000","last_fullsized":"2023-09-23T19:50:56.773415+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765786+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:50:56.765786+0000","last_peered":"2023-09-23T19:50:56.765786+0000","last_clean":"2023-09-23T19:50:56.765786+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:50:56.765786+0000","last_undegraded":"2023-09-23T19:50:56.765786+0000","last_fullsized":"2023-09-23T19:50:56.765786+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774219+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:50:56.774219+0000","last_peered":"2023-09-23T19:50:56.774219+0000","last_clean":"2023-09-23T19:50:56.774219+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:50:56.774219+0000","last_undegraded":"2023-09-23T19:50:56.774219+0000","last_fullsized":"2023-09-23T19:50:56.774219+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789346+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:50:57.789346+0000","last_peered":"2023-09-23T19:50:57.789346+0000","last_clean":"2023-09-23T19:50:57.789346+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:50:57.789346+0000","last_undegraded":"2023-09-23T19:50:57.789346+0000","last_fullsized":"2023-09-23T19:50:57.789346+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770341+0000","last_change":"2023-09-23T19:48:37.567607+0000","last_active":"2023-09-23T19:50:56.770341+0000","last_peered":"2023-09-23T19:50:56.770341+0000","last_clean":"2023-09-23T19:50:56.770341+0000","last_became_active":"2023-09-23T19:48:37.567300+0000","last_became_peered":"2023-09-23T19:48:37.567300+0000","last_unstale":"2023-09-23T19:50:56.770341+0000","last_undegraded":"2023-09-23T19:50:56.770341+0000","last_fullsized":"2023-09-23T19:50:56.770341+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.088452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793018+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:50:57.793018+0000","last_peered":"2023-09-23T19:50:57.793018+0000","last_clean":"2023-09-23T19:50:57.793018+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:50:57.793018+0000","last_undegraded":"2023-09-23T19:50:57.793018+0000","last_fullsized":"2023-09-23T19:50:57.793018+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.771560+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:50:56.771560+0000","last_peered":"2023-09-23T19:50:56.771560+0000","last_clean":"2023-09-23T19:50:56.771560+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:50:56.771560+0000","last_undegraded":"2023-09-23T19:50:56.771560+0000","last_fullsized":"2023-09-23T19:50:56.771560+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790293+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:50:57.790293+0000","last_peered":"2023-09-23T19:50:57.790293+0000","last_clean":"2023-09-23T19:50:57.790293+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:50:57.790293+0000","last_undegraded":"2023-09-23T19:50:57.790293+0000","last_fullsized":"2023-09-23T19:50:57.790293+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.788133+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:50:57.788133+0000","last_peered":"2023-09-23T19:50:57.788133+0000","last_clean":"2023-09-23T19:50:57.788133+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:50:57.788133+0000","last_undegraded":"2023-09-23T19:50:57.788133+0000","last_fullsized":"2023-09-23T19:50:57.788133+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790499+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:50:57.790499+0000","last_peered":"2023-09-23T19:50:57.790499+0000","last_clean":"2023-09-23T19:50:57.790499+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:50:57.790499+0000","last_undegraded":"2023-09-23T19:50:57.790499+0000","last_fullsized":"2023-09-23T19:50:57.790499+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.773321+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:50:56.773321+0000","last_peered":"2023-09-23T19:50:56.773321+0000","last_clean":"2023-09-23T19:50:56.773321+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:50:56.773321+0000","last_undegraded":"2023-09-23T19:50:56.773321+0000","last_fullsized":"2023-09-23T19:50:56.773321+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764907+0000","last_change":"2023-09-23T19:49:51.379282+0000","last_active":"2023-09-23T19:50:56.764907+0000","last_peered":"2023-09-23T19:50:56.764907+0000","last_clean":"2023-09-23T19:50:56.764907+0000","last_became_active":"2023-09-23T19:49:51.378849+0000","last_became_peered":"2023-09-23T19:49:51.378849+0000","last_unstale":"2023-09-23T19:50:56.764907+0000","last_undegraded":"2023-09-23T19:50:56.764907+0000","last_fullsized":"2023-09-23T19:50:56.764907+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:00.093034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766139+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:50:56.766139+0000","last_peered":"2023-09-23T19:50:56.766139+0000","last_clean":"2023-09-23T19:50:56.766139+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:50:56.766139+0000","last_undegraded":"2023-09-23T19:50:56.766139+0000","last_fullsized":"2023-09-23T19:50:56.766139+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770911+0000","last_change":"2023-09-23T19:50:23.998186+0000","last_active":"2023-09-23T19:50:56.770911+0000","last_peered":"2023-09-23T19:50:56.770911+0000","last_clean":"2023-09-23T19:50:56.770911+0000","last_became_active":"2023-09-23T19:50:23.997897+0000","last_became_peered":"2023-09-23T19:50:23.997897+0000","last_unstale":"2023-09-23T19:50:56.770911+0000","last_undegraded":"2023-09-23T19:50:56.770911+0000","last_fullsized":"2023-09-23T19:50:56.770911+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:57.792861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793984+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:50:57.793984+0000","last_peered":"2023-09-23T19:50:57.793984+0000","last_clean":"2023-09-23T19:50:57.793984+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:50:57.793984+0000","last_undegraded":"2023-09-23T19:50:57.793984+0000","last_fullsized":"2023-09-23T19:50:57.793984+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789044+0000","last_change":"2023-09-23T19:50:57.789044+0000","last_active":"2023-09-23T19:50:57.789044+0000","last_peered":"2023-09-23T19:50:57.789044+0000","last_clean":"2023-09-23T19:50:57.789044+0000","last_became_active":"2023-09-23T19:50:57.788730+0000","last_became_peered":"2023-09-23T19:50:57.788730+0000","last_unstale":"2023-09-23T19:50:57.789044+0000","last_undegraded":"2023-09-23T19:50:57.789044+0000","last_fullsized":"2023-09-23T19:50:57.789044+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:23.585694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765794+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:50:56.765794+0000","last_peered":"2023-09-23T19:50:56.765794+0000","last_clean":"2023-09-23T19:50:56.765794+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:50:56.765794+0000","last_undegraded":"2023-09-23T19:50:56.765794+0000","last_fullsized":"2023-09-23T19:50:56.765794+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789294+0000","last_change":"2023-09-23T19:50:57.789294+0000","last_active":"2023-09-23T19:50:57.789294+0000","last_peered":"2023-09-23T19:50:57.789294+0000","last_clean":"2023-09-23T19:50:57.789294+0000","last_became_active":"2023-09-23T19:50:57.788922+0000","last_became_peered":"2023-09-23T19:50:57.788922+0000","last_unstale":"2023-09-23T19:50:57.789294+0000","last_undegraded":"2023-09-23T19:50:57.789294+0000","last_fullsized":"2023-09-23T19:50:57.789294+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:11.829374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789127+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:50:57.789127+0000","last_peered":"2023-09-23T19:50:57.789127+0000","last_clean":"2023-09-23T19:50:57.789127+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:50:57.789127+0000","last_undegraded":"2023-09-23T19:50:57.789127+0000","last_fullsized":"2023-09-23T19:50:57.789127+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770478+0000","last_change":"2023-09-23T19:49:51.374353+0000","last_active":"2023-09-23T19:50:56.770478+0000","last_peered":"2023-09-23T19:50:56.770478+0000","last_clean":"2023-09-23T19:50:56.770478+0000","last_became_active":"2023-09-23T19:49:51.373992+0000","last_became_peered":"2023-09-23T19:49:51.373992+0000","last_unstale":"2023-09-23T19:50:56.770478+0000","last_undegraded":"2023-09-23T19:50:56.770478+0000","last_fullsized":"2023-09-23T19:50:56.770478+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.344350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789164+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:50:57.789164+0000","last_peered":"2023-09-23T19:50:57.789164+0000","last_clean":"2023-09-23T19:50:57.789164+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:50:57.789164+0000","last_undegraded":"2023-09-23T19:50:57.789164+0000","last_fullsized":"2023-09-23T19:50:57.789164+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.791240+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:50:57.791240+0000","last_peered":"2023-09-23T19:50:57.791240+0000","last_clean":"2023-09-23T19:50:57.791240+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:50:57.791240+0000","last_undegraded":"2023-09-23T19:50:57.791240+0000","last_fullsized":"2023-09-23T19:50:57.791240+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794931+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:50:57.794931+0000","last_peered":"2023-09-23T19:50:57.794931+0000","last_clean":"2023-09-23T19:50:57.794931+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:50:57.794931+0000","last_undegraded":"2023-09-23T19:50:57.794931+0000","last_fullsized":"2023-09-23T19:50:57.794931+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.792804+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:50:57.792804+0000","last_peered":"2023-09-23T19:50:57.792804+0000","last_clean":"2023-09-23T19:50:57.792804+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:50:57.792804+0000","last_undegraded":"2023-09-23T19:50:57.792804+0000","last_fullsized":"2023-09-23T19:50:57.792804+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764710+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:50:56.764710+0000","last_peered":"2023-09-23T19:50:56.764710+0000","last_clean":"2023-09-23T19:50:56.764710+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:50:56.764710+0000","last_undegraded":"2023-09-23T19:50:56.764710+0000","last_fullsized":"2023-09-23T19:50:56.764710+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234129,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17132,"kb_used_data":5796,"kb_used_omap":0,"kb_used_meta":11328,"kb_avail":93728020,"statfs":{"total":95995035648,"available":95977492480,"internally_reserved":0,"allocated":5935104,"data_stored":5405311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11599872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.61199999999999999,"15min":0.59299999999999997},"min":{"1min":0.36899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.9850000000000003,"5min":4.9850000000000003,"15min":4.9850000000000003},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":0.65100000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":5.1600000000000001,"5min":5.1600000000000001,"15min":5.1600000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.61699999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59099999999999997,"15min":0.53300000000000003},"min":{"1min":0.45300000000000001,"5min":0.39800000000000002,"15min":0.36299999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":1.546},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.54500000000000004},"min":{"1min":0.379,"5min":0.35899999999999999,"15min":0.317},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.44700000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.504,"15min":0.47199999999999998},"min":{"1min":0.36499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73999999999999999,"5min":0.85299999999999998,"15min":0.878},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.56599999999999995,"15min":0.51900000000000002},"min":{"1min":0.39800000000000002,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":0.79200000000000004,"5min":0.93300000000000005,"15min":1.47},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.51800000000000002},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":0.77000000000000002,"5min":0.87,"15min":0.89100000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58199999999999996,"15min":0.53600000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69699999999999995,"5min":0.80400000000000005,"15min":1.3500000000000001},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.61099999999999999,"15min":0.56200000000000006},"min":{"1min":0.41499999999999998,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":0.77600000000000002,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.64700000000000002,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.82599999999999996,"5min":0.86199999999999999,"15min":1.52},"last":0.745}]}]},{"osd":6,"up_from":46,"seq":197568495766,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8352,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93736800,"statfs":{"total":95995035648,"available":95986483200,"internally_reserved":0,"allocated":1728512,"data_stored":1210673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.57299999999999995,"15min":0.59299999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.60599999999999998,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.63900000000000001,"15min":0.65500000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.71999999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.63100000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:51:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58199999999999996,"15min":0.55400000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.32400000000000001},"max":{"1min":0.76800000000000002,"5min":0.83399999999999996,"15min":0.996},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.39600000000000002,"5min":0.34999999999999998,"15min":0.32100000000000001},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":2.0579999999999998},"last":0.54600000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.503,"15min":0.47799999999999998},"min":{"1min":0.373,"5min":0.33800000000000002,"15min":0.28999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.84699999999999998},"last":0.436},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.52200000000000002,"15min":0.5},"min":{"1min":0.38500000000000001,"5min":0.34100000000000003,"15min":0.26500000000000001},"max":{"1min":0.73199999999999998,"5min":0.75700000000000001,"15min":0.93799999999999994},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52300000000000002,"15min":0.48699999999999999},"min":{"1min":0.38200000000000001,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":0.69899999999999995,"5min":0.70799999999999996,"15min":0.76500000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.56399999999999995,"15min":0.54100000000000004},"min":{"1min":0.373,"5min":0.32200000000000001,"15min":0.30499999999999999},"max":{"1min":0.72499999999999998,"5min":0.79200000000000004,"15min":0.92400000000000004},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:51:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.57999999999999996,"15min":0.56899999999999995},"min":{"1min":0.42399999999999999,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":0.877,"5min":0.877,"15min":0.89700000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.63500000000000001,"15min":0.60699999999999998},"min":{"1min":0.45200000000000001,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":0.82999999999999996,"5min":0.86499999999999999,"15min":1.0029999999999999},"last":0.68799999999999994}]}]},{"osd":1,"up_from":99,"seq":425201762331,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32832,"kb_used_data":5240,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93712320,"statfs":{"total":95995035648,"available":95961415680,"internally_reserved":0,"allocated":5365760,"data_stored":4818922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":28238650},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.35399999999999998,"15min":0.35799999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.55100000000000005,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.36299999999999999,"15min":0.36399999999999999},"min":{"1min":0.252,"5min":0.25,"15min":0.25},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.32200000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.33300000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28799999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.72699999999999998,"5min":1.161,"15min":1.161},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.441,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.81299999999999994,"5min":1.107,"15min":1.107},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.625,"15min":0.63500000000000001},"min":{"1min":0.439,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70599999999999996,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.441,"5min":0.375,"15min":0.375},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66100000000000003,"15min":0.66000000000000003},"min":{"1min":0.44900000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.622,"5min":0.68100000000000005,"15min":0.69099999999999995},"min":{"1min":0.46700000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81499999999999995,"5min":1.214,"15min":1.214},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64900000000000002},"min":{"1min":0.47599999999999998,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.65900000000000003,"15min":0.66700000000000004},"min":{"1min":0.44700000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80400000000000005,"5min":1.246,"15min":1.246},"last":0.54600000000000004}]}]},{"osd":0,"up_from":94,"seq":403726925865,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33416,"kb_used_data":5824,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93711736,"statfs":{"total":95995035648,"available":95960817664,"internally_reserved":0,"allocated":5963776,"data_stored":5409717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":28237409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8029999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.311}]},{"osd":3,"last update":"Sat Sep 23 19:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.47999999999999998,"15min":0.498},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.55400000000000005,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.47599999999999998,"15min":0.48799999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.64400000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.623,"15min":0.64300000000000002},"min":{"1min":0.41699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.496},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.64500000000000002,"15min":0.67700000000000005},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79000000000000004,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67800000000000005,"15min":0.70199999999999996},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.66900000000000004,"15min":0.70199999999999996},"min":{"1min":0.433,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.66700000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.68000000000000005,"15min":0.70999999999999996},"min":{"1min":0.46200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67600000000000005,"15min":0.70399999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.63300000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.68500000000000005,"15min":0.71899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.69699999999999995}]}]},{"osd":2,"up_from":104,"seq":446676598804,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28664,"kb_used_data":1136,"kb_used_omap":12,"kb_used_meta":27507,"kb_avail":93716488,"statfs":{"total":95995035648,"available":95965683712,"internally_reserved":0,"allocated":1163264,"data_stored":622590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":28168190},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.32200000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.35499999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34000000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.1950000000000001,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.42599999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.53500000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1120,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1146880,"data_stored":620545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":28175469},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52700000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58999999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69899999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019039,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7844,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":6720,"kb_avail":93737308,"statfs":{"total":95995035648,"available":95987003392,"internally_reserved":0,"allocated":1142784,"data_stored":620627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6881280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58599999999999997,"15min":0.59999999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78000000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.626,"15min":0.63800000000000001},"min":{"1min":0.47399999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82999999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.442}]},{"osd":1,"last update":"Sat Sep 23 19:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.56499999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:50:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56200000000000006,"15min":0.52900000000000003},"min":{"1min":0.39600000000000002,"5min":0.315,"15min":0.315},"max":{"1min":1.012,"5min":1.012,"15min":1.2430000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.58399999999999996,"15min":0.54500000000000004},"min":{"1min":0.42699999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":1.2090000000000001},"last":0.627}]},{"osd":5,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.502,"15min":0.45200000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":0.68500000000000005,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.47899999999999998},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":0.66100000000000003,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:50:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.53500000000000003,"15min":0.48299999999999998},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.27100000000000002},"max":{"1min":0.85499999999999998,"5min":0.86399999999999999,"15min":1.0589999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.56499999999999995,"15min":0.51400000000000001},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.309},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.073},"last":0.61599999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59199999999999997,"15min":0.56699999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.29599999999999999},"max":{"1min":0.79400000000000004,"5min":1.117,"15min":1.117},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.63,"15min":0.58599999999999997},"min":{"1min":0.51500000000000001,"5min":0.376,"15min":0.35799999999999998},"max":{"1min":0.79200000000000004,"5min":1.016,"15min":1.016},"last":0.64000000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757403,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7448,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":6336,"kb_avail":93737704,"statfs":{"total":95995035648,"available":95987408896,"internally_reserved":0,"allocated":1130496,"data_stored":616332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6488064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.54200000000000004,"15min":0.56699999999999995},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.70499999999999996,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73599999999999999,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.55700000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:51:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.68400000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.59299999999999997,"15min":0.55100000000000005},"min":{"1min":0.35399999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78900000000000003,"5min":1.6599999999999999,"15min":2.3519999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58599999999999997,"15min":0.58799999999999997},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.86899999999999999,"5min":0.88200000000000001,"15min":4.008},"last":0.70099999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.46700000000000003,"15min":0.44400000000000001},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.23699999999999999},"max":{"1min":0.78900000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.50700000000000001,"15min":0.47599999999999998},"min":{"1min":0.39400000000000002,"5min":0.28599999999999998,"15min":0.27500000000000002},"max":{"1min":0.753,"5min":0.76800000000000002,"15min":0.86399999999999999},"last":0.76600000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52100000000000002,"15min":0.48599999999999999},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":0.72299999999999998,"5min":0.84199999999999997,"15min":0.86599999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.54100000000000004},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.307},"max":{"1min":0.75700000000000001,"5min":0.80400000000000005,"15min":0.82799999999999996},"last":0.79000000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:51:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.55200000000000005,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":1.0309999999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.55800000000000005,"15min":0.55500000000000005},"min":{"1min":0.46000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.86499999999999999},"last":0.71399999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:51:56.628 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T19:51:56.628 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T19:51:56.628 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T19:51:56.628 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T19:51:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:56 smithi144 ceph-mon[194833]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:56.980 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:51:57.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:57 smithi135 ceph-mon[221113]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:51:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:57 smithi135 ceph-mon[213871]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:51:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:57 smithi144 ceph-mon[194833]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:51:58.342 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:58 smithi135 ceph-mon[221113]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:58.342 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:58 smithi135 ceph-mon[213871]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:58 smithi144 ceph-mon[194833]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:51:58.779 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:51:58.779 INFO:teuthology.orchestra.run.smithi135.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T19:51:59.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:51:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/877075133' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:51:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:51:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/877075133' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:51:59.548 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T19:51:59.549 INFO:teuthology.run_tasks:Running task print... 2023-09-23T19:51:59.558 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T19:51:59.558 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T19:51:59.571 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T19:51:59.571 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T19:51:59.571 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T19:51:59.572 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T19:51:59.572 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T19:51:59.573 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T19:51:59.573 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T19:51:59.573 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T19:51:59.573 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:51:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:51:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/877075133' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T19:51:59.918 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:00 smithi135 ceph-mon[221113]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:00 smithi135 ceph-mon[213871]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:00 smithi144 ceph-mon[194833]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:01.360 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:01 smithi135 ceph-mon[221113]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:01.360 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:01 smithi135 ceph-mon[213871]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:01 smithi144 ceph-mon[194833]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:01.778 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:52:01.780 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:52:02.549 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":215,"stamp":"2023-09-23T19:52:01.109417+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164336,"kb_used_data":23024,"kb_used_omap":32,"kb_used_meta":141215,"kb_avail":749796880,"statfs":{"total":767960285184,"available":767792005120,"internally_reserved":0,"allocated":23576576,"data_stored":19324717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":144604678},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002632"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764512+0000","last_change":"2023-09-23T19:49:51.378856+0000","last_active":"2023-09-23T19:50:56.764512+0000","last_peered":"2023-09-23T19:50:56.764512+0000","last_clean":"2023-09-23T19:50:56.764512+0000","last_became_active":"2023-09-23T19:49:51.378451+0000","last_became_peered":"2023-09-23T19:49:51.378451+0000","last_unstale":"2023-09-23T19:50:56.764512+0000","last_undegraded":"2023-09-23T19:50:56.764512+0000","last_fullsized":"2023-09-23T19:50:56.764512+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:03.276740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770013+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:50:56.770013+0000","last_peered":"2023-09-23T19:50:56.770013+0000","last_clean":"2023-09-23T19:50:56.770013+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:50:56.770013+0000","last_undegraded":"2023-09-23T19:50:56.770013+0000","last_fullsized":"2023-09-23T19:50:56.770013+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.785311+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:50:57.785311+0000","last_peered":"2023-09-23T19:50:57.785311+0000","last_clean":"2023-09-23T19:50:57.785311+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:50:57.785311+0000","last_undegraded":"2023-09-23T19:50:57.785311+0000","last_fullsized":"2023-09-23T19:50:57.785311+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786561+0000","last_change":"2023-09-23T19:50:57.786561+0000","last_active":"2023-09-23T19:50:57.786561+0000","last_peered":"2023-09-23T19:50:57.786561+0000","last_clean":"2023-09-23T19:50:57.786561+0000","last_became_active":"2023-09-23T19:50:57.786231+0000","last_became_peered":"2023-09-23T19:50:57.786231+0000","last_unstale":"2023-09-23T19:50:57.786561+0000","last_undegraded":"2023-09-23T19:50:57.786561+0000","last_fullsized":"2023-09-23T19:50:57.786561+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:27.327075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790948+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:50:57.790948+0000","last_peered":"2023-09-23T19:50:57.790948+0000","last_clean":"2023-09-23T19:50:57.790948+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:50:57.790948+0000","last_undegraded":"2023-09-23T19:50:57.790948+0000","last_fullsized":"2023-09-23T19:50:57.790948+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768019+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:50:56.768019+0000","last_peered":"2023-09-23T19:50:56.768019+0000","last_clean":"2023-09-23T19:50:56.768019+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:50:56.768019+0000","last_undegraded":"2023-09-23T19:50:56.768019+0000","last_fullsized":"2023-09-23T19:50:56.768019+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789973+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:50:57.789973+0000","last_peered":"2023-09-23T19:50:57.789973+0000","last_clean":"2023-09-23T19:50:57.789973+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:50:57.789973+0000","last_undegraded":"2023-09-23T19:50:57.789973+0000","last_fullsized":"2023-09-23T19:50:57.789973+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768916+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:50:56.768916+0000","last_peered":"2023-09-23T19:50:56.768916+0000","last_clean":"2023-09-23T19:50:56.768916+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:50:56.768916+0000","last_undegraded":"2023-09-23T19:50:56.768916+0000","last_fullsized":"2023-09-23T19:50:56.768916+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790667+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:50:57.790667+0000","last_peered":"2023-09-23T19:50:57.790667+0000","last_clean":"2023-09-23T19:50:57.790667+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:50:57.790667+0000","last_undegraded":"2023-09-23T19:50:57.790667+0000","last_fullsized":"2023-09-23T19:50:57.790667+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.777637+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:50:57.777637+0000","last_peered":"2023-09-23T19:50:57.777637+0000","last_clean":"2023-09-23T19:50:57.777637+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:50:57.777637+0000","last_undegraded":"2023-09-23T19:50:57.777637+0000","last_fullsized":"2023-09-23T19:50:57.777637+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765709+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:50:56.765709+0000","last_peered":"2023-09-23T19:50:56.765709+0000","last_clean":"2023-09-23T19:50:56.765709+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:50:56.765709+0000","last_undegraded":"2023-09-23T19:50:56.765709+0000","last_fullsized":"2023-09-23T19:50:56.765709+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.777187+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:50:56.777187+0000","last_peered":"2023-09-23T19:50:56.777187+0000","last_clean":"2023-09-23T19:50:56.777187+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:50:56.777187+0000","last_undegraded":"2023-09-23T19:50:56.777187+0000","last_fullsized":"2023-09-23T19:50:56.777187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":342,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770704+0000","last_change":"2023-09-23T19:50:23.999651+0000","last_active":"2023-09-23T19:50:56.770704+0000","last_peered":"2023-09-23T19:50:56.770704+0000","last_clean":"2023-09-23T19:50:56.770704+0000","last_became_active":"2023-09-23T19:50:23.999184+0000","last_became_peered":"2023-09-23T19:50:23.999184+0000","last_unstale":"2023-09-23T19:50:56.770704+0000","last_undegraded":"2023-09-23T19:50:56.770704+0000","last_fullsized":"2023-09-23T19:50:56.770704+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:41.010259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793288+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:50:57.793288+0000","last_peered":"2023-09-23T19:50:57.793288+0000","last_clean":"2023-09-23T19:50:57.793288+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:50:57.793288+0000","last_undegraded":"2023-09-23T19:50:57.793288+0000","last_fullsized":"2023-09-23T19:50:57.793288+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":371,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.781464+0000","last_change":"2023-09-23T19:49:51.380375+0000","last_active":"2023-09-23T19:50:57.781464+0000","last_peered":"2023-09-23T19:50:57.781464+0000","last_clean":"2023-09-23T19:50:57.781464+0000","last_became_active":"2023-09-23T19:49:51.379793+0000","last_became_peered":"2023-09-23T19:49:51.379793+0000","last_unstale":"2023-09-23T19:50:57.781464+0000","last_undegraded":"2023-09-23T19:50:57.781464+0000","last_fullsized":"2023-09-23T19:50:57.781464+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:26.355152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789467+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:50:57.789467+0000","last_peered":"2023-09-23T19:50:57.789467+0000","last_clean":"2023-09-23T19:50:57.789467+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:50:57.789467+0000","last_undegraded":"2023-09-23T19:50:57.789467+0000","last_fullsized":"2023-09-23T19:50:57.789467+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.767556+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:50:56.767556+0000","last_peered":"2023-09-23T19:50:56.767556+0000","last_clean":"2023-09-23T19:50:56.767556+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:50:56.767556+0000","last_undegraded":"2023-09-23T19:50:56.767556+0000","last_fullsized":"2023-09-23T19:50:56.767556+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787777+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:50:57.787777+0000","last_peered":"2023-09-23T19:50:57.787777+0000","last_clean":"2023-09-23T19:50:57.787777+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:50:57.787777+0000","last_undegraded":"2023-09-23T19:50:57.787777+0000","last_fullsized":"2023-09-23T19:50:57.787777+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":188,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774652+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:50:56.774652+0000","last_peered":"2023-09-23T19:50:56.774652+0000","last_clean":"2023-09-23T19:50:56.774652+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:50:56.774652+0000","last_undegraded":"2023-09-23T19:50:56.774652+0000","last_fullsized":"2023-09-23T19:50:56.774652+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.777263+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:50:56.777263+0000","last_peered":"2023-09-23T19:50:56.777263+0000","last_clean":"2023-09-23T19:50:56.777263+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:50:56.777263+0000","last_undegraded":"2023-09-23T19:50:56.777263+0000","last_fullsized":"2023-09-23T19:50:56.777263+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794256+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:50:57.794256+0000","last_peered":"2023-09-23T19:50:57.794256+0000","last_clean":"2023-09-23T19:50:57.794256+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:50:57.794256+0000","last_undegraded":"2023-09-23T19:50:57.794256+0000","last_fullsized":"2023-09-23T19:50:57.794256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":828,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.772649+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:50:56.772649+0000","last_peered":"2023-09-23T19:50:56.772649+0000","last_clean":"2023-09-23T19:50:56.772649+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:50:56.772649+0000","last_undegraded":"2023-09-23T19:50:56.772649+0000","last_fullsized":"2023-09-23T19:50:56.772649+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770724+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:50:56.770724+0000","last_peered":"2023-09-23T19:50:56.770724+0000","last_clean":"2023-09-23T19:50:56.770724+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:50:56.770724+0000","last_undegraded":"2023-09-23T19:50:56.770724+0000","last_fullsized":"2023-09-23T19:50:56.770724+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787513+0000","last_change":"2023-09-23T19:50:57.787513+0000","last_active":"2023-09-23T19:50:57.787513+0000","last_peered":"2023-09-23T19:50:57.787513+0000","last_clean":"2023-09-23T19:50:57.787513+0000","last_became_active":"2023-09-23T19:50:57.787175+0000","last_became_peered":"2023-09-23T19:50:57.787175+0000","last_unstale":"2023-09-23T19:50:57.787513+0000","last_undegraded":"2023-09-23T19:50:57.787513+0000","last_fullsized":"2023-09-23T19:50:57.787513+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:19.081986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":450,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.772552+0000","last_change":"2023-09-23T19:48:37.575593+0000","last_active":"2023-09-23T19:50:56.772552+0000","last_peered":"2023-09-23T19:50:56.772552+0000","last_clean":"2023-09-23T19:50:56.772552+0000","last_became_active":"2023-09-23T19:48:37.575214+0000","last_became_peered":"2023-09-23T19:48:37.575214+0000","last_unstale":"2023-09-23T19:50:56.772552+0000","last_undegraded":"2023-09-23T19:50:56.772552+0000","last_fullsized":"2023-09-23T19:50:56.772552+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:13.335201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":919,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766899+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:50:56.766899+0000","last_peered":"2023-09-23T19:50:56.766899+0000","last_clean":"2023-09-23T19:50:56.766899+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:50:56.766899+0000","last_undegraded":"2023-09-23T19:50:56.766899+0000","last_fullsized":"2023-09-23T19:50:56.766899+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.782011+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:50:57.782011+0000","last_peered":"2023-09-23T19:50:57.782011+0000","last_clean":"2023-09-23T19:50:57.782011+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:50:57.782011+0000","last_undegraded":"2023-09-23T19:50:57.782011+0000","last_fullsized":"2023-09-23T19:50:57.782011+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786740+0000","last_change":"2023-09-23T19:50:23.997623+0000","last_active":"2023-09-23T19:50:57.786740+0000","last_peered":"2023-09-23T19:50:57.786740+0000","last_clean":"2023-09-23T19:50:57.786740+0000","last_became_active":"2023-09-23T19:50:23.997428+0000","last_became_peered":"2023-09-23T19:50:23.997428+0000","last_unstale":"2023-09-23T19:50:57.786740+0000","last_undegraded":"2023-09-23T19:50:57.786740+0000","last_fullsized":"2023-09-23T19:50:57.786740+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:50.529980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":594,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786571+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:50:57.786571+0000","last_peered":"2023-09-23T19:50:57.786571+0000","last_clean":"2023-09-23T19:50:57.786571+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:50:57.786571+0000","last_undegraded":"2023-09-23T19:50:57.786571+0000","last_fullsized":"2023-09-23T19:50:57.786571+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794391+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:50:57.794391+0000","last_peered":"2023-09-23T19:50:57.794391+0000","last_clean":"2023-09-23T19:50:57.794391+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:50:57.794391+0000","last_undegraded":"2023-09-23T19:50:57.794391+0000","last_fullsized":"2023-09-23T19:50:57.794391+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765277+0000","last_change":"2023-09-23T19:48:37.573541+0000","last_active":"2023-09-23T19:50:56.765277+0000","last_peered":"2023-09-23T19:50:56.765277+0000","last_clean":"2023-09-23T19:50:56.765277+0000","last_became_active":"2023-09-23T19:48:37.573213+0000","last_became_peered":"2023-09-23T19:48:37.573213+0000","last_unstale":"2023-09-23T19:50:56.765277+0000","last_undegraded":"2023-09-23T19:50:56.765277+0000","last_fullsized":"2023-09-23T19:50:56.765277+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:16.981072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":634,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766698+0000","last_change":"2023-09-23T19:49:51.382830+0000","last_active":"2023-09-23T19:50:56.766698+0000","last_peered":"2023-09-23T19:50:56.766698+0000","last_clean":"2023-09-23T19:50:56.766698+0000","last_became_active":"2023-09-23T19:49:51.382367+0000","last_became_peered":"2023-09-23T19:49:51.382367+0000","last_unstale":"2023-09-23T19:50:56.766698+0000","last_undegraded":"2023-09-23T19:50:56.766698+0000","last_fullsized":"2023-09-23T19:50:56.766698+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:48.082643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.781187+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:50:57.781187+0000","last_peered":"2023-09-23T19:50:57.781187+0000","last_clean":"2023-09-23T19:50:57.781187+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:50:57.781187+0000","last_undegraded":"2023-09-23T19:50:57.781187+0000","last_fullsized":"2023-09-23T19:50:57.781187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787126+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:50:57.787126+0000","last_peered":"2023-09-23T19:50:57.787126+0000","last_clean":"2023-09-23T19:50:57.787126+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:50:57.787126+0000","last_undegraded":"2023-09-23T19:50:57.787126+0000","last_fullsized":"2023-09-23T19:50:57.787126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":445,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787693+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:50:57.787693+0000","last_peered":"2023-09-23T19:50:57.787693+0000","last_clean":"2023-09-23T19:50:57.787693+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:50:57.787693+0000","last_undegraded":"2023-09-23T19:50:57.787693+0000","last_fullsized":"2023-09-23T19:50:57.787693+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790403+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:50:57.790403+0000","last_peered":"2023-09-23T19:50:57.790403+0000","last_clean":"2023-09-23T19:50:57.790403+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:50:57.790403+0000","last_undegraded":"2023-09-23T19:50:57.790403+0000","last_fullsized":"2023-09-23T19:50:57.790403+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793102+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:50:57.793102+0000","last_peered":"2023-09-23T19:50:57.793102+0000","last_clean":"2023-09-23T19:50:57.793102+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:50:57.793102+0000","last_undegraded":"2023-09-23T19:50:57.793102+0000","last_fullsized":"2023-09-23T19:50:57.793102+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765182+0000","last_change":"2023-09-23T19:50:23.998753+0000","last_active":"2023-09-23T19:50:56.765182+0000","last_peered":"2023-09-23T19:50:56.765182+0000","last_clean":"2023-09-23T19:50:56.765182+0000","last_became_active":"2023-09-23T19:50:23.998417+0000","last_became_peered":"2023-09-23T19:50:23.998417+0000","last_unstale":"2023-09-23T19:50:56.765182+0000","last_undegraded":"2023-09-23T19:50:56.765182+0000","last_fullsized":"2023-09-23T19:50:56.765182+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:11.002531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765806+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:50:56.765806+0000","last_peered":"2023-09-23T19:50:56.765806+0000","last_clean":"2023-09-23T19:50:56.765806+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:50:56.765806+0000","last_undegraded":"2023-09-23T19:50:56.765806+0000","last_fullsized":"2023-09-23T19:50:56.765806+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766899+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:50:56.766899+0000","last_peered":"2023-09-23T19:50:56.766899+0000","last_clean":"2023-09-23T19:50:56.766899+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:50:56.766899+0000","last_undegraded":"2023-09-23T19:50:56.766899+0000","last_fullsized":"2023-09-23T19:50:56.766899+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770764+0000","last_change":"2023-09-23T19:49:51.380925+0000","last_active":"2023-09-23T19:50:56.770764+0000","last_peered":"2023-09-23T19:50:56.770764+0000","last_clean":"2023-09-23T19:50:56.770764+0000","last_became_active":"2023-09-23T19:49:51.380427+0000","last_became_peered":"2023-09-23T19:49:51.380427+0000","last_unstale":"2023-09-23T19:50:56.770764+0000","last_undegraded":"2023-09-23T19:50:56.770764+0000","last_fullsized":"2023-09-23T19:50:56.770764+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:49.879887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790231+0000","last_change":"2023-09-23T19:50:57.790231+0000","last_active":"2023-09-23T19:50:57.790231+0000","last_peered":"2023-09-23T19:50:57.790231+0000","last_clean":"2023-09-23T19:50:57.790231+0000","last_became_active":"2023-09-23T19:50:57.789446+0000","last_became_peered":"2023-09-23T19:50:57.789446+0000","last_unstale":"2023-09-23T19:50:57.790231+0000","last_undegraded":"2023-09-23T19:50:57.790231+0000","last_fullsized":"2023-09-23T19:50:57.790231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:41.313276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793011+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:50:57.793011+0000","last_peered":"2023-09-23T19:50:57.793011+0000","last_clean":"2023-09-23T19:50:57.793011+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:50:57.793011+0000","last_undegraded":"2023-09-23T19:50:57.793011+0000","last_fullsized":"2023-09-23T19:50:57.793011+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768198+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:50:56.768198+0000","last_peered":"2023-09-23T19:50:56.768198+0000","last_clean":"2023-09-23T19:50:56.768198+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:50:56.768198+0000","last_undegraded":"2023-09-23T19:50:56.768198+0000","last_fullsized":"2023-09-23T19:50:56.768198+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.788389+0000","last_change":"2023-09-23T19:50:57.788389+0000","last_active":"2023-09-23T19:50:57.788389+0000","last_peered":"2023-09-23T19:50:57.788389+0000","last_clean":"2023-09-23T19:50:57.788389+0000","last_became_active":"2023-09-23T19:50:57.788075+0000","last_became_peered":"2023-09-23T19:50:57.788075+0000","last_unstale":"2023-09-23T19:50:57.788389+0000","last_undegraded":"2023-09-23T19:50:57.788389+0000","last_fullsized":"2023-09-23T19:50:57.788389+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:06.180629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770475+0000","last_change":"2023-09-23T19:49:51.380726+0000","last_active":"2023-09-23T19:50:56.770475+0000","last_peered":"2023-09-23T19:50:56.770475+0000","last_clean":"2023-09-23T19:50:56.770475+0000","last_became_active":"2023-09-23T19:49:51.380021+0000","last_became_peered":"2023-09-23T19:49:51.380021+0000","last_unstale":"2023-09-23T19:50:56.770475+0000","last_undegraded":"2023-09-23T19:50:56.770475+0000","last_fullsized":"2023-09-23T19:50:56.770475+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:16.892029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774672+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:50:56.774672+0000","last_peered":"2023-09-23T19:50:56.774672+0000","last_clean":"2023-09-23T19:50:56.774672+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:50:56.774672+0000","last_undegraded":"2023-09-23T19:50:56.774672+0000","last_fullsized":"2023-09-23T19:50:56.774672+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.795290+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:50:57.795290+0000","last_peered":"2023-09-23T19:50:57.795290+0000","last_clean":"2023-09-23T19:50:57.795290+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:50:57.795290+0000","last_undegraded":"2023-09-23T19:50:57.795290+0000","last_fullsized":"2023-09-23T19:50:57.795290+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.773415+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:50:56.773415+0000","last_peered":"2023-09-23T19:50:56.773415+0000","last_clean":"2023-09-23T19:50:56.773415+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:50:56.773415+0000","last_undegraded":"2023-09-23T19:50:56.773415+0000","last_fullsized":"2023-09-23T19:50:56.773415+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765786+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:50:56.765786+0000","last_peered":"2023-09-23T19:50:56.765786+0000","last_clean":"2023-09-23T19:50:56.765786+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:50:56.765786+0000","last_undegraded":"2023-09-23T19:50:56.765786+0000","last_fullsized":"2023-09-23T19:50:56.765786+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774219+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:50:56.774219+0000","last_peered":"2023-09-23T19:50:56.774219+0000","last_clean":"2023-09-23T19:50:56.774219+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:50:56.774219+0000","last_undegraded":"2023-09-23T19:50:56.774219+0000","last_fullsized":"2023-09-23T19:50:56.774219+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789346+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:50:57.789346+0000","last_peered":"2023-09-23T19:50:57.789346+0000","last_clean":"2023-09-23T19:50:57.789346+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:50:57.789346+0000","last_undegraded":"2023-09-23T19:50:57.789346+0000","last_fullsized":"2023-09-23T19:50:57.789346+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770341+0000","last_change":"2023-09-23T19:48:37.567607+0000","last_active":"2023-09-23T19:50:56.770341+0000","last_peered":"2023-09-23T19:50:56.770341+0000","last_clean":"2023-09-23T19:50:56.770341+0000","last_became_active":"2023-09-23T19:48:37.567300+0000","last_became_peered":"2023-09-23T19:48:37.567300+0000","last_unstale":"2023-09-23T19:50:56.770341+0000","last_undegraded":"2023-09-23T19:50:56.770341+0000","last_fullsized":"2023-09-23T19:50:56.770341+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.088452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793018+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:50:57.793018+0000","last_peered":"2023-09-23T19:50:57.793018+0000","last_clean":"2023-09-23T19:50:57.793018+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:50:57.793018+0000","last_undegraded":"2023-09-23T19:50:57.793018+0000","last_fullsized":"2023-09-23T19:50:57.793018+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.771560+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:50:56.771560+0000","last_peered":"2023-09-23T19:50:56.771560+0000","last_clean":"2023-09-23T19:50:56.771560+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:50:56.771560+0000","last_undegraded":"2023-09-23T19:50:56.771560+0000","last_fullsized":"2023-09-23T19:50:56.771560+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790293+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:50:57.790293+0000","last_peered":"2023-09-23T19:50:57.790293+0000","last_clean":"2023-09-23T19:50:57.790293+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:50:57.790293+0000","last_undegraded":"2023-09-23T19:50:57.790293+0000","last_fullsized":"2023-09-23T19:50:57.790293+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.788133+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:50:57.788133+0000","last_peered":"2023-09-23T19:50:57.788133+0000","last_clean":"2023-09-23T19:50:57.788133+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:50:57.788133+0000","last_undegraded":"2023-09-23T19:50:57.788133+0000","last_fullsized":"2023-09-23T19:50:57.788133+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790499+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:50:57.790499+0000","last_peered":"2023-09-23T19:50:57.790499+0000","last_clean":"2023-09-23T19:50:57.790499+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:50:57.790499+0000","last_undegraded":"2023-09-23T19:50:57.790499+0000","last_fullsized":"2023-09-23T19:50:57.790499+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.773321+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:50:56.773321+0000","last_peered":"2023-09-23T19:50:56.773321+0000","last_clean":"2023-09-23T19:50:56.773321+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:50:56.773321+0000","last_undegraded":"2023-09-23T19:50:56.773321+0000","last_fullsized":"2023-09-23T19:50:56.773321+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764907+0000","last_change":"2023-09-23T19:49:51.379282+0000","last_active":"2023-09-23T19:50:56.764907+0000","last_peered":"2023-09-23T19:50:56.764907+0000","last_clean":"2023-09-23T19:50:56.764907+0000","last_became_active":"2023-09-23T19:49:51.378849+0000","last_became_peered":"2023-09-23T19:49:51.378849+0000","last_unstale":"2023-09-23T19:50:56.764907+0000","last_undegraded":"2023-09-23T19:50:56.764907+0000","last_fullsized":"2023-09-23T19:50:56.764907+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:00.093034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766139+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:50:56.766139+0000","last_peered":"2023-09-23T19:50:56.766139+0000","last_clean":"2023-09-23T19:50:56.766139+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:50:56.766139+0000","last_undegraded":"2023-09-23T19:50:56.766139+0000","last_fullsized":"2023-09-23T19:50:56.766139+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770911+0000","last_change":"2023-09-23T19:50:23.998186+0000","last_active":"2023-09-23T19:50:56.770911+0000","last_peered":"2023-09-23T19:50:56.770911+0000","last_clean":"2023-09-23T19:50:56.770911+0000","last_became_active":"2023-09-23T19:50:23.997897+0000","last_became_peered":"2023-09-23T19:50:23.997897+0000","last_unstale":"2023-09-23T19:50:56.770911+0000","last_undegraded":"2023-09-23T19:50:56.770911+0000","last_fullsized":"2023-09-23T19:50:56.770911+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:57.792861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793984+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:50:57.793984+0000","last_peered":"2023-09-23T19:50:57.793984+0000","last_clean":"2023-09-23T19:50:57.793984+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:50:57.793984+0000","last_undegraded":"2023-09-23T19:50:57.793984+0000","last_fullsized":"2023-09-23T19:50:57.793984+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789044+0000","last_change":"2023-09-23T19:50:57.789044+0000","last_active":"2023-09-23T19:50:57.789044+0000","last_peered":"2023-09-23T19:50:57.789044+0000","last_clean":"2023-09-23T19:50:57.789044+0000","last_became_active":"2023-09-23T19:50:57.788730+0000","last_became_peered":"2023-09-23T19:50:57.788730+0000","last_unstale":"2023-09-23T19:50:57.789044+0000","last_undegraded":"2023-09-23T19:50:57.789044+0000","last_fullsized":"2023-09-23T19:50:57.789044+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:23.585694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765794+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:50:56.765794+0000","last_peered":"2023-09-23T19:50:56.765794+0000","last_clean":"2023-09-23T19:50:56.765794+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:50:56.765794+0000","last_undegraded":"2023-09-23T19:50:56.765794+0000","last_fullsized":"2023-09-23T19:50:56.765794+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789294+0000","last_change":"2023-09-23T19:50:57.789294+0000","last_active":"2023-09-23T19:50:57.789294+0000","last_peered":"2023-09-23T19:50:57.789294+0000","last_clean":"2023-09-23T19:50:57.789294+0000","last_became_active":"2023-09-23T19:50:57.788922+0000","last_became_peered":"2023-09-23T19:50:57.788922+0000","last_unstale":"2023-09-23T19:50:57.789294+0000","last_undegraded":"2023-09-23T19:50:57.789294+0000","last_fullsized":"2023-09-23T19:50:57.789294+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:11.829374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789127+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:50:57.789127+0000","last_peered":"2023-09-23T19:50:57.789127+0000","last_clean":"2023-09-23T19:50:57.789127+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:50:57.789127+0000","last_undegraded":"2023-09-23T19:50:57.789127+0000","last_fullsized":"2023-09-23T19:50:57.789127+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770478+0000","last_change":"2023-09-23T19:49:51.374353+0000","last_active":"2023-09-23T19:50:56.770478+0000","last_peered":"2023-09-23T19:50:56.770478+0000","last_clean":"2023-09-23T19:50:56.770478+0000","last_became_active":"2023-09-23T19:49:51.373992+0000","last_became_peered":"2023-09-23T19:49:51.373992+0000","last_unstale":"2023-09-23T19:50:56.770478+0000","last_undegraded":"2023-09-23T19:50:56.770478+0000","last_fullsized":"2023-09-23T19:50:56.770478+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.344350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789164+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:50:57.789164+0000","last_peered":"2023-09-23T19:50:57.789164+0000","last_clean":"2023-09-23T19:50:57.789164+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:50:57.789164+0000","last_undegraded":"2023-09-23T19:50:57.789164+0000","last_fullsized":"2023-09-23T19:50:57.789164+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.791240+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:50:57.791240+0000","last_peered":"2023-09-23T19:50:57.791240+0000","last_clean":"2023-09-23T19:50:57.791240+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:50:57.791240+0000","last_undegraded":"2023-09-23T19:50:57.791240+0000","last_fullsized":"2023-09-23T19:50:57.791240+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794931+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:50:57.794931+0000","last_peered":"2023-09-23T19:50:57.794931+0000","last_clean":"2023-09-23T19:50:57.794931+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:50:57.794931+0000","last_undegraded":"2023-09-23T19:50:57.794931+0000","last_fullsized":"2023-09-23T19:50:57.794931+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.792804+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:50:57.792804+0000","last_peered":"2023-09-23T19:50:57.792804+0000","last_clean":"2023-09-23T19:50:57.792804+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:50:57.792804+0000","last_undegraded":"2023-09-23T19:50:57.792804+0000","last_fullsized":"2023-09-23T19:50:57.792804+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764710+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:50:56.764710+0000","last_peered":"2023-09-23T19:50:56.764710+0000","last_clean":"2023-09-23T19:50:56.764710+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:50:56.764710+0000","last_undegraded":"2023-09-23T19:50:56.764710+0000","last_fullsized":"2023-09-23T19:50:56.764710+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234130,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17132,"kb_used_data":5796,"kb_used_omap":0,"kb_used_meta":11328,"kb_avail":93728020,"statfs":{"total":95995035648,"available":95977492480,"internally_reserved":0,"allocated":5935104,"data_stored":5405311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11599872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.61199999999999999,"15min":0.59299999999999997},"min":{"1min":0.36899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.9850000000000003,"5min":4.9850000000000003,"15min":4.9850000000000003},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":0.80300000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.61399999999999999,"15min":0.58899999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0510000000000002,"5min":5.0510000000000002,"15min":5.0510000000000002},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.60799999999999998,"15min":0.57899999999999996},"min":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.1200000000000001,"5min":5.1200000000000001,"15min":5.1200000000000001},"last":0.69599999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":5.1600000000000001,"5min":5.1600000000000001,"15min":5.1600000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.77500000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59099999999999997,"15min":0.53300000000000003},"min":{"1min":0.45300000000000001,"5min":0.39800000000000002,"15min":0.36299999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":1.546},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.54500000000000004},"min":{"1min":0.379,"5min":0.35899999999999999,"15min":0.317},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.73899999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.504,"15min":0.47199999999999998},"min":{"1min":0.36499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73999999999999999,"5min":0.85299999999999998,"15min":0.878},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.56599999999999995,"15min":0.51900000000000002},"min":{"1min":0.39800000000000002,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":0.79200000000000004,"5min":0.93300000000000005,"15min":1.47},"last":0.45100000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.51800000000000002},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":0.77000000000000002,"5min":0.87,"15min":0.89100000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58199999999999996,"15min":0.53600000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69699999999999995,"5min":0.80400000000000005,"15min":1.3500000000000001},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.61099999999999999,"15min":0.56200000000000006},"min":{"1min":0.41499999999999998,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":0.77600000000000002,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.64700000000000002,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.82599999999999996,"5min":0.86199999999999999,"15min":1.52},"last":0.61099999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495767,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8352,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93736800,"statfs":{"total":95995035648,"available":95986483200,"internally_reserved":0,"allocated":1728512,"data_stored":1210673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.57299999999999995,"15min":0.59299999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.60599999999999998,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.63900000000000001,"15min":0.65500000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.71999999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.67400000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.63100000000000001,"15min":0.625},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.5259999999999998,"5min":2.5259999999999998,"15min":2.5259999999999998},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.57399999999999995,"15min":0.56799999999999995},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.6160000000000001,"5min":2.6160000000000001,"15min":2.6160000000000001},"last":0.371}]},{"osd":2,"last update":"Sat Sep 23 19:51:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58199999999999996,"15min":0.55400000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.32400000000000001},"max":{"1min":0.76800000000000002,"5min":0.83399999999999996,"15min":0.996},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.39600000000000002,"5min":0.34999999999999998,"15min":0.32100000000000001},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":2.0579999999999998},"last":0.53700000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.51200000000000001,"15min":0.48899999999999999},"min":{"1min":0.40899999999999997,"5min":0.33800000000000002,"15min":0.28999999999999998},"max":{"1min":0.72399999999999998,"5min":0.78600000000000003,"15min":0.84699999999999998},"last":0.629},{"interface":"front","average":{"1min":0.625,"5min":0.53700000000000003,"15min":0.51100000000000001},"min":{"1min":0.41499999999999998,"5min":0.34100000000000003,"15min":0.26500000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.93799999999999994},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.53300000000000003,"15min":0.497},"min":{"1min":0.44400000000000001,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.57899999999999996,"15min":0.55100000000000005},"min":{"1min":0.47599999999999998,"5min":0.32200000000000001,"15min":0.30499999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.92400000000000004},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:51:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.57999999999999996,"15min":0.56899999999999995},"min":{"1min":0.42399999999999999,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":0.877,"5min":0.877,"15min":0.89700000000000002},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.63500000000000001,"15min":0.60699999999999998},"min":{"1min":0.45200000000000001,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":0.82999999999999996,"5min":0.86499999999999999,"15min":1.0029999999999999},"last":0.70599999999999996}]}]},{"osd":1,"up_from":99,"seq":425201762332,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32832,"kb_used_data":5240,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93712320,"statfs":{"total":95995035648,"available":95961415680,"internally_reserved":0,"allocated":5365760,"data_stored":4818922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":28238650},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.35399999999999998,"15min":0.35799999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.55100000000000005,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.36299999999999999,"15min":0.36399999999999999},"min":{"1min":0.252,"5min":0.25,"15min":0.25},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.36699999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.72699999999999998,"5min":1.161,"15min":1.161},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.441,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.81299999999999994,"5min":1.107,"15min":1.107},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.625,"15min":0.63500000000000001},"min":{"1min":0.439,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70599999999999996,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.441,"5min":0.375,"15min":0.375},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66100000000000003,"15min":0.66000000000000003},"min":{"1min":0.44900000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.622,"5min":0.68100000000000005,"15min":0.69099999999999995},"min":{"1min":0.46700000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81499999999999995,"5min":1.214,"15min":1.214},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64900000000000002},"min":{"1min":0.47599999999999998,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.65900000000000003,"15min":0.66700000000000004},"min":{"1min":0.44700000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80400000000000005,"5min":1.246,"15min":1.246},"last":0.63300000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925867,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33416,"kb_used_data":5824,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93711736,"statfs":{"total":95995035648,"available":95960817664,"internally_reserved":0,"allocated":5963776,"data_stored":5409717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":28237409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.32100000000000001,"15min":0.32300000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.42299999999999999,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.313},{"interface":"front","average":{"1min":0.32800000000000001,"5min":0.33600000000000002,"15min":0.33800000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8029999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.29199999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.374},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.623,"15min":0.64300000000000002},"min":{"1min":0.41699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.433},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.64500000000000002,"15min":0.67700000000000005},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79000000000000004,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67800000000000005,"15min":0.70199999999999996},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":0.501},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.66900000000000004,"15min":0.70199999999999996},"min":{"1min":0.433,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.68000000000000005,"15min":0.70999999999999996},"min":{"1min":0.46200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67600000000000005,"15min":0.70399999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.68500000000000005,"15min":0.71899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.52100000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598805,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28664,"kb_used_data":1136,"kb_used_omap":12,"kb_used_meta":27507,"kb_avail":93716488,"statfs":{"total":95995035648,"available":95965683712,"internally_reserved":0,"allocated":1163264,"data_stored":622590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":28168190},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.375},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.29899999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.28100000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.1950000000000001,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.77500000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.71999999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.66000000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435279,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1120,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1146880,"data_stored":620545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":28175469},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.67400000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019040,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7844,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":6720,"kb_avail":93737308,"statfs":{"total":95995035648,"available":95987003392,"internally_reserved":0,"allocated":1142784,"data_stored":620627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6881280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58599999999999997,"15min":0.59999999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78000000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.374},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.626,"15min":0.63800000000000001},"min":{"1min":0.47399999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82999999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.64600000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81399999999999995,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56299999999999994,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.72599999999999998,"5min":1.036,"15min":1.036},"last":0.51500000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:51:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.61199999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.502,"15min":0.45200000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":0.68500000000000005,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.47899999999999998},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":0.66100000000000003,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.66500000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:51:57 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55100000000000005,"15min":0.49099999999999999},"min":{"1min":0.38600000000000001,"5min":0.32700000000000001,"15min":0.27100000000000002},"max":{"1min":0.71299999999999997,"5min":0.86399999999999999,"15min":1.0589999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.57999999999999996,"15min":0.52300000000000002},"min":{"1min":0.441,"5min":0.32000000000000001,"15min":0.309},"max":{"1min":0.73399999999999999,"5min":1.0660000000000001,"15min":1.073},"last":0.68500000000000005}]},{"osd":7,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59199999999999997,"15min":0.56699999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.29599999999999999},"max":{"1min":0.79400000000000004,"5min":1.117,"15min":1.117},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.63,"15min":0.58599999999999997},"min":{"1min":0.51500000000000001,"5min":0.376,"15min":0.35799999999999998},"max":{"1min":0.79200000000000004,"5min":1.016,"15min":1.016},"last":0.57599999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757404,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7448,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":6336,"kb_avail":93737704,"statfs":{"total":95995035648,"available":95987408896,"internally_reserved":0,"allocated":1130496,"data_stored":616332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6488064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.54200000000000004,"15min":0.56699999999999995},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.70499999999999996,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73599999999999999,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.54100000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52200000000000002,"15min":0.52400000000000002},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.64300000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52600000000000002,"15min":0.52500000000000002},"min":{"1min":0.42999999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76400000000000001,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:51:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.37},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.50800000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.46700000000000003,"15min":0.44400000000000001},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.23699999999999999},"max":{"1min":0.78900000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.50700000000000001,"15min":0.47599999999999998},"min":{"1min":0.39400000000000002,"5min":0.28599999999999998,"15min":0.27500000000000002},"max":{"1min":0.753,"5min":0.76800000000000002,"15min":0.86399999999999999},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 19:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52100000000000002,"15min":0.48599999999999999},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":0.72299999999999998,"5min":0.84199999999999997,"15min":0.86599999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.54100000000000004},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.307},"max":{"1min":0.75700000000000001,"5min":0.80400000000000005,"15min":0.82799999999999996},"last":0.64200000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:51:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.55200000000000005,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":1.0309999999999999},"last":0.629},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.55800000000000005,"15min":0.55500000000000005},"min":{"1min":0.46000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.86499999999999999},"last":0.61499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:52:02.554 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:52:02.907 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:03.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:03 smithi144 ceph-mon[194833]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:03.457 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:03 smithi135 ceph-mon[221113]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:03.457 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:03 smithi135 ceph-mon[213871]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:03.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:52:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:52:03] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.43.0" 2023-09-23T19:52:04.293 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:04 smithi135 ceph-mon[213871]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:04.293 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:04 smithi135 ceph-mon[221113]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:04 smithi144 ceph-mon[194833]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:04.690 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:52:04.691 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:52:05.497 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":216,"stamp":"2023-09-23T19:52:03.110048+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164336,"kb_used_data":23024,"kb_used_omap":32,"kb_used_meta":141215,"kb_avail":749796880,"statfs":{"total":767960285184,"available":767792005120,"internally_reserved":0,"allocated":23576576,"data_stored":19324717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":144604678},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002843"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764512+0000","last_change":"2023-09-23T19:49:51.378856+0000","last_active":"2023-09-23T19:50:56.764512+0000","last_peered":"2023-09-23T19:50:56.764512+0000","last_clean":"2023-09-23T19:50:56.764512+0000","last_became_active":"2023-09-23T19:49:51.378451+0000","last_became_peered":"2023-09-23T19:49:51.378451+0000","last_unstale":"2023-09-23T19:50:56.764512+0000","last_undegraded":"2023-09-23T19:50:56.764512+0000","last_fullsized":"2023-09-23T19:50:56.764512+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:03.276740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770013+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:50:56.770013+0000","last_peered":"2023-09-23T19:50:56.770013+0000","last_clean":"2023-09-23T19:50:56.770013+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:50:56.770013+0000","last_undegraded":"2023-09-23T19:50:56.770013+0000","last_fullsized":"2023-09-23T19:50:56.770013+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.785311+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:50:57.785311+0000","last_peered":"2023-09-23T19:50:57.785311+0000","last_clean":"2023-09-23T19:50:57.785311+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:50:57.785311+0000","last_undegraded":"2023-09-23T19:50:57.785311+0000","last_fullsized":"2023-09-23T19:50:57.785311+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786561+0000","last_change":"2023-09-23T19:50:57.786561+0000","last_active":"2023-09-23T19:50:57.786561+0000","last_peered":"2023-09-23T19:50:57.786561+0000","last_clean":"2023-09-23T19:50:57.786561+0000","last_became_active":"2023-09-23T19:50:57.786231+0000","last_became_peered":"2023-09-23T19:50:57.786231+0000","last_unstale":"2023-09-23T19:50:57.786561+0000","last_undegraded":"2023-09-23T19:50:57.786561+0000","last_fullsized":"2023-09-23T19:50:57.786561+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:27.327075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790948+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:50:57.790948+0000","last_peered":"2023-09-23T19:50:57.790948+0000","last_clean":"2023-09-23T19:50:57.790948+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:50:57.790948+0000","last_undegraded":"2023-09-23T19:50:57.790948+0000","last_fullsized":"2023-09-23T19:50:57.790948+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768019+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:50:56.768019+0000","last_peered":"2023-09-23T19:50:56.768019+0000","last_clean":"2023-09-23T19:50:56.768019+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:50:56.768019+0000","last_undegraded":"2023-09-23T19:50:56.768019+0000","last_fullsized":"2023-09-23T19:50:56.768019+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789973+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:50:57.789973+0000","last_peered":"2023-09-23T19:50:57.789973+0000","last_clean":"2023-09-23T19:50:57.789973+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:50:57.789973+0000","last_undegraded":"2023-09-23T19:50:57.789973+0000","last_fullsized":"2023-09-23T19:50:57.789973+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768916+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:50:56.768916+0000","last_peered":"2023-09-23T19:50:56.768916+0000","last_clean":"2023-09-23T19:50:56.768916+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:50:56.768916+0000","last_undegraded":"2023-09-23T19:50:56.768916+0000","last_fullsized":"2023-09-23T19:50:56.768916+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790667+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:50:57.790667+0000","last_peered":"2023-09-23T19:50:57.790667+0000","last_clean":"2023-09-23T19:50:57.790667+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:50:57.790667+0000","last_undegraded":"2023-09-23T19:50:57.790667+0000","last_fullsized":"2023-09-23T19:50:57.790667+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.777637+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:50:57.777637+0000","last_peered":"2023-09-23T19:50:57.777637+0000","last_clean":"2023-09-23T19:50:57.777637+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:50:57.777637+0000","last_undegraded":"2023-09-23T19:50:57.777637+0000","last_fullsized":"2023-09-23T19:50:57.777637+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765709+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:50:56.765709+0000","last_peered":"2023-09-23T19:50:56.765709+0000","last_clean":"2023-09-23T19:50:56.765709+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:50:56.765709+0000","last_undegraded":"2023-09-23T19:50:56.765709+0000","last_fullsized":"2023-09-23T19:50:56.765709+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.777187+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:50:56.777187+0000","last_peered":"2023-09-23T19:50:56.777187+0000","last_clean":"2023-09-23T19:50:56.777187+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:50:56.777187+0000","last_undegraded":"2023-09-23T19:50:56.777187+0000","last_fullsized":"2023-09-23T19:50:56.777187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":342,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770704+0000","last_change":"2023-09-23T19:50:23.999651+0000","last_active":"2023-09-23T19:50:56.770704+0000","last_peered":"2023-09-23T19:50:56.770704+0000","last_clean":"2023-09-23T19:50:56.770704+0000","last_became_active":"2023-09-23T19:50:23.999184+0000","last_became_peered":"2023-09-23T19:50:23.999184+0000","last_unstale":"2023-09-23T19:50:56.770704+0000","last_undegraded":"2023-09-23T19:50:56.770704+0000","last_fullsized":"2023-09-23T19:50:56.770704+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:41.010259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793288+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:50:57.793288+0000","last_peered":"2023-09-23T19:50:57.793288+0000","last_clean":"2023-09-23T19:50:57.793288+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:50:57.793288+0000","last_undegraded":"2023-09-23T19:50:57.793288+0000","last_fullsized":"2023-09-23T19:50:57.793288+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":371,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.781464+0000","last_change":"2023-09-23T19:49:51.380375+0000","last_active":"2023-09-23T19:50:57.781464+0000","last_peered":"2023-09-23T19:50:57.781464+0000","last_clean":"2023-09-23T19:50:57.781464+0000","last_became_active":"2023-09-23T19:49:51.379793+0000","last_became_peered":"2023-09-23T19:49:51.379793+0000","last_unstale":"2023-09-23T19:50:57.781464+0000","last_undegraded":"2023-09-23T19:50:57.781464+0000","last_fullsized":"2023-09-23T19:50:57.781464+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:26.355152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789467+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:50:57.789467+0000","last_peered":"2023-09-23T19:50:57.789467+0000","last_clean":"2023-09-23T19:50:57.789467+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:50:57.789467+0000","last_undegraded":"2023-09-23T19:50:57.789467+0000","last_fullsized":"2023-09-23T19:50:57.789467+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.767556+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:50:56.767556+0000","last_peered":"2023-09-23T19:50:56.767556+0000","last_clean":"2023-09-23T19:50:56.767556+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:50:56.767556+0000","last_undegraded":"2023-09-23T19:50:56.767556+0000","last_fullsized":"2023-09-23T19:50:56.767556+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787777+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:50:57.787777+0000","last_peered":"2023-09-23T19:50:57.787777+0000","last_clean":"2023-09-23T19:50:57.787777+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:50:57.787777+0000","last_undegraded":"2023-09-23T19:50:57.787777+0000","last_fullsized":"2023-09-23T19:50:57.787777+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":188,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774652+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:50:56.774652+0000","last_peered":"2023-09-23T19:50:56.774652+0000","last_clean":"2023-09-23T19:50:56.774652+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:50:56.774652+0000","last_undegraded":"2023-09-23T19:50:56.774652+0000","last_fullsized":"2023-09-23T19:50:56.774652+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.777263+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:50:56.777263+0000","last_peered":"2023-09-23T19:50:56.777263+0000","last_clean":"2023-09-23T19:50:56.777263+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:50:56.777263+0000","last_undegraded":"2023-09-23T19:50:56.777263+0000","last_fullsized":"2023-09-23T19:50:56.777263+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794256+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:50:57.794256+0000","last_peered":"2023-09-23T19:50:57.794256+0000","last_clean":"2023-09-23T19:50:57.794256+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:50:57.794256+0000","last_undegraded":"2023-09-23T19:50:57.794256+0000","last_fullsized":"2023-09-23T19:50:57.794256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":828,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.772649+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:50:56.772649+0000","last_peered":"2023-09-23T19:50:56.772649+0000","last_clean":"2023-09-23T19:50:56.772649+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:50:56.772649+0000","last_undegraded":"2023-09-23T19:50:56.772649+0000","last_fullsized":"2023-09-23T19:50:56.772649+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770724+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:50:56.770724+0000","last_peered":"2023-09-23T19:50:56.770724+0000","last_clean":"2023-09-23T19:50:56.770724+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:50:56.770724+0000","last_undegraded":"2023-09-23T19:50:56.770724+0000","last_fullsized":"2023-09-23T19:50:56.770724+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787513+0000","last_change":"2023-09-23T19:50:57.787513+0000","last_active":"2023-09-23T19:50:57.787513+0000","last_peered":"2023-09-23T19:50:57.787513+0000","last_clean":"2023-09-23T19:50:57.787513+0000","last_became_active":"2023-09-23T19:50:57.787175+0000","last_became_peered":"2023-09-23T19:50:57.787175+0000","last_unstale":"2023-09-23T19:50:57.787513+0000","last_undegraded":"2023-09-23T19:50:57.787513+0000","last_fullsized":"2023-09-23T19:50:57.787513+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:19.081986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":450,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.772552+0000","last_change":"2023-09-23T19:48:37.575593+0000","last_active":"2023-09-23T19:50:56.772552+0000","last_peered":"2023-09-23T19:50:56.772552+0000","last_clean":"2023-09-23T19:50:56.772552+0000","last_became_active":"2023-09-23T19:48:37.575214+0000","last_became_peered":"2023-09-23T19:48:37.575214+0000","last_unstale":"2023-09-23T19:50:56.772552+0000","last_undegraded":"2023-09-23T19:50:56.772552+0000","last_fullsized":"2023-09-23T19:50:56.772552+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:13.335201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":919,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766899+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:50:56.766899+0000","last_peered":"2023-09-23T19:50:56.766899+0000","last_clean":"2023-09-23T19:50:56.766899+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:50:56.766899+0000","last_undegraded":"2023-09-23T19:50:56.766899+0000","last_fullsized":"2023-09-23T19:50:56.766899+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.782011+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:50:57.782011+0000","last_peered":"2023-09-23T19:50:57.782011+0000","last_clean":"2023-09-23T19:50:57.782011+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:50:57.782011+0000","last_undegraded":"2023-09-23T19:50:57.782011+0000","last_fullsized":"2023-09-23T19:50:57.782011+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786740+0000","last_change":"2023-09-23T19:50:23.997623+0000","last_active":"2023-09-23T19:50:57.786740+0000","last_peered":"2023-09-23T19:50:57.786740+0000","last_clean":"2023-09-23T19:50:57.786740+0000","last_became_active":"2023-09-23T19:50:23.997428+0000","last_became_peered":"2023-09-23T19:50:23.997428+0000","last_unstale":"2023-09-23T19:50:57.786740+0000","last_undegraded":"2023-09-23T19:50:57.786740+0000","last_fullsized":"2023-09-23T19:50:57.786740+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:50.529980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":594,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.786571+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:50:57.786571+0000","last_peered":"2023-09-23T19:50:57.786571+0000","last_clean":"2023-09-23T19:50:57.786571+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:50:57.786571+0000","last_undegraded":"2023-09-23T19:50:57.786571+0000","last_fullsized":"2023-09-23T19:50:57.786571+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794391+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:50:57.794391+0000","last_peered":"2023-09-23T19:50:57.794391+0000","last_clean":"2023-09-23T19:50:57.794391+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:50:57.794391+0000","last_undegraded":"2023-09-23T19:50:57.794391+0000","last_fullsized":"2023-09-23T19:50:57.794391+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765277+0000","last_change":"2023-09-23T19:48:37.573541+0000","last_active":"2023-09-23T19:50:56.765277+0000","last_peered":"2023-09-23T19:50:56.765277+0000","last_clean":"2023-09-23T19:50:56.765277+0000","last_became_active":"2023-09-23T19:48:37.573213+0000","last_became_peered":"2023-09-23T19:48:37.573213+0000","last_unstale":"2023-09-23T19:50:56.765277+0000","last_undegraded":"2023-09-23T19:50:56.765277+0000","last_fullsized":"2023-09-23T19:50:56.765277+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:16.981072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":634,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766698+0000","last_change":"2023-09-23T19:49:51.382830+0000","last_active":"2023-09-23T19:50:56.766698+0000","last_peered":"2023-09-23T19:50:56.766698+0000","last_clean":"2023-09-23T19:50:56.766698+0000","last_became_active":"2023-09-23T19:49:51.382367+0000","last_became_peered":"2023-09-23T19:49:51.382367+0000","last_unstale":"2023-09-23T19:50:56.766698+0000","last_undegraded":"2023-09-23T19:50:56.766698+0000","last_fullsized":"2023-09-23T19:50:56.766698+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:48.082643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.781187+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:50:57.781187+0000","last_peered":"2023-09-23T19:50:57.781187+0000","last_clean":"2023-09-23T19:50:57.781187+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:50:57.781187+0000","last_undegraded":"2023-09-23T19:50:57.781187+0000","last_fullsized":"2023-09-23T19:50:57.781187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787126+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:50:57.787126+0000","last_peered":"2023-09-23T19:50:57.787126+0000","last_clean":"2023-09-23T19:50:57.787126+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:50:57.787126+0000","last_undegraded":"2023-09-23T19:50:57.787126+0000","last_fullsized":"2023-09-23T19:50:57.787126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":445,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.787693+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:50:57.787693+0000","last_peered":"2023-09-23T19:50:57.787693+0000","last_clean":"2023-09-23T19:50:57.787693+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:50:57.787693+0000","last_undegraded":"2023-09-23T19:50:57.787693+0000","last_fullsized":"2023-09-23T19:50:57.787693+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790403+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:50:57.790403+0000","last_peered":"2023-09-23T19:50:57.790403+0000","last_clean":"2023-09-23T19:50:57.790403+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:50:57.790403+0000","last_undegraded":"2023-09-23T19:50:57.790403+0000","last_fullsized":"2023-09-23T19:50:57.790403+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793102+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:50:57.793102+0000","last_peered":"2023-09-23T19:50:57.793102+0000","last_clean":"2023-09-23T19:50:57.793102+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:50:57.793102+0000","last_undegraded":"2023-09-23T19:50:57.793102+0000","last_fullsized":"2023-09-23T19:50:57.793102+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765182+0000","last_change":"2023-09-23T19:50:23.998753+0000","last_active":"2023-09-23T19:50:56.765182+0000","last_peered":"2023-09-23T19:50:56.765182+0000","last_clean":"2023-09-23T19:50:56.765182+0000","last_became_active":"2023-09-23T19:50:23.998417+0000","last_became_peered":"2023-09-23T19:50:23.998417+0000","last_unstale":"2023-09-23T19:50:56.765182+0000","last_undegraded":"2023-09-23T19:50:56.765182+0000","last_fullsized":"2023-09-23T19:50:56.765182+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:11.002531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765806+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:50:56.765806+0000","last_peered":"2023-09-23T19:50:56.765806+0000","last_clean":"2023-09-23T19:50:56.765806+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:50:56.765806+0000","last_undegraded":"2023-09-23T19:50:56.765806+0000","last_fullsized":"2023-09-23T19:50:56.765806+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766899+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:50:56.766899+0000","last_peered":"2023-09-23T19:50:56.766899+0000","last_clean":"2023-09-23T19:50:56.766899+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:50:56.766899+0000","last_undegraded":"2023-09-23T19:50:56.766899+0000","last_fullsized":"2023-09-23T19:50:56.766899+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770764+0000","last_change":"2023-09-23T19:49:51.380925+0000","last_active":"2023-09-23T19:50:56.770764+0000","last_peered":"2023-09-23T19:50:56.770764+0000","last_clean":"2023-09-23T19:50:56.770764+0000","last_became_active":"2023-09-23T19:49:51.380427+0000","last_became_peered":"2023-09-23T19:49:51.380427+0000","last_unstale":"2023-09-23T19:50:56.770764+0000","last_undegraded":"2023-09-23T19:50:56.770764+0000","last_fullsized":"2023-09-23T19:50:56.770764+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:49.879887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790231+0000","last_change":"2023-09-23T19:50:57.790231+0000","last_active":"2023-09-23T19:50:57.790231+0000","last_peered":"2023-09-23T19:50:57.790231+0000","last_clean":"2023-09-23T19:50:57.790231+0000","last_became_active":"2023-09-23T19:50:57.789446+0000","last_became_peered":"2023-09-23T19:50:57.789446+0000","last_unstale":"2023-09-23T19:50:57.790231+0000","last_undegraded":"2023-09-23T19:50:57.790231+0000","last_fullsized":"2023-09-23T19:50:57.790231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:41.313276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793011+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:50:57.793011+0000","last_peered":"2023-09-23T19:50:57.793011+0000","last_clean":"2023-09-23T19:50:57.793011+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:50:57.793011+0000","last_undegraded":"2023-09-23T19:50:57.793011+0000","last_fullsized":"2023-09-23T19:50:57.793011+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.768198+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:50:56.768198+0000","last_peered":"2023-09-23T19:50:56.768198+0000","last_clean":"2023-09-23T19:50:56.768198+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:50:56.768198+0000","last_undegraded":"2023-09-23T19:50:56.768198+0000","last_fullsized":"2023-09-23T19:50:56.768198+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.788389+0000","last_change":"2023-09-23T19:50:57.788389+0000","last_active":"2023-09-23T19:50:57.788389+0000","last_peered":"2023-09-23T19:50:57.788389+0000","last_clean":"2023-09-23T19:50:57.788389+0000","last_became_active":"2023-09-23T19:50:57.788075+0000","last_became_peered":"2023-09-23T19:50:57.788075+0000","last_unstale":"2023-09-23T19:50:57.788389+0000","last_undegraded":"2023-09-23T19:50:57.788389+0000","last_fullsized":"2023-09-23T19:50:57.788389+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:06.180629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770475+0000","last_change":"2023-09-23T19:49:51.380726+0000","last_active":"2023-09-23T19:50:56.770475+0000","last_peered":"2023-09-23T19:50:56.770475+0000","last_clean":"2023-09-23T19:50:56.770475+0000","last_became_active":"2023-09-23T19:49:51.380021+0000","last_became_peered":"2023-09-23T19:49:51.380021+0000","last_unstale":"2023-09-23T19:50:56.770475+0000","last_undegraded":"2023-09-23T19:50:56.770475+0000","last_fullsized":"2023-09-23T19:50:56.770475+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:16.892029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774672+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:50:56.774672+0000","last_peered":"2023-09-23T19:50:56.774672+0000","last_clean":"2023-09-23T19:50:56.774672+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:50:56.774672+0000","last_undegraded":"2023-09-23T19:50:56.774672+0000","last_fullsized":"2023-09-23T19:50:56.774672+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.795290+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:50:57.795290+0000","last_peered":"2023-09-23T19:50:57.795290+0000","last_clean":"2023-09-23T19:50:57.795290+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:50:57.795290+0000","last_undegraded":"2023-09-23T19:50:57.795290+0000","last_fullsized":"2023-09-23T19:50:57.795290+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.773415+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:50:56.773415+0000","last_peered":"2023-09-23T19:50:56.773415+0000","last_clean":"2023-09-23T19:50:56.773415+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:50:56.773415+0000","last_undegraded":"2023-09-23T19:50:56.773415+0000","last_fullsized":"2023-09-23T19:50:56.773415+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765786+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:50:56.765786+0000","last_peered":"2023-09-23T19:50:56.765786+0000","last_clean":"2023-09-23T19:50:56.765786+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:50:56.765786+0000","last_undegraded":"2023-09-23T19:50:56.765786+0000","last_fullsized":"2023-09-23T19:50:56.765786+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.774219+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:50:56.774219+0000","last_peered":"2023-09-23T19:50:56.774219+0000","last_clean":"2023-09-23T19:50:56.774219+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:50:56.774219+0000","last_undegraded":"2023-09-23T19:50:56.774219+0000","last_fullsized":"2023-09-23T19:50:56.774219+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789346+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:50:57.789346+0000","last_peered":"2023-09-23T19:50:57.789346+0000","last_clean":"2023-09-23T19:50:57.789346+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:50:57.789346+0000","last_undegraded":"2023-09-23T19:50:57.789346+0000","last_fullsized":"2023-09-23T19:50:57.789346+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770341+0000","last_change":"2023-09-23T19:48:37.567607+0000","last_active":"2023-09-23T19:50:56.770341+0000","last_peered":"2023-09-23T19:50:56.770341+0000","last_clean":"2023-09-23T19:50:56.770341+0000","last_became_active":"2023-09-23T19:48:37.567300+0000","last_became_peered":"2023-09-23T19:48:37.567300+0000","last_unstale":"2023-09-23T19:50:56.770341+0000","last_undegraded":"2023-09-23T19:50:56.770341+0000","last_fullsized":"2023-09-23T19:50:56.770341+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.088452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793018+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:50:57.793018+0000","last_peered":"2023-09-23T19:50:57.793018+0000","last_clean":"2023-09-23T19:50:57.793018+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:50:57.793018+0000","last_undegraded":"2023-09-23T19:50:57.793018+0000","last_fullsized":"2023-09-23T19:50:57.793018+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.771560+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:50:56.771560+0000","last_peered":"2023-09-23T19:50:56.771560+0000","last_clean":"2023-09-23T19:50:56.771560+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:50:56.771560+0000","last_undegraded":"2023-09-23T19:50:56.771560+0000","last_fullsized":"2023-09-23T19:50:56.771560+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790293+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:50:57.790293+0000","last_peered":"2023-09-23T19:50:57.790293+0000","last_clean":"2023-09-23T19:50:57.790293+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:50:57.790293+0000","last_undegraded":"2023-09-23T19:50:57.790293+0000","last_fullsized":"2023-09-23T19:50:57.790293+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.788133+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:50:57.788133+0000","last_peered":"2023-09-23T19:50:57.788133+0000","last_clean":"2023-09-23T19:50:57.788133+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:50:57.788133+0000","last_undegraded":"2023-09-23T19:50:57.788133+0000","last_fullsized":"2023-09-23T19:50:57.788133+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.790499+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:50:57.790499+0000","last_peered":"2023-09-23T19:50:57.790499+0000","last_clean":"2023-09-23T19:50:57.790499+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:50:57.790499+0000","last_undegraded":"2023-09-23T19:50:57.790499+0000","last_fullsized":"2023-09-23T19:50:57.790499+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.773321+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:50:56.773321+0000","last_peered":"2023-09-23T19:50:56.773321+0000","last_clean":"2023-09-23T19:50:56.773321+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:50:56.773321+0000","last_undegraded":"2023-09-23T19:50:56.773321+0000","last_fullsized":"2023-09-23T19:50:56.773321+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764907+0000","last_change":"2023-09-23T19:49:51.379282+0000","last_active":"2023-09-23T19:50:56.764907+0000","last_peered":"2023-09-23T19:50:56.764907+0000","last_clean":"2023-09-23T19:50:56.764907+0000","last_became_active":"2023-09-23T19:49:51.378849+0000","last_became_peered":"2023-09-23T19:49:51.378849+0000","last_unstale":"2023-09-23T19:50:56.764907+0000","last_undegraded":"2023-09-23T19:50:56.764907+0000","last_fullsized":"2023-09-23T19:50:56.764907+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:00.093034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.766139+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:50:56.766139+0000","last_peered":"2023-09-23T19:50:56.766139+0000","last_clean":"2023-09-23T19:50:56.766139+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:50:56.766139+0000","last_undegraded":"2023-09-23T19:50:56.766139+0000","last_fullsized":"2023-09-23T19:50:56.766139+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770911+0000","last_change":"2023-09-23T19:50:23.998186+0000","last_active":"2023-09-23T19:50:56.770911+0000","last_peered":"2023-09-23T19:50:56.770911+0000","last_clean":"2023-09-23T19:50:56.770911+0000","last_became_active":"2023-09-23T19:50:23.997897+0000","last_became_peered":"2023-09-23T19:50:23.997897+0000","last_unstale":"2023-09-23T19:50:56.770911+0000","last_undegraded":"2023-09-23T19:50:56.770911+0000","last_fullsized":"2023-09-23T19:50:56.770911+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:57.792861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.793984+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:50:57.793984+0000","last_peered":"2023-09-23T19:50:57.793984+0000","last_clean":"2023-09-23T19:50:57.793984+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:50:57.793984+0000","last_undegraded":"2023-09-23T19:50:57.793984+0000","last_fullsized":"2023-09-23T19:50:57.793984+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789044+0000","last_change":"2023-09-23T19:50:57.789044+0000","last_active":"2023-09-23T19:50:57.789044+0000","last_peered":"2023-09-23T19:50:57.789044+0000","last_clean":"2023-09-23T19:50:57.789044+0000","last_became_active":"2023-09-23T19:50:57.788730+0000","last_became_peered":"2023-09-23T19:50:57.788730+0000","last_unstale":"2023-09-23T19:50:57.789044+0000","last_undegraded":"2023-09-23T19:50:57.789044+0000","last_fullsized":"2023-09-23T19:50:57.789044+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:23.585694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.765794+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:50:56.765794+0000","last_peered":"2023-09-23T19:50:56.765794+0000","last_clean":"2023-09-23T19:50:56.765794+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:50:56.765794+0000","last_undegraded":"2023-09-23T19:50:56.765794+0000","last_fullsized":"2023-09-23T19:50:56.765794+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789294+0000","last_change":"2023-09-23T19:50:57.789294+0000","last_active":"2023-09-23T19:50:57.789294+0000","last_peered":"2023-09-23T19:50:57.789294+0000","last_clean":"2023-09-23T19:50:57.789294+0000","last_became_active":"2023-09-23T19:50:57.788922+0000","last_became_peered":"2023-09-23T19:50:57.788922+0000","last_unstale":"2023-09-23T19:50:57.789294+0000","last_undegraded":"2023-09-23T19:50:57.789294+0000","last_fullsized":"2023-09-23T19:50:57.789294+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:11.829374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789127+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:50:57.789127+0000","last_peered":"2023-09-23T19:50:57.789127+0000","last_clean":"2023-09-23T19:50:57.789127+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:50:57.789127+0000","last_undegraded":"2023-09-23T19:50:57.789127+0000","last_fullsized":"2023-09-23T19:50:57.789127+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.770478+0000","last_change":"2023-09-23T19:49:51.374353+0000","last_active":"2023-09-23T19:50:56.770478+0000","last_peered":"2023-09-23T19:50:56.770478+0000","last_clean":"2023-09-23T19:50:56.770478+0000","last_became_active":"2023-09-23T19:49:51.373992+0000","last_became_peered":"2023-09-23T19:49:51.373992+0000","last_unstale":"2023-09-23T19:50:56.770478+0000","last_undegraded":"2023-09-23T19:50:56.770478+0000","last_fullsized":"2023-09-23T19:50:56.770478+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.344350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.789164+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:50:57.789164+0000","last_peered":"2023-09-23T19:50:57.789164+0000","last_clean":"2023-09-23T19:50:57.789164+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:50:57.789164+0000","last_undegraded":"2023-09-23T19:50:57.789164+0000","last_fullsized":"2023-09-23T19:50:57.789164+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.791240+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:50:57.791240+0000","last_peered":"2023-09-23T19:50:57.791240+0000","last_clean":"2023-09-23T19:50:57.791240+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:50:57.791240+0000","last_undegraded":"2023-09-23T19:50:57.791240+0000","last_fullsized":"2023-09-23T19:50:57.791240+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.794931+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:50:57.794931+0000","last_peered":"2023-09-23T19:50:57.794931+0000","last_clean":"2023-09-23T19:50:57.794931+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:50:57.794931+0000","last_undegraded":"2023-09-23T19:50:57.794931+0000","last_fullsized":"2023-09-23T19:50:57.794931+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T19:50:57.792804+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:50:57.792804+0000","last_peered":"2023-09-23T19:50:57.792804+0000","last_clean":"2023-09-23T19:50:57.792804+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:50:57.792804+0000","last_undegraded":"2023-09-23T19:50:57.792804+0000","last_fullsized":"2023-09-23T19:50:57.792804+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T19:50:56.764710+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:50:56.764710+0000","last_peered":"2023-09-23T19:50:56.764710+0000","last_clean":"2023-09-23T19:50:56.764710+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:50:56.764710+0000","last_undegraded":"2023-09-23T19:50:56.764710+0000","last_fullsized":"2023-09-23T19:50:56.764710+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234131,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17132,"kb_used_data":5796,"kb_used_omap":0,"kb_used_meta":11328,"kb_avail":93728020,"statfs":{"total":95995035648,"available":95977492480,"internally_reserved":0,"allocated":5935104,"data_stored":5405311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11599872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.61199999999999999,"15min":0.59299999999999997},"min":{"1min":0.36899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.9850000000000003,"5min":4.9850000000000003,"15min":4.9850000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":0.53900000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.61399999999999999,"15min":0.58899999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0510000000000002,"5min":5.0510000000000002,"15min":5.0510000000000002},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.60799999999999998,"15min":0.57899999999999996},"min":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.1200000000000001,"5min":5.1200000000000001,"15min":5.1200000000000001},"last":0.69199999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":5.1600000000000001,"5min":5.1600000000000001,"15min":5.1600000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.65600000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":5.1980000000000004,"5min":5.1980000000000004,"15min":5.1980000000000004},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.504,"15min":0.47199999999999998},"min":{"1min":0.36499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73999999999999999,"5min":0.85299999999999998,"15min":0.878},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.56599999999999995,"15min":0.51900000000000002},"min":{"1min":0.39800000000000002,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":0.79200000000000004,"5min":0.93300000000000005,"15min":1.47},"last":0.39900000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.51800000000000002},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":0.77000000000000002,"5min":0.87,"15min":0.89100000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58199999999999996,"15min":0.53600000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69699999999999995,"5min":0.80400000000000005,"15min":1.3500000000000001},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 19:51:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.61099999999999999,"15min":0.56200000000000006},"min":{"1min":0.41499999999999998,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":0.77600000000000002,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.64700000000000002,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.82599999999999996,"5min":0.86199999999999999,"15min":1.52},"last":0.63200000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495768,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8352,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93736800,"statfs":{"total":95995035648,"available":95986483200,"internally_reserved":0,"allocated":1728512,"data_stored":1210673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.57299999999999995,"15min":0.59299999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.60599999999999998,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.63900000000000001,"15min":0.65500000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.71999999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.505}]},{"osd":1,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.63100000000000001,"15min":0.625},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.5259999999999998,"5min":2.5259999999999998,"15min":2.5259999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.57399999999999995,"15min":0.56799999999999995},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.6160000000000001,"5min":2.6160000000000001,"15min":2.6160000000000001},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 19:51:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":2.794,"5min":2.794,"15min":2.794},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.60899999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.51200000000000001,"15min":0.48899999999999999},"min":{"1min":0.40899999999999997,"5min":0.33800000000000002,"15min":0.28999999999999998},"max":{"1min":0.72399999999999998,"5min":0.78600000000000003,"15min":0.84699999999999998},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.625,"5min":0.53700000000000003,"15min":0.51100000000000001},"min":{"1min":0.41499999999999998,"5min":0.34100000000000003,"15min":0.26500000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.93799999999999994},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.53300000000000003,"15min":0.497},"min":{"1min":0.44400000000000001,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.57899999999999996,"15min":0.55100000000000005},"min":{"1min":0.47599999999999998,"5min":0.32200000000000001,"15min":0.30499999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.92400000000000004},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:51:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.57999999999999996,"15min":0.56899999999999995},"min":{"1min":0.42399999999999999,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":0.877,"5min":0.877,"15min":0.89700000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.63500000000000001,"15min":0.60699999999999998},"min":{"1min":0.45200000000000001,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":0.82999999999999996,"5min":0.86499999999999999,"15min":1.0029999999999999},"last":0.57699999999999996}]}]},{"osd":1,"up_from":99,"seq":425201762332,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32832,"kb_used_data":5240,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93712320,"statfs":{"total":95995035648,"available":95961415680,"internally_reserved":0,"allocated":5365760,"data_stored":4818922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":28238650},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.35399999999999998,"15min":0.35799999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.55100000000000005,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.36299999999999999,"15min":0.36399999999999999},"min":{"1min":0.252,"5min":0.25,"15min":0.25},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:51:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.36699999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.72699999999999998,"5min":1.161,"15min":1.161},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.441,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.81299999999999994,"5min":1.107,"15min":1.107},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.625,"15min":0.63500000000000001},"min":{"1min":0.439,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70599999999999996,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.441,"5min":0.375,"15min":0.375},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66100000000000003,"15min":0.66000000000000003},"min":{"1min":0.44900000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.622,"5min":0.68100000000000005,"15min":0.69099999999999995},"min":{"1min":0.46700000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81499999999999995,"5min":1.214,"15min":1.214},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64900000000000002},"min":{"1min":0.47599999999999998,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.65900000000000003,"15min":0.66700000000000004},"min":{"1min":0.44700000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80400000000000005,"5min":1.246,"15min":1.246},"last":0.63300000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925867,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33416,"kb_used_data":5824,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93711736,"statfs":{"total":95995035648,"available":95960817664,"internally_reserved":0,"allocated":5963776,"data_stored":5409717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":28237409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.32100000000000001,"15min":0.32300000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.42299999999999999,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.313},{"interface":"front","average":{"1min":0.32800000000000001,"5min":0.33600000000000002,"15min":0.33800000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8029999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.29199999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.374},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.623,"15min":0.64300000000000002},"min":{"1min":0.41699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.433},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.64500000000000002,"15min":0.67700000000000005},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79000000000000004,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67800000000000005,"15min":0.70199999999999996},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":0.501},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.66900000000000004,"15min":0.70199999999999996},"min":{"1min":0.433,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.68000000000000005,"15min":0.70999999999999996},"min":{"1min":0.46200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67600000000000005,"15min":0.70399999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.68500000000000005,"15min":0.71899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.52100000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598806,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28664,"kb_used_data":1136,"kb_used_omap":12,"kb_used_meta":27507,"kb_avail":93716488,"statfs":{"total":95995035648,"available":95965683712,"internally_reserved":0,"allocated":1163264,"data_stored":622590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":28168190},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.26000000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.30499999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.32100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.64800000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.1950000000000001,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.63},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.623},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.61799999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435279,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1120,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1146880,"data_stored":620545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":28175469},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.67400000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019041,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7844,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":6720,"kb_avail":93737308,"statfs":{"total":95995035648,"available":95987003392,"internally_reserved":0,"allocated":1142784,"data_stored":620627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6881280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58599999999999997,"15min":0.59999999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78000000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.626,"15min":0.63800000000000001},"min":{"1min":0.47399999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82999999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.59199999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81399999999999995,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56299999999999994,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.72599999999999998,"5min":1.036,"15min":1.036},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.64300000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:51:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.502,"15min":0.45200000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":0.68500000000000005,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.47899999999999998},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":0.66100000000000003,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:51:57 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55100000000000005,"15min":0.49099999999999999},"min":{"1min":0.38600000000000001,"5min":0.32700000000000001,"15min":0.27100000000000002},"max":{"1min":0.71299999999999997,"5min":0.86399999999999999,"15min":1.0589999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.57999999999999996,"15min":0.52300000000000002},"min":{"1min":0.441,"5min":0.32000000000000001,"15min":0.309},"max":{"1min":0.73399999999999999,"5min":1.0660000000000001,"15min":1.073},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59199999999999997,"15min":0.56699999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.29599999999999999},"max":{"1min":0.79400000000000004,"5min":1.117,"15min":1.117},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.63,"15min":0.58599999999999997},"min":{"1min":0.51500000000000001,"5min":0.376,"15min":0.35799999999999998},"max":{"1min":0.79200000000000004,"5min":1.016,"15min":1.016},"last":0.625}]}]},{"osd":5,"up_from":38,"seq":163208757405,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7448,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":6336,"kb_avail":93737704,"statfs":{"total":95995035648,"available":95987408896,"internally_reserved":0,"allocated":1130496,"data_stored":616332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6488064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.54200000000000004,"15min":0.56699999999999995},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.70499999999999996,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73599999999999999,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52200000000000002,"15min":0.52400000000000002},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.64300000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52600000000000002,"15min":0.52500000000000002},"min":{"1min":0.42999999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76400000000000001,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.60399999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:51:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.46700000000000003,"15min":0.44400000000000001},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.23699999999999999},"max":{"1min":0.78900000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.50700000000000001,"15min":0.47599999999999998},"min":{"1min":0.39400000000000002,"5min":0.28599999999999998,"15min":0.27500000000000002},"max":{"1min":0.753,"5min":0.76800000000000002,"15min":0.86399999999999999},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52100000000000002,"15min":0.48599999999999999},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":0.72299999999999998,"5min":0.84199999999999997,"15min":0.86599999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.54100000000000004},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.307},"max":{"1min":0.75700000000000001,"5min":0.80400000000000005,"15min":0.82799999999999996},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:51:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.55200000000000005,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":1.0309999999999999},"last":0.621},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.55800000000000005,"15min":0.55500000000000005},"min":{"1min":0.46000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.86499999999999999},"last":0.64700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:52:05.503 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T19:52:05.503 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T19:52:05.847 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:06 smithi144 ceph-mon[194833]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:06 smithi144 ceph-mon[194833]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:06.423 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:06 smithi135 ceph-mon[221113]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:06.423 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:06 smithi135 ceph-mon[221113]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:06.423 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:06 smithi135 ceph-mon[213871]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:06.424 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:06 smithi135 ceph-mon[213871]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:07.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:07.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:07.632 INFO:teuthology.orchestra.run.smithi135.stdout:epoch 110 2023-09-23T19:52:07.632 INFO:teuthology.orchestra.run.smithi135.stdout:fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 2023-09-23T19:52:07.632 INFO:teuthology.orchestra.run.smithi135.stdout:created 2023-09-23T19:35:08.772740+0000 2023-09-23T19:52:07.632 INFO:teuthology.orchestra.run.smithi135.stdout:modified 2023-09-23T19:50:57.767213+0000 2023-09-23T19:52:07.632 INFO:teuthology.orchestra.run.smithi135.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:crush_version 22 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:full_ratio 0.95 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:backfillfull_ratio 0.9 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:nearfull_ratio 0.85 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:require_min_compat_client luminous 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:min_compat_client luminous 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:require_osd_release quincy 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:stretch_mode_enabled false 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 85 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T19:52:07.633 INFO:teuthology.orchestra.run.smithi135.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:max_osd 8 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857] [v2:172.21.15.135:6804/100269857,v1:172.21.15.135:6805/100269857] exists,up 369a8e37-43f6-4c41-a626-2b159656f305 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760] [v2:172.21.15.135:6812/2160782760,v1:172.21.15.135:6813/2160782760] exists,up ccf1949a-3059-45d2-a46a-07dfbbc68fb5 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510] [v2:172.21.15.135:6820/209403510,v1:172.21.15.135:6821/209403510] exists,up 2a8e9267-6c97-4c1f-b56b-7ca1844a1334 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678] [v2:172.21.15.135:6828/2232272678,v1:172.21.15.135:6829/2232272678] exists,up 21fd337a-25fa-42ee-9f4c-130d8882de6c 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977] [v2:172.21.15.144:6802/1825747977,v1:172.21.15.144:6803/1825747977] exists,up de72ce1b-4c64-4f34-b1ba-563833e9fcf4 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 up in weight 1 up_from 38 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223] [v2:172.21.15.144:6810/2025173223,v1:172.21.15.144:6811/2025173223] exists,up bd0e910c-7c37-4ee6-97e8-031834dadc71 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 up in weight 1 up_from 46 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554] [v2:172.21.15.144:6818/1530810554,v1:172.21.15.144:6819/1530810554] exists,up 84e8261b-0520-44c9-aeae-c25256356c3f 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 up in weight 1 up_from 54 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059] [v2:172.21.15.144:6826/3376336059,v1:172.21.15.144:6827/3376336059] exists,up 030b85ad-503a-4291-a267-97a0e0f2db6e 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:pg_upmap_items 4.6 [1,5] 2023-09-23T19:52:07.634 INFO:teuthology.orchestra.run.smithi135.stdout:pg_upmap_items 4.e [1,2] 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:pg_upmap_items 4.1d [1,2] 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6801/782979070 expires 2023-09-24T19:35:36.550885+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6800/782979070 expires 2023-09-24T19:35:36.550885+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6801/1867076080 expires 2023-09-24T19:45:36.991484+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.144:6834/1742442049 expires 2023-09-24T19:44:56.542176+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/1611423028 expires 2023-09-24T19:35:36.550885+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.144:0/3940572722 expires 2023-09-24T19:44:56.542176+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/456433686 expires 2023-09-24T19:45:36.991484+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.144:0/2226038972 expires 2023-09-24T19:44:56.542176+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/1191504160 expires 2023-09-24T19:36:00.784922+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/1005337745 expires 2023-09-24T19:40:37.220934+0000 2023-09-23T19:52:07.635 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6800/1867076080 expires 2023-09-24T19:45:36.991484+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6800/2392681307 expires 2023-09-24T19:40:37.220934+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/4110552877 expires 2023-09-24T19:36:00.784922+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.144:6835/1742442049 expires 2023-09-24T19:44:56.542176+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.144:0/3293023812 expires 2023-09-24T19:44:56.542176+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6801/4224665887 expires 2023-09-24T19:36:00.784922+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/173145882 expires 2023-09-24T19:40:37.220934+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/4159692573 expires 2023-09-24T19:45:36.991484+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/775791172 expires 2023-09-24T19:40:37.220934+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/3361160768 expires 2023-09-24T19:40:37.220934+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/2123602186 expires 2023-09-24T19:45:36.991484+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6800/4224665887 expires 2023-09-24T19:36:00.784922+0000 2023-09-23T19:52:07.636 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6801/2392681307 expires 2023-09-24T19:40:37.220934+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/2749491827 expires 2023-09-24T19:45:36.991484+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.144:0/3756303050 expires 2023-09-24T19:44:56.542176+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/3668294279 expires 2023-09-24T19:44:16.725201+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6801/1642850725 expires 2023-09-24T19:44:16.725201+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/4238089144 expires 2023-09-24T19:44:16.725201+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/1017551752 expires 2023-09-24T19:45:36.991484+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/1625057187 expires 2023-09-24T19:45:36.991484+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:6800/1642850725 expires 2023-09-24T19:44:16.725201+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/3693941440 expires 2023-09-24T19:44:16.725201+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.144:0/1990744051 expires 2023-09-24T19:44:56.542176+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/645490139 expires 2023-09-24T19:35:36.550885+0000 2023-09-23T19:52:07.637 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.144:0/2993818522 expires 2023-09-24T19:44:56.542176+0000 2023-09-23T19:52:07.638 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/3516425720 expires 2023-09-24T19:44:16.725201+0000 2023-09-23T19:52:07.638 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/1336192479 expires 2023-09-24T19:36:00.784922+0000 2023-09-23T19:52:07.638 INFO:teuthology.orchestra.run.smithi135.stdout:blocklist 172.21.15.135:0/1910257642 expires 2023-09-24T19:35:36.550885+0000 2023-09-23T19:52:08.433 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:08 smithi135 ceph-mon[221113]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:08.433 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:08 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2567872682' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:52:08.434 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:08 smithi135 ceph-mon[213871]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:08.434 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:08 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2567872682' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:52:08.448 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.135:6802/100269857,v1:172.21.15.135:6803/100269857] [v2:172.21.15.135:6804/100269857,v1:172.21.15.135:6805/100269857] exists,up 369a8e37-43f6-4c41-a626-2b159656f305', 'osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.135:6810/2160782760,v1:172.21.15.135:6811/2160782760] [v2:172.21.15.135:6812/2160782760,v1:172.21.15.135:6813/2160782760] exists,up ccf1949a-3059-45d2-a46a-07dfbbc68fb5', 'osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.135:6818/209403510,v1:172.21.15.135:6819/209403510] [v2:172.21.15.135:6820/209403510,v1:172.21.15.135:6821/209403510] exists,up 2a8e9267-6c97-4c1f-b56b-7ca1844a1334', 'osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.135:6826/2232272678,v1:172.21.15.135:6827/2232272678] [v2:172.21.15.135:6828/2232272678,v1:172.21.15.135:6829/2232272678] exists,up 21fd337a-25fa-42ee-9f4c-130d8882de6c', 'osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.144:6800/1825747977,v1:172.21.15.144:6801/1825747977] [v2:172.21.15.144:6802/1825747977,v1:172.21.15.144:6803/1825747977] exists,up de72ce1b-4c64-4f34-b1ba-563833e9fcf4', 'osd.5 up in weight 1 up_from 38 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.144:6808/2025173223,v1:172.21.15.144:6809/2025173223] [v2:172.21.15.144:6810/2025173223,v1:172.21.15.144:6811/2025173223] exists,up bd0e910c-7c37-4ee6-97e8-031834dadc71', 'osd.6 up in weight 1 up_from 46 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.144:6816/1530810554,v1:172.21.15.144:6817/1530810554] [v2:172.21.15.144:6818/1530810554,v1:172.21.15.144:6819/1530810554] exists,up 84e8261b-0520-44c9-aeae-c25256356c3f', 'osd.7 up in weight 1 up_from 54 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.144:6824/3376336059,v1:172.21.15.144:6825/3376336059] [v2:172.21.15.144:6826/3376336059,v1:172.21.15.144:6827/3376336059] exists,up 030b85ad-503a-4291-a267-97a0e0f2db6e'] 2023-09-23T19:52:08.449 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T19:52:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:08 smithi144 ceph-mon[194833]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:08 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2567872682' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:52:08.793 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:10.217 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:10 smithi135 ceph-mon[221113]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:10.218 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:10 smithi135 ceph-mon[213871]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:10.431 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "name": "mon.a", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "cluster": "ceph", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "admin_socket_mode": "", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_client_required": "cephx, none", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_cluster_required": "cephx", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_debug": "true", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T19:52:10.432 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_service_required": "cephx", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "auth_supported": "", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_aio": "true", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_aio_reap_max": "16", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_async_discard": "false", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_block_size": "4096", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_debug_aio": "false", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T19:52:10.433 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_enable_discard": "false", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_flock_retry": "3", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_inject_crash": "0", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_ioring": "false", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_ioring_hipri": "false", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T19:52:10.434 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bdev_type": "", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_allocator": "hybrid", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_buffered_io": "true", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T19:52:10.435 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_replay_recovery": "false", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluefs_sync_write": "false", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T19:52:10.436 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_allocator": "hybrid", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_create": "true", 2023-09-23T19:52:10.437 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_db_create": "false", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_db_path": "", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_db_size": "0", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_path": "", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_size": "107374182400", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_wal_create": "false", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_wal_path": "", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_bluefs": "true", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T19:52:10.438 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_autotune": "true", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T19:52:10.439 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_size": "0", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cache_type": "2q", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_clone_cow": "true", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T19:52:10.440 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_mode": "none", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_freelist": "false", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T19:52:10.441 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T19:52:10.442 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T19:52:10.443 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fail_eio": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T19:52:10.444 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T19:52:10.445 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_max_blob_size": "0", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T19:52:10.446 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T19:52:10.447 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_spdk_mem": "512", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_tracing": "false", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T19:52:10.448 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T19:52:10.449 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephx_cluster_require_version": "2", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephx_require_signatures": "false", 2023-09-23T19:52:10.450 INFO:teuthology.orchestra.run.smithi135.stdout: "cephx_require_version": "2", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "cephx_service_require_signatures": "false", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "cephx_service_require_version": "2", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "cephx_sign_messages": "true", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "chdir": "", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "client_acl_type": "", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "client_asio_thread_count": "2", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "client_cache_mid": "0.750000", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "client_cache_size": "16384", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "client_caps_release_delay": "5", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "client_check_pool_perm": "true", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T19:52:10.451 INFO:teuthology.orchestra.run.smithi135.stdout: "client_debug_force_sync_read": "false", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_debug_getattr_caps": "false", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_die_on_failed_remount": "false", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_dirsize_rbytes": "true", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_force_lazyio": "false", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_fs": "", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_inject_release_failure": "false", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_max_inline_size": "4096", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T19:52:10.452 INFO:teuthology.orchestra.run.smithi135.stdout: "client_mds_namespace": "", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_metadata": "", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_mount_gid": "-1", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_mount_timeout": "300", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_mount_uid": "-1", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_mountpoint": "/", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_notify_timeout": "10", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_oc": "true", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_oc_max_objects": "1000", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_oc_size": "209715200", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T19:52:10.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_permissions": "true", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_quota": "true", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_quota_df": "true", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_readahead_max_bytes": "0", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_readahead_max_periods": "4", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_readahead_min": "131072", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_reconnect_stale": "false", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_shutdown_timeout": "30", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_snapdir": ".snap", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_tick_interval": "1", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_trace": "", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T19:52:10.454 INFO:teuthology.orchestra.run.smithi135.stdout: "client_use_faked_inos": "false", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "client_use_random_mds": "false", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "clog_to_graylog": "false", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "clog_to_graylog_port": "12201", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "clog_to_monitors": "default=true", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "clog_to_syslog": "false", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "clog_to_syslog_level": "info", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "cluster_addr": "-", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "cluster_network": "", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "cluster_network_interface": "", 2023-09-23T19:52:10.455 INFO:teuthology.orchestra.run.smithi135.stdout: "compressor_zlib_isal": "false", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "compressor_zlib_level": "5", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "compressor_zstd_level": "1", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "crimson_seastar_smp": "1", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "crush_location": "", 2023-09-23T19:52:10.456 INFO:teuthology.orchestra.run.smithi135.stdout: "crush_location_hook": "", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "crush_location_hook_timeout": "10", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "daemonize": "false", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "daos_pool": "tank", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_alienstore": "0/5", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_asok": "1/5", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_asok_assert_abort": "false", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T19:52:10.457 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_auth": "1/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_bdev": "1/3", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_bluefs": "1/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_bluestore": "1/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_buffer": "0/1", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_cephsqlite": "0/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_client": "0/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_compressor": "1/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_context": "0/1", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_crush": "1/1", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_crypto": "1/5", 2023-09-23T19:52:10.458 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_cyanstore": "0/5", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_dpdk": "1/5", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_eventtrace": "1/5", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_filer": "0/1", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_filestore": "1/3", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_finisher": "1/1", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_fuse": "1/5", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T19:52:10.459 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_javaclient": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_journal": "1/3", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_journaler": "0/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_kstore": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_leveldb": "4/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_lockdep": "0/1", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mclock": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mds": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mds_balancer": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mds_locker": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mds_log": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mds_migrator": "1/5", 2023-09-23T19:52:10.460 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_memstore": "1/5", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mgr": "2/5", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mgrc": "1/5", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_mon": "20/20", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_monc": "0/10", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_ms": "1/1", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_none": "0/5", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_objclass": "0/5", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_objectcacher": "0/5", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_objecter": "0/1", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_optracker": "0/5", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_osd": "1/5", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_paxos": "20/20", 2023-09-23T19:52:10.461 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_perfcounter": "1/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_prioritycache": "1/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rados": "0/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rbd": "0/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rbd_replay": "0/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_refs": "0/0", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_reserver": "1/1", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rgw": "1/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rgw_access": "1/5", 2023-09-23T19:52:10.462 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rgw_flight": "1/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rgw_sync": "1/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_rocksdb": "4/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore": "0/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_backref": "0/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_cache": "0/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_device": "0/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_epm": "0/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T19:52:10.463 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_journal": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_lba": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_odata": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_omap": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_onode": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_t": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_seastore_tm": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_striper": "0/1", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_test": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_throttle": "1/1", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_timer": "0/1", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "debug_tp": "0/5", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "device_failure_prediction_mode": "none", 2023-09-23T19:52:10.464 INFO:teuthology.orchestra.run.smithi135.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "err_to_graylog": "false", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "err_to_journald": "false", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "err_to_stderr": "true", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "err_to_syslog": "false", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event_tracing": "false", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "exporter_http_port": "9926", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "exporter_prio_limit": "5", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T19:52:10.465 INFO:teuthology.orchestra.run.smithi135.stdout: "exporter_sort_metrics": "true", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "exporter_stats_period": "5", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "fake_statfs_for_testing": "0", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "fatal_signal_handlers": "true", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "filer_max_purge_ops": "10", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "filer_max_truncate_ops": "128", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_blackhole": "false", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_btrfs_snap": "true", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_caller_concurrency": "10", 2023-09-23T19:52:10.466 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_debug_omap_check": "false", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_debug_verify_split": "false", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_dump_file": "", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_fadvise": "true", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_fail_eio": "true", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T19:52:10.467 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_fd_cache_size": "128", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_fiemap": "false", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_inject_stall": "0", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_journal_parallel": "false", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_journal_trailing": "false", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_journal_writeahead": "false", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_kill_at": "0", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T19:52:10.468 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T19:52:10.469 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_merge_threshold": "-10", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_odsync_write": "false", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_omap_backend_path": "", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_op_threads": "2", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_punch_hole": "false", 2023-09-23T19:52:10.470 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_queue_max_ops": "50", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_seek_data_hole": "false", 2023-09-23T19:52:10.471 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_sloppy_crc": "false", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_splice": "false", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_split_multiple": "2", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_split_rand_factor": "20", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_update_to": "1000", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T19:52:10.472 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_xfs_extsize": "false", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "filestore_zfs_snap": "false", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "fio_dir": "/tmp/fio", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "fsid": "2f95c858-5a48-11ee-8db4-212e2dc638e7", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_allow_other": "true", 2023-09-23T19:52:10.473 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_big_writes": "true", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_debug": "false", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_default_permissions": "false", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_disable_pagecache": "false", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_max_write": "0", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_multithreaded": "true", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_require_active_mds": "true", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_set_user_groups": "true", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_splice_move": "true", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_splice_read": "true", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_splice_write": "true", 2023-09-23T19:52:10.474 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "gss_target_name": "ceph", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "heartbeat_file": "", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "heartbeat_inject_failure": "0", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "heartbeat_interval": "5", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "host": "smithi135", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T19:52:10.475 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "inject_early_sigterm": "false", 2023-09-23T19:52:10.476 INFO:teuthology.orchestra.run.smithi135.stdout: "jaeger_agent_port": "6799", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "jaeger_tracing_enable": "false", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_aio": "true", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_align_min_size": "65536", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_block_align": "true", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_block_size": "4096", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_dio": "true", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_discard": "false", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_force_aio": "false", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_ignore_corruption": "false", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_max_write_entries": "100", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_replay_from": "0", 2023-09-23T19:52:10.477 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_write_header_frequency": "0", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "journal_zero_on_create": "false", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "journaler_prefetch_periods": "10", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "journaler_prezero_periods": "5", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "journaler_write_head_interval": "15", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "key": "", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "keyfile": "", 2023-09-23T19:52:10.478 INFO:teuthology.orchestra.run.smithi135.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_backend": "rocksdb", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_max_bytes": "67108864", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_max_ops": "512", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_onode_map_size": "1024", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "kstore_sync_transaction": "false", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_block_size": "65536", 2023-09-23T19:52:10.479 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_bloom_size": "0", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_cache_size": "536870912", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_compression": "false", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_log": "", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_max_open_files": "0", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_paranoid": "false", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "librados_thread_count": "2", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "lockdep": "false", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "lockdep_force_backtrace": "false", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "log_coarse_timestamps": "true", 2023-09-23T19:52:10.480 INFO:teuthology.orchestra.run.smithi135.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_flush_on_exit": "false", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_graylog_port": "12201", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_max_new": "1000", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_max_recent": "10000", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_stderr_prefix": "", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_to_file": "true", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_to_graylog": "false", 2023-09-23T19:52:10.481 INFO:teuthology.orchestra.run.smithi135.stdout: "log_to_journald": "false", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "log_to_stderr": "false", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "log_to_syslog": "false", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_action_on_write_error": "1", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_alternate_name_max": "8192", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_asio_thread_count": "2", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_export_pin": "true", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T19:52:10.482 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_interval": "10", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_max": "-1", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_max_until": "-1", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_merge_size": "50", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_mode": "0", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T19:52:10.483 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_split_bits": "3", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_split_size": "10000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cache_mid": "0.700000", 2023-09-23T19:52:10.484 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cache_trim_interval": "1", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T19:52:10.485 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_debug_auth_pins": "false", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_debug_frag": "false", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_debug_scatterstat": "false", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_debug_subtrees": "false", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_default_dir_hash": "2", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_defer_session_stale": "true", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T19:52:10.486 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dir_max_entries": "0", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dir_prefetch": "true", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_early_reply": "true", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_enable_op_tracker": "true", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_enforce_unique_name": "true", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T19:52:10.487 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T19:52:10.488 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_inject_health_dummy": "false", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_join_fs": "", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_journal_format": "1", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_create_at": "0", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_export_at": "0", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_import_at": "0", 2023-09-23T19:52:10.489 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_journal_at": "0", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_link_at": "0", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_openc_at": "0", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_rename_at": "0", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_log_max_events": "-1", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_log_max_segments": "128", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_log_pause": "false", 2023-09-23T19:52:10.490 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_log_segment_size": "0", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_completed_requests": "100000", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_export_size": "20971520", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_file_recover": "32", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_purge_files": "64", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_purge_ops": "8192", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T19:52:10.491 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_metrics_update_interval": "2", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_min_caps_per_client": "100", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_numa_node": "-1", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_op_history_duration": "600", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_op_history_size": "20", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_op_log_threshold": "5", 2023-09-23T19:52:10.492 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_ping_grace": "15", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_ping_interval": "5", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_recall_max_caps": "30000", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_replay_interval": "1.000000", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T19:52:10.493 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_root_ino_gid": "0", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_root_ino_uid": "0", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T19:52:10.494 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_shutdown_check": "0", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_skip_ino": "0", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_snap_min_uid": "0", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_snap_rstat": "false", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_symlink_recovery": "true", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_thrash_exports": "0", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_thrash_fragments": "0", 2023-09-23T19:52:10.495 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_tick_interval": "5.000000", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_valgrind_exit": "false", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_verify_backtrace": "1", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_verify_scatter": "false", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "mds_wipe_sessions": "false", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "mempool_debug": "false", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "memstore_page_set": "false", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "memstore_page_size": "65536", 2023-09-23T19:52:10.496 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_client_bytes": "134217728", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_client_messages": "512", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_disabled_modules": "", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_mds_messages": "128", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T19:52:10.497 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_mon_messages": "128", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_osd_messages": "8192", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_pool": "true", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_standby_modules": "true", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_stats_period": "5", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_stats_threshold": "5", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_tick_period": "2", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T19:52:10.498 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_allow_pool_delete": "true", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_bytes": "104857600", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T19:52:10.499 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T19:52:10.500 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_compact_on_start": "false", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_compact_on_trim": "true", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T19:52:10.501 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_cpu_threads": "4", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_data_avail_crit": "5", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_data_avail_warn": "5", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_dump_json": "false", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T19:52:10.502 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_extra_checks": "false", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_election_default_strategy": "1", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_election_timeout": "5.000000", 2023-09-23T19:52:10.503 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_enable_op_tracker": "true", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_fake_pool_delete": "false", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_force_quorum_join": "false", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_health_log_update_period": "5", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_health_max_detail": "50", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_health_to_clog": "true", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T19:52:10.504 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_host": "[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] [v2:172.21.15.135:3301/0,v1:172.21.15.135:6790/0]", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_host_override": "", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_initial_members": "", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_lease": "5.000000", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T19:52:10.505 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_log_full_interval": "50", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_log_max": "10000", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_log_max_summary": "50", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_max_log_epochs": "500", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_max_osd": "10000", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T19:52:10.506 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_memory_autotune": "true", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_memory_target": "2147483648", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mgr_digest_period": "5", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T19:52:10.507 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_op_complaint_time": "30", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_op_history_duration": "600", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_op_history_size": "20", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_op_log_threshold": "5", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T19:52:10.508 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_cache_size": "500", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T19:52:10.509 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T19:52:10.510 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_report_timeout": "900", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T19:52:10.511 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_reweight_max_osds": "4", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T19:52:10.512 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_scrub_interval": "86400", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_scrub_max_keys": "100", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_scrub_timeout": "300", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_session_timeout": "300", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_smart_report_timeout": "5", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_stretch_pool_size": "4", 2023-09-23T19:52:10.513 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_sync_debug": "false", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_tick_interval": "5", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T19:52:10.514 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T19:52:10.515 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_older_version": "true", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "monmap": "", 2023-09-23T19:52:10.516 INFO:teuthology.orchestra.run.smithi135.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "motr_tracing_enabled": "false", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_op_threads": "3", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_cm": "false", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_device_name": "", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T19:52:10.517 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_sl": "3", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_rdma_type": "ib", 2023-09-23T19:52:10.518 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_async_reap_threshold": "5", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_before_connect": "false", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_ipv4": "true", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_ipv6": "false", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_msgr1": "true", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_msgr2": "true", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_port_max": "7568", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_port_min": "6800", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_retry_count": "3", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_bind_retry_delay": "5", 2023-09-23T19:52:10.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_blackhole_client": "false", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_blackhole_mds": "false", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_blackhole_mgr": "false", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_blackhole_mon": "false", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_blackhole_osd": "false", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_client_mode": "crc secure", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_cluster_type": "", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_compress_secure": "false", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T19:52:10.520 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_crc_data": "true", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_crc_header": "true", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_die_on_bug": "true", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_die_on_old_message": "true", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T19:52:10.521 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_hugepages": "", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_lro": "true", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_pmd": "", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_port_id": "0", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T19:52:10.522 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_dump_on_send": "false", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_inject_delay_type": "", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_inject_network_congestion": "0", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_inject_socket_failures": "0", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_max_accept_failures": "4", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_max_backoff": "15.000000", 2023-09-23T19:52:10.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_osd_compress_mode": "none", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_pq_min_cost": "65536", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_public_type": "", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_service_mode": "crc secure", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_tcp_nodelay": "true", 2023-09-23T19:52:10.524 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "ms_type": "async+posix", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "no_config_file": "false", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objecter_inflight_ops": "1024", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objecter_timeout": "10.000000", 2023-09-23T19:52:10.525 INFO:teuthology.orchestra.run.smithi135.stdout: "objectstore_blackhole": "false", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "openssl_engine_opts": "", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_agent_max_ops": "4", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_agent_slop": "0.020000", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T19:52:10.526 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_asio_thread_count": "2", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_backfill_scan_max": "512", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_backfill_scan_min": "64", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_backoff_on_peering": "false", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_beacon_report_interval": "300", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_bench_duration": "30", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T19:52:10.527 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_blkin_trace_all": "false", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T19:52:10.528 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_class_update_on_start": "true", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_client_message_cap": "256", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_client_op_priority": "63", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_client_watch_timeout": "30", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_command_max_records": "256", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_compact_on_start": "false", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_crush_update_on_start": "true", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T19:52:10.529 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T19:52:10.530 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_op_order": "false", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_shutdown": "false", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T19:52:10.531 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T19:52:10.532 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_default_notify_timeout": "30", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_enable_op_tracker": "true", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_fast_info": "true", 2023-09-23T19:52:10.533 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_fast_shutdown": "true", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_function_tracing": "false", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_heartbeat_grace": "20", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_heartbeat_interval": "6", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T19:52:10.534 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_heartbeat_stale": "600", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_journal_size": "5120", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_kill_backfill_at": "0", 2023-09-23T19:52:10.535 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_map_cache_size": "50", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_map_dedup": "true", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_map_message_max": "40", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_attr_name_len": "100", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_attr_size": "0", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_backfills": "1", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_markdown_count": "5", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_markdown_period": "600", 2023-09-23T19:52:10.536 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_object_name_len": "2048", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_object_size": "134217728", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_pgls": "1024", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_push_cost": "8388608", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_push_objects": "10", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_scrubs": "1", 2023-09-23T19:52:10.537 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_max_write_size": "90", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T19:52:10.538 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_profile": "balanced", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T19:52:10.539 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_memory_base": "805306368", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_memory_target": "4294967296", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_memory_target_autotune": "false", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_min_recovery_priority": "0", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T19:52:10.540 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mon_report_interval": "5", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_num_cache_shards": "32", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_numa_node": "-1", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_objecter_finishers": "1", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_objectstore": "bluestore", 2023-09-23T19:52:10.541 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_objectstore_fuse": "false", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_objectstore_tracing": "false", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_history_duration": "600", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_history_size": "20", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_log_threshold": "5", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_num_shards": "0", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T19:52:10.542 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_op_thread_timeout": "15", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_open_classes_on_start": "true", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_os_flags": "0", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_peering_op_priority": "255", 2023-09-23T19:52:10.543 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_delete_priority": "5", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T19:52:10.544 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_crimson": "false", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T19:52:10.545 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_flags": "0", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_min_size": "0", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_size": "3", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_default_type": "replicated", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T19:52:10.546 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_cost": "20971520", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_max_active": "0", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T19:52:10.547 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_op_priority": "3", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_priority": "5", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_repair_during_recovery": "false", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T19:52:10.548 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_cost": "52428800", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T19:52:10.549 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_end_hour": "0", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_priority": "5", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T19:52:10.550 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_skip_data_digest": "false", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_smart_report_timeout": "5", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_snap_trim_priority": "5", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T19:52:10.551 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_target_transaction_size": "30", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T19:52:10.552 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_tracing": "false", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_use_stale_snap": "false", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_kill_at": "0", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_max_join_drift": "10", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_min": "500", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_min_wait": "0.050000", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T19:52:10.553 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_service_trim_max": "500", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_service_trim_min": "250", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_stash_full_interval": "25", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_trim_max": "500", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "paxos_trim_min": "250", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "perf": "true", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "pid_file": "", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "public_addr": "-", 2023-09-23T19:52:10.554 INFO:teuthology.orchestra.run.smithi135.stdout: "public_addrv": "", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "public_bind_addr": "-", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "public_network": "172.21.0.0/20", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "public_network_interface": "", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "qat_compressor_enabled": "false", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "rados_mon_op_timeout": "0", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "rados_osd_op_timeout": "0", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "rados_tracing": "false", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_atime_update_interval": "60", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T19:52:10.555 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache": "true", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache_policy": "writearound", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache_size": "33554432", 2023-09-23T19:52:10.556 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_compression_hint": "none", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_clone_format": "auto", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_data_pool": "", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_features": "61", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_format": "2", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_map_options": "", 2023-09-23T19:52:10.557 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_order": "22", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_pool": "rbd", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_stripe_count": "0", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_io_scheduler": "simple", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T19:52:10.558 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_order": "24", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_pool": "", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_journal_splay_width": "4", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T19:52:10.559 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T19:52:10.560 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T19:52:10.561 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T19:52:10.562 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T19:52:10.563 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_op_threads": "1", 2023-09-23T19:52:10.563 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T19:52:10.565 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T19:52:10.565 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T19:52:10.565 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T19:52:10.565 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_plugins": "", 2023-09-23T19:52:10.565 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T19:52:10.565 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T19:52:10.565 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T19:52:10.565 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T19:52:10.566 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T19:52:10.567 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_tracing": "false", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_validate_names": "true", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rbd_validate_pool": "true", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "restapi_base_url": "", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "restapi_log_level": "", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_admin_entry": "admin", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_backend_store": "rados", 2023-09-23T19:52:10.568 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_barbican_url": "", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_beast_enable_async": "true", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_cache_enabled": "true", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T19:52:10.569 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_config_store": "rados", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_content_length_compat": "false", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_cross_domain_policy": "", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T19:52:10.570 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T19:52:10.571 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T19:52:10.572 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T19:52:10.573 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T19:52:10.574 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_data_log_window": "30", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T19:52:10.575 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T19:52:10.576 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dns_name": "", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dns_s3website_name": "", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T19:52:10.577 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_enable_ops_log": "false", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_enable_static_website": "false", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_enable_usage_log": "false", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_expose_bucket": "false", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_extended_http_attrs": "", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_filter": "none", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T19:52:10.578 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_max_objs": "32", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T19:52:10.579 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_init_timeout": "300", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_admin_password": "", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_admin_project": "", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T19:52:10.580 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_admin_token": "", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_admin_user": "", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_api_version": "2", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T19:52:10.581 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_url": "", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_lc_max_objs": "32", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_lc_max_worker": "3", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T19:52:10.582 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_log_http_headers": "", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T19:52:10.583 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_attr_size": "0", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_listing_results": "1000", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_notify_retries": "10", 2023-09-23T19:52:10.584 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T19:52:10.585 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T19:52:10.586 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_num_control_oids": "8", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_numa_node": "-1", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T19:52:10.587 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_op_tracing": "false", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_opa_token": "", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_opa_url": "", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ops_log_rados": "false", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T19:52:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_print_continue": "true", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T19:52:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_rados_tracing": "false", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_realm": "", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_realm_id": "", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_region": "", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_request_uri": "", 2023-09-23T19:52:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_resolve_cname": "false", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_run_sync_thread": "true", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T19:52:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_script_uri": "", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_service_provider_name": "", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sts_client_id": "", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sts_client_secret": "", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sts_entry": "sts", 2023-09-23T19:52:10.592 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sts_key": "sts", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_auth_url": "", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_custom_header": "", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_need_stats": "true", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_tenant_name": "", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T19:52:10.593 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_url": "", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_lease_period": "120", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T19:52:10.594 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_thread_pool_size": "512", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_torrent_comment": "", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_torrent_createby": "", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_torrent_encoding": "", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_torrent_flag": "false", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_torrent_origin": "", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_torrent_tracker": "", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T19:52:10.595 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_usage_max_shards": "32", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_use_opa_authz": "false", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T19:52:10.596 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_user_unique_email": "true", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_verify_ssl": "true", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_zone": "", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_zone_id": "", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_zonegroup": "", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_zonegroup_id": "", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T19:52:10.597 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_block_size": "4096", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T19:52:10.598 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_partition_filters": "false", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_perf": "false", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T19:52:10.599 INFO:teuthology.orchestra.run.smithi135.stdout: "run_dir": "/var/run/ceph", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_block_create": "true", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_device_size": "53687091200", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T19:52:10.600 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_main_device_type": "SSD", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "seastore_segment_size": "67108864", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "set_keepcaps": "false", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "setgroup": "ceph", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "setuser": "ceph", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "setuser_match_path": "", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T19:52:10.601 INFO:teuthology.orchestra.run.smithi135.stdout: "thp": "false", 2023-09-23T19:52:10.602 INFO:teuthology.orchestra.run.smithi135.stdout: "threadpool_default_timeout": "60", 2023-09-23T19:52:10.602 INFO:teuthology.orchestra.run.smithi135.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T19:52:10.602 INFO:teuthology.orchestra.run.smithi135.stdout: "throttler_perf_counter": "true" 2023-09-23T19:52:10.602 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:52:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:10 smithi144 ceph-mon[194833]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:11.273 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T19:52:11.612 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:12.146 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:12 smithi135 ceph-mon[213871]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:12 smithi135 ceph-mon[221113]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:12 smithi144 ceph-mon[194833]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:13.202 INFO:teuthology.orchestra.run.smithi135.stdout:mon.a: {} 2023-09-23T19:52:13.211 INFO:teuthology.orchestra.run.smithi135.stdout:mon.b: {} 2023-09-23T19:52:13.221 INFO:teuthology.orchestra.run.smithi135.stdout:mon.c: {} 2023-09-23T19:52:13.222 INFO:teuthology.orchestra.run.smithi135.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T19:52:13.222 INFO:teuthology.orchestra.run.smithi135.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T19:52:13.222 INFO:teuthology.orchestra.run.smithi135.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T19:52:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:52:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:52:13] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2023-09-23T19:52:13.996 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T19:52:13.996 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T19:52:13.996 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:52:13.998 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:52:13.998 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T19:52:13.998 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T19:52:13.999 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T19:52:13.999 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:52:13.999 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:52:13.999 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T19:52:13.999 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:13.999 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T19:52:14.000 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:52:14.000 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T19:52:14.000 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:52:14.000 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T19:52:14.001 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:52:14.051 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:14.103 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:52:14.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:14.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:14.208 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:14.241 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:14 smithi135 ceph-mon[213871]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:14.241 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/350012632' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:52:14.241 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:14 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:52:14.241 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:14 smithi135 ceph-mon[221113]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:14.242 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/350012632' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:52:14.242 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:14 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:52:14.244 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:14.311 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:14.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:14.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:14 smithi144 ceph-mon[194833]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/350012632' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:52:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:14 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:52:14.417 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:14.445 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:14.479 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:14.489 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:14.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:14.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:14.496 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:14.595 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:14.622 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:14.664 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:14.666 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:14.777 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:14.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:14.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:14.880 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:14.907 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:15.049 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:52:15.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:15.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:15.154 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:15.184 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:16.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:16 smithi135 ceph-mon[213871]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:16 smithi135 ceph-mon[221113]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:16.484 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:16.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:16.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:16.589 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:16.614 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:16 smithi144 ceph-mon[194833]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:16.617 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:16.816 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:52:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:16.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:16.921 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:16.949 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:17.350 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:17.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:17.454 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:17.480 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:17.794 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:17.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:17.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:17.898 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:17.926 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:18 smithi135 ceph-mon[221113]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:18 smithi135 ceph-mon[213871]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:18.429 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:18.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:18.533 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:18.562 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:18 smithi144 ceph-mon[194833]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:18.562 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:18.563 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:18.566 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:18.616 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:52:18.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:18.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:18.722 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:18.753 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:19.174 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:52:19.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/672702090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:19.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:19.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/672702090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:19.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:19.302 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:19.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:19.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:19.406 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:19.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/672702090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:19.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:52:19.440 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:19.489 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:52:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:19.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:19.501 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:52:19.501 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T19:52:19.501 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T19:52:19.501 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:52:19.594 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:19.626 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:19.628 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:52:19.651 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:52:19.673 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:52:19.693 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:52:19.716 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:52:19.731 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:52:19.750 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:52:19.768 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:52:19.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:52:19.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:52:19.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:52:19.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:52:19.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:52:19.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:52:19.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:52:19.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:52:20.171 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_0' created 2023-09-23T19:52:20.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[221113]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:20.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:20.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[221113]: failed to encode map e111 with expected crc 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[221113]: failed to encode map e111 with expected crc 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[221113]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3585151588' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3664178987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[213871]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[213871]: failed to encode map e111 with expected crc 2023-09-23T19:52:20.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[213871]: failed to encode map e111 with expected crc 2023-09-23T19:52:20.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:52:20.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[213871]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T19:52:20.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3585151588' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:52:20.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3664178987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:20 smithi144 ceph-mon[194833]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:20 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:20 smithi144 ceph-mon[194833]: failed to encode map e111 with expected crc 2023-09-23T19:52:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:20 smithi144 ceph-mon[194833]: failed to encode map e111 with expected crc 2023-09-23T19:52:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:20 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:52:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:20 smithi144 ceph-mon[194833]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T19:52:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3585151588' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T19:52:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3664178987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:21.055 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:52:21.232 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:21.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:21.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:21.336 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:21.368 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:21.418 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:52:21.421 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:21 smithi144 ceph-mon[194833]: failed to encode map e111 with expected crc 2023-09-23T19:52:21.421 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3585151588' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T19:52:21.421 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:21 smithi144 ceph-mon[194833]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T19:52:21.421 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:21 smithi144 ceph-mon[194833]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.421 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:21 smithi144 ceph-mon[194833]: failed to encode map e111 with expected crc 2023-09-23T19:52:21.421 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:21 smithi144 ceph-mon[194833]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:21 smithi144 ceph-mon[194833]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:21.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[221113]: failed to encode map e111 with expected crc 2023-09-23T19:52:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3585151588' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T19:52:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[221113]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T19:52:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[221113]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[221113]: failed to encode map e111 with expected crc 2023-09-23T19:52:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[221113]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[221113]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[213871]: failed to encode map e111 with expected crc 2023-09-23T19:52:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3585151588' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T19:52:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[213871]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T19:52:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[213871]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[213871]: failed to encode map e111 with expected crc 2023-09-23T19:52:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[213871]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:21 smithi135 ceph-mon[213871]: failed to encode map e112 with expected crc 2023-09-23T19:52:21.524 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:21.551 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:21.720 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T19:52:21.974 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:21.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:21.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:22.079 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:22.106 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:22.114 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:22.165 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:52:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:22.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:22.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[221113]: pgmap v227: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:22.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[221113]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[221113]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T19:52:22.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[221113]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[221113]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[221113]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[213871]: pgmap v227: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[213871]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[213871]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[213871]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[213871]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[213871]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:22.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:22.234 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:22.269 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:22.301 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:22.349 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:52:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:22.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:22.456 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:22.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:22 smithi144 ceph-mon[194833]: pgmap v227: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:22.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:22 smithi144 ceph-mon[194833]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:22 smithi144 ceph-mon[194833]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T19:52:22.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:22 smithi144 ceph-mon[194833]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:22 smithi144 ceph-mon[194833]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:22 smithi144 ceph-mon[194833]: failed to encode map e113 with expected crc 2023-09-23T19:52:22.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:22.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:22.484 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:22.529 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:52:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:22.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:22.634 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:22.661 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:22.705 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:52:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:22.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:22.811 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:22.840 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:23.088 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:23.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:23.192 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:23.219 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:23.261 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:23.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:23.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:23.368 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:23.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:23 smithi135 ceph-mon[221113]: failed to encode map e114 with expected crc 2023-09-23T19:52:23.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:23 smithi135 ceph-mon[221113]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T19:52:23.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:23 smithi135 ceph-mon[221113]: failed to encode map e114 with expected crc 2023-09-23T19:52:23.393 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:23 smithi135 ceph-mon[213871]: failed to encode map e114 with expected crc 2023-09-23T19:52:23.393 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:23 smithi135 ceph-mon[213871]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T19:52:23.394 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:23 smithi135 ceph-mon[213871]: failed to encode map e114 with expected crc 2023-09-23T19:52:23.396 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:23 smithi144 ceph-mon[194833]: failed to encode map e114 with expected crc 2023-09-23T19:52:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:23 smithi144 ceph-mon[194833]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T19:52:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:23 smithi144 ceph-mon[194833]: failed to encode map e114 with expected crc 2023-09-23T19:52:23.713 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:23.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:23.817 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:23.846 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:23.891 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:23.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:23.953 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:52:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:52:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:52:23] "GET /metrics HTTP/1.1" 200 36252 "" "Prometheus/2.43.0" 2023-09-23T19:52:23.977 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:52:23.997 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:24.046 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:24.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:24 smithi135 ceph-mon[221113]: failed to encode map e114 with expected crc 2023-09-23T19:52:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:24 smithi135 ceph-mon[221113]: failed to encode map e114 with expected crc 2023-09-23T19:52:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:24 smithi135 ceph-mon[221113]: pgmap v230: 89 pgs: 3 creating+peering, 1 unknown, 85 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:24 smithi135 ceph-mon[213871]: failed to encode map e114 with expected crc 2023-09-23T19:52:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:24 smithi135 ceph-mon[213871]: failed to encode map e114 with expected crc 2023-09-23T19:52:24.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:24 smithi135 ceph-mon[213871]: pgmap v230: 89 pgs: 3 creating+peering, 1 unknown, 85 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:24 smithi144 ceph-mon[194833]: failed to encode map e114 with expected crc 2023-09-23T19:52:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:24 smithi144 ceph-mon[194833]: failed to encode map e114 with expected crc 2023-09-23T19:52:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:24 smithi144 ceph-mon[194833]: pgmap v230: 89 pgs: 3 creating+peering, 1 unknown, 85 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:24.765 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:24.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:24.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:24.870 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:24.900 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:24.964 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:24.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:24.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:25.068 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:25.102 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:25.152 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:52:25.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:25.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:25.201 INFO:teuthology.orchestra.run.smithi135.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T19:52:25.231 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:25.258 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:25.293 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:25.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3940251367' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:52:25.438 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:25.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:25.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:25.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3940251367' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:52:25.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3940251367' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:52:25.542 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:25.570 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:26.294 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:26.296 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:52:26.296 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:52:26.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:26.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:26.398 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:26.426 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[221113]: pgmap v231: 89 pgs: 3 creating+peering, 86 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[221113]: failed to encode map e115 with expected crc 2023-09-23T19:52:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3940251367' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:52:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[221113]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T19:52:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[221113]: failed to encode map e115 with expected crc 2023-09-23T19:52:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[213871]: pgmap v231: 89 pgs: 3 creating+peering, 86 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[213871]: failed to encode map e115 with expected crc 2023-09-23T19:52:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3940251367' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:52:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[213871]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T19:52:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[213871]: failed to encode map e115 with expected crc 2023-09-23T19:52:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:26.573 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:26.623 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:52:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:26.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:26 smithi144 ceph-mon[194833]: pgmap v231: 89 pgs: 3 creating+peering, 86 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:26 smithi144 ceph-mon[194833]: failed to encode map e115 with expected crc 2023-09-23T19:52:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3940251367' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:52:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:26 smithi144 ceph-mon[194833]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T19:52:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:26 smithi144 ceph-mon[194833]: failed to encode map e115 with expected crc 2023-09-23T19:52:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:26.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:26.728 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:26.755 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:26.803 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:26.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:26.907 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:26.937 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:26.985 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:52:26.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:26.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:27.091 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:27.117 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[221113]: failed to encode map e115 with expected crc 2023-09-23T19:52:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[221113]: failed to encode map e115 with expected crc 2023-09-23T19:52:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:52:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:52:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[213871]: failed to encode map e115 with expected crc 2023-09-23T19:52:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[213871]: failed to encode map e115 with expected crc 2023-09-23T19:52:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:52:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:52:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:27.611 DEBUG:teuthology.orchestra.run.smithi135:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T19:52:27.612 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T19:52:27.612 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:52:27.614 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:52:27.614 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:52:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:27 smithi144 ceph-mon[194833]: failed to encode map e115 with expected crc 2023-09-23T19:52:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:27 smithi144 ceph-mon[194833]: failed to encode map e115 with expected crc 2023-09-23T19:52:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:52:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:52:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:27.660 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:hints = 1 2023-09-23T19:52:27.661 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:52:27.661 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object prefix: benchmark_data_smithi135_284570 2023-09-23T19:52:27.661 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:52:27.661 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:52:27.861 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:27.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:27.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:27.966 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:28.026 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:28.121 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:52:28.203 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:28.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:28.311 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:28.346 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:28.422 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:28.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:28.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:28 smithi135 ceph-mon[213871]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:28 smithi135 ceph-mon[221113]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:28.528 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:28.566 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:28 smithi144 ceph-mon[194833]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T19:52:28.570 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:28.648 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:28.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:28.661 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 1 16 3351 3335 208.426 208.438 0.0026493 0.0047825 2023-09-23T19:52:28.756 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:28.809 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:29.662 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 2 16 7413 7397 231.067 253.875 0.00404176 0.00431865 2023-09-23T19:52:30.452 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:30.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:30.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:30 smithi135 ceph-mon[221113]: pgmap v234: 89 pgs: 89 active+clean; 44 MiB data, 199 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s wr, 79 op/s 2023-09-23T19:52:30.459 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:30 smithi135 ceph-mon[213871]: pgmap v234: 89 pgs: 89 active+clean; 44 MiB data, 199 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s wr, 79 op/s 2023-09-23T19:52:30.560 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:30.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:30 smithi144 ceph-mon[194833]: pgmap v234: 89 pgs: 89 active+clean; 44 MiB data, 199 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s wr, 79 op/s 2023-09-23T19:52:30.601 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:30.662 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 3 16 10917 10901 227.039 219 0.0102854 0.00438911 2023-09-23T19:52:30.686 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:30.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:30.734 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:30.796 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:30.858 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:30.884 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:31.267 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:31.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:31.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:31.391 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:31.451 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:31.481 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:52:31.528 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:52:31.560 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:52:31.603 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:52:31.640 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:52:31.662 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 4 16 13421 13405 209.404 156.5 0.00453636 0.00476683 2023-09-23T19:52:31.677 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:52:31.715 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:52:31.753 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:52:31.760 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:52:31.761 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:52:31.761 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:52:31.761 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:52:31.761 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:52:31.761 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:52:31.761 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:52:31.761 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:52:32.099 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:32.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:32.206 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:32.226 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:52:32.237 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[221113]: pgmap v235: 89 pgs: 89 active+clean; 200 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s wr, 348 op/s 2023-09-23T19:52:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2286175227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3856020087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:32.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[213871]: pgmap v235: 89 pgs: 89 active+clean; 200 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s wr, 348 op/s 2023-09-23T19:52:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2286175227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3856020087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:32 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:32 smithi144 ceph-mon[194833]: pgmap v235: 89 pgs: 89 active+clean; 200 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s wr, 348 op/s 2023-09-23T19:52:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2286175227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:32.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:32 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:32.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3856020087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:32.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:32 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:52:32.664 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 5 16 15767 15751 196.782 146.625 0.0072817 0.00507548 2023-09-23T19:52:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[213871]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T19:52:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[213871]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[213871]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[213871]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[213871]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[213871]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:33.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:33.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[221113]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T19:52:33.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[221113]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[221113]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[221113]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[221113]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:33 smithi135 ceph-mon[221113]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:33 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:33 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:33 smithi144 ceph-mon[194833]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T19:52:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:33 smithi144 ceph-mon[194833]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:33 smithi144 ceph-mon[194833]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:33 smithi144 ceph-mon[194833]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:33 smithi144 ceph-mon[194833]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:33 smithi144 ceph-mon[194833]: failed to encode map e116 with expected crc 2023-09-23T19:52:33.664 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 6 16 18813 18797 195.712 190.375 0.00337244 0.00510376 2023-09-23T19:52:33.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:52:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:52:33] "GET /metrics HTTP/1.1" 200 36252 "" "Prometheus/2.43.0" 2023-09-23T19:52:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:34 smithi135 ceph-mon[221113]: failed to encode map e116 with expected crc 2023-09-23T19:52:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:34 smithi135 ceph-mon[221113]: failed to encode map e116 with expected crc 2023-09-23T19:52:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:34 smithi135 ceph-mon[221113]: pgmap v237: 89 pgs: 89 active+clean; 899 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:52:34.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:34 smithi135 ceph-mon[213871]: failed to encode map e116 with expected crc 2023-09-23T19:52:34.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:34 smithi135 ceph-mon[213871]: failed to encode map e116 with expected crc 2023-09-23T19:52:34.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:34 smithi135 ceph-mon[213871]: pgmap v237: 89 pgs: 89 active+clean; 899 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:52:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:34 smithi144 ceph-mon[194833]: failed to encode map e116 with expected crc 2023-09-23T19:52:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:34 smithi144 ceph-mon[194833]: failed to encode map e116 with expected crc 2023-09-23T19:52:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:34 smithi144 ceph-mon[194833]: pgmap v237: 89 pgs: 89 active+clean; 899 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T19:52:34.665 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 7 16 21208 21192 189.091 149.688 0.00582033 0.00528297 2023-09-23T19:52:35.666 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 8 16 23047 23031 179.823 114.938 0.00649997 0.00555606 2023-09-23T19:52:35.918 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:52:35.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:35.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:36.023 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:36.087 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:36.352 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:36.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:36.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:36.457 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:36.486 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:36 smithi135 ceph-mon[221113]: pgmap v238: 89 pgs: 89 active+clean; 954 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:36.486 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:36 smithi135 ceph-mon[213871]: pgmap v238: 89 pgs: 89 active+clean; 954 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:36.501 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:36 smithi144 ceph-mon[194833]: pgmap v238: 89 pgs: 89 active+clean; 954 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T19:52:36.507 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:36.601 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:52:36.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:36.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:36.667 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 9 16 25425 25409 176.331 148.625 0.00482551 0.00566602 2023-09-23T19:52:36.709 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:36.765 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:37.240 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:52:37.247 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:52:37.364 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:52:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:37.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:37.467 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:37.503 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:37.503 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:37 smithi144 ceph-mon[194833]: pgmap v239: 89 pgs: 89 active+clean; 1.3 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-09-23T19:52:37.503 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:37 smithi144 ceph-mon[194833]: from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:37.505 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:37.548 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:52:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:37.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:37.653 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:37.667 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 10 16 27323 27307 170.562 118.625 0.00878737 0.00585867 2023-09-23T19:52:37.695 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:37 smithi135 ceph-mon[221113]: pgmap v239: 89 pgs: 89 active+clean; 1.3 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-09-23T19:52:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:37 smithi135 ceph-mon[221113]: from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:37 smithi135 ceph-mon[213871]: pgmap v239: 89 pgs: 89 active+clean; 1.3 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-09-23T19:52:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:37 smithi135 ceph-mon[213871]: from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:52:37.777 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:52:37.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:37.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:37.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:37.936 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:38.105 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:52:38.563 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:38.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:38.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:38.667 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 11 16 29148 29132 165.427 114.062 0.0028842 0.00601677 2023-09-23T19:52:38.668 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:38.693 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:39.210 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:39.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:39.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:39.320 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:39.365 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:52:39.366 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:52:39.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:52:39.366 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:52:39.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.366 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:14787 5.5 5:a5355356:::benchmark_data_smithi135_284570_object14786:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T19:52:39.366 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:32.155803+0000", 2023-09-23T19:52:39.366 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.20708634, 2023-09-23T19:52:39.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.042819511999999997, 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 14787 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.155803+0000", 2023-09-23T19:52:39.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.155803+0000", 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.155802+0000", 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.155824+0000", 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1895999999999998e-05 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.155837+0000", 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2813000000000001e-05 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.155844+0000", 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7139999999999993e-06 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.155892+0000", 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7919e-05 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.155993+0000", 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101005 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.156072+0000", 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9048000000000006e-05 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.157098+0000", 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001025665 2023-09-23T19:52:39.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.157168+0000", 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9864999999999996e-05 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.158282+0000", 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0011139629999999999 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.158324+0000", 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2234999999999998e-05 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.198623+0000", 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.040298885 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.373 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:15268 5.5 5:a0057611:::benchmark_data_smithi135_284570_object15267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:32.441887+0000", 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 6.9210031680000004, 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.041850843999999998, 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 15268 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.374 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.441887+0000", 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.441887+0000", 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.441885+0000", 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.441918+0000", 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2249000000000002e-05 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.441930+0000", 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.198e-05 2023-09-23T19:52:39.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.441937+0000", 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4560000000000003e-06 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.441971+0000", 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3626e-05 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.442055+0000", 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4135000000000007e-05 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.442159+0000", 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000104268 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.443080+0000", 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00092050499999999996 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.443774+0000", 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00069417199999999995 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.483655+0000", 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.039881512000000001 2023-09-23T19:52:39.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.483700+0000", 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4705e-05 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:32.483737+0000", 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7428999999999997e-05 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.380 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:22015 5.5 5:ae704a38:::benchmark_data_smithi135_284570_object22014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:34.947371+0000", 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 4.4155190959999997, 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.090950813000000005, 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.381 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 22015 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947371+0000", 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947371+0000", 2023-09-23T19:52:39.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947369+0000", 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947394+0000", 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.482e-05 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947408+0000", 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3701e-05 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947417+0000", 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.73e-06 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947470+0000", 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2924999999999999e-05 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947614+0000", 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000143707 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.947767+0000", 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015302999999999999 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.948560+0000", 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00079318300000000004 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.950078+0000", 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015174489999999999 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.038230+0000", 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.088152567000000001 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.038284+0000", 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3386e-05 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.038322+0000", 2023-09-23T19:52:39.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8016000000000003e-05 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:22479 5.5 5:a764f105:::benchmark_data_smithi135_284570_object22478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:35.363410+0000", 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.9994793930000001, 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.103832044, 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.388 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 22479 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363410+0000", 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363410+0000", 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363409+0000", 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363443+0000", 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4251000000000001e-05 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363454+0000", 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0998e-05 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363458+0000", 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2130000000000002e-06 2023-09-23T19:52:39.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363494+0000", 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5574000000000002e-05 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363629+0000", 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000134831 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:52:39.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363818+0000", 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018937000000000001 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364722+0000", 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00090445200000000003 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.467117+0000", 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.10239448499999999 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.467156+0000", 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8797000000000001e-05 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.467202+0000", 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6042999999999999e-05 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.467242+0000", 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0704999999999997e-05 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.395 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:22480 5.3 5:c1922d15:::benchmark_data_smithi135_284570_object22479:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:35.363470+0000", 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.9994202579999998, 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.105547378, 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 22480 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363470+0000", 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363470+0000", 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363469+0000", 2023-09-23T19:52:39.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:52:39.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363493+0000", 2023-09-23T19:52:39.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.404e-05 2023-09-23T19:52:39.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363500+0000", 2023-09-23T19:52:39.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6209999999999998e-06 2023-09-23T19:52:39.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363505+0000", 2023-09-23T19:52:39.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8969999999999999e-06 2023-09-23T19:52:39.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363548+0000", 2023-09-23T19:52:39.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3133000000000002e-05 2023-09-23T19:52:39.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363670+0000", 2023-09-23T19:52:39.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012258399999999999 2023-09-23T19:52:39.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:52:39.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363796+0000", 2023-09-23T19:52:39.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012557599999999999 2023-09-23T19:52:39.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364997+0000", 2023-09-23T19:52:39.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001201248 2023-09-23T19:52:39.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.468118+0000", 2023-09-23T19:52:39.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.10312101799999999 2023-09-23T19:52:39.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.468944+0000", 2023-09-23T19:52:39.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00082578399999999998 2023-09-23T19:52:39.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.468989+0000", 2023-09-23T19:52:39.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4885000000000001e-05 2023-09-23T19:52:39.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.469017+0000", 2023-09-23T19:52:39.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8112000000000001e-05 2023-09-23T19:52:39.406 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.406 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.406 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.406 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:22483 5.b e116/112 5:d0a0af89:::benchmark_data_smithi135_284570_object22482:head v 116'1346, mlcod=116'1344)", 2023-09-23T19:52:39.406 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:35.363628+0000", 2023-09-23T19:52:39.407 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.9992621079999999, 2023-09-23T19:52:39.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.105272881, 2023-09-23T19:52:39.407 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.407 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.407 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363628+0000", 2023-09-23T19:52:39.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363628+0000", 2023-09-23T19:52:39.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363627+0000", 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363646+0000", 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8760999999999999e-05 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.410 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363652+0000", 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1510000000000001e-06 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.363657+0000", 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.916e-06 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364222+0000", 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00056471699999999995 2023-09-23T19:52:39.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364241+0000", 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8569000000000001e-05 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.468856+0000", 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.10461535299999999 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.468901+0000", 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4685999999999999e-05 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:22481 5.2 e116/112 5:4316e361:::benchmark_data_smithi135_284570_object22480:head v 116'1419, mlcod=116'1416)", 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:35.364513+0000", 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.998377031, 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.102694086, 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.413 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364513+0000", 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364513+0000", 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364512+0000", 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364543+0000", 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0967999999999999e-05 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364551+0000", 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2050000000000002e-06 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364558+0000", 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3629999999999999e-06 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364609+0000", 2023-09-23T19:52:39.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1029999999999998e-05 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.364627+0000", 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7889000000000001e-05 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.467164+0000", 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.10253741 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.467207+0000", 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2935000000000001e-05 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:22485 5.3 5:c914a579:::benchmark_data_smithi135_284570_object22484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:35.365137+0000", 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.997752433, 2023-09-23T19:52:39.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.108444473, 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 22485 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.419 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365137+0000", 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365137+0000", 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365136+0000", 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365157+0000", 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1143999999999999e-05 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365168+0000", 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0869e-05 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365175+0000", 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0809999999999996e-06 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466881+0000", 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.101706299 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.467045+0000", 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016351199999999999 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.467203+0000", 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015821800000000001 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.469667+0000", 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024641620000000002 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.469698+0000", 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0914999999999998e-05 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.472900+0000", 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032017809999999999 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.473541+0000", 2023-09-23T19:52:39.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00064087699999999998 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.473582+0000", 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0918999999999999e-05 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:22484 5.b e116/112 5:dcd9f232:::benchmark_data_smithi135_284570_object22483:head v 116'1347, mlcod=116'1344)", 2023-09-23T19:52:39.425 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:35.365579+0000", 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.9973109920000001, 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.10612390400000001, 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365579+0000", 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365579+0000", 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365578+0000", 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365601+0000", 2023-09-23T19:52:39.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3059999999999999e-05 2023-09-23T19:52:39.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466723+0000", 2023-09-23T19:52:39.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.101121641 2023-09-23T19:52:39.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466737+0000", 2023-09-23T19:52:39.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4303e-05 2023-09-23T19:52:39.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466914+0000", 2023-09-23T19:52:39.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017711100000000001 2023-09-23T19:52:39.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.431 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466935+0000", 2023-09-23T19:52:39.431 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0409999999999999e-05 2023-09-23T19:52:39.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.431 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.471665+0000", 2023-09-23T19:52:39.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047301970000000002 2023-09-23T19:52:39.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.432 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.471703+0000", 2023-09-23T19:52:39.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8009000000000001e-05 2023-09-23T19:52:39.432 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.432 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:22487 5.7 e116/112 5:ed120754:::benchmark_data_smithi135_284570_object22486:head v 116'1375, mlcod=116'1373)", 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:35.365922+0000", 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.9969682139999998, 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.10275587899999999, 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.433 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.434 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365922+0000", 2023-09-23T19:52:39.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365922+0000", 2023-09-23T19:52:39.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.365921+0000", 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.366012+0000", 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1435999999999997e-05 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466721+0000", 2023-09-23T19:52:39.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.10070923 2023-09-23T19:52:39.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466738+0000", 2023-09-23T19:52:39.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6052000000000001e-05 2023-09-23T19:52:39.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466786+0000", 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8945e-05 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.466812+0000", 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5843e-05 2023-09-23T19:52:39.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.468635+0000", 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001823131 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.468677+0000", 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2061000000000003e-05 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.439 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:26971 5.5 5:a28365c3:::benchmark_data_smithi135_284570_object26970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:37.347048+0000", 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.0158419489999999, 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.075075841000000004, 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.440 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 26971 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.347048+0000", 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.347048+0000", 2023-09-23T19:52:39.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.347047+0000", 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.347065+0000", 2023-09-23T19:52:39.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8210999999999999e-05 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.347074+0000", 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3620000000000006e-06 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.347079+0000", 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0470000000000001e-06 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.347112+0000", 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2848000000000001e-05 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.418451+0000", 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.071339713999999999 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.418621+0000", 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000170068 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.420631+0000", 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020100090000000001 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.421692+0000", 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001060673 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.422041+0000", 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000348578 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.422087+0000", 2023-09-23T19:52:39.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6901999999999999e-05 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.422124+0000", 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6219e-05 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:26974 5.c e116/112 5:3840748b:::benchmark_data_smithi135_284570_object26973:head v 116'1675, mlcod=116'1673)", 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:37.348455+0000", 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.0144351469999999, 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.073475595000000005, 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.348455+0000", 2023-09-23T19:52:39.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.348455+0000", 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.348454+0000", 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.348473+0000", 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8638999999999999e-05 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.418691+0000", 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.070218056000000001 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.418703+0000", 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2383e-05 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.419076+0000", 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037246399999999998 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.419096+0000", 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0917999999999999e-05 2023-09-23T19:52:39.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.421890+0000", 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027936200000000001 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:37.421930+0000", 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.019e-05 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:28882 5.3 5:c68616b0:::benchmark_data_smithi135_284570_object28881:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:38.353071+0000", 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 1.0098192850000001, 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.078421703999999995, 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 28882 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.453 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353071+0000", 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353071+0000", 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353069+0000", 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353091+0000", 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.196e-05 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353107+0000", 2023-09-23T19:52:39.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5383000000000002e-05 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353115+0000", 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3149999999999992e-06 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353178+0000", 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3239999999999998e-05 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353312+0000", 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013377800000000001 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.353439+0000", 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012703599999999999 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354691+0000", 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012517800000000001 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.356168+0000", 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001477632 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.458 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431414+0000", 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.075245384999999998 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431451+0000", 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7706000000000001e-05 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431492+0000", 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0843999999999997e-05 2023-09-23T19:52:39.459 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:28886 5.3 5:c5699a52:::benchmark_data_smithi135_284570_object28885:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:38.354751+0000", 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 1.008138483, 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.076896897000000006, 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.460 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 28886 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354751+0000", 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.461 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354751+0000", 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354750+0000", 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354769+0000", 2023-09-23T19:52:39.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9029999999999999e-05 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354779+0000", 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4350000000000003e-06 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354784+0000", 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6160000000000001e-06 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.463 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354815+0000", 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0620000000000002e-05 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.354940+0000", 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000124771 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.355077+0000", 2023-09-23T19:52:39.464 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000137162 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.356219+0000", 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001141902 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.358969+0000", 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027500979999999999 2023-09-23T19:52:39.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431553+0000", 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.072583833 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431611+0000", 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7979999999999997e-05 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.466 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431648+0000", 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7549999999999998e-05 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:28893 5.3 5:cb1e0f23:::benchmark_data_smithi135_284570_object28892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:38.357535+0000", 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 1.005354525, 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.074263757, 2023-09-23T19:52:39.467 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.468 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.468 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.468 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.468 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.468 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 28893 2023-09-23T19:52:39.468 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.468 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.468 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.469 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357535+0000", 2023-09-23T19:52:39.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.469 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.469 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357535+0000", 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357534+0000", 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357554+0000", 2023-09-23T19:52:39.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0333999999999999e-05 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357566+0000", 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1735e-05 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357573+0000", 2023-09-23T19:52:39.471 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.824e-06 2023-09-23T19:52:39.473 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.473 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.473 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.473 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357625+0000", 2023-09-23T19:52:39.473 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1774000000000003e-05 2023-09-23T19:52:39.473 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.473 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.473 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357777+0000", 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000151927 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.357957+0000", 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018018299999999999 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.359082+0000", 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0011250769999999999 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.359300+0000", 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00021714099999999999 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431712+0000", 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.072412299999999999 2023-09-23T19:52:39.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431764+0000", 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.168e-05 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.431799+0000", 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5508e-05 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.476 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:28988 5.5 5:a5fe06fb:::benchmark_data_smithi135_284570_object28987:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:38.463132+0000", 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.89975733199999997, 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.088588418000000002, 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.477 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 28988 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463132+0000", 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.478 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463132+0000", 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463131+0000", 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.480 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463161+0000", 2023-09-23T19:52:39.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9743999999999998e-05 2023-09-23T19:52:39.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.480 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.480 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463177+0000", 2023-09-23T19:52:39.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5911999999999999e-05 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463185+0000", 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4400000000000005e-06 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463263+0000", 2023-09-23T19:52:39.481 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8055999999999999e-05 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463379+0000", 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011593 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.463475+0000", 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5431000000000002e-05 2023-09-23T19:52:39.482 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.464813+0000", 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001338215 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.468168+0000", 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033551549999999999 2023-09-23T19:52:39.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.484 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.484 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.484 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.551632+0000", 2023-09-23T19:52:39.484 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.083464514000000004 2023-09-23T19:52:39.484 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.484 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.484 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.485 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.551683+0000", 2023-09-23T19:52:39.485 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0158000000000003e-05 2023-09-23T19:52:39.485 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.485 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.485 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.485 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.551721+0000", 2023-09-23T19:52:39.485 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8349000000000002e-05 2023-09-23T19:52:39.486 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.486 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.486 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.486 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.486 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.486 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:29001 5.5 5:a7b183a3:::benchmark_data_smithi135_284570_object29000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.486 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:38.470520+0000", 2023-09-23T19:52:39.487 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.89237017500000004, 2023-09-23T19:52:39.487 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.082896453999999994, 2023-09-23T19:52:39.487 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.487 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.487 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.487 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.488 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.488 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 29001 2023-09-23T19:52:39.488 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.488 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.488 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.488 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.489 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470520+0000", 2023-09-23T19:52:39.489 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.489 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.489 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.489 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.489 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470520+0000", 2023-09-23T19:52:39.489 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.490 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.490 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.490 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.490 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470518+0000", 2023-09-23T19:52:39.490 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:39.490 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.490 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.491 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.491 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470536+0000", 2023-09-23T19:52:39.491 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8090000000000001e-05 2023-09-23T19:52:39.491 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.491 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.491 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.492 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470553+0000", 2023-09-23T19:52:39.492 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6748999999999998e-05 2023-09-23T19:52:39.492 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.492 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.492 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.492 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470561+0000", 2023-09-23T19:52:39.493 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5120000000000008e-06 2023-09-23T19:52:39.493 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.493 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.493 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.493 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470607+0000", 2023-09-23T19:52:39.493 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6032999999999998e-05 2023-09-23T19:52:39.493 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470739+0000", 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000131218 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.470875+0000", 2023-09-23T19:52:39.494 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013688799999999999 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.472060+0000", 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001184844 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474245+0000", 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0021845060000000001 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.495 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.496 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":239,"stamp":"2023-09-23T19:52:37.120064+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1392318387,"num_objects":21200,"num_object_clones":0,"num_object_copies":63600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21200,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":22318,"num_write_kb":1370149,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22364,"ondisk_log_size":22364,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3906504,"kb_used_data":3681800,"kb_used_omap":32,"kb_used_meta":224543,"kb_avail":746054712,"statfs":{"total":767960285184,"available":763960025088,"internally_reserved":0,"allocated":3770163200,"data_stored":3765823563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":229932550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,4,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1387528192,"num_objects":21172,"num_object_clones":0,"num_object_copies":63516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21172,"num_write_kb":1355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002834"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":95,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806376+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:52:25.806376+0000","last_peered":"2023-09-23T19:52:25.806376+0000","last_clean":"2023-09-23T19:52:25.806376+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:52:25.806376+0000","last_undegraded":"2023-09-23T19:52:25.806376+0000","last_fullsized":"2023-09-23T19:52:25.806376+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":91,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196788+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:52:25.196788+0000","last_peered":"2023-09-23T19:52:25.196788+0000","last_clean":"2023-09-23T19:52:25.196788+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:52:25.196788+0000","last_undegraded":"2023-09-23T19:52:25.196788+0000","last_fullsized":"2023-09-23T19:52:25.196788+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.099851+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:52:26.099851+0000","last_peered":"2023-09-23T19:52:26.099851+0000","last_clean":"2023-09-23T19:52:26.099851+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:52:26.099851+0000","last_undegraded":"2023-09-23T19:52:26.099851+0000","last_fullsized":"2023-09-23T19:52:26.099851+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:32.306501+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:52:32.306501+0000","last_peered":"2023-09-23T19:52:32.306501+0000","last_clean":"2023-09-23T19:52:32.306501+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:52:32.306501+0000","last_undegraded":"2023-09-23T19:52:32.306501+0000","last_fullsized":"2023-09-23T19:52:32.306501+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":95,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196327+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:52:25.196327+0000","last_peered":"2023-09-23T19:52:25.196327+0000","last_clean":"2023-09-23T19:52:25.196327+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:52:25.196327+0000","last_undegraded":"2023-09-23T19:52:25.196327+0000","last_fullsized":"2023-09-23T19:52:25.196327+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.200031+0000","last_change":"2023-09-23T19:49:51.374353+0000","last_active":"2023-09-23T19:52:25.200031+0000","last_peered":"2023-09-23T19:52:25.200031+0000","last_clean":"2023-09-23T19:52:25.200031+0000","last_became_active":"2023-09-23T19:49:51.373992+0000","last_became_peered":"2023-09-23T19:49:51.373992+0000","last_unstale":"2023-09-23T19:52:25.200031+0000","last_undegraded":"2023-09-23T19:52:25.200031+0000","last_fullsized":"2023-09-23T19:52:25.200031+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.344350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":57,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198782+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:52:26.198782+0000","last_peered":"2023-09-23T19:52:26.198782+0000","last_clean":"2023-09-23T19:52:26.198782+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:52:26.198782+0000","last_undegraded":"2023-09-23T19:52:26.198782+0000","last_fullsized":"2023-09-23T19:52:26.198782+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196197+0000","last_change":"2023-09-23T19:50:57.789294+0000","last_active":"2023-09-23T19:52:25.196197+0000","last_peered":"2023-09-23T19:52:25.196197+0000","last_clean":"2023-09-23T19:52:25.196197+0000","last_became_active":"2023-09-23T19:50:57.788922+0000","last_became_peered":"2023-09-23T19:50:57.788922+0000","last_unstale":"2023-09-23T19:52:25.196197+0000","last_undegraded":"2023-09-23T19:52:25.196197+0000","last_fullsized":"2023-09-23T19:52:25.196197+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:11.829374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.199157+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:52:26.199157+0000","last_peered":"2023-09-23T19:52:26.199157+0000","last_clean":"2023-09-23T19:52:26.199157+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:52:26.199157+0000","last_undegraded":"2023-09-23T19:52:26.199157+0000","last_fullsized":"2023-09-23T19:52:26.199157+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":28,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196348+0000","last_change":"2023-09-23T19:50:57.789044+0000","last_active":"2023-09-23T19:52:25.196348+0000","last_peered":"2023-09-23T19:52:25.196348+0000","last_clean":"2023-09-23T19:52:25.196348+0000","last_became_active":"2023-09-23T19:50:57.788730+0000","last_became_peered":"2023-09-23T19:50:57.788730+0000","last_unstale":"2023-09-23T19:52:25.196348+0000","last_undegraded":"2023-09-23T19:52:25.196348+0000","last_fullsized":"2023-09-23T19:52:25.196348+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:23.585694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.070395+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:52:26.070395+0000","last_peered":"2023-09-23T19:52:26.070395+0000","last_clean":"2023-09-23T19:52:26.070395+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:52:26.070395+0000","last_undegraded":"2023-09-23T19:52:26.070395+0000","last_fullsized":"2023-09-23T19:52:26.070395+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.100700+0000","last_change":"2023-09-23T19:50:23.998186+0000","last_active":"2023-09-23T19:52:26.100700+0000","last_peered":"2023-09-23T19:52:26.100700+0000","last_clean":"2023-09-23T19:52:26.100700+0000","last_became_active":"2023-09-23T19:50:23.997897+0000","last_became_peered":"2023-09-23T19:50:23.997897+0000","last_unstale":"2023-09-23T19:52:26.100700+0000","last_undegraded":"2023-09-23T19:52:26.100700+0000","last_fullsized":"2023-09-23T19:52:26.100700+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:57.792861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":82,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806603+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:52:25.806603+0000","last_peered":"2023-09-23T19:52:25.806603+0000","last_clean":"2023-09-23T19:52:25.806603+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:52:25.806603+0000","last_undegraded":"2023-09-23T19:52:25.806603+0000","last_fullsized":"2023-09-23T19:52:25.806603+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806441+0000","last_change":"2023-09-23T19:49:51.379282+0000","last_active":"2023-09-23T19:52:25.806441+0000","last_peered":"2023-09-23T19:52:25.806441+0000","last_clean":"2023-09-23T19:52:25.806441+0000","last_became_active":"2023-09-23T19:49:51.378849+0000","last_became_peered":"2023-09-23T19:49:51.378849+0000","last_unstale":"2023-09-23T19:52:25.806441+0000","last_undegraded":"2023-09-23T19:52:25.806441+0000","last_fullsized":"2023-09-23T19:52:25.806441+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:00.093034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.070797+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:52:26.070797+0000","last_peered":"2023-09-23T19:52:26.070797+0000","last_clean":"2023-09-23T19:52:26.070797+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:52:26.070797+0000","last_undegraded":"2023-09-23T19:52:26.070797+0000","last_fullsized":"2023-09-23T19:52:26.070797+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196700+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:52:25.196700+0000","last_peered":"2023-09-23T19:52:25.196700+0000","last_clean":"2023-09-23T19:52:25.196700+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:52:25.196700+0000","last_undegraded":"2023-09-23T19:52:25.196700+0000","last_fullsized":"2023-09-23T19:52:25.196700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":57,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198576+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:52:26.198576+0000","last_peered":"2023-09-23T19:52:26.198576+0000","last_clean":"2023-09-23T19:52:26.198576+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:52:26.198576+0000","last_undegraded":"2023-09-23T19:52:26.198576+0000","last_fullsized":"2023-09-23T19:52:26.198576+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198542+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:52:26.198542+0000","last_peered":"2023-09-23T19:52:26.198542+0000","last_clean":"2023-09-23T19:52:26.198542+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:52:26.198542+0000","last_undegraded":"2023-09-23T19:52:26.198542+0000","last_fullsized":"2023-09-23T19:52:26.198542+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":59,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.070714+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:52:26.070714+0000","last_peered":"2023-09-23T19:52:26.070714+0000","last_clean":"2023-09-23T19:52:26.070714+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:52:26.070714+0000","last_undegraded":"2023-09-23T19:52:26.070714+0000","last_fullsized":"2023-09-23T19:52:26.070714+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":58,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198758+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:52:26.198758+0000","last_peered":"2023-09-23T19:52:26.198758+0000","last_clean":"2023-09-23T19:52:26.198758+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:52:26.198758+0000","last_undegraded":"2023-09-23T19:52:26.198758+0000","last_fullsized":"2023-09-23T19:52:26.198758+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":84,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.199664+0000","last_change":"2023-09-23T19:48:37.567607+0000","last_active":"2023-09-23T19:52:25.199664+0000","last_peered":"2023-09-23T19:52:25.199664+0000","last_clean":"2023-09-23T19:52:25.199664+0000","last_became_active":"2023-09-23T19:48:37.567300+0000","last_became_peered":"2023-09-23T19:48:37.567300+0000","last_unstale":"2023-09-23T19:52:25.199664+0000","last_undegraded":"2023-09-23T19:52:25.199664+0000","last_fullsized":"2023-09-23T19:52:25.199664+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.088452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196912+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:52:25.196912+0000","last_peered":"2023-09-23T19:52:25.196912+0000","last_clean":"2023-09-23T19:52:25.196912+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:52:25.196912+0000","last_undegraded":"2023-09-23T19:52:25.196912+0000","last_fullsized":"2023-09-23T19:52:25.196912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"116'1025","reported_seq":1037,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:33.044189+0000","last_change":"2023-09-23T19:52:21.187755+0000","last_active":"2023-09-23T19:52:33.044189+0000","last_peered":"2023-09-23T19:52:33.044189+0000","last_clean":"2023-09-23T19:52:33.044189+0000","last_became_active":"2023-09-23T19:52:21.187425+0000","last_became_peered":"2023-09-23T19:52:21.187425+0000","last_unstale":"2023-09-23T19:52:33.044189+0000","last_undegraded":"2023-09-23T19:52:33.044189+0000","last_fullsized":"2023-09-23T19:52:33.044189+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1025,"log_dups_size":0,"ondisk_log_size":1025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:24.504315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67174400,"num_objects":1025,"num_object_clones":0,"num_object_copies":3075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1025,"num_write_kb":65600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.071091+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:52:26.071091+0000","last_peered":"2023-09-23T19:52:26.071091+0000","last_clean":"2023-09-23T19:52:26.071091+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:52:26.071091+0000","last_undegraded":"2023-09-23T19:52:26.071091+0000","last_fullsized":"2023-09-23T19:52:26.071091+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":58,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198641+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:52:26.198641+0000","last_peered":"2023-09-23T19:52:26.198641+0000","last_clean":"2023-09-23T19:52:26.198641+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:52:26.198641+0000","last_undegraded":"2023-09-23T19:52:26.198641+0000","last_fullsized":"2023-09-23T19:52:26.198641+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"116'1577","reported_seq":1589,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:37.008451+0000","last_change":"2023-09-23T19:52:21.181571+0000","last_active":"2023-09-23T19:52:37.008451+0000","last_peered":"2023-09-23T19:52:37.008451+0000","last_clean":"2023-09-23T19:52:37.008451+0000","last_became_active":"2023-09-23T19:52:21.181249+0000","last_became_peered":"2023-09-23T19:52:21.181249+0000","last_unstale":"2023-09-23T19:52:37.008451+0000","last_undegraded":"2023-09-23T19:52:37.008451+0000","last_fullsized":"2023-09-23T19:52:37.008451+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1577,"log_dups_size":0,"ondisk_log_size":1577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:02.562956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103350272,"num_objects":1577,"num_object_clones":0,"num_object_copies":4731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1577,"num_write_kb":100928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.071079+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:52:26.071079+0000","last_peered":"2023-09-23T19:52:26.071079+0000","last_clean":"2023-09-23T19:52:26.071079+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:52:26.071079+0000","last_undegraded":"2023-09-23T19:52:26.071079+0000","last_fullsized":"2023-09-23T19:52:26.071079+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":93,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806006+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:52:25.806006+0000","last_peered":"2023-09-23T19:52:25.806006+0000","last_clean":"2023-09-23T19:52:25.806006+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:52:25.806006+0000","last_undegraded":"2023-09-23T19:52:25.806006+0000","last_fullsized":"2023-09-23T19:52:25.806006+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"116'1598","reported_seq":1608,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:37.011748+0000","last_change":"2023-09-23T19:52:21.183245+0000","last_active":"2023-09-23T19:52:37.011748+0000","last_peered":"2023-09-23T19:52:37.011748+0000","last_clean":"2023-09-23T19:52:37.011748+0000","last_became_active":"2023-09-23T19:52:21.182965+0000","last_became_peered":"2023-09-23T19:52:21.182965+0000","last_unstale":"2023-09-23T19:52:37.011748+0000","last_undegraded":"2023-09-23T19:52:37.011748+0000","last_fullsized":"2023-09-23T19:52:37.011748+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1598,"log_dups_size":0,"ondisk_log_size":1598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:14.836347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104726528,"num_objects":1598,"num_object_clones":0,"num_object_copies":4794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1598,"num_write_kb":102272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":60,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.070995+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:52:26.070995+0000","last_peered":"2023-09-23T19:52:26.070995+0000","last_clean":"2023-09-23T19:52:26.070995+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:52:26.070995+0000","last_undegraded":"2023-09-23T19:52:26.070995+0000","last_fullsized":"2023-09-23T19:52:26.070995+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.199751+0000","last_change":"2023-09-23T19:49:51.380726+0000","last_active":"2023-09-23T19:52:25.199751+0000","last_peered":"2023-09-23T19:52:25.199751+0000","last_clean":"2023-09-23T19:52:25.199751+0000","last_became_active":"2023-09-23T19:49:51.380021+0000","last_became_peered":"2023-09-23T19:49:51.380021+0000","last_unstale":"2023-09-23T19:52:25.199751+0000","last_undegraded":"2023-09-23T19:52:25.199751+0000","last_fullsized":"2023-09-23T19:52:25.199751+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:16.892029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"116'1585","reported_seq":1597,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:37.000002+0000","last_change":"2023-09-23T19:52:21.183448+0000","last_active":"2023-09-23T19:52:37.000002+0000","last_peered":"2023-09-23T19:52:37.000002+0000","last_clean":"2023-09-23T19:52:37.000002+0000","last_became_active":"2023-09-23T19:52:21.183139+0000","last_became_peered":"2023-09-23T19:52:21.183139+0000","last_unstale":"2023-09-23T19:52:37.000002+0000","last_undegraded":"2023-09-23T19:52:37.000002+0000","last_fullsized":"2023-09-23T19:52:37.000002+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1585,"log_dups_size":0,"ondisk_log_size":1585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:28.123472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103874560,"num_objects":1585,"num_object_clones":0,"num_object_copies":4755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1585,"num_write_kb":101440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198499+0000","last_change":"2023-09-23T19:50:57.788389+0000","last_active":"2023-09-23T19:52:26.198499+0000","last_peered":"2023-09-23T19:52:26.198499+0000","last_clean":"2023-09-23T19:52:26.198499+0000","last_became_active":"2023-09-23T19:50:57.788075+0000","last_became_peered":"2023-09-23T19:50:57.788075+0000","last_unstale":"2023-09-23T19:52:26.198499+0000","last_undegraded":"2023-09-23T19:52:26.198499+0000","last_fullsized":"2023-09-23T19:52:26.198499+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:06.180629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":58,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.199157+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:52:26.199157+0000","last_peered":"2023-09-23T19:52:26.199157+0000","last_clean":"2023-09-23T19:52:26.199157+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:52:26.199157+0000","last_undegraded":"2023-09-23T19:52:26.199157+0000","last_fullsized":"2023-09-23T19:52:26.199157+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"116'1655","reported_seq":1668,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:37.008358+0000","last_change":"2023-09-23T19:52:21.186399+0000","last_active":"2023-09-23T19:52:37.008358+0000","last_peered":"2023-09-23T19:52:37.008358+0000","last_clean":"2023-09-23T19:52:37.008358+0000","last_became_active":"2023-09-23T19:52:21.186119+0000","last_became_peered":"2023-09-23T19:52:21.186119+0000","last_unstale":"2023-09-23T19:52:37.008358+0000","last_undegraded":"2023-09-23T19:52:37.008358+0000","last_fullsized":"2023-09-23T19:52:37.008358+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1655,"log_dups_size":0,"ondisk_log_size":1655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:05.981956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108462080,"num_objects":1655,"num_object_clones":0,"num_object_copies":4965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1655,"num_write_kb":105920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196547+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:52:25.196547+0000","last_peered":"2023-09-23T19:52:25.196547+0000","last_clean":"2023-09-23T19:52:25.196547+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:52:25.196547+0000","last_undegraded":"2023-09-23T19:52:25.196547+0000","last_fullsized":"2023-09-23T19:52:25.196547+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806033+0000","last_change":"2023-09-23T19:50:57.790231+0000","last_active":"2023-09-23T19:52:25.806033+0000","last_peered":"2023-09-23T19:52:25.806033+0000","last_clean":"2023-09-23T19:52:25.806033+0000","last_became_active":"2023-09-23T19:50:57.789446+0000","last_became_peered":"2023-09-23T19:50:57.789446+0000","last_unstale":"2023-09-23T19:52:25.806033+0000","last_undegraded":"2023-09-23T19:52:25.806033+0000","last_fullsized":"2023-09-23T19:52:25.806033+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:41.313276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"116'1280","reported_seq":1293,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:34.380238+0000","last_change":"2023-09-23T19:52:21.179841+0000","last_active":"2023-09-23T19:52:34.380238+0000","last_peered":"2023-09-23T19:52:34.380238+0000","last_clean":"2023-09-23T19:52:34.380238+0000","last_became_active":"2023-09-23T19:52:21.179598+0000","last_became_peered":"2023-09-23T19:52:21.179598+0000","last_unstale":"2023-09-23T19:52:34.380238+0000","last_undegraded":"2023-09-23T19:52:34.380238+0000","last_fullsized":"2023-09-23T19:52:34.380238+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1280,"log_dups_size":0,"ondisk_log_size":1280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:39.463431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83886080,"num_objects":1280,"num_object_clones":0,"num_object_copies":3840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1280,"num_write_kb":81920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.199860+0000","last_change":"2023-09-23T19:49:51.380925+0000","last_active":"2023-09-23T19:52:25.199860+0000","last_peered":"2023-09-23T19:52:25.199860+0000","last_clean":"2023-09-23T19:52:25.199860+0000","last_became_active":"2023-09-23T19:49:51.380427+0000","last_became_peered":"2023-09-23T19:49:51.380427+0000","last_unstale":"2023-09-23T19:52:25.199860+0000","last_undegraded":"2023-09-23T19:52:25.199860+0000","last_fullsized":"2023-09-23T19:52:25.199860+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:49.879887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":93,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806605+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:52:25.806605+0000","last_peered":"2023-09-23T19:52:25.806605+0000","last_clean":"2023-09-23T19:52:25.806605+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:52:25.806605+0000","last_undegraded":"2023-09-23T19:52:25.806605+0000","last_fullsized":"2023-09-23T19:52:25.806605+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"116'1048","reported_seq":1060,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:33.048461+0000","last_change":"2023-09-23T19:52:21.187234+0000","last_active":"2023-09-23T19:52:33.048461+0000","last_peered":"2023-09-23T19:52:33.048461+0000","last_clean":"2023-09-23T19:52:33.048461+0000","last_became_active":"2023-09-23T19:52:21.186449+0000","last_became_peered":"2023-09-23T19:52:21.186449+0000","last_unstale":"2023-09-23T19:52:33.048461+0000","last_undegraded":"2023-09-23T19:52:33.048461+0000","last_fullsized":"2023-09-23T19:52:33.048461+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1048,"log_dups_size":0,"ondisk_log_size":1048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:15.473024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68681728,"num_objects":1048,"num_object_clones":0,"num_object_copies":3144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1048,"num_write_kb":67072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806269+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:52:25.806269+0000","last_peered":"2023-09-23T19:52:25.806269+0000","last_clean":"2023-09-23T19:52:25.806269+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:52:25.806269+0000","last_undegraded":"2023-09-23T19:52:25.806269+0000","last_fullsized":"2023-09-23T19:52:25.806269+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":86,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806803+0000","last_change":"2023-09-23T19:50:23.998753+0000","last_active":"2023-09-23T19:52:25.806803+0000","last_peered":"2023-09-23T19:52:25.806803+0000","last_clean":"2023-09-23T19:52:25.806803+0000","last_became_active":"2023-09-23T19:50:23.998417+0000","last_became_peered":"2023-09-23T19:50:23.998417+0000","last_unstale":"2023-09-23T19:52:25.806803+0000","last_undegraded":"2023-09-23T19:52:25.806803+0000","last_fullsized":"2023-09-23T19:52:25.806803+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:11.002531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"116'1044","reported_seq":1056,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:33.044166+0000","last_change":"2023-09-23T19:52:21.184411+0000","last_active":"2023-09-23T19:52:33.044166+0000","last_peered":"2023-09-23T19:52:33.044166+0000","last_clean":"2023-09-23T19:52:33.044166+0000","last_became_active":"2023-09-23T19:52:21.184090+0000","last_became_peered":"2023-09-23T19:52:21.184090+0000","last_unstale":"2023-09-23T19:52:33.044166+0000","last_undegraded":"2023-09-23T19:52:33.044166+0000","last_fullsized":"2023-09-23T19:52:33.044166+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1044,"log_dups_size":0,"ondisk_log_size":1044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:46.571567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68419584,"num_objects":1044,"num_object_clones":0,"num_object_copies":3132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1044,"num_write_kb":66816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196384+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:52:25.196384+0000","last_peered":"2023-09-23T19:52:25.196384+0000","last_clean":"2023-09-23T19:52:25.196384+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:52:25.196384+0000","last_undegraded":"2023-09-23T19:52:25.196384+0000","last_fullsized":"2023-09-23T19:52:25.196384+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.099883+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:52:26.099883+0000","last_peered":"2023-09-23T19:52:26.099883+0000","last_clean":"2023-09-23T19:52:26.099883+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:52:26.099883+0000","last_undegraded":"2023-09-23T19:52:26.099883+0000","last_fullsized":"2023-09-23T19:52:26.099883+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":450,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.100743+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:52:26.100743+0000","last_peered":"2023-09-23T19:52:26.100743+0000","last_clean":"2023-09-23T19:52:26.100743+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:52:26.100743+0000","last_undegraded":"2023-09-23T19:52:26.100743+0000","last_fullsized":"2023-09-23T19:52:26.100743+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":93,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198514+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:52:26.198514+0000","last_peered":"2023-09-23T19:52:26.198514+0000","last_clean":"2023-09-23T19:52:26.198514+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:52:26.198514+0000","last_undegraded":"2023-09-23T19:52:26.198514+0000","last_fullsized":"2023-09-23T19:52:26.198514+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":39,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.100675+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:52:26.100675+0000","last_peered":"2023-09-23T19:52:26.100675+0000","last_clean":"2023-09-23T19:52:26.100675+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:52:26.100675+0000","last_undegraded":"2023-09-23T19:52:26.100675+0000","last_fullsized":"2023-09-23T19:52:26.100675+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"116'1499","reported_seq":1512,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:35.586874+0000","last_change":"2023-09-23T19:52:21.185632+0000","last_active":"2023-09-23T19:52:35.586874+0000","last_peered":"2023-09-23T19:52:35.586874+0000","last_clean":"2023-09-23T19:52:35.586874+0000","last_became_active":"2023-09-23T19:52:21.185348+0000","last_became_peered":"2023-09-23T19:52:21.185348+0000","last_unstale":"2023-09-23T19:52:35.586874+0000","last_undegraded":"2023-09-23T19:52:35.586874+0000","last_fullsized":"2023-09-23T19:52:35.586874+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1499,"log_dups_size":0,"ondisk_log_size":1499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:30.337130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1499,"num_write_kb":95936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"116'1379","reported_seq":1392,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:35.584337+0000","last_change":"2023-09-23T19:52:21.181440+0000","last_active":"2023-09-23T19:52:35.584337+0000","last_peered":"2023-09-23T19:52:35.584337+0000","last_clean":"2023-09-23T19:52:35.584337+0000","last_became_active":"2023-09-23T19:52:21.181162+0000","last_became_peered":"2023-09-23T19:52:21.181162+0000","last_unstale":"2023-09-23T19:52:35.584337+0000","last_undegraded":"2023-09-23T19:52:35.584337+0000","last_fullsized":"2023-09-23T19:52:35.584337+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1379,"log_dups_size":0,"ondisk_log_size":1379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:18.844043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90374144,"num_objects":1379,"num_object_clones":0,"num_object_copies":4137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1379,"num_write_kb":88256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":640,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198761+0000","last_change":"2023-09-23T19:49:51.382830+0000","last_active":"2023-09-23T19:52:26.198761+0000","last_peered":"2023-09-23T19:52:26.198761+0000","last_clean":"2023-09-23T19:52:26.198761+0000","last_became_active":"2023-09-23T19:49:51.382367+0000","last_became_peered":"2023-09-23T19:49:51.382367+0000","last_unstale":"2023-09-23T19:52:26.198761+0000","last_undegraded":"2023-09-23T19:52:26.198761+0000","last_fullsized":"2023-09-23T19:52:26.198761+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:48.082643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806543+0000","last_change":"2023-09-23T19:48:37.573541+0000","last_active":"2023-09-23T19:52:25.806543+0000","last_peered":"2023-09-23T19:52:25.806543+0000","last_clean":"2023-09-23T19:52:25.806543+0000","last_became_active":"2023-09-23T19:48:37.573213+0000","last_became_peered":"2023-09-23T19:48:37.573213+0000","last_unstale":"2023-09-23T19:52:25.806543+0000","last_undegraded":"2023-09-23T19:52:25.806543+0000","last_fullsized":"2023-09-23T19:52:25.806543+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:16.981072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":69,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.099992+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:52:26.099992+0000","last_peered":"2023-09-23T19:52:26.099992+0000","last_clean":"2023-09-23T19:52:26.099992+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:52:26.099992+0000","last_undegraded":"2023-09-23T19:52:26.099992+0000","last_fullsized":"2023-09-23T19:52:26.099992+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"116'1696","reported_seq":1709,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:36.996899+0000","last_change":"2023-09-23T19:52:21.183302+0000","last_active":"2023-09-23T19:52:36.996899+0000","last_peered":"2023-09-23T19:52:36.996899+0000","last_clean":"2023-09-23T19:52:36.996899+0000","last_became_active":"2023-09-23T19:52:21.183010+0000","last_became_peered":"2023-09-23T19:52:21.183010+0000","last_unstale":"2023-09-23T19:52:36.996899+0000","last_undegraded":"2023-09-23T19:52:36.996899+0000","last_fullsized":"2023-09-23T19:52:36.996899+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1696,"log_dups_size":0,"ondisk_log_size":1696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:01.323672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111149056,"num_objects":1696,"num_object_clones":0,"num_object_copies":5088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1696,"num_write_kb":108544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":600,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:27.007976+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:52:27.007976+0000","last_peered":"2023-09-23T19:52:27.007976+0000","last_clean":"2023-09-23T19:52:27.007976+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:52:27.007976+0000","last_undegraded":"2023-09-23T19:52:27.007976+0000","last_fullsized":"2023-09-23T19:52:27.007976+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":53,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.100417+0000","last_change":"2023-09-23T19:50:23.997623+0000","last_active":"2023-09-23T19:52:26.100417+0000","last_peered":"2023-09-23T19:52:26.100417+0000","last_clean":"2023-09-23T19:52:26.100417+0000","last_became_active":"2023-09-23T19:50:23.997428+0000","last_became_peered":"2023-09-23T19:50:23.997428+0000","last_unstale":"2023-09-23T19:52:26.100417+0000","last_undegraded":"2023-09-23T19:52:26.100417+0000","last_fullsized":"2023-09-23T19:52:26.100417+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:50.529980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":83,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.100858+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:52:26.100858+0000","last_peered":"2023-09-23T19:52:26.100858+0000","last_clean":"2023-09-23T19:52:26.100858+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:52:26.100858+0000","last_undegraded":"2023-09-23T19:52:26.100858+0000","last_fullsized":"2023-09-23T19:52:26.100858+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":925,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198809+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:52:26.198809+0000","last_peered":"2023-09-23T19:52:26.198809+0000","last_clean":"2023-09-23T19:52:26.198809+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:52:26.198809+0000","last_undegraded":"2023-09-23T19:52:26.198809+0000","last_fullsized":"2023-09-23T19:52:26.198809+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":456,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.070635+0000","last_change":"2023-09-23T19:48:37.575593+0000","last_active":"2023-09-23T19:52:26.070635+0000","last_peered":"2023-09-23T19:52:26.070635+0000","last_clean":"2023-09-23T19:52:26.070635+0000","last_became_active":"2023-09-23T19:48:37.575214+0000","last_became_peered":"2023-09-23T19:48:37.575214+0000","last_unstale":"2023-09-23T19:52:26.070635+0000","last_undegraded":"2023-09-23T19:52:26.070635+0000","last_fullsized":"2023-09-23T19:52:26.070635+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:38:09.132356+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:13.335201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":27,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196332+0000","last_change":"2023-09-23T19:50:57.787513+0000","last_active":"2023-09-23T19:52:25.196332+0000","last_peered":"2023-09-23T19:52:25.196332+0000","last_clean":"2023-09-23T19:52:25.196332+0000","last_became_active":"2023-09-23T19:50:57.787175+0000","last_became_peered":"2023-09-23T19:50:57.787175+0000","last_unstale":"2023-09-23T19:52:25.196332+0000","last_undegraded":"2023-09-23T19:52:25.196332+0000","last_fullsized":"2023-09-23T19:52:25.196332+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:19.081986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":58,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.199936+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:52:25.199936+0000","last_peered":"2023-09-23T19:52:25.199936+0000","last_clean":"2023-09-23T19:52:25.199936+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:52:25.199936+0000","last_undegraded":"2023-09-23T19:52:25.199936+0000","last_fullsized":"2023-09-23T19:52:25.199936+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"116'1048","reported_seq":1060,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:33.043401+0000","last_change":"2023-09-23T19:52:21.183523+0000","last_active":"2023-09-23T19:52:33.043401+0000","last_peered":"2023-09-23T19:52:33.043401+0000","last_clean":"2023-09-23T19:52:33.043401+0000","last_became_active":"2023-09-23T19:52:21.183198+0000","last_became_peered":"2023-09-23T19:52:21.183198+0000","last_unstale":"2023-09-23T19:52:33.043401+0000","last_undegraded":"2023-09-23T19:52:33.043401+0000","last_fullsized":"2023-09-23T19:52:33.043401+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1048,"log_dups_size":0,"ondisk_log_size":1048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:07.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68681728,"num_objects":1048,"num_object_clones":0,"num_object_copies":3144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1048,"num_write_kb":67072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"116'1043","reported_seq":1055,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:33.044194+0000","last_change":"2023-09-23T19:52:21.181927+0000","last_active":"2023-09-23T19:52:33.044194+0000","last_peered":"2023-09-23T19:52:33.044194+0000","last_clean":"2023-09-23T19:52:33.044194+0000","last_became_active":"2023-09-23T19:52:21.181527+0000","last_became_peered":"2023-09-23T19:52:21.181527+0000","last_unstale":"2023-09-23T19:52:33.044194+0000","last_undegraded":"2023-09-23T19:52:33.044194+0000","last_fullsized":"2023-09-23T19:52:33.044194+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1043,"log_dups_size":0,"ondisk_log_size":1043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.841652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68354048,"num_objects":1043,"num_object_clones":0,"num_object_copies":3129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1043,"num_write_kb":66752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":834,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.070694+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:52:26.070694+0000","last_peered":"2023-09-23T19:52:26.070694+0000","last_clean":"2023-09-23T19:52:26.070694+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:52:26.070694+0000","last_undegraded":"2023-09-23T19:52:26.070694+0000","last_fullsized":"2023-09-23T19:52:26.070694+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":57,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.198597+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:52:26.198597+0000","last_peered":"2023-09-23T19:52:26.198597+0000","last_clean":"2023-09-23T19:52:26.198597+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:52:26.198597+0000","last_undegraded":"2023-09-23T19:52:26.198597+0000","last_fullsized":"2023-09-23T19:52:26.198597+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":71,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.199426+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:52:26.199426+0000","last_peered":"2023-09-23T19:52:26.199426+0000","last_clean":"2023-09-23T19:52:26.199426+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:52:26.199426+0000","last_undegraded":"2023-09-23T19:52:26.199426+0000","last_fullsized":"2023-09-23T19:52:26.199426+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":194,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.070990+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:52:26.070990+0000","last_peered":"2023-09-23T19:52:26.070990+0000","last_clean":"2023-09-23T19:52:26.070990+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:52:26.070990+0000","last_undegraded":"2023-09-23T19:52:26.070990+0000","last_fullsized":"2023-09-23T19:52:26.070990+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":53,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.100836+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:52:26.100836+0000","last_peered":"2023-09-23T19:52:26.100836+0000","last_clean":"2023-09-23T19:52:26.100836+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:52:26.100836+0000","last_undegraded":"2023-09-23T19:52:26.100836+0000","last_fullsized":"2023-09-23T19:52:26.100836+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":43,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.199233+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:52:26.199233+0000","last_peered":"2023-09-23T19:52:26.199233+0000","last_clean":"2023-09-23T19:52:26.199233+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:52:26.199233+0000","last_undegraded":"2023-09-23T19:52:26.199233+0000","last_fullsized":"2023-09-23T19:52:26.199233+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"116'1074","reported_seq":1086,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:33.043062+0000","last_change":"2023-09-23T19:52:21.188633+0000","last_active":"2023-09-23T19:52:33.043062+0000","last_peered":"2023-09-23T19:52:33.043062+0000","last_clean":"2023-09-23T19:52:33.043062+0000","last_became_active":"2023-09-23T19:52:21.188310+0000","last_became_peered":"2023-09-23T19:52:21.188310+0000","last_unstale":"2023-09-23T19:52:33.043062+0000","last_undegraded":"2023-09-23T19:52:33.043062+0000","last_fullsized":"2023-09-23T19:52:33.043062+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1074,"log_dups_size":0,"ondisk_log_size":1074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:56.470945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70385664,"num_objects":1074,"num_object_clones":0,"num_object_copies":3222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1074,"num_write_kb":68736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196977+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:52:25.196977+0000","last_peered":"2023-09-23T19:52:25.196977+0000","last_clean":"2023-09-23T19:52:25.196977+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:52:25.196977+0000","last_undegraded":"2023-09-23T19:52:25.196977+0000","last_fullsized":"2023-09-23T19:52:25.196977+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":376,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.100797+0000","last_change":"2023-09-23T19:49:51.380375+0000","last_active":"2023-09-23T19:52:26.100797+0000","last_peered":"2023-09-23T19:52:26.100797+0000","last_clean":"2023-09-23T19:52:26.100797+0000","last_became_active":"2023-09-23T19:49:51.379793+0000","last_became_peered":"2023-09-23T19:49:51.379793+0000","last_unstale":"2023-09-23T19:52:26.100797+0000","last_undegraded":"2023-09-23T19:52:26.100797+0000","last_fullsized":"2023-09-23T19:52:26.100797+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:26.355152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196295+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:52:25.196295+0000","last_peered":"2023-09-23T19:52:25.196295+0000","last_clean":"2023-09-23T19:52:25.196295+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:52:25.196295+0000","last_undegraded":"2023-09-23T19:52:25.196295+0000","last_fullsized":"2023-09-23T19:52:25.196295+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"116'1122","reported_seq":1134,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:33.045142+0000","last_change":"2023-09-23T19:52:21.183759+0000","last_active":"2023-09-23T19:52:33.045142+0000","last_peered":"2023-09-23T19:52:33.045142+0000","last_clean":"2023-09-23T19:52:33.045142+0000","last_became_active":"2023-09-23T19:52:21.183451+0000","last_became_peered":"2023-09-23T19:52:21.183451+0000","last_unstale":"2023-09-23T19:52:33.045142+0000","last_undegraded":"2023-09-23T19:52:33.045142+0000","last_fullsized":"2023-09-23T19:52:33.045142+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1122,"log_dups_size":0,"ondisk_log_size":1122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:43.359001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73531392,"num_objects":1122,"num_object_clones":0,"num_object_copies":3366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1122,"num_write_kb":71808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":347,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.200004+0000","last_change":"2023-09-23T19:50:23.999651+0000","last_active":"2023-09-23T19:52:25.200004+0000","last_peered":"2023-09-23T19:52:25.200004+0000","last_clean":"2023-09-23T19:52:25.200004+0000","last_became_active":"2023-09-23T19:50:23.999184+0000","last_became_peered":"2023-09-23T19:50:23.999184+0000","last_unstale":"2023-09-23T19:52:25.200004+0000","last_undegraded":"2023-09-23T19:52:25.200004+0000","last_fullsized":"2023-09-23T19:52:25.200004+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:41.010259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":61,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.199433+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:52:26.199433+0000","last_peered":"2023-09-23T19:52:26.199433+0000","last_clean":"2023-09-23T19:52:26.199433+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:52:26.199433+0000","last_undegraded":"2023-09-23T19:52:26.199433+0000","last_fullsized":"2023-09-23T19:52:26.199433+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":59,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806312+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:52:25.806312+0000","last_peered":"2023-09-23T19:52:25.806312+0000","last_clean":"2023-09-23T19:52:25.806312+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:52:25.806312+0000","last_undegraded":"2023-09-23T19:52:25.806312+0000","last_fullsized":"2023-09-23T19:52:25.806312+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"116'1499","reported_seq":1512,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T19:52:35.785397+0000","last_change":"2023-09-23T19:52:21.180582+0000","last_active":"2023-09-23T19:52:35.785397+0000","last_peered":"2023-09-23T19:52:35.785397+0000","last_clean":"2023-09-23T19:52:35.785397+0000","last_became_active":"2023-09-23T19:52:21.180255+0000","last_became_peered":"2023-09-23T19:52:21.180255+0000","last_unstale":"2023-09-23T19:52:35.785397+0000","last_undegraded":"2023-09-23T19:52:35.785397+0000","last_fullsized":"2023-09-23T19:52:35.785397+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":1499,"log_dups_size":0,"ondisk_log_size":1499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:51.310686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1499,"num_write_kb":95936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":46,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.199241+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:52:26.199241+0000","last_peered":"2023-09-23T19:52:26.199241+0000","last_clean":"2023-09-23T19:52:26.199241+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:52:26.199241+0000","last_undegraded":"2023-09-23T19:52:26.199241+0000","last_fullsized":"2023-09-23T19:52:26.199241+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":97,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.855004+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:52:25.855004+0000","last_peered":"2023-09-23T19:52:25.855004+0000","last_clean":"2023-09-23T19:52:25.855004+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:52:25.855004+0000","last_undegraded":"2023-09-23T19:52:25.855004+0000","last_fullsized":"2023-09-23T19:52:25.855004+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.199000+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:52:26.199000+0000","last_peered":"2023-09-23T19:52:26.199000+0000","last_clean":"2023-09-23T19:52:26.199000+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:52:26.199000+0000","last_undegraded":"2023-09-23T19:52:26.199000+0000","last_fullsized":"2023-09-23T19:52:26.199000+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":93,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806038+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:52:25.806038+0000","last_peered":"2023-09-23T19:52:25.806038+0000","last_clean":"2023-09-23T19:52:25.806038+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:52:25.806038+0000","last_undegraded":"2023-09-23T19:52:25.806038+0000","last_fullsized":"2023-09-23T19:52:25.806038+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":61,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.199086+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:52:26.199086+0000","last_peered":"2023-09-23T19:52:26.199086+0000","last_clean":"2023-09-23T19:52:26.199086+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:52:26.199086+0000","last_undegraded":"2023-09-23T19:52:26.199086+0000","last_fullsized":"2023-09-23T19:52:26.199086+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":93,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806009+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:52:25.806009+0000","last_peered":"2023-09-23T19:52:25.806009+0000","last_clean":"2023-09-23T19:52:25.806009+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:52:25.806009+0000","last_undegraded":"2023-09-23T19:52:25.806009+0000","last_fullsized":"2023-09-23T19:52:25.806009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":83,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196819+0000","last_change":"2023-09-23T19:50:57.786561+0000","last_active":"2023-09-23T19:52:25.196819+0000","last_peered":"2023-09-23T19:52:25.196819+0000","last_clean":"2023-09-23T19:52:25.196819+0000","last_became_active":"2023-09-23T19:50:57.786231+0000","last_became_peered":"2023-09-23T19:50:57.786231+0000","last_unstale":"2023-09-23T19:52:25.196819+0000","last_undegraded":"2023-09-23T19:52:25.196819+0000","last_fullsized":"2023-09-23T19:52:25.196819+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:27.327075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":98,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.196347+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:52:25.196347+0000","last_peered":"2023-09-23T19:52:25.196347+0000","last_clean":"2023-09-23T19:52:25.196347+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:52:25.196347+0000","last_undegraded":"2023-09-23T19:52:25.196347+0000","last_fullsized":"2023-09-23T19:52:25.196347+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":61,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:26.212604+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:52:26.212604+0000","last_peered":"2023-09-23T19:52:26.212604+0000","last_clean":"2023-09-23T19:52:26.212604+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:52:26.212604+0000","last_undegraded":"2023-09-23T19:52:26.212604+0000","last_fullsized":"2023-09-23T19:52:26.212604+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T19:52:25.806164+0000","last_change":"2023-09-23T19:49:51.378856+0000","last_active":"2023-09-23T19:52:25.806164+0000","last_peered":"2023-09-23T19:52:25.806164+0000","last_clean":"2023-09-23T19:52:25.806164+0000","last_became_active":"2023-09-23T19:49:51.378451+0000","last_became_peered":"2023-09-23T19:49:51.378451+0000","last_unstale":"2023-09-23T19:52:25.806164+0000","last_undegraded":"2023-09-23T19:52:25.806164+0000","last_fullsized":"2023-09-23T19:52:25.806164+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:03.276740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1387528192,"num_objects":21172,"num_object_clones":0,"num_object_copies":63516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21172,"num_write_kb":1355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3895721984,"data_stored":3895721984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21172,"ondisk_log_size":21172,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234137,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":349020,"kb_used_data":332628,"kb_used_omap":0,"kb_used_meta":16384,"kb_avail":93396132,"statfs":{"total":95995035648,"available":95637639168,"internally_reserved":0,"allocated":340611072,"data_stored":340080163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16777216},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.61199999999999999,"15min":0.59299999999999997},"min":{"1min":0.36899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.9850000000000003,"5min":4.9850000000000003,"15min":4.9850000000000003},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":0.32600000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.61399999999999999,"15min":0.58899999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0510000000000002,"5min":5.0510000000000002,"15min":5.0510000000000002},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.60799999999999998,"15min":0.57899999999999996},"min":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.1200000000000001,"5min":5.1200000000000001,"15min":5.1200000000000001},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.79700000000000004,"15min":0.83399999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.74399999999999999,"5min":5.1600000000000001,"15min":5.1600000000000001},"last":0.435},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.58699999999999997,"15min":0.58599999999999997},"min":{"1min":0.375,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.83599999999999997,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.46700000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":5.1980000000000004,"5min":5.1980000000000004,"15min":5.1980000000000004},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:52:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.499,"15min":0.47499999999999998},"min":{"1min":0.35399999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.72899999999999998,"5min":0.85299999999999998,"15min":0.878},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.56999999999999995,"15min":0.52800000000000002},"min":{"1min":0.32600000000000001,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.36299999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:52:30 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.56299999999999994,"15min":0.52400000000000002},"min":{"1min":0.35999999999999999,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.56799999999999995,"15min":0.53800000000000003},"min":{"1min":0.42099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.67600000000000005,"5min":0.80400000000000005,"15min":1.3500000000000001},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 19:52:30 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.60999999999999999,"15min":0.56899999999999995},"min":{"1min":0.43099999999999999,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.65500000000000003,"15min":0.60099999999999998},"min":{"1min":0.39100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.468,"5min":1.468,"15min":1.52},"last":0.45400000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495774,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":264912,"kb_used_data":254344,"kb_used_omap":0,"kb_used_meta":10560,"kb_avail":93480240,"statfs":{"total":95995035648,"available":95723765760,"internally_reserved":0,"allocated":260448256,"data_stored":259929301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":10813440},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.57299999999999995,"15min":0.59299999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.60599999999999998,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.502},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.63900000000000001,"15min":0.65500000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.71999999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.32400000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.63100000000000001,"15min":0.625},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.5259999999999998,"5min":2.5259999999999998,"15min":2.5259999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.57399999999999995,"15min":0.56799999999999995},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.6160000000000001,"5min":2.6160000000000001,"15min":2.6160000000000001},"last":0.78000000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:52:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.59199999999999997,"15min":0.58499999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":2.004,"5min":2.004,"15min":2.004},"last":2.004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.68700000000000006,"15min":0.71299999999999997},"min":{"1min":0.36399999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76000000000000001,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":2.794,"5min":2.794,"15min":2.794},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.629}]},{"osd":4,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.51200000000000001,"15min":0.48899999999999999},"min":{"1min":0.40899999999999997,"5min":0.33800000000000002,"15min":0.28999999999999998},"max":{"1min":0.72399999999999998,"5min":0.78600000000000003,"15min":0.84699999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.53700000000000003,"15min":0.51100000000000001},"min":{"1min":0.41499999999999998,"5min":0.34100000000000003,"15min":0.26500000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.93799999999999994},"last":0.54900000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.53300000000000003,"15min":0.497},"min":{"1min":0.44400000000000001,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.57899999999999996,"15min":0.55100000000000005},"min":{"1min":0.47599999999999998,"5min":0.32200000000000001,"15min":0.30499999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.92400000000000004},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:51:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.57999999999999996,"15min":0.56899999999999995},"min":{"1min":0.42399999999999999,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":0.877,"5min":0.877,"15min":0.89700000000000002},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.63500000000000001,"15min":0.60699999999999998},"min":{"1min":0.45200000000000001,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":0.82999999999999996,"5min":0.86499999999999999,"15min":1.0029999999999999},"last":0.64600000000000002}]}]},{"osd":1,"up_from":99,"seq":425201762339,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":430536,"kb_used_data":398848,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93314616,"statfs":{"total":95995035648,"available":95554166784,"internally_reserved":0,"allocated":408420352,"data_stored":407850891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":32432954},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.35399999999999998,"15min":0.35799999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.55100000000000005,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.36299999999999999,"15min":0.36399999999999999},"min":{"1min":0.252,"5min":0.25,"15min":0.25},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.47799999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.35699999999999998,"15min":0.35099999999999998},"min":{"1min":0.23699999999999999,"5min":0.192,"15min":0.192},"max":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.35799999999999998,"15min":0.35399999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59299999999999997,"5min":0.752,"15min":0.752},"last":0.39800000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.377}]},{"osd":4,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.72699999999999998,"5min":1.161,"15min":1.161},"last":1.286},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.441,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.81299999999999994,"5min":1.107,"15min":1.107},"last":0.71999999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.625,"15min":0.63500000000000001},"min":{"1min":0.439,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70599999999999996,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.441,"5min":0.375,"15min":0.375},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66100000000000003,"15min":0.66000000000000003},"min":{"1min":0.44900000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.622,"5min":0.68100000000000005,"15min":0.69099999999999995},"min":{"1min":0.46700000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81499999999999995,"5min":1.214,"15min":1.214},"last":1.2270000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64900000000000002},"min":{"1min":0.47599999999999998,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.65900000000000003,"15min":0.66700000000000004},"min":{"1min":0.44700000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80400000000000005,"5min":1.246,"15min":1.246},"last":1.26}]}]},{"osd":0,"up_from":94,"seq":403726925874,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":744080,"kb_used_data":716488,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93001072,"statfs":{"total":95995035648,"available":95233097728,"internally_reserved":0,"allocated":733683712,"data_stored":733107030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":28237409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.32100000000000001,"15min":0.32300000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.42299999999999999,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.32800000000000001,"5min":0.33600000000000002,"15min":0.33800000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.40699999999999997,"15min":0.41899999999999998},"min":{"1min":0.26500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.47999999999999998,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.36799999999999999,"15min":0.374},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.435,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.85399999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.623,"15min":0.64300000000000002},"min":{"1min":0.41699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.64500000000000002,"15min":0.67700000000000005},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79000000000000004,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67800000000000005,"15min":0.70199999999999996},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.66900000000000004,"15min":0.70199999999999996},"min":{"1min":0.433,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":3.286}]},{"osd":6,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.68000000000000005,"15min":0.70999999999999996},"min":{"1min":0.46200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":0.437},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67600000000000005,"15min":0.70399999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":1.093}]},{"osd":7,"last update":"Sat Sep 23 19:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.68500000000000005,"15min":0.71899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":3.323}]}]},{"osd":2,"up_from":104,"seq":446676598813,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":655672,"kb_used_data":615856,"kb_used_omap":12,"kb_used_meta":39795,"kb_avail":93089480,"statfs":{"total":95995035648,"available":95323627520,"internally_reserved":0,"allocated":630636544,"data_stored":630080425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":40751102},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.38100000000000001,"15min":0.36099999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":2.3130000000000002,"5min":2.3130000000000002,"15min":2.3130000000000002},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.40100000000000002,"15min":0.38200000000000001},"min":{"1min":0.253,"5min":0.252,"15min":0.252},"max":{"1min":2.3519999999999999,"5min":2.3519999999999999,"15min":2.3519999999999999},"last":0.624}]},{"osd":1,"last update":"Sat Sep 23 19:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.23799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.38400000000000001,"15min":0.38600000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.63,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.78700000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.61799999999999999,"15min":0.629},"min":{"1min":0.44400000000000001,"5min":0.44,"15min":0.44},"max":{"1min":0.71299999999999997,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.627,"5min":0.623,"15min":0.622},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.95299999999999996,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.57899999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.63,"15min":0.63300000000000001},"min":{"1min":0.441,"5min":0.378,"15min":0.378},"max":{"1min":1.173,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.751},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.625,"15min":0.627},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.95899999999999996,"5min":1.161,"15min":1.161},"last":0.51100000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.436,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":2.081},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45400000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0269999999999999,"5min":1.052,"15min":1.052},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 19:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.437,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.66000000000000003,"15min":0.66700000000000004},"min":{"1min":0.48299999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.82399999999999995,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":1.6160000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435286,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":309680,"kb_used_data":265704,"kb_used_omap":4,"kb_used_meta":43899,"kb_avail":93435472,"statfs":{"total":95995035648,"available":95677923328,"internally_reserved":0,"allocated":272080896,"data_stored":271531938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":44952685},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":1.3759999999999999},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.30099999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.437}]},{"osd":4,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.94099999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.68700000000000006}]},{"osd":6,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.96799999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:52:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.71399999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019047,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":551252,"kb_used_data":524108,"kb_used_omap":0,"kb_used_meta":27136,"kb_avail":93193900,"statfs":{"total":95995035648,"available":95430553600,"internally_reserved":0,"allocated":536686592,"data_stored":536163319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27787264},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58599999999999997,"15min":0.59999999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78000000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.626,"15min":0.63800000000000001},"min":{"1min":0.47399999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82999999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.32000000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81399999999999995,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.216},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56299999999999994,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.72599999999999998,"5min":1.036,"15min":1.036},"last":0.41399999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:52:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41999999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.55200000000000005,"15min":0.54800000000000004},"min":{"1min":0.39700000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73399999999999999,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:51:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.36199999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:51:46 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.502,"15min":0.45200000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":0.68500000000000005,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.47899999999999998},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":0.66100000000000003,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.29499999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:51:57 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55100000000000005,"15min":0.49099999999999999},"min":{"1min":0.38600000000000001,"5min":0.32700000000000001,"15min":0.27100000000000002},"max":{"1min":0.71299999999999997,"5min":0.86399999999999999,"15min":1.0589999999999999},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.57999999999999996,"15min":0.52300000000000002},"min":{"1min":0.441,"5min":0.32000000000000001,"15min":0.309},"max":{"1min":0.73399999999999999,"5min":1.0660000000000001,"15min":1.073},"last":0.38200000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60499999999999998,"15min":0.56899999999999995},"min":{"1min":0.48899999999999999,"5min":0.372,"15min":0.29599999999999999},"max":{"1min":0.73399999999999999,"5min":1.117,"15min":1.117},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.64100000000000001,"15min":0.58999999999999997},"min":{"1min":0.47599999999999998,"5min":0.44600000000000001,"15min":0.35799999999999998},"max":{"1min":0.79100000000000004,"5min":1.016,"15min":1.016},"last":0.89800000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757411,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":601352,"kb_used_data":573824,"kb_used_omap":0,"kb_used_meta":27520,"kb_avail":93143800,"statfs":{"total":95995035648,"available":95379251200,"internally_reserved":0,"allocated":587595776,"data_stored":587080496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":28180480},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.54200000000000004,"15min":0.56699999999999995},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.70499999999999996,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73599999999999999,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.98799999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52200000000000002,"15min":0.52400000000000002},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.64300000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52600000000000002,"15min":0.52500000000000002},"min":{"1min":0.42999999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76400000000000001,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.45900000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:52:29 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":2.0150000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":1.982}]},{"osd":3,"last update":"Sat Sep 23 19:51:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.46700000000000003,"15min":0.44400000000000001},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.23699999999999999},"max":{"1min":0.78900000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.50700000000000001,"15min":0.47599999999999998},"min":{"1min":0.39400000000000002,"5min":0.28599999999999998,"15min":0.27500000000000002},"max":{"1min":0.753,"5min":0.76800000000000002,"15min":0.86399999999999999},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:52:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.53600000000000003,"15min":0.497},"min":{"1min":0.41299999999999998,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.86599999999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.59399999999999997,"15min":0.55200000000000005},"min":{"1min":0.52200000000000002,"5min":0.34300000000000003,"15min":0.307},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.82799999999999996},"last":1.2370000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.56899999999999995,"15min":0.55000000000000004},"min":{"1min":0.41299999999999998,"5min":0.36299999999999999,"15min":0.29999999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":1.0309999999999999},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.57299999999999995,"15min":0.56000000000000005},"min":{"1min":0.42499999999999999,"5min":0.379,"15min":0.30099999999999999},"max":{"1min":0.76400000000000001,"5min":0.82299999999999995,"15min":0.86499999999999999},"last":1.5389999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":739180544,"data_stored":739180544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":416743424,"data_stored":416743424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":639827968,"data_stored":639827968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":288161792,"data_stored":288161792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":552861696,"data_stored":552861696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":627900416,"data_stored":627900416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":279576576,"data_stored":279576576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":351469568,"data_stored":351469568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:52:39.502 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T19:52:39.503 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-23T19:52:39.503 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.503 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.553317+0000", 2023-09-23T19:52:39.503 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.079071849 2023-09-23T19:52:39.503 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.503 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.503 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.553369+0000", 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1993999999999997e-05 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.553416+0000", 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7413000000000002e-05 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.504 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:29008 5.5 5:aec4145b:::benchmark_data_smithi135_284570_object29007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:38.474325+0000", 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.88856445699999997, 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.079248809000000003, 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 29008 2023-09-23T19:52:39.505 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474325+0000", 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474325+0000", 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.506 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474324+0000", 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474351+0000", 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6636999999999999e-05 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.507 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474370+0000", 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9661e-05 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474378+0000", 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4240000000000004e-06 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474420+0000", 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1813000000000001e-05 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.508 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474510+0000", 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0959999999999996e-05 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.474618+0000", 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010747999999999999 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:39.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.475684+0000", 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001065711 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.476016+0000", 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00033226499999999999 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.553489+0000", 2023-09-23T19:52:39.510 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.077472608999999998 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.553540+0000", 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1341999999999997e-05 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.553574+0000", 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4267999999999998e-05 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.511 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:29143 5.7 e116/112 5:efe1bbac:::benchmark_data_smithi135_284570_object29142:head v 116'1809, mlcod=116'1807)", 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:38.626401+0000", 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.73648847699999997, 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13672536399999999, 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.512 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.626401+0000", 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.626401+0000", 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.626401+0000", 2023-09-23T19:52:39.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.626509+0000", 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107834 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.761576+0000", 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13506721199999999 2023-09-23T19:52:39.514 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.761587+0000", 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1208e-05 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.761627+0000", 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0030999999999997e-05 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.515 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.761652+0000", 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5236e-05 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.763090+0000", 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014373739999999999 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.763127+0000", 2023-09-23T19:52:39.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7125000000000001e-05 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:29148 5.e e116/112 5:7c07793d:::benchmark_data_smithi135_284570_object29147:head v 116'1772, mlcod=116'1770)", 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:38.629215+0000", 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.73367506500000002, 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13412099099999999, 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:52:39.517 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.629215+0000", 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.629215+0000", 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.518 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.629213+0000", 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.629274+0000", 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0522000000000001e-05 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:39.519 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.761573+0000", 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.132298737 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.761585+0000", 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2225999999999999e-05 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.761627+0000", 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2193000000000001e-05 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.520 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.761652+0000", 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5109999999999998e-05 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.763302+0000", 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001650243 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:38.763336+0000", 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3300000000000003e-05 2023-09-23T19:52:39.521 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.522 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.522 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.522 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:39.522 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:39.522 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:52:39.522 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:39.534 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:39.669 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 12 16 31048 31032 161.519 118.75 0.00569626 0.0061863 2023-09-23T19:52:39.682 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:52:39.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:39.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:39.792 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:39.859 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:40 smithi135 ceph-mon[221113]: pgmap v240: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s 2023-09-23T19:52:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:40 smithi135 ceph-mon[213871]: pgmap v240: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s 2023-09-23T19:52:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:40 smithi144 ceph-mon[194833]: pgmap v240: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s 2023-09-23T19:52:40.666 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:40.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:40.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:40.669 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 13 16 32637 32621 156.736 99.3125 0.00213707 0.00636398 2023-09-23T19:52:40.770 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:40.843 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:52:40.847 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:40.974 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:52:41.016 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:41.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:41.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:41.130 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:41.253 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:41.303 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:41.546 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:41.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:41.649 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:41.669 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 14 16 34386 34370 153.35 109.312 0.00564907 0.00651642 2023-09-23T19:52:41.700 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:42.311 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:42.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:42.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:42.417 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:42.463 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:42 smithi135 ceph-mon[213871]: pgmap v241: 89 pgs: 89 active+clean; 1.7 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2023-09-23T19:52:42.463 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:42 smithi135 ceph-mon[221113]: pgmap v241: 89 pgs: 89 active+clean; 1.7 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2023-09-23T19:52:42.469 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:42 smithi144 ceph-mon[194833]: pgmap v241: 89 pgs: 89 active+clean; 1.7 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2023-09-23T19:52:42.669 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 15 16 36368 36352 151.385 123.875 0.00682974 0.00660144 2023-09-23T19:52:42.678 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:42.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:42.786 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:42.856 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:42.949 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:43.478 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:43.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:43.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:43.584 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:43.638 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:43.669 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 16 16 38180 38164 149.002 113.25 0.005089 0.00670627 2023-09-23T19:52:43.685 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:52:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:52:43] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.43.0" 2023-09-23T19:52:44.138 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:44.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:44.206 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:44 smithi135 ceph-mon[213871]: pgmap v242: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2023-09-23T19:52:44.206 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:44 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1706886331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:44.206 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:44 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:44.246 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:44.271 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:44.310 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:52:44.314 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:44.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:44.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:44.424 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:44 smithi135 ceph-mon[221113]: pgmap v242: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2023-09-23T19:52:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:44 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1706886331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:44 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:44.481 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:44 smithi144 ceph-mon[194833]: pgmap v242: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2023-09-23T19:52:44.481 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:44 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1706886331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:44.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:44 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:52:44.484 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:44.547 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:52:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:44.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:44.654 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:44.669 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 17 16 39694 39678 145.804 94.625 0.0021493 0.00683905 2023-09-23T19:52:44.681 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:45.248 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:45.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:45.353 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:45.398 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:45.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:45 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:45.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:45 smithi135 ceph-mon[221113]: failed to encode map e117 with expected crc 2023-09-23T19:52:45.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:45 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:52:45.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:45 smithi135 ceph-mon[221113]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T19:52:45.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:45 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:45.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:45 smithi135 ceph-mon[213871]: failed to encode map e117 with expected crc 2023-09-23T19:52:45.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:45 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:52:45.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:45 smithi135 ceph-mon[213871]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T19:52:45.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:45 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:52:45.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:45 smithi144 ceph-mon[194833]: failed to encode map e117 with expected crc 2023-09-23T19:52:45.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:45 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:52:45.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:45 smithi144 ceph-mon[194833]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T19:52:45.669 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 18 16 41334 41318 143.398 102.5 0.00998294 0.00696994 2023-09-23T19:52:46.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:46 smithi135 ceph-mon[213871]: pgmap v244: 89 pgs: 89 active+clean; 2.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:52:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:46 smithi135 ceph-mon[221113]: pgmap v244: 89 pgs: 89 active+clean; 2.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:52:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:46 smithi144 ceph-mon[194833]: pgmap v244: 89 pgs: 89 active+clean; 2.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T19:52:46.671 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 19 16 42570 42554 139.907 77.25 0.0056644 0.00714199 2023-09-23T19:52:46.682 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:46.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:46.717 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:46.791 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:46.832 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:46.950 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:52:46.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:46.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:47.061 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:47.137 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:47.610 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:47.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:47.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:47.671 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T19:52:47.672144+0000 min lat: 0.000994654 max lat: 0.22432 avg lat: 0.0072407 2023-09-23T19:52:47.671 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:52:47.671 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 20 16 44200 44184 138.006 101.875 0.00685529 0.0072407 2023-09-23T19:52:47.716 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:47.776 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:44245 5.c 5:381a135f:::benchmark_data_smithi135_284570_object44244:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:47.703669+0000", 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0049031630000000003, 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049445740000000002, 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:52:47.787 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 44245 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.703669+0000", 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.703669+0000", 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.703667+0000", 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.703696+0000", 2023-09-23T19:52:47.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8623e-05 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.703715+0000", 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9244000000000001e-05 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.703734+0000", 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9239999999999999e-05 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.703801+0000", 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6867999999999997e-05 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.703966+0000", 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016522600000000001 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:52:47.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.704120+0000", 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015369400000000001 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706384+0000", 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002263653 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706758+0000", 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037423600000000001 2023-09-23T19:52:47.792 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:44252 5.7 e117/112 5:ec4e5e88:::benchmark_data_smithi135_284570_object44251:head v 117'2738, mlcod=117'2736)", 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:47.706396+0000", 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.002176056, 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023256650000000002, 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:52:47.793 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706396+0000", 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706396+0000", 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.794 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706394+0000", 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706432+0000", 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7985000000000003e-05 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706450+0000", 2023-09-23T19:52:47.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7357e-05 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706458+0000", 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1640000000000001e-06 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706574+0000", 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011589799999999999 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:52:47.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:47.706596+0000", 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2472000000000001e-05 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 2 2023-09-23T19:52:47.797 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:52:48.138 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:48 smithi135 ceph-mon[213871]: pgmap v245: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T19:52:48.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:48 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/383557513' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:52:48.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:48 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:52:48.244 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 1 pg_num to 11 2023-09-23T19:52:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:48 smithi135 ceph-mon[221113]: pgmap v245: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T19:52:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:48 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/383557513' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:52:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:48 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:52:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:48 smithi144 ceph-mon[194833]: pgmap v245: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T19:52:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:48 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/383557513' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:52:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:48 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T19:52:48.671 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 21 16 45815 45799 136.241 100.938 0.00826701 0.00733507 2023-09-23T19:52:48.697 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:52:49.618 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T19:52:49.618 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T19:52:49.618 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.618 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.618 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.618 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.618 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: pgmap v247: 89 pgs: 89 active+clean; 2.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T19:52:49.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T19:52:49.619 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T19:52:49.619 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T19:52:49.619 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.619 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.619 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.620 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.620 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.620 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: pgmap v247: 89 pgs: 89 active+clean; 2.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T19:52:49.620 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:49 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T19:52:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T19:52:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T19:52:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: failed to encode map e118 with expected crc 2023-09-23T19:52:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: pgmap v247: 89 pgs: 89 active+clean; 2.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T19:52:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:49 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T19:52:49.671 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 22 16 47415 47399 134.594 100 0.00244742 0.00741975 2023-09-23T19:52:50.171 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:50.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:50.277 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:50.325 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:50.366 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:52:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:50.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:50 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T19:52:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:50 smithi135 ceph-mon[221113]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T19:52:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:50 smithi135 ceph-mon[221113]: failed to encode map e119 with expected crc 2023-09-23T19:52:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:50 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T19:52:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:50 smithi135 ceph-mon[213871]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T19:52:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:50 smithi135 ceph-mon[213871]: failed to encode map e119 with expected crc 2023-09-23T19:52:50.471 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:50.497 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:50 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T19:52:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:50 smithi144 ceph-mon[194833]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T19:52:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:50 smithi144 ceph-mon[194833]: failed to encode map e119 with expected crc 2023-09-23T19:52:50.672 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 23 16 48554 48538 131.83 71.1875 0.0098202 0.00757988 2023-09-23T19:52:50.821 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:52:50.948 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:50.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:50.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:51.056 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:52:51.093 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:51.158 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:52:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:51.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:52:51.228 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:51.269 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:51.344 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:51.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:51 smithi144 ceph-mon[194833]: failed to encode map e120 with expected crc 2023-09-23T19:52:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:51 smithi144 ceph-mon[194833]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T19:52:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:51 smithi144 ceph-mon[194833]: failed to encode map e120 with expected crc 2023-09-23T19:52:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:51 smithi144 ceph-mon[194833]: pgmap v250: 99 pgs: 10 unknown, 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:52:51.448 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:51.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:51.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:51.497 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:51 smithi135 ceph-mon[221113]: failed to encode map e120 with expected crc 2023-09-23T19:52:51.497 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:51 smithi135 ceph-mon[221113]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T19:52:51.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:51 smithi135 ceph-mon[221113]: failed to encode map e120 with expected crc 2023-09-23T19:52:51.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:51 smithi135 ceph-mon[221113]: pgmap v250: 99 pgs: 10 unknown, 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:52:51.498 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:51 smithi135 ceph-mon[213871]: failed to encode map e120 with expected crc 2023-09-23T19:52:51.498 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:51 smithi135 ceph-mon[213871]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T19:52:51.498 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:51 smithi135 ceph-mon[213871]: failed to encode map e120 with expected crc 2023-09-23T19:52:51.498 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:51 smithi135 ceph-mon[213871]: pgmap v250: 99 pgs: 10 unknown, 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T19:52:51.554 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:51.611 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:51.673 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 24 16 50051 50035 130.235 93.5625 0.00514619 0.00767272 2023-09-23T19:52:51.848 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:51.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:51.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:51.862 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:52:51.902 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:52:51.939 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:52:51.954 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:51.971 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:52:52.008 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:52.021 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:52:52.063 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:52:52.082 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:52.102 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:52:52.144 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:52:52.161 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:52:52.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:52:52.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:52:52.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:52:52.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:52:52.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:52:52.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:52:52.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:52:52.201 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:52:52.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:52.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:52.308 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:52.364 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:52.372 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:52 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T19:52:52.372 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:52 smithi135 ceph-mon[221113]: 1.0 scrub ok 2023-09-23T19:52:52.373 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4269587856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:52.373 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:52.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:52 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T19:52:52.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:52 smithi135 ceph-mon[213871]: 1.0 scrub ok 2023-09-23T19:52:52.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4269587856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:52.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:52.474 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:52.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:52.586 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:52 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T19:52:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:52 smithi144 ceph-mon[194833]: 1.0 scrub ok 2023-09-23T19:52:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4269587856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:52:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:52:52.635 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:52.673 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 25 16 51561 51545 128.801 94.375 0.0107409 0.00775975 2023-09-23T19:52:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:53 smithi144 ceph-mon[194833]: pgmap v251: 99 pgs: 99 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T19:52:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T19:52:53.673 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 26 16 52888 52872 127.038 82.9375 0.0145512 0.00786681 2023-09-23T19:52:53.685 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:52:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:52:53] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.43.0" 2023-09-23T19:52:53.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:53 smithi135 ceph-mon[221113]: pgmap v251: 99 pgs: 99 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T19:52:53.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T19:52:53.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:53 smithi135 ceph-mon[213871]: pgmap v251: 99 pgs: 99 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T19:52:53.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T19:52:54.292 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:52:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:54.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:54.397 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:52:54.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[213871]: 1.9 scrub starts 2023-09-23T19:52:54.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[213871]: 1.9 scrub ok 2023-09-23T19:52:54.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[213871]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[213871]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T19:52:54.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[213871]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T19:52:54.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[213871]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[213871]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[221113]: 1.9 scrub starts 2023-09-23T19:52:54.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[221113]: 1.9 scrub ok 2023-09-23T19:52:54.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[221113]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[221113]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T19:52:54.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[221113]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T19:52:54.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[221113]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:54 smithi135 ceph-mon[221113]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.478 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:54.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:54 smithi144 ceph-mon[194833]: 1.9 scrub starts 2023-09-23T19:52:54.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:54 smithi144 ceph-mon[194833]: 1.9 scrub ok 2023-09-23T19:52:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:54 smithi144 ceph-mon[194833]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:54 smithi144 ceph-mon[194833]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:54 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T19:52:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:54 smithi144 ceph-mon[194833]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T19:52:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:54 smithi144 ceph-mon[194833]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:54 smithi144 ceph-mon[194833]: failed to encode map e121 with expected crc 2023-09-23T19:52:54.658 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:52:54.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:54.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:52:54.673 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 27 16 54305 54289 125.613 88.5625 0.00594673 0.00795688 2023-09-23T19:52:54.765 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:52:54.838 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:55.488 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:52:55.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:55.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:52:55.592 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:55.615 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:55 smithi144 ceph-mon[194833]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T19:52:55.616 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:55 smithi144 ceph-mon[194833]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.616 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:55 smithi144 ceph-mon[194833]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.616 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:55 smithi144 ceph-mon[194833]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.616 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:55 smithi144 ceph-mon[194833]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.616 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:55 smithi144 ceph-mon[194833]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.616 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:55 smithi144 ceph-mon[194833]: pgmap v254: 99 pgs: 99 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-23T19:52:55.617 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:55.661 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:55.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:55.673 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 28 16 55526 55510 123.852 76.3125 0.00515953 0.00801399 2023-09-23T19:52:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[221113]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T19:52:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[221113]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[221113]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[221113]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[221113]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[221113]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[221113]: pgmap v254: 99 pgs: 99 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[213871]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[213871]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[213871]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[213871]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[213871]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[213871]: failed to encode map e122 with expected crc 2023-09-23T19:52:55.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:55 smithi135 ceph-mon[213871]: pgmap v254: 99 pgs: 99 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-23T19:52:55.764 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:52:55.825 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:55.855 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:52:55.855 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:52:55.855 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T19:52:55.855 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:52:55.855 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:52:55.856 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T19:52:55.916 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:52:55.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:55.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:52:56.028 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:56.104 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:56.234 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:56.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:56.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:56.248 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:52:56.339 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:52:56.345 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:56.410 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[213871]: failed to encode map e122 with expected crc 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[213871]: failed to encode map e123 with expected crc 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[213871]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[213871]: failed to encode map e123 with expected crc 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4050483772' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[221113]: failed to encode map e122 with expected crc 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[221113]: failed to encode map e123 with expected crc 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[221113]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[221113]: failed to encode map e123 with expected crc 2023-09-23T19:52:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4050483772' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:56.491 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:52:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:56 smithi144 ceph-mon[194833]: failed to encode map e122 with expected crc 2023-09-23T19:52:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:56 smithi144 ceph-mon[194833]: failed to encode map e123 with expected crc 2023-09-23T19:52:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:56 smithi144 ceph-mon[194833]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T19:52:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:56 smithi144 ceph-mon[194833]: failed to encode map e123 with expected crc 2023-09-23T19:52:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4050483772' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:52:56.678 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 29 16 56823 56807 122.358 81.0625 0.00816447 0.00816824 2023-09-23T19:52:56.702 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:56.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:56.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:56.819 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:52:56.885 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:56.991 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:52:57.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:57.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:52:57.101 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:57.154 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:57.417 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:52:57.418 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:52:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:52:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T19:52:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4050483772' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[213871]: pgmap v257: 99 pgs: 6 active+undersized+remapped, 93 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 144 KiB/s, 0 objects/s recovering 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4050483772' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T19:52:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:57 smithi135 ceph-mon[221113]: pgmap v257: 99 pgs: 6 active+undersized+remapped, 93 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 144 KiB/s, 0 objects/s recovering 2023-09-23T19:52:57.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:52:57.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T19:52:57.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4050483772' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:52:57.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T19:52:57.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: failed to encode map e124 with expected crc 2023-09-23T19:52:57.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:57 smithi144 ceph-mon[194833]: pgmap v257: 99 pgs: 6 active+undersized+remapped, 93 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 144 KiB/s, 0 objects/s recovering 2023-09-23T19:52:57.678 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 30 16 58100 58084 120.94 79.8125 0.00379524 0.00826373 2023-09-23T19:52:57.919 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:57.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:58.028 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:58.064 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:58.594 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:58.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: 1.1 scrub starts 2023-09-23T19:52:58.601 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[221113]: 1.1 scrub ok 2023-09-23T19:52:58.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:58.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: 1.1 scrub starts 2023-09-23T19:52:58.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:58 smithi135 ceph-mon[213871]: 1.1 scrub ok 2023-09-23T19:52:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T19:52:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: failed to encode map e125 with expected crc 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: 1.1 scrub starts 2023-09-23T19:52:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:58 smithi144 ceph-mon[194833]: 1.1 scrub ok 2023-09-23T19:52:58.678 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 31 16 59280 59264 119.419 73.75 0.00808268 0.00836923 2023-09-23T19:52:58.703 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:58.727 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:52:58.754 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:58.824 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:58.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:58.832 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:52:58.928 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:59.018 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:59.213 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:59.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:59.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:52:59.320 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:52:59.370 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:59.468 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:52:59.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:59.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:52:59.577 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:52:59.627 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T19:52:59.627 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[213871]: 1.2 scrub ok 2023-09-23T19:52:59.627 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[213871]: 1.8 scrub starts 2023-09-23T19:52:59.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[213871]: 1.8 scrub ok 2023-09-23T19:52:59.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[213871]: pgmap v259: 99 pgs: 2 active+undersized+remapped, 97 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 54 KiB/s, 0 objects/s recovering 2023-09-23T19:52:59.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T19:52:59.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[221113]: 1.2 scrub ok 2023-09-23T19:52:59.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[221113]: 1.8 scrub starts 2023-09-23T19:52:59.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[221113]: 1.8 scrub ok 2023-09-23T19:52:59.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:52:59 smithi135 ceph-mon[221113]: pgmap v259: 99 pgs: 2 active+undersized+remapped, 97 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 54 KiB/s, 0 objects/s recovering 2023-09-23T19:52:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:59 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T19:52:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:59 smithi144 ceph-mon[194833]: 1.2 scrub ok 2023-09-23T19:52:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:59 smithi144 ceph-mon[194833]: 1.8 scrub starts 2023-09-23T19:52:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:59 smithi144 ceph-mon[194833]: 1.8 scrub ok 2023-09-23T19:52:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:52:59 smithi144 ceph-mon[194833]: pgmap v259: 99 pgs: 2 active+undersized+remapped, 97 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 54 KiB/s, 0 objects/s recovering 2023-09-23T19:52:59.632 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:59.678 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 32 16 60624 60608 118.312 84 0.0096678 0.00844658 2023-09-23T19:52:59.725 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:52:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:59.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:52:59.829 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:52:59.884 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:52:59.970 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:52:59.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:52:59.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:00.073 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:00.099 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:00.141 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:00.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:00.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:00.244 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:00.277 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:00.375 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:00.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:00.482 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:00.485 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:53:00.551 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:00.617 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:00.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:00 smithi144 ceph-mon[194833]: 1.7 deep-scrub starts 2023-09-23T19:53:00.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:00 smithi144 ceph-mon[194833]: 1.7 deep-scrub ok 2023-09-23T19:53:00.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:00 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T19:53:00.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:00 smithi144 ceph-mon[194833]: 1.5 scrub ok 2023-09-23T19:53:00.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:00 smithi144 ceph-mon[194833]: 1.a deep-scrub starts 2023-09-23T19:53:00.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:00 smithi144 ceph-mon[194833]: 1.a deep-scrub ok 2023-09-23T19:53:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:00.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:00.678 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 33 16 61599 61583 116.574 60.9375 0.0139442 0.00857397 2023-09-23T19:53:00.720 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[221113]: 1.7 deep-scrub starts 2023-09-23T19:53:00.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[221113]: 1.7 deep-scrub ok 2023-09-23T19:53:00.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T19:53:00.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[221113]: 1.5 scrub ok 2023-09-23T19:53:00.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[221113]: 1.a deep-scrub starts 2023-09-23T19:53:00.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[221113]: 1.a deep-scrub ok 2023-09-23T19:53:00.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[213871]: 1.7 deep-scrub starts 2023-09-23T19:53:00.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[213871]: 1.7 deep-scrub ok 2023-09-23T19:53:00.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T19:53:00.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[213871]: 1.5 scrub ok 2023-09-23T19:53:00.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[213871]: 1.a deep-scrub starts 2023-09-23T19:53:00.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:00 smithi135 ceph-mon[213871]: 1.a deep-scrub ok 2023-09-23T19:53:00.776 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:00.778 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:00.856 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:00.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:00.961 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:01.011 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:01.170 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:53:01.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:01.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:01.277 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:01.350 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:01.411 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:01 smithi135 ceph-mon[213871]: 1.4 deep-scrub starts 2023-09-23T19:53:01.411 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:01 smithi135 ceph-mon[213871]: 1.4 deep-scrub ok 2023-09-23T19:53:01.411 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:01 smithi135 ceph-mon[213871]: pgmap v260: 99 pgs: 99 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:53:01.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:01 smithi135 ceph-mon[221113]: 1.4 deep-scrub starts 2023-09-23T19:53:01.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:01 smithi135 ceph-mon[221113]: 1.4 deep-scrub ok 2023-09-23T19:53:01.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:01 smithi135 ceph-mon[221113]: pgmap v260: 99 pgs: 99 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:53:01.417 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:01 smithi144 ceph-mon[194833]: 1.4 deep-scrub starts 2023-09-23T19:53:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:01 smithi144 ceph-mon[194833]: 1.4 deep-scrub ok 2023-09-23T19:53:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:01 smithi144 ceph-mon[194833]: pgmap v260: 99 pgs: 99 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T19:53:01.649 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:01.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:01.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:01.678 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 34 16 62901 62885 115.539 81.375 0.0101435 0.00865019 2023-09-23T19:53:01.740 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:53:01.755 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:01.776 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:53:01.805 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:01.812 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:53:01.840 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:53:01.878 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:53:01.915 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:53:01.949 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:53:01.987 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:53:01.996 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:53:01.996 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:53:01.996 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:53:01.996 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:53:01.996 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:53:01.996 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:53:01.996 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:53:01.996 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:53:02.000 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:02.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:02.119 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:02.169 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:02.244 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:02.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:02.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:02.348 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:02.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1974472038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:02.378 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:02.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:02.481 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:02.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:02.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:02.586 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:02.621 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:02.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1974472038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:02.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:02.678 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 35 16 64128 64112 114.429 76.6875 0.0167566 0.00873435 2023-09-23T19:53:02.679 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:53:02.679 INFO:teuthology.orchestra.run.smithi144.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:53:02.679 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:53:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1974472038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:03.200 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:03.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:03.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:03.304 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:03.371 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:03.427 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:03.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:03.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:03.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:03 smithi135 ceph-mon[221113]: pgmap v261: 99 pgs: 99 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.57k op/s 2023-09-23T19:53:03.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:03 smithi135 ceph-mon[213871]: pgmap v261: 99 pgs: 99 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.57k op/s 2023-09-23T19:53:03.530 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:03.582 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:03 smithi144 ceph-mon[194833]: pgmap v261: 99 pgs: 99 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.57k op/s 2023-09-23T19:53:03.678 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 36 16 65163 65147 113.047 64.6875 0.01218 0.00884116 2023-09-23T19:53:03.738 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:53:03.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:03.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:03.842 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:53:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:53:03] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.43.0" 2023-09-23T19:53:03.846 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:03.883 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:03.976 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:03.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:03.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:04.083 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:04.153 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:04.251 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:04.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:04.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:04.356 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:04.401 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:04.679 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 37 16 66333 66317 111.968 73.125 0.00617265 0.00890745 2023-09-23T19:53:05.110 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:05.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:05.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:05.214 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:05.264 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:05.359 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:05.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:05.473 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:05.518 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:53:05.518 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:21986 5.7 e116/112 5:ed93fdd0:::benchmark_data_smithi135_284570_object21985:head v 116'1350, mlcod=116'1346)", 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:34.941956+0000", 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 30.572046495999999, 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.096114014999999997, 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.519 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.941956+0000", 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.941956+0000", 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.520 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.521 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.941955+0000", 2023-09-23T19:53:05.521 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:53:05.521 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.521 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.521 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.521 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.942821+0000", 2023-09-23T19:53:05.522 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00086589800000000004 2023-09-23T19:53:05.522 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.522 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.522 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.522 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036204+0000", 2023-09-23T19:53:05.522 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.093382978000000005 2023-09-23T19:53:05.522 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.522 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036238+0000", 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4054000000000003e-05 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036359+0000", 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012066 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.523 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036418+0000", 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9355999999999999e-05 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.038026+0000", 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016079239999999999 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.524 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.038070+0000", 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3939000000000001e-05 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:22001 5.0 e116/112 5:07afb5fa:::benchmark_data_smithi135_284570_object22000:head v 116'1410, mlcod=116'1408)", 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:34.943745+0000", 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 30.570257818000002, 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.095301495999999999, 2023-09-23T19:53:05.525 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.943745+0000", 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.943745+0000", 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.526 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.943744+0000", 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.943766+0000", 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1861e-05 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.527 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036214+0000", 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.092447854999999995 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036235+0000", 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1393e-05 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036353+0000", 2023-09-23T19:53:05.528 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000118354 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036418+0000", 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4762000000000001e-05 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.039007+0000", 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025887499999999999 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.529 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.039046+0000", 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9279000000000001e-05 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:22003 5.c 5:3ded1178:::benchmark_data_smithi135_284570_object22002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:34.943856+0000", 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 30.570146518000001, 2023-09-23T19:53:05.530 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.095890762000000004, 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 22003 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.531 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.943856+0000", 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.943856+0000", 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.943854+0000", 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:53:05.532 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:34.943879+0000", 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5184000000000001e-05 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036207+0000", 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.092327459000000001 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.533 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036246+0000", 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8658999999999997e-05 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036368+0000", 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012250100000000001 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.534 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036644+0000", 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00027605400000000002 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.036805+0000", 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000160779 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.039586+0000", 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027815240000000001 2023-09-23T19:53:05.535 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.536 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.536 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.536 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.039649+0000", 2023-09-23T19:53:05.536 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2488000000000002e-05 2023-09-23T19:53:05.536 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.536 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.536 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.536 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.039657+0000", 2023-09-23T19:53:05.537 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1319999999999994e-06 2023-09-23T19:53:05.537 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.537 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.537 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.537 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.039709+0000", 2023-09-23T19:53:05.537 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1564000000000003e-05 2023-09-23T19:53:05.538 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.538 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.538 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.538 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:35.039747+0000", 2023-09-23T19:53:05.538 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8123000000000001e-05 2023-09-23T19:53:05.538 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.538 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.538 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:39668 5.c 5:342dc35e:::benchmark_data_smithi135_284570_object39667:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:44.624749+0000", 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 20.889253111999999, 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.151622689, 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.539 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 39668 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624749+0000", 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624749+0000", 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624748+0000", 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624771+0000", 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2762e-05 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.541 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624787+0000", 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6019999999999999e-05 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624795+0000", 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.507e-06 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.542 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624849+0000", 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3761999999999998e-05 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624994+0000", 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014497699999999999 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.625116+0000", 2023-09-23T19:53:05.543 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012239099999999999 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.627655+0000", 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002538485 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.630018+0000", 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023631350000000001 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.544 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.776281+0000", 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14626317799999999 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.776333+0000", 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2383999999999999e-05 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.545 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.776372+0000", 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8757999999999997e-05 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:39669 5.c 5:3070bdd7:::benchmark_data_smithi135_284570_object39668:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:44.624810+0000", 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 20.889192081000001, 2023-09-23T19:53:05.546 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.151665311, 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 39669 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624810+0000", 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.547 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624810+0000", 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624810+0000", 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.548 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624830+0000", 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0370999999999999e-05 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624838+0000", 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6210000000000004e-06 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.624844+0000", 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3640000000000004e-06 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.626247+0000", 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014028389999999999 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.626377+0000", 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000129405 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.626529+0000", 2023-09-23T19:53:05.550 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015218499999999999 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.627764+0000", 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001235009 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.630237+0000", 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024732640000000002 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.551 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.776408+0000", 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.146171253 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.776445+0000", 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6643999999999997e-05 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:44.776476+0000", 2023-09-23T19:53:05.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0885e-05 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:41031 5.c 5:34db2ac1:::benchmark_data_smithi135_284570_object41030:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:45.450098+0000", 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 20.063904175000001, 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.100763387, 2023-09-23T19:53:05.553 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 41031 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450098+0000", 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.554 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450098+0000", 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450097+0000", 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.555 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450123+0000", 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.586e-05 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450137+0000", 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4116e-05 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450145+0000", 2023-09-23T19:53:05.556 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2549999999999997e-06 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450197+0000", 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1366999999999998e-05 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450338+0000", 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014165599999999999 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.450476+0000", 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000137434 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.452300+0000", 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018238869999999999 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.454334+0000", 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020345799999999998 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.458359+0000", 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0040247240000000004 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.550781+0000", 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.092422116999999998 2023-09-23T19:53:05.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.550862+0000", 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0778000000000005e-05 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:41053 5.8 e117/112 5:18d4b527:::benchmark_data_smithi135_284570_object41052:head v 117'2474, mlcod=117'2472)", 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:45.458808+0000", 2023-09-23T19:53:05.560 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 20.055194406999998, 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.096673949999999995, 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.458808+0000", 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.458808+0000", 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.458807+0000", 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.458830+0000", 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2668999999999999e-05 2023-09-23T19:53:05.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.550679+0000", 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.091848900999999997 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.550703+0000", 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3839999999999999e-05 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.550834+0000", 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013155299999999999 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.550858+0000", 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3353000000000002e-05 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.555435+0000", 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045777600000000002 2023-09-23T19:53:05.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.555482+0000", 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6597999999999997e-05 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:41603 5.c 5:30da6b20:::benchmark_data_smithi135_284570_object41602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:45.988879+0000", 2023-09-23T19:53:05.565 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 19.525123542999999, 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.121065939, 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 41603 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.988879+0000", 2023-09-23T19:53:05.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.988879+0000", 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.988877+0000", 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.988901+0000", 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4559e-05 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.988944+0000", 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2453000000000002e-05 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.988952+0000", 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9160000000000001e-06 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.988999+0000", 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6937000000000001e-05 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.569 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.989120+0000", 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000121649 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.989253+0000", 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013285400000000001 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.991955+0000", 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027021879999999999 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.992986+0000", 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001030462 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.109839+0000", 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.11685330400000001 2023-09-23T19:53:05.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.109894+0000", 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5081999999999999e-05 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.109945+0000", 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0639000000000001e-05 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.572 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:41611 5.c 5:33de63ba:::benchmark_data_smithi135_284570_object41610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:45.993099+0000", 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 19.520903359999998, 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.12240596099999999, 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.573 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 41611 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993099+0000", 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993099+0000", 2023-09-23T19:53:05.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993098+0000", 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993119+0000", 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.143e-05 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993131+0000", 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1505000000000001e-05 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993138+0000", 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2840000000000001e-06 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993191+0000", 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2917999999999997e-05 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993302+0000", 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110716 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.993388+0000", 2023-09-23T19:53:05.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6488000000000003e-05 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.994818+0000", 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001429898 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995102+0000", 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028438900000000002 2023-09-23T19:53:05.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.115413+0000", 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.120310653 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.115468+0000", 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4657000000000002e-05 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.115505+0000", 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7455e-05 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:41613 5.c 5:30fbefac:::benchmark_data_smithi135_284570_object41612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:45.995021+0000", 2023-09-23T19:53:05.580 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 19.518981277000002, 2023-09-23T19:53:05.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.12061555, 2023-09-23T19:53:05.581 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.581 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.581 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.581 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.581 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 41613 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995021+0000", 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995021+0000", 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995020+0000", 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995045+0000", 2023-09-23T19:53:05.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5109e-05 2023-09-23T19:53:05.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995057+0000", 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2551999999999999e-05 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995065+0000", 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4379999999999998e-06 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995124+0000", 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9632000000000001e-05 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995237+0000", 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000112268 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995363+0000", 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012642999999999999 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.100863+0000", 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.10549966400000001 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.101951+0000", 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001088526 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.115554+0000", 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013602201 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.115603+0000", 2023-09-23T19:53:05.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9561999999999997e-05 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.115637+0000", 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3733999999999999e-05 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:41616 5.c 5:32747799:::benchmark_data_smithi135_284570_object41615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:53:05.589 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:45.995775+0000", 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 19.518227195000001, 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.121742828, 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 41616 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995775+0000", 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995775+0000", 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995774+0000", 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995794+0000", 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9423e-05 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995805+0000", 2023-09-23T19:53:05.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1406999999999999e-05 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.995813+0000", 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5599999999999996e-06 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996679+0000", 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00086617200000000001 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.099650+0000", 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.102971496 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.099748+0000", 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7707999999999994e-05 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.101962+0000", 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00221426 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.102006+0000", 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3965999999999999e-05 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.117425+0000", 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015418708 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.117480+0000", 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5222000000000001e-05 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.117518+0000", 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7817000000000001e-05 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:41615 5.0 e117/112 5:04a33d53:::benchmark_data_smithi135_284570_object41614:head v 117'2621, mlcod=117'2619)", 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:45.996066+0000", 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 19.517936191, 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.103537718, 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996066+0000", 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996066+0000", 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996066+0000", 2023-09-23T19:53:05.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996088+0000", 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2293000000000002e-05 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996098+0000", 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0185e-05 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996104+0000", 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9839999999999998e-06 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996144+0000", 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9511000000000002e-05 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996163+0000", 2023-09-23T19:53:05.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9228999999999998e-05 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.099568+0000", 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.103405028 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.099604+0000", 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6198999999999997e-05 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.601 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:41613 5.c e117/112 ondisk, result = 0)", 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:52:45.996809+0000", 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 19.517193532, 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.104057395, 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996809+0000", 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996809+0000", 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996808+0000", 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:53:05.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:45.996817+0000", 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2660000000000001e-06 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.099523+0000", 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.102706531 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.099556+0000", 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2409000000000001e-05 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.100853+0000", 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012972109999999999 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.100859+0000", 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1619999999999996e-06 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:52:46.100866+0000", 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3420000000000002e-06 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:64612 5.c 5:3e93a94a:::benchmark_data_smithi135_284570_object64611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:03.065927+0000", 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.4480757990000002, 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14606184999999999, 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 64612 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.607 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.065927+0000", 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.065927+0000", 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.065925+0000", 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.065947+0000", 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2354999999999999e-05 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.065959+0000", 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1598e-05 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.065966+0000", 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1609999999999997e-06 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.066012+0000", 2023-09-23T19:53:05.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6221999999999999e-05 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.066115+0000", 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010265299999999999 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.066249+0000", 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000133452 2023-09-23T19:53:05.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.068394+0000", 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00214584 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.071734+0000", 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033393110000000002 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.211880+0000", 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14014669199999999 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.211947+0000", 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6180999999999995e-05 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:03.211989+0000", 2023-09-23T19:53:05.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2021999999999998e-05 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:66279 5.c 5:3c5de298:::benchmark_data_smithi135_284570_object66278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:04.527055+0000", 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.98694750799999997, 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14414137499999999, 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.614 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 66279 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.527055+0000", 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.527055+0000", 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.527055+0000", 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.527083+0000", 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7971e-05 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.527097+0000", 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4831000000000001e-05 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.527103+0000", 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.665e-06 2023-09-23T19:53:05.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.529329+0000", 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022255310000000002 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.529452+0000", 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012327599999999999 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.529585+0000", 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013312999999999999 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.531657+0000", 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020721070000000001 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.531760+0000", 2023-09-23T19:53:05.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000102419 2023-09-23T19:53:05.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:53:05.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671099+0000", 2023-09-23T19:53:05.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13933984599999999 2023-09-23T19:53:05.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671155+0000", 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5429000000000001e-05 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671196+0000", 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1581000000000001e-05 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.621 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:66285 5.7 e125/112 5:e52a0984:::benchmark_data_smithi135_284570_object66284:head v 125'4141, mlcod=125'4139)", 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:04.533720+0000", 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.98028203800000002, 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14449987, 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.622 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533720+0000", 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533720+0000", 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533720+0000", 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533778+0000", 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8400000000000003e-05 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533791+0000", 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2602999999999999e-05 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533799+0000", 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5979999999999997e-06 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533841+0000", 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1480999999999998e-05 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533862+0000", 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1386000000000002e-05 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.678169+0000", 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14430731699999999 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.678220+0000", 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0918999999999998e-05 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.627 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:66286 5.7 e125/112 5:ed900595:::benchmark_data_smithi135_284570_object66285:head v 125'4142, mlcod=125'4139)", 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:04.533820+0000", 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.98018296999999999, 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14594294599999999, 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533820+0000", 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533820+0000", 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533819+0000", 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533894+0000", 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4836e-05 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533901+0000", 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3289999999999996e-06 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.533917+0000", 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5614e-05 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.673106+0000", 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13918941600000001 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.673128+0000", 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1180999999999999e-05 2023-09-23T19:53:05.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.679727+0000", 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065995350000000001 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.679763+0000", 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5385000000000001e-05 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.632 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:66279 5.c e125/112 ondisk, result = 0)", 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:04.536881+0000", 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.97712133300000004, 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13432264699999999, 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.536881+0000", 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.536881+0000", 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.536880+0000", 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.536888+0000", 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7300000000000005e-06 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671029+0000", 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.134140803 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671039+0000", 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0899e-05 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671082+0000", 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2635000000000001e-05 2023-09-23T19:53:05.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671092+0000", 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0339999999999999e-05 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671204+0000", 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111414 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.637 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:66287 5.7 e125/112 5:ebbd689e:::benchmark_data_smithi135_284570_object66286:head v 125'4143, mlcod=125'4139)", 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:04.537377+0000", 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.97662557299999997, 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.142415028, 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.537377+0000", 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.537377+0000", 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.537376+0000", 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.537425+0000", 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9076000000000002e-05 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671031+0000", 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13360620000000001 2023-09-23T19:53:05.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671052+0000", 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0670999999999999e-05 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.675043+0000", 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0039910199999999996 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.675061+0000", 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7540000000000001e-05 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.679765+0000", 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047048840000000003 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.679792+0000", 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6543e-05 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:66298 5.0 e125/112 5:0101bba5:::benchmark_data_smithi135_284570_object66297:head v 125'4175, mlcod=125'4173)", 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:04.537646+0000", 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.97635610500000003, 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.142971986, 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:53:05.643 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.537646+0000", 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.537646+0000", 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.537646+0000", 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.537675+0000", 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9527999999999999e-05 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:53:05.645 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671042+0000", 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13336690900000001 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671052+0000", 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5699999999999999e-06 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:53:05.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671127+0000", 2023-09-23T19:53:05.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4980000000000004e-05 2023-09-23T19:53:05.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:53:05.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.671151+0000", 2023-09-23T19:53:05.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4219999999999999e-05 2023-09-23T19:53:05.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:53:05.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.680570+0000", 2023-09-23T19:53:05.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094184820000000006 2023-09-23T19:53:05.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:53:05.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:53:05.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:53:05.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:04.680618+0000", 2023-09-23T19:53:05.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8869000000000003e-05 2023-09-23T19:53:05.649 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.649 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.649 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.649 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:53:05.650 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:53:05.650 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:53:05.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:05.652 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:05.652 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:05.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:05.679 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 38 16 67615 67599 111.13 80.125 0.012118 0.0089942 2023-09-23T19:53:05.752 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:05.812 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:05.842 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:53:05.889 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:05.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:05.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:06.001 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:06.027 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:06.143 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:06.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:06.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:06 smithi135 ceph-mon[213871]: pgmap v262: 99 pgs: 99 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.28k op/s 2023-09-23T19:53:06.192 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:06 smithi135 ceph-mon[221113]: pgmap v262: 99 pgs: 99 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.28k op/s 2023-09-23T19:53:06.257 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:06.310 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:06.400 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:06.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:06.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:06.411 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:06 smithi144 ceph-mon[194833]: pgmap v262: 99 pgs: 99 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.28k op/s 2023-09-23T19:53:06.512 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:06.560 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:06.655 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:06.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:06.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:06.679 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 39 16 68737 68721 110.079 70.125 0.00380749 0.00907949 2023-09-23T19:53:06.765 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:06.836 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:06.928 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:06.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:06.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:07.001 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:53:07.037 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:07.130 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:07.141 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:07.230 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:07 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/793879620' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:07.230 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:07 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:07.230 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:07.230 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:07.231 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:07 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/793879620' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:07.231 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:07 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:07.231 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:07.231 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:07.254 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:53:07.302 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:07.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:07.415 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:07.460 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:07.566 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:07 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/793879620' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:07.566 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:07 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:07.566 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:07.566 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:07.653 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:07.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:07.680 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T19:53:07.681212+0000 min lat: 0.000994654 max lat: 0.330188 avg lat: 0.00915699 2023-09-23T19:53:07.680 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:53:07.680 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 40 16 69909 69893 109.156 73.25 0.00753193 0.00915699 2023-09-23T19:53:07.760 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:07.804 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: 1.6 scrub ok 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: pgmap v263: 99 pgs: 99 active+clean; 4.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[221113]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T19:53:08.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: 1.6 scrub ok 2023-09-23T19:53:08.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: pgmap v263: 99 pgs: 99 active+clean; 4.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2023-09-23T19:53:08.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:08.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:53:08.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T19:53:08.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:08 smithi135 ceph-mon[213871]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.585 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:08.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: 1.6 scrub ok 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: pgmap v263: 99 pgs: 99 active+clean; 4.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:08 smithi144 ceph-mon[194833]: failed to encode map e126 with expected crc 2023-09-23T19:53:08.680 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 41 16 70997 70981 108.152 68 0.00852219 0.00924129 2023-09-23T19:53:08.690 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:08.760 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:08.844 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:53:08.844 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:53:08.856 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:08.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:08.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:08.963 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:08.989 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:09.404 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:09.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:09.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:09.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:09 smithi135 ceph-mon[213871]: pgmap v265: 99 pgs: 99 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.17k op/s 2023-09-23T19:53:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:09 smithi135 ceph-mon[221113]: pgmap v265: 99 pgs: 99 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.17k op/s 2023-09-23T19:53:09.508 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:09.551 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:09.572 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:09.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:09 smithi144 ceph-mon[194833]: pgmap v265: 99 pgs: 99 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.17k op/s 2023-09-23T19:53:09.674 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:09.680 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 42 16 71941 71925 106.982 59 0.0123301 0.00934244 2023-09-23T19:53:09.737 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:09.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:09.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:09.841 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:09.900 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:09.989 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:10.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:10.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:10.101 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:10.178 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:10.461 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:53:10.499 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:53:10.531 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:10.537 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:53:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:10.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:10.538 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4123203900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:10.539 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4123203900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:10.574 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:53:10.610 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:53:10.639 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:10.651 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:53:10.682 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 43 16 73154 73138 106.253 75.8125 0.011607 0.00940676 2023-09-23T19:53:10.687 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:53:10.700 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:10.728 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:53:10.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:53:10.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:53:10.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:53:10.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:53:10.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:53:10.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:53:10.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:53:10.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:53:10.777 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:53:10.799 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:10.804 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4123203900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:10.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:10.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:10.906 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:10.975 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:11.683 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 44 16 74116 74100 105.204 60.125 0.0431567 0.00949345 2023-09-23T19:53:11.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:11 smithi144 ceph-mon[194833]: pgmap v266: 99 pgs: 99 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.15k op/s 2023-09-23T19:53:11.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:11 smithi135 ceph-mon[221113]: pgmap v266: 99 pgs: 99 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.15k op/s 2023-09-23T19:53:11.937 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:11 smithi135 ceph-mon[213871]: pgmap v266: 99 pgs: 99 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.15k op/s 2023-09-23T19:53:12.478 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:12.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:12.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:12.586 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:12.636 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:12.683 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 45 16 75237 75221 104.423 70.0625 0.0114876 0.00957178 2023-09-23T19:53:12.731 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:12.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:12.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:12.844 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:12.903 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:12.997 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:13.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:13.108 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:13.120 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:13.155 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:13.234 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:13.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:13.345 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:13.390 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:13.493 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:13.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:13.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:13.602 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:13.663 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:13.683 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 46 16 76384 76368 103.711 71.6875 0.0131458 0.0096371 2023-09-23T19:53:13.694 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:53:13.694 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:53:13.734 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:13.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:13.844 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:13.912 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:13.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:53:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:53:13] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.43.0" 2023-09-23T19:53:14.000 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:14.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:14.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:14.112 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:14.164 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:14.180 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:53:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:14 smithi135 ceph-mon[221113]: pgmap v267: 99 pgs: 99 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:53:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2390012947' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:53:14.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:14 smithi135 ceph-mon[213871]: pgmap v267: 99 pgs: 99 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:53:14.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2390012947' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:53:14.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:14 smithi144 ceph-mon[194833]: pgmap v267: 99 pgs: 99 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T19:53:14.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2390012947' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:53:14.683 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 47 16 77392 77376 102.845 63 0.0145882 0.00971922 2023-09-23T19:53:14.892 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:53:14.938 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:14.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:15.047 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:15.098 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:15.175 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:15.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:15.185 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2390012947' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:53:15.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:15 smithi135 ceph-mon[213871]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T19:53:15.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:15 smithi135 ceph-mon[213871]: failed to encode map e127 with expected crc 2023-09-23T19:53:15.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:15 smithi135 ceph-mon[213871]: failed to encode map e127 with expected crc 2023-09-23T19:53:15.285 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:15.339 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:15.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2390012947' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:53:15.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:15 smithi144 ceph-mon[194833]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T19:53:15.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:15 smithi144 ceph-mon[194833]: failed to encode map e127 with expected crc 2023-09-23T19:53:15.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:15 smithi144 ceph-mon[194833]: failed to encode map e127 with expected crc 2023-09-23T19:53:15.412 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:15.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2390012947' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:53:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:15 smithi135 ceph-mon[221113]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T19:53:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:15 smithi135 ceph-mon[221113]: failed to encode map e127 with expected crc 2023-09-23T19:53:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:15 smithi135 ceph-mon[221113]: failed to encode map e127 with expected crc 2023-09-23T19:53:15.519 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:15.575 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:15.685 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:15.686 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 48 16 78395 78379 102.003 62.6875 0.0123581 0.00979851 2023-09-23T19:53:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:15.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:15.795 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:15.858 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:15.909 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:15.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:15.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:16.017 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:16.086 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:16.325 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:16 smithi135 ceph-mon[221113]: failed to encode map e127 with expected crc 2023-09-23T19:53:16.325 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:16 smithi135 ceph-mon[221113]: pgmap v269: 99 pgs: 99 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:53:16.326 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:16 smithi135 ceph-mon[213871]: failed to encode map e127 with expected crc 2023-09-23T19:53:16.326 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:16 smithi135 ceph-mon[213871]: pgmap v269: 99 pgs: 99 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:53:16.354 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:53:16.566 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:53:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:16 smithi144 ceph-mon[194833]: failed to encode map e127 with expected crc 2023-09-23T19:53:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:16 smithi144 ceph-mon[194833]: pgmap v269: 99 pgs: 99 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T19:53:16.635 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:16.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:16.686 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 49 16 79484 79468 101.311 68.0625 0.0091813 0.00986544 2023-09-23T19:53:16.744 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:16.754 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:16.794 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:16.863 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:16.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:16.970 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:17.033 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:17.345 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:17.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:17.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:17.459 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:17.500 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:17.504 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:17.557 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:17.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:17.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:17.666 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:17.687 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 50 16 80528 80512 100.588 65.25 0.00696101 0.00993744 2023-09-23T19:53:17.719 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:17.812 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:17.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:17.919 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:17.987 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:18.080 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:18.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:18.188 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:18.210 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:53:18.220 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:18 smithi144 ceph-mon[194833]: pgmap v270: 99 pgs: 99 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:53:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3105752339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:18.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:18 smithi135 ceph-mon[213871]: pgmap v270: 99 pgs: 99 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:53:18.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3105752339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:18.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:18.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:18 smithi135 ceph-mon[221113]: pgmap v270: 99 pgs: 99 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T19:53:18.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3105752339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:18.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:18.687 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 51 16 81400 81384 99.6847 54.5 0.0079345 0.010027 2023-09-23T19:53:18.738 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:53:18.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:18.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:18.846 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:18.893 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:18.949 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:19.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:19.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[213871]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:19.207 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[213871]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T19:53:19.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[213871]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[213871]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[213871]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[221113]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[221113]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T19:53:19.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[221113]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[221113]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:19 smithi135 ceph-mon[221113]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:19 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:19 smithi144 ceph-mon[194833]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:19 smithi144 ceph-mon[194833]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T19:53:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:19 smithi144 ceph-mon[194833]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:19 smithi144 ceph-mon[194833]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:19 smithi144 ceph-mon[194833]: failed to encode map e128 with expected crc 2023-09-23T19:53:19.681 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:53:19.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:19.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:19.688 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 52 16 82270 82254 98.8137 54.375 0.0113347 0.0101149 2023-09-23T19:53:19.788 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:19.840 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:20.382 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:20 smithi135 ceph-mon[213871]: pgmap v272: 99 pgs: 99 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T19:53:20.383 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:20 smithi135 ceph-mon[221113]: pgmap v272: 99 pgs: 99 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T19:53:20.637 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:20 smithi144 ceph-mon[194833]: pgmap v272: 99 pgs: 99 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T19:53:20.687 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 53 16 83375 83359 98.2524 69.0625 0.0119279 0.0101731 2023-09-23T19:53:20.966 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:53:20.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:53:20.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:53:20.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:53:20.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:53:20.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:53:20.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:53:20.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:53:20.967 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:53:20.990 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:21.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:21.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:21.108 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:21.175 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2560279060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:21.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2560279060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:21.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2560279060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:21.687 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 54 16 84337 84321 97.5466 60.125 0.00812738 0.010247 2023-09-23T19:53:21.894 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:53:22.088 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:22.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:22.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:22.197 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:22.222 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:22 smithi135 ceph-mon[221113]: pgmap v273: 99 pgs: 99 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-09-23T19:53:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:22.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:22 smithi135 ceph-mon[213871]: pgmap v273: 99 pgs: 99 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-09-23T19:53:22.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:22.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:22 smithi144 ceph-mon[194833]: pgmap v273: 99 pgs: 99 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-09-23T19:53:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:22.687 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 55 16 85147 85131 96.6937 50.625 0.010259 0.0103373 2023-09-23T19:53:22.804 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:22.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:22.912 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:22.977 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:23.034 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:23.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:23.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:23.097 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:23.144 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:23.192 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:23.269 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:23.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:23.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:23.374 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:23.430 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:23.522 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:23.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:23.634 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:23.685 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:53:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:53:23] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T19:53:23.687 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:23.696 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 56 16 86207 86191 96.1339 66.25 0.0175291 0.0103971 2023-09-23T19:53:23.772 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:23.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:23.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:23.888 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:23.959 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:24.095 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:24.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:24.206 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:24.215 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:53:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T19:53:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T19:53:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T19:53:24.215 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:53:24.271 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:53:24.298 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:24 smithi144 ceph-mon[194833]: pgmap v274: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T19:53:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3488638896' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:53:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:24 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:53:24.697 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 57 16 87218 87202 95.5561 63.1875 0.0127738 0.0104606 2023-09-23T19:53:24.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:24 smithi135 ceph-mon[221113]: pgmap v274: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T19:53:24.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3488638896' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:53:24.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:24 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:53:24.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:24 smithi135 ceph-mon[213871]: pgmap v274: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T19:53:24.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3488638896' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:53:24.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:24 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:53:25.023 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:53:25.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:53:25.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[221113]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T19:53:25.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[221113]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[221113]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[221113]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[221113]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[221113]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[221113]: pgmap v276: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T19:53:25.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:53:25.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[213871]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T19:53:25.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[213871]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[213871]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[213871]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[213871]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[213871]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:25 smithi135 ceph-mon[213871]: pgmap v276: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T19:53:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:25 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:53:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:25 smithi144 ceph-mon[194833]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T19:53:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:25 smithi144 ceph-mon[194833]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:25 smithi144 ceph-mon[194833]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:25 smithi144 ceph-mon[194833]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:25 smithi144 ceph-mon[194833]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:25 smithi144 ceph-mon[194833]: failed to encode map e129 with expected crc 2023-09-23T19:53:25.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:25 smithi144 ceph-mon[194833]: pgmap v276: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T19:53:25.697 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 58 16 88097 88081 94.8561 54.9375 0.0117875 0.0105371 2023-09-23T19:53:25.757 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:25.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:25.863 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:25.924 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:26.234 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:26.237 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:53:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:26.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:26.345 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:26.465 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:26.697 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 59 16 89181 89165 94.3968 67.75 0.0137312 0.0105891 2023-09-23T19:53:27.017 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:53:27.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:27.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:27.121 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:27.168 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:53:27.168 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:26 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:53:27.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:26 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:53:27.212 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:27.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:26 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:53:27.506 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:27.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:27.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:27.523 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:27.524 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:27.619 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:27.659 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:27.697 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T19:53:27.698225+0000 min lat: 0.000994654 max lat: 0.330188 avg lat: 0.0106603 2023-09-23T19:53:27.697 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:53:27.697 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 60 16 90082 90066 93.7624 56.3125 0.0304141 0.0106603 2023-09-23T19:53:27.812 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:27.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:27.917 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:28.006 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:28 smithi135 ceph-mon[213871]: pgmap v277: 99 pgs: 99 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-09-23T19:53:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2328710628' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:28 smithi135 ceph-mon[221113]: pgmap v277: 99 pgs: 99 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-09-23T19:53:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2328710628' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:28.486 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:28.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:28.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:28.598 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:28 smithi144 ceph-mon[194833]: pgmap v277: 99 pgs: 99 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-09-23T19:53:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2328710628' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:53:28.651 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:28.700 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 61 15 91023 91008 93.1851 58.875 0.0118748 0.0107264 2023-09-23T19:53:28.921 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:53:29.162 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:29 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:29.163 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2328710628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:53:29.163 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:29 smithi135 ceph-mon[213871]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T19:53:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:29 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2328710628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:53:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:29 smithi135 ceph-mon[221113]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T19:53:29.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:29 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:29.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2328710628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:53:29.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:29 smithi144 ceph-mon[194833]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T19:53:29.701 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 62 16 91892 91876 92.5574 54.25 0.0657395 0.0107941 2023-09-23T19:53:30.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:30 smithi135 ceph-mon[213871]: failed to encode map e130 with expected crc 2023-09-23T19:53:30.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:30 smithi135 ceph-mon[213871]: pgmap v279: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:53:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:30 smithi135 ceph-mon[221113]: failed to encode map e130 with expected crc 2023-09-23T19:53:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:30 smithi135 ceph-mon[221113]: pgmap v279: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:53:30.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:30 smithi144 ceph-mon[194833]: failed to encode map e130 with expected crc 2023-09-23T19:53:30.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:30 smithi144 ceph-mon[194833]: pgmap v279: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T19:53:30.701 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 63 16 92860 92844 92.0487 60.5 0.006163 0.0108599 2023-09-23T19:53:31.492 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:31.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:31.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:31.517 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:31.601 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:31.686 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:31.701 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 64 16 93616 93600 91.3491 47.25 0.00823178 0.0109429 2023-09-23T19:53:31.997 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:32.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:32.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:32.085 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:53:32.102 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:32.129 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:53:32.182 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:53:32.210 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:32.234 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:53:32.274 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:53:32.292 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:32 smithi135 ceph-mon[221113]: pgmap v280: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s 2023-09-23T19:53:32.292 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3017644321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:32.297 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:32 smithi135 ceph-mon[213871]: pgmap v280: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s 2023-09-23T19:53:32.297 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3017644321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:32.313 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:53:32.349 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:53:32.394 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:32.396 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:53:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:32.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:32.398 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:53:32.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:53:32.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:53:32.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:53:32.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:53:32.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:53:32.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:53:32.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:53:32.448 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:53:32.449 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:53:32.499 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:32.591 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:32.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:32 smithi144 ceph-mon[194833]: pgmap v280: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s 2023-09-23T19:53:32.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3017644321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:32.701 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 65 16 94512 94496 90.8054 56 0.0129637 0.011008 2023-09-23T19:53:33.406 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:53:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:33 smithi144 ceph-mon[194833]: pgmap v281: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:53:33.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:33.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:33.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:33 smithi135 ceph-mon[213871]: pgmap v281: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:53:33.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:33.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:33.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:33 smithi135 ceph-mon[221113]: pgmap v281: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T19:53:33.701 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 66 16 95395 95379 90.266 55.1875 0.0250052 0.0110696 2023-09-23T19:53:33.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:53:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:53:33] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T19:53:34.701 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 67 16 96285 96269 89.7492 55.625 0.0102513 0.0111373 2023-09-23T19:53:34.809 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:34.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:34.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:34.916 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:34.970 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:35.027 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:53:35.035 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:35.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:35.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:35.101 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:34 smithi135 ceph-mon[213871]: 1.3 scrub starts 2023-09-23T19:53:35.101 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:34 smithi135 ceph-mon[213871]: 1.3 scrub ok 2023-09-23T19:53:35.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:34 smithi135 ceph-mon[221113]: 1.3 scrub starts 2023-09-23T19:53:35.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:34 smithi135 ceph-mon[221113]: 1.3 scrub ok 2023-09-23T19:53:35.145 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:35.180 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:34 smithi144 ceph-mon[194833]: 1.3 scrub starts 2023-09-23T19:53:35.181 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:34 smithi144 ceph-mon[194833]: 1.3 scrub ok 2023-09-23T19:53:35.184 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:35.235 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:35.270 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:35.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:35.387 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:35.423 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:35.522 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:35.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:35.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:35.641 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:35.681 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:35.706 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 68 16 97197 97181 89.2617 57 0.0217986 0.0111976 2023-09-23T19:53:35.744 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:35.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:35.852 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:35.950 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:35 smithi135 ceph-mon[221113]: pgmap v282: 99 pgs: 99 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T19:53:35.950 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:35 smithi135 ceph-mon[213871]: pgmap v282: 99 pgs: 99 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T19:53:35.951 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:35 smithi144 ceph-mon[194833]: pgmap v282: 99 pgs: 99 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T19:53:36.706 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 69 16 98086 98070 88.7735 55.5625 0.00403274 0.0112568 2023-09-23T19:53:36.927 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:53:36.946 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:53:37.216 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1156953446' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:53:37.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1156953446' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:53:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:37.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:37.326 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1156953446' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:53:37.414 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:37.708 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 70 16 99014 98998 88.332 58 0.0149001 0.0113167 2023-09-23T19:53:38.042 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:38.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:38.149 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:38.190 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1156953446' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:38.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[221113]: pgmap v284: 99 pgs: 99 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T19:53:38.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1156953446' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:53:38.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T19:53:38.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.193 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:38.193 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:37 smithi135 ceph-mon[213871]: pgmap v284: 99 pgs: 99 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T19:53:38.216 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1156953446' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:53:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T19:53:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: failed to encode map e131 with expected crc 2023-09-23T19:53:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:38.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:37 smithi144 ceph-mon[194833]: pgmap v284: 99 pgs: 99 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T19:53:38.514 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:38.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:38.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:38.623 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:38.670 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:38.708 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 71 16 99937 99921 87.9006 57.6875 0.0272964 0.0113711 2023-09-23T19:53:38.767 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:38.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:38.855 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:38.880 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:38.910 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:39.014 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:39.155 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:53:39.239 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:39.240 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/360135008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.240 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:39 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.241 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/360135008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.241 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:39 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/360135008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.636 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:39 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:53:39.708 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 72 16 100856 100840 87.4777 57.4375 0.0239754 0.011427 2023-09-23T19:53:40.151 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:53:40.151 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T19:53:40.151 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:40.151 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T19:53:40.152 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.261 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:40.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:40.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:40.368 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: pgmap v286: 99 pgs: 99 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.427 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:40 smithi144 ceph-mon[194833]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.430 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: pgmap v286: 99 pgs: 99 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T19:53:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[213871]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:53:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T19:53:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:53:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T19:53:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: pgmap v286: 99 pgs: 99 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T19:53:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:40 smithi135 ceph-mon[221113]: failed to encode map e132 with expected crc 2023-09-23T19:53:40.559 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:40.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:40.671 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:40.709 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 73 16 101641 101625 86.9516 49.0625 0.012645 0.0114954 2023-09-23T19:53:40.756 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:41.369 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:41.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:41.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:41.482 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:41.557 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:41.666 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:41.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:41.708 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 74 16 102607 102591 86.5926 60.375 0.0164196 0.0115437 2023-09-23T19:53:41.776 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:41.836 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:41.938 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:41.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:42.052 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:42.117 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:42 smithi135 ceph-mon[221113]: pgmap v287: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2023-09-23T19:53:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:42 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:42 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:42 smithi135 ceph-mon[213871]: pgmap v287: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2023-09-23T19:53:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:42 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:42 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:42 smithi144 ceph-mon[194833]: pgmap v287: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2023-09-23T19:53:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:42 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:42 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:42.708 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 75 16 103459 103443 86.1482 53.25 0.0115451 0.0116034 2023-09-23T19:53:42.762 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:53:42.840 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:42.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:42.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:42.949 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:43.027 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:43.122 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:43.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:43.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:43.167 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:43 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:53:43.168 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:43 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:53:43.168 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:43 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:43.235 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:43.268 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:43.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:43 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:53:43.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:43 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:53:43.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:43 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:43.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:43 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:53:43.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:43 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:53:43.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:43 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:53:43.482 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:43.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:43.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:43.593 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:43.681 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:43.713 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 76 16 104377 104361 85.7641 57.375 0.0307153 0.0116547 2023-09-23T19:53:43.801 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:43.923 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:53:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:53:43] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.43.0" 2023-09-23T19:53:43.939 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:43.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:43.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:44.048 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:44.132 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:44.180 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:44 smithi135 ceph-mon[213871]: pgmap v288: 99 pgs: 99 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T19:53:44.221 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:44.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:44.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:44.295 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:53:44.295 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:53:44.328 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:44 smithi144 ceph-mon[194833]: pgmap v288: 99 pgs: 99 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T19:53:44.410 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:44.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:44 smithi135 ceph-mon[221113]: pgmap v288: 99 pgs: 99 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T19:53:44.713 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 77 16 105291 105275 85.3923 57.125 0.0135681 0.0117067 2023-09-23T19:53:45.470 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:45.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:45.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:45.580 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:45.606 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:45.648 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:45.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:45.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:45.713 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 78 16 105998 105982 84.8643 44.1875 0.171399 0.0117704 2023-09-23T19:53:45.754 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:45.837 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:45.975 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:53:46.123 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:53:46.164 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:53:46.209 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:53:46.220 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:53:46.226 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:46.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:46.261 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:53:46.300 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:53:46.331 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:53:46.335 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:46.371 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:53:46.393 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:46.417 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:53:46.425 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:53:46.425 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:53:46.425 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:53:46.425 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:53:46.425 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:53:46.425 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:53:46.425 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:53:46.425 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:53:46.450 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:46 smithi135 ceph-mon[213871]: pgmap v289: 99 pgs: 99 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T19:53:46.450 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3825498413' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:53:46.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:53:46.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2903490119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:46.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:46 smithi135 ceph-mon[221113]: pgmap v289: 99 pgs: 99 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T19:53:46.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3825498413' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:53:46.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:53:46.451 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2903490119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:46.493 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:46.511 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:46 smithi144 ceph-mon[194833]: pgmap v289: 99 pgs: 99 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T19:53:46.511 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3825498413' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:53:46.511 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:53:46.511 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2903490119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:46.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:46.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:46.613 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:46.661 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:46.714 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 79 16 106900 106884 84.5038 56.375 0.0108839 0.011829 2023-09-23T19:53:46.747 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:46.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:46.858 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:46.920 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:47.219 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:53:47.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[213871]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T19:53:47.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[213871]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[213871]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[213871]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:53:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[221113]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T19:53:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[221113]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[221113]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:47 smithi135 ceph-mon[221113]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:47 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:53:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:47 smithi144 ceph-mon[194833]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T19:53:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:47 smithi144 ceph-mon[194833]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:47 smithi144 ceph-mon[194833]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:47 smithi144 ceph-mon[194833]: failed to encode map e133 with expected crc 2023-09-23T19:53:47.713 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T19:53:47.714870+0000 min lat: 0.000994654 max lat: 0.330188 avg lat: 0.0118837 2023-09-23T19:53:47.714 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:53:47.714 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 80 16 107684 107668 84.0602 49 0.00335603 0.0118837 2023-09-23T19:53:47.796 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:53:47.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:47.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:47.901 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:47.953 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:48.264 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T19:53:48.269 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:48.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:48.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:48.374 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:48.454 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:48.520 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:48 smithi135 ceph-mon[213871]: pgmap v291: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-09-23T19:53:48.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:48 smithi135 ceph-mon[221113]: pgmap v291: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-09-23T19:53:48.583 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:48.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:48.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:48 smithi144 ceph-mon[194833]: pgmap v291: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-09-23T19:53:48.695 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:48.707 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:48.713 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 81 16 108482 108466 83.6384 49.875 0.0193232 0.0119514 2023-09-23T19:53:48.733 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:48.797 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:48.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:48.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:48.907 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:48.952 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:49.042 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:49.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:49.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:49.151 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:49.212 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:49.260 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:53:49.293 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:49.301 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:49.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:49.319 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:49 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3225552459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:49.418 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:49.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:49 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3225552459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:49.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:49 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3225552459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:53:49.478 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:49.568 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:49.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:49.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:49.681 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:49.716 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 82 16 109348 109332 83.2771 54.125 0.0171366 0.0120031 2023-09-23T19:53:49.731 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:50.028 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:53:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[221113]: pgmap v292: 99 pgs: 99 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T19:53:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3225552459' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:53:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[221113]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T19:53:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[221113]: failed to encode map e134 with expected crc 2023-09-23T19:53:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[221113]: failed to encode map e134 with expected crc 2023-09-23T19:53:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[213871]: pgmap v292: 99 pgs: 99 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T19:53:50.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:50.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3225552459' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:53:50.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[213871]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T19:53:50.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[213871]: failed to encode map e134 with expected crc 2023-09-23T19:53:50.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:50 smithi135 ceph-mon[213871]: failed to encode map e134 with expected crc 2023-09-23T19:53:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:50 smithi144 ceph-mon[194833]: pgmap v292: 99 pgs: 99 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T19:53:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:50 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:53:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3225552459' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:53:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:50 smithi144 ceph-mon[194833]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T19:53:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:50 smithi144 ceph-mon[194833]: failed to encode map e134 with expected crc 2023-09-23T19:53:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:50 smithi144 ceph-mon[194833]: failed to encode map e134 with expected crc 2023-09-23T19:53:50.715 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 83 16 110158 110142 82.8839 50.625 0.00430824 0.0120439 2023-09-23T19:53:51.005 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:53:51.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:51.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:51.112 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:51.167 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:51.249 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:51.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:51.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:51.354 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:51.415 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:51.500 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:51.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:51.613 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:51.680 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:51.716 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 84 16 111023 111007 82.5409 54.0625 0.0243907 0.0121109 2023-09-23T19:53:51.776 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:51.778 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:53:51.778 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T19:53:51.778 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T19:53:51.779 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T19:53:51.779 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:53:51.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:51.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:51.898 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:51.954 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:52.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:52 smithi144 ceph-mon[194833]: pgmap v294: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s 2023-09-23T19:53:52.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:52.716 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 85 16 111848 111832 82.1766 51.5625 0.00685125 0.0121643 2023-09-23T19:53:52.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:52 smithi135 ceph-mon[221113]: pgmap v294: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s 2023-09-23T19:53:52.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:52.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:52 smithi135 ceph-mon[213871]: pgmap v294: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s 2023-09-23T19:53:52.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:53:52.821 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:52.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:52.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:52.929 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:52.995 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:53.099 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:53.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:53.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:53.177 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:53:53.207 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:53.265 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:53.327 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:53.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:53.376 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:53:53.412 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:53.435 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:53.484 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:53.561 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:53.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:53.669 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:53.718 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 86 16 112706 112690 81.8428 53.625 0.0178289 0.012214 2023-09-23T19:53:53.734 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:53.799 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:53.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:53.810 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:53:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:53:53] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.43.0" 2023-09-23T19:53:53.906 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:53:53.951 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:54.028 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:53:54.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:54.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:53:54.140 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:53:54.238 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:54.412 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:53:54.430 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:54 smithi135 ceph-mon[213871]: pgmap v295: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:53:54.431 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:54 smithi135 ceph-mon[221113]: pgmap v295: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:53:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:54.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:53:54.532 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:53:54.602 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:54 smithi144 ceph-mon[194833]: pgmap v295: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T19:53:54.663 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:53:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:54.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:53:54.718 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 87 16 113472 113456 81.4525 47.875 0.00434868 0.0122624 2023-09-23T19:53:54.766 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:54.801 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:54.891 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:54.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:55.007 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:53:55.079 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:55.192 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:53:55.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:55.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:53:55.242 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:55.312 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:55.332 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T19:53:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:55 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1183602340' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:53:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:55 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:53:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:55 smithi144 ceph-mon[194833]: pgmap v296: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T19:53:55.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:55.698 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:55 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1183602340' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:53:55.699 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:55 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:53:55.699 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:55 smithi135 ceph-mon[213871]: pgmap v296: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T19:53:55.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:55 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1183602340' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:53:55.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:55 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:53:55.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:55 smithi135 ceph-mon[221113]: pgmap v296: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T19:53:55.718 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 88 16 114286 114270 81.1052 50.875 0.014601 0.0123244 2023-09-23T19:53:55.780 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:55.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:55.885 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:53:55.944 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:56.060 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:53:56.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:56.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:53:56.173 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:56.200 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:56.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:56 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:53:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:56 smithi135 ceph-mon[221113]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T19:53:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:56 smithi135 ceph-mon[221113]: failed to encode map e135 with expected crc 2023-09-23T19:53:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:56 smithi135 ceph-mon[221113]: failed to encode map e135 with expected crc 2023-09-23T19:53:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:56 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:53:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:56 smithi135 ceph-mon[213871]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T19:53:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:56 smithi135 ceph-mon[213871]: failed to encode map e135 with expected crc 2023-09-23T19:53:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:56 smithi135 ceph-mon[213871]: failed to encode map e135 with expected crc 2023-09-23T19:53:56.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:56 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:53:56.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:56 smithi144 ceph-mon[194833]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T19:53:56.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:56 smithi144 ceph-mon[194833]: failed to encode map e135 with expected crc 2023-09-23T19:53:56.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:56 smithi144 ceph-mon[194833]: failed to encode map e135 with expected crc 2023-09-23T19:53:56.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 89 16 115075 115059 80.7411 49.3125 0.0220465 0.0123809 2023-09-23T19:53:56.801 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:56.837 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T19:53:57.547 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:53:57.584 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:53:57.628 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:53:57.667 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:53:57.689 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:53:57.708 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total time run: 90.0104 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total writes made: 115781 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Write size: 65536 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object size: 65536 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Bandwidth (MB/sec): 80.3942 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Bandwidth: 39.6065 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max bandwidth (MB/sec): 253.875 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min bandwidth (MB/sec): 44.1875 2023-09-23T19:53:57.726 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average IOPS: 1286 2023-09-23T19:53:57.727 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev IOPS: 633.704 2023-09-23T19:53:57.727 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max IOPS: 4062 2023-09-23T19:53:57.727 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min IOPS: 707 2023-09-23T19:53:57.727 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average Latency(s): 0.0124348 2023-09-23T19:53:57.727 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Latency(s): 0.0150918 2023-09-23T19:53:57.727 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max latency(s): 0.330188 2023-09-23T19:53:57.727 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min latency(s): 0.000994654 2023-09-23T19:53:57.728 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:53:57.731 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:53:57.759 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:53:57.774 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:53:57.774 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:53:57.774 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:53:57.774 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:53:57.774 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:53:57.774 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:53:57.774 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:53:57.774 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:53:57.782 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:53:57.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:57.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:53:57.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:57.965 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:58.422 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:58 smithi135 ceph-mon[221113]: pgmap v298: 99 pgs: 99 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-09-23T19:53:58.423 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:53:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4282188085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:58.423 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:58 smithi135 ceph-mon[213871]: pgmap v298: 99 pgs: 99 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-09-23T19:53:58.423 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:53:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4282188085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:58 smithi144 ceph-mon[194833]: pgmap v298: 99 pgs: 99 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-09-23T19:53:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:53:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4282188085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:53:58.771 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:58.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:58.877 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:53:58.926 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:59.019 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:53:59.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:59.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:53:59.124 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:53:59.153 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:53:59.741 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:53:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:53:59.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:53:59.853 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:53:59.874 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:53:59.947 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:00.193 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:00 smithi135 ceph-mon[213871]: pgmap v299: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 833 op/s 2023-09-23T19:54:00.194 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:00.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:00.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:00.308 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:00.376 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:00.450 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:00.468 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:00 smithi144 ceph-mon[194833]: pgmap v299: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 833 op/s 2023-09-23T19:54:00.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:00 smithi135 ceph-mon[221113]: pgmap v299: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 833 op/s 2023-09-23T19:54:00.554 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:00.554 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:00.569 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:00.649 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:00.750 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:00.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:00.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:00.855 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:00.908 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:01.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:01 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/453992881' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:01.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:01 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:01.224 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:54:01.379 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:54:01.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:01 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/453992881' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:01.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:01 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:01 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/453992881' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:01 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[221113]: pgmap v300: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 819 op/s 2023-09-23T19:54:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[221113]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[221113]: failed to encode map e136 with expected crc 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[221113]: failed to encode map e136 with expected crc 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[213871]: pgmap v300: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 819 op/s 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[213871]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[213871]: failed to encode map e136 with expected crc 2023-09-23T19:54:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:02 smithi135 ceph-mon[213871]: failed to encode map e136 with expected crc 2023-09-23T19:54:02.520 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:54:02.575 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:02.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:02.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:02.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:02 smithi144 ceph-mon[194833]: pgmap v300: 99 pgs: 99 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 819 op/s 2023-09-23T19:54:02.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:02 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:02.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:02 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:02.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:02 smithi144 ceph-mon[194833]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T19:54:02.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:02 smithi144 ceph-mon[194833]: failed to encode map e136 with expected crc 2023-09-23T19:54:02.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:02 smithi144 ceph-mon[194833]: failed to encode map e136 with expected crc 2023-09-23T19:54:02.679 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:02.706 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:03.097 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:03.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:03.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:03.194 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:03.203 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:03.258 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:03.373 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:03 smithi135 ceph-mon[221113]: failed to encode map e136 with expected crc 2023-09-23T19:54:03.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:03 smithi135 ceph-mon[213871]: failed to encode map e136 with expected crc 2023-09-23T19:54:03.455 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:03.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:03.505 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:03 smithi144 ceph-mon[194833]: failed to encode map e136 with expected crc 2023-09-23T19:54:03.560 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:03.566 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:03.617 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:03.692 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:54:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:54:03] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.43.0" 2023-09-23T19:54:03.766 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:03.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:03.871 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:03.946 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:04.246 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T19:54:04.260 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:04.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:04.372 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:04.440 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:04.485 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:04.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:04 smithi144 ceph-mon[194833]: pgmap v302: 99 pgs: 99 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 33 MiB/s wr, 961 op/s 2023-09-23T19:54:04.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2158895281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:54:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:04.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:04.588 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:04.674 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:04.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:04 smithi135 ceph-mon[213871]: pgmap v302: 99 pgs: 99 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 33 MiB/s wr, 961 op/s 2023-09-23T19:54:04.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2158895281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:54:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:04 smithi135 ceph-mon[221113]: pgmap v302: 99 pgs: 99 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 33 MiB/s wr, 961 op/s 2023-09-23T19:54:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2158895281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:54:04.790 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:04.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:04.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:04.900 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:04.928 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:05.291 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:54:05.399 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:05.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:05.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:05.511 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:05.576 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2158895281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:54:05.576 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:05 smithi144 ceph-mon[194833]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T19:54:05.576 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:05 smithi144 ceph-mon[194833]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.576 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:05 smithi144 ceph-mon[194833]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.577 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:05 smithi144 ceph-mon[194833]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.577 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:05 smithi144 ceph-mon[194833]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.577 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:05 smithi144 ceph-mon[194833]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.577 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:05 smithi144 ceph-mon[194833]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.585 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:05.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2158895281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:54:05.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[213871]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T19:54:05.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[213871]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[213871]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[213871]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[213871]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[213871]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[213871]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2158895281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[221113]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[221113]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[221113]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[221113]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[221113]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[221113]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:05 smithi135 ceph-mon[221113]: failed to encode map e137 with expected crc 2023-09-23T19:54:05.774 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:05.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:05.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:05.884 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:05.947 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:05.969 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T19:54:06.090 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:06.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:06.197 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:06.214 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:06.214 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:06.265 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:06.309 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:06 smithi135 ceph-mon[221113]: pgmap v304: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 894 op/s 2023-09-23T19:54:06.309 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:06 smithi135 ceph-mon[213871]: pgmap v304: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 894 op/s 2023-09-23T19:54:06.460 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:06.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:06.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:06.569 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:06.620 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:54:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:06 smithi144 ceph-mon[194833]: pgmap v304: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 894 op/s 2023-09-23T19:54:06.655 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:54:06.657 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:06.674 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:06.697 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:54:06.718 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:06.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:54:06.737 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:06.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:06.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:06.748 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:54:06.842 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:06.869 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:06.910 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:54:06.955 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:54:06.992 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:54:07.007 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:54:07.007 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:54:07.008 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:54:07.008 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:54:07.008 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:54:07.008 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:54:07.008 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:54:07.008 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:54:07.157 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:07.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:07.261 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:07.287 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:54:07.354 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:07.417 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:07 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/928736177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:07.417 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:07 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/928736177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:07 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/928736177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:07.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:07.501 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:54:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:07.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:07.609 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:07.638 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:07.869 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:08.268 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:08.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:08.372 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:08.415 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:08.435 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:08 smithi144 ceph-mon[194833]: pgmap v305: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 4.1 MiB/s wr, 617 op/s 2023-09-23T19:54:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:08 smithi135 ceph-mon[221113]: pgmap v305: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 4.1 MiB/s wr, 617 op/s 2023-09-23T19:54:08.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:08 smithi135 ceph-mon[213871]: pgmap v305: 99 pgs: 99 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 4.1 MiB/s wr, 617 op/s 2023-09-23T19:54:09.573 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:09.578 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:09 smithi135 ceph-mon[221113]: pgmap v306: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 832 KiB/s wr, 847 op/s 2023-09-23T19:54:09.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:09.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:09.579 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:09 smithi135 ceph-mon[213871]: pgmap v306: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 832 KiB/s wr, 847 op/s 2023-09-23T19:54:09.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:09 smithi144 ceph-mon[194833]: pgmap v306: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 832 KiB/s wr, 847 op/s 2023-09-23T19:54:09.679 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:09.728 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:09.819 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:09.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:09.924 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:10.034 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:10.146 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:54:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:10.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:10.176 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:54:10.176 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T19:54:10.176 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T19:54:10.176 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T19:54:10.176 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:54:10.252 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:10.308 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:10.469 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:10.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:10.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:10.584 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:10.700 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:10.763 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T19:54:10.774 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:10.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:10.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:10.878 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:10.904 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:10.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3890843355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:54:10.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:54:10.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3890843355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:54:10.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:54:10.988 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:10.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:11.093 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:11.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3890843355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:54:11.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:54:11.168 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:11.251 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:11.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:11.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:11.262 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:54:11.286 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:54:11.357 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:11.383 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:11.426 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:54:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:11.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:11.532 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:11.622 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:11.759 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:11.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:11 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:54:11.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:11 smithi144 ceph-mon[194833]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T19:54:11.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:11 smithi144 ceph-mon[194833]: failed to encode map e138 with expected crc 2023-09-23T19:54:11.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:11 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1169201049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:11.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:11 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:11.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:11 smithi144 ceph-mon[194833]: pgmap v308: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T19:54:11.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:11.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:11.807 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:54:11.880 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:11.885 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:11.941 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:12.047 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:54:12.047 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[213871]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T19:54:12.047 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[213871]: failed to encode map e138 with expected crc 2023-09-23T19:54:12.047 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1169201049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:12.047 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:12.047 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[213871]: pgmap v308: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T19:54:12.052 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:54:12.052 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[221113]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T19:54:12.052 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[221113]: failed to encode map e138 with expected crc 2023-09-23T19:54:12.052 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1169201049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:12.052 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:12.052 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:11 smithi135 ceph-mon[221113]: pgmap v308: 99 pgs: 99 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T19:54:12.073 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T19:54:12.077 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:12.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:12.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:12.190 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:12.256 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:12.370 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:12.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:12.480 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:12.527 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:12.795 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:12.795 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[213871]: failed to encode map e139 with expected crc 2023-09-23T19:54:12.796 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:12.796 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[213871]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T19:54:12.796 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[213871]: failed to encode map e139 with expected crc 2023-09-23T19:54:12.808 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:12.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:12.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:12.891 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:12.916 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:12.941 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:12.945 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:13.033 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:13.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:13.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:13.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:12 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:13.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:12 smithi144 ceph-mon[194833]: failed to encode map e139 with expected crc 2023-09-23T19:54:13.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:12 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:13.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:12 smithi144 ceph-mon[194833]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T19:54:13.136 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:12 smithi144 ceph-mon[194833]: failed to encode map e139 with expected crc 2023-09-23T19:54:13.136 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:13.148 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:13.148 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[221113]: failed to encode map e139 with expected crc 2023-09-23T19:54:13.148 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:13.148 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[221113]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T19:54:13.148 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:12 smithi135 ceph-mon[221113]: failed to encode map e139 with expected crc 2023-09-23T19:54:13.188 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:13.467 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:13.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:13.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:13.581 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:13.638 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:13.802 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:54:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:54:13] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T19:54:13.803 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:13 smithi135 ceph-mon[213871]: pgmap v310: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:54:13.805 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:13 smithi144 ceph-mon[194833]: pgmap v310: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:54:13.896 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:13.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:14.003 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:14.053 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:14.168 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:54:14.168 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T19:54:14.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T19:54:14.168 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T19:54:14.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.168 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:39683 5.e 5:734e9013:::benchmark_data_smithi135_284570_object39682:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T19:54:14.168 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:52:44.631488+0000", 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 89.527331990999997, 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.143358392, 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 39683 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.169 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631488+0000", 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631488+0000", 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631487+0000", 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631551+0000", 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3367999999999994e-05 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631557+0000", 2023-09-23T19:54:14.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6120000000000004e-06 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631561+0000", 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6059999999999999e-06 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631606+0000", 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.511e-05 2023-09-23T19:54:14.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631737+0000", 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013041300000000001 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.631888+0000", 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000151626 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.173 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.634547+0000", 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002659317 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.634566+0000", 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8893999999999999e-05 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.774745+0000", 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.14017885299999999 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.774804+0000", 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8740999999999999e-05 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:44.774846+0000", 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2225000000000003e-05 2023-09-23T19:54:14.175 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:124372 5.1 5:8e24453f:::benchmark_data_smithi135_284570_object8588:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.841063+0000", 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3177570630000002, 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17748480699999999, 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.176 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 124372 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841063+0000", 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841063+0000", 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841062+0000", 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841065+0000", 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.039e-06 2023-09-23T19:54:14.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841072+0000", 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4400000000000002e-06 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841087+0000", 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4958e-05 2023-09-23T19:54:14.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841220+0000", 2023-09-23T19:54:14.181 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013344400000000001 2023-09-23T19:54:14.181 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841299+0000", 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9046000000000002e-05 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841371+0000", 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1734000000000001e-05 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852740+0000", 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011369354999999999 2023-09-23T19:54:14.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855892+0000", 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031518729999999999 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018456+0000", 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16256414599999999 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018502+0000", 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5352000000000002e-05 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018548+0000", 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5843000000000001e-05 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:124373 5.1 5:83af82cc:::benchmark_data_smithi135_284570_object8589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.841099+0000", 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3177205159999996, 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17769863899999999, 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 124373 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.185 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841099+0000", 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841099+0000", 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841099+0000", 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841102+0000", 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2140000000000001e-06 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841108+0000", 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3849999999999999e-06 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841113+0000", 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8040000000000001e-06 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.844737+0000", 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0036240970000000002 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.844837+0000", 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9901000000000005e-05 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.844961+0000", 2023-09-23T19:54:14.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012399399999999999 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855897+0000", 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010936094 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018595+0000", 2023-09-23T19:54:14.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16269724799999999 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018721+0000", 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012601100000000001 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018762+0000", 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0890000000000003e-05 2023-09-23T19:54:14.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018798+0000", 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6399000000000002e-05 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:124374 5.1 5:89dbf51d:::benchmark_data_smithi135_284570_object8590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.841120+0000", 2023-09-23T19:54:14.192 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3177000510000001, 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17778350000000001, 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 124374 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841120+0000", 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841120+0000", 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841119+0000", 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841122+0000", 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.356e-06 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841128+0000", 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0100000000000001e-06 2023-09-23T19:54:14.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841132+0000", 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.189e-06 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848302+0000", 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0071699090000000004 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848416+0000", 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113943 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848931+0000", 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00051576699999999998 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855902+0000", 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.006970777 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018638+0000", 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.162735621 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018828+0000", 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000190423 2023-09-23T19:54:14.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018871+0000", 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2926999999999997e-05 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018903+0000", 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2113000000000002e-05 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.199 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:124377 5.e 5:74dcb83f:::benchmark_data_smithi135_284570_object8593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.841143+0000", 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3176770610000004, 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.177999927, 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.200 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 124377 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841143+0000", 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841143+0000", 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841142+0000", 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841145+0000", 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6690000000000002e-06 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841151+0000", 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9909999999999999e-06 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841155+0000", 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2010000000000001e-06 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.844711+0000", 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035559670000000002 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.844819+0000", 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107877 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.844961+0000", 2023-09-23T19:54:14.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014230800000000001 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855507+0000", 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010546125 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018588+0000", 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16308066099999999 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.019073+0000", 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00048536299999999999 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.019108+0000", 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4360999999999998e-05 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.019143+0000", 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4802000000000003e-05 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:124375 5.f e137/112 5:fa220f90:::benchmark_data_smithi135_284570_object8591:head v 137'7784, mlcod=137'7782)", 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.841496+0000", 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3173240220000002, 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.176962534, 2023-09-23T19:54:14.207 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841496+0000", 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841496+0000", 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841495+0000", 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841501+0000", 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3460000000000001e-06 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841509+0000", 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8310000000000007e-06 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841515+0000", 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.181e-06 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852153+0000", 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010638395 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852172+0000", 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8683000000000001e-05 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018422+0000", 2023-09-23T19:54:14.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.166250443 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018458+0000", 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6145e-05 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.212 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:124383 5.e 5:7865c7fe:::benchmark_data_smithi135_284570_object8599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.851950+0000", 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3068693769999999, 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16728146899999999, 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 124383 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.213 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851950+0000", 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851950+0000", 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851949+0000", 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851954+0000", 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1529999999999996e-06 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851964+0000", 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6250000000000002e-06 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851969+0000", 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2109999999999997e-06 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852009+0000", 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0252e-05 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852127+0000", 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000118306 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852243+0000", 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115869 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855646+0000", 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0034021479999999998 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018628+0000", 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.162982037 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.019171+0000", 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00054386899999999999 2023-09-23T19:54:14.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.019204+0000", 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2523e-05 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.019232+0000", 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7946e-05 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.219 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:124385 5.1 5:85f9a840:::benchmark_data_smithi135_284570_object8601:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.851974+0000", 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3068455449999998, 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.167035674, 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.220 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 124385 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851974+0000", 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851974+0000", 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851974+0000", 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851976+0000", 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5119999999999998e-06 2023-09-23T19:54:14.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851982+0000", 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2560000000000002e-06 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851986+0000", 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9029999999999997e-06 2023-09-23T19:54:14.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852753+0000", 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00076736299999999996 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852817+0000", 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4253999999999998e-05 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852895+0000", 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7987999999999999e-05 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855907+0000", 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0030117630000000002 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018691+0000", 2023-09-23T19:54:14.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16278451699999999 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018934+0000", 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00024294 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018978+0000", 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3856999999999997e-05 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.019010+0000", 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1702000000000002e-05 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:124384 5.f e137/112 5:f8f237e4:::benchmark_data_smithi135_284570_object8600:head v 137'7785, mlcod=137'7782)", 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.852090+0000", 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3067299300000004, 2023-09-23T19:54:14.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.166398825, 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852090+0000", 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852090+0000", 2023-09-23T19:54:14.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852089+0000", 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852094+0000", 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.442e-06 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852101+0000", 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3250000000000002e-06 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.852106+0000", 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9350000000000002e-06 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855606+0000", 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0034995909999999998 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855617+0000", 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0956e-05 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.231 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018473+0000", 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16285586399999999 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018489+0000", 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.613e-05 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.232 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:124386 5.a e137/112 5:5fe0704e:::benchmark_data_smithi135_284570_object8602:head v 137'7682, mlcod=137'7680)", 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.854899+0000", 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3039208149999997, 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16355482099999999, 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.233 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.854899+0000", 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.854899+0000", 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.854898+0000", 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:54:14.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.854904+0000", 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4520000000000002e-06 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.854916+0000", 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1579999999999999e-05 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.235 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.854922+0000", 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.575e-06 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.854989+0000", 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7484999999999995e-05 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855010+0000", 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0568e-05 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018411+0000", 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16340165000000001 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018454+0000", 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2588e-05 2023-09-23T19:54:14.237 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:124388 5.c e137/112 5:33065abd:::benchmark_data_smithi135_284570_object8604:head v 137'7723, mlcod=137'7720)", 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.856871+0000", 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3019487700000001, 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.168369673, 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.238 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.856871+0000", 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.856871+0000", 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.239 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.856870+0000", 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.856876+0000", 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.631e-06 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.856887+0000", 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.047e-05 2023-09-23T19:54:14.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.856892+0000", 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1889999999999998e-06 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.856928+0000", 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6089e-05 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.856944+0000", 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6152000000000001e-05 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.025198+0000", 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16825395400000001 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.025241+0000", 2023-09-23T19:54:14.242 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2809000000000003e-05 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop_reply(client.44591.0:124377 5.e e137/112 ondisk, result = 0)", 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.857554+0000", 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.3012661139999997, 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.161042876, 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.243 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.857554+0000", 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.857554+0000", 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.857553+0000", 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.857557+0000", 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3649999999999997e-06 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018399+0000", 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.160841084 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018405+0000", 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9859999999999999e-06 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018574+0000", 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000169524 2023-09-23T19:54:14.246 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018582+0000", 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.729e-06 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018597+0000", 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4769e-05 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.247 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop_reply(client.44591.0:124372 5.1 e137/112 ondisk, result = 0)", 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.863899+0000", 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 5.2949211600000003, 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15465610699999999, 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.248 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.863899+0000", 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.863899+0000", 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.863898+0000", 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:54:14.249 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.863904+0000", 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0170000000000002e-06 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018380+0000", 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15447654799999999 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.250 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018392+0000", 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2129e-05 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018441+0000", 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8769e-05 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.251 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018450+0000", 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2719999999999997e-06 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018555+0000", 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105213 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.252 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:125817 5.0 e137/112 5:007c7e62:::benchmark_data_smithi135_284570_object10033:head v 137'7963, mlcod=137'7959)", 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:10.727040+0000", 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 3.4317794199999998, 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.20207420300000001, 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727040+0000", 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.253 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727040+0000", 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727040+0000", 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.254 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727048+0000", 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.653e-06 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727057+0000", 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1770000000000004e-06 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727062+0000", 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7079999999999996e-06 2023-09-23T19:54:14.255 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918255+0000", 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.191192998 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918282+0000", 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7053e-05 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.256 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.929082+0000", 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010800172 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.929115+0000", 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2199999999999997e-05 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.257 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:125823 5.c e137/112 5:3be4aa05:::benchmark_data_smithi135_284570_object10039:head v 137'7817, mlcod=137'7815)", 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:10.733899+0000", 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 3.4249204820000001, 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19391745499999999, 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.258 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.733899+0000", 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.733899+0000", 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.733899+0000", 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:14.259 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.733906+0000", 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8900000000000001e-06 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918216+0000", 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.184310378 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.260 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918232+0000", 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5702999999999999e-05 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918287+0000", 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5600999999999998e-05 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918309+0000", 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2082999999999998e-05 2023-09-23T19:54:14.261 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.927716+0000", 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0094066219999999999 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.927817+0000", 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010086000000000001 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.262 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:125826 5.e 5:77df08cc:::benchmark_data_smithi135_284570_object10042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:10.736113+0000", 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 3.4227066810000002, 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19501096800000001, 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.263 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 125826 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.736113+0000", 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.736113+0000", 2023-09-23T19:54:14.264 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.736111+0000", 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.736123+0000", 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.226e-05 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.265 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918216+0000", 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18209268300000001 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918236+0000", 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0194999999999999e-05 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.266 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918286+0000", 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0318999999999997e-05 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918429+0000", 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014285799999999999 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918555+0000", 2023-09-23T19:54:14.267 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012537900000000001 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.925579+0000", 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070240140000000003 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.927711+0000", 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002131984 2023-09-23T19:54:14.268 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.931038+0000", 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033274670000000002 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.931084+0000", 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5541000000000003e-05 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.269 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.931124+0000", 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0367999999999997e-05 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:127287 5.e 5:7f91015f:::benchmark_data_smithi135_284570_object11503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.892771+0000", 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 1.2660490280000001, 2023-09-23T19:54:14.270 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19526686700000001, 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127287 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892771+0000", 2023-09-23T19:54:14.271 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892771+0000", 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892769+0000", 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:14.272 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892777+0000", 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9500000000000001e-06 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892791+0000", 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4039000000000001e-05 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.273 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892799+0000", 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6499999999999996e-06 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892845+0000", 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6211000000000002e-05 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.274 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892958+0000", 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113736 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.893064+0000", 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105321 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898810+0000", 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0057458459999999998 2023-09-23T19:54:14.275 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.084220+0000", 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.185410361 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.087948+0000", 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037277460000000001 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.276 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.087996+0000", 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8072000000000003e-05 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.088038+0000", 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1841000000000001e-05 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.277 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:127283 5.0 e139/112 5:0b4522b3:::benchmark_data_smithi135_284570_object11499:head v 139'8063, mlcod=139'8061)", 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.897143+0000", 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 1.2616763770000001, 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.179679438, 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897143+0000", 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.278 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897143+0000", 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897143+0000", 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.279 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897151+0000", 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9959999999999993e-06 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897161+0000", 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0457e-05 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897167+0000", 2023-09-23T19:54:14.280 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1689999999999997e-06 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897249+0000", 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1717000000000004e-05 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897268+0000", 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8640000000000001e-05 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.281 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076783+0000", 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17951484500000001 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076823+0000", 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0333999999999997e-05 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.282 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:127284 5.5 e139/112 5:a1c4128d:::benchmark_data_smithi135_284570_object11500:head v 139'7968, mlcod=139'7966)", 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.899283+0000", 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 1.2595369590000001, 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18496454900000001, 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.283 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899283+0000", 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899283+0000", 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899282+0000", 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:54:14.284 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899288+0000", 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2049999999999999e-06 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899297+0000", 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7430000000000008e-06 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.285 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899301+0000", 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5959999999999997e-06 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899339+0000", 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8058000000000001e-05 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.286 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899353+0000", 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3337e-05 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.084215+0000", 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18486185999999999 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.084247+0000", 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2809999999999999e-05 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:127294 5.e 5:7c49cd13:::benchmark_data_smithi135_284570_object11510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.904248+0000", 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 1.254571925, 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.194697595, 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:14.288 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127294 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.904248+0000", 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.904248+0000", 2023-09-23T19:54:14.289 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.904246+0000", 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.904253+0000", 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7120000000000003e-06 2023-09-23T19:54:14.290 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076794+0000", 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17254071600000001 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076807+0000", 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2687999999999999e-05 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.291 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076854+0000", 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7380999999999999e-05 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076999+0000", 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000145432 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:54:14.292 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.077128+0000", 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012904100000000001 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.084542+0000", 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074134409999999998 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.084552+0000", 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6919999999999997e-06 2023-09-23T19:54:14.293 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.098879+0000", 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014327174999999999 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.098910+0000", 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1044000000000003e-05 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:14.294 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.098945+0000", 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5649999999999999e-05 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:14.295 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:54:14.296 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:13 smithi135 ceph-mon[221113]: pgmap v310: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-09-23T19:54:14.343 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:14.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:14.451 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:14.495 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:14.578 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:14.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:14.681 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:14.730 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:14.815 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:54:14.993 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:15.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:15.045 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:54:15.102 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:15.191 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:15.252 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:15.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:15.360 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:15.434 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:15.741 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:15.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:15.752 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:15.848 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:15.948 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:16.088 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:16.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:16.142 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:16.159 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:54:16.192 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:54:16.194 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:16.227 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:54:16.255 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:16.265 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:54:16.325 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:54:16.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:16 smithi135 ceph-mon[221113]: pgmap v311: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-09-23T19:54:16.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2749128709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:16.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:16 smithi135 ceph-mon[213871]: pgmap v311: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-09-23T19:54:16.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2749128709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:16.363 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:54:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:16 smithi144 ceph-mon[194833]: pgmap v311: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-09-23T19:54:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2749128709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:16.394 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:54:16.438 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:54:16.438 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:54:16.438 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:54:16.438 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:54:16.438 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:54:16.438 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:54:16.439 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:54:16.439 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:54:16.439 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:54:16.452 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:16.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:16.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:16.559 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:16.642 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:16.768 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:16.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:16.871 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:16.949 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:17.014 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:17.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:17.119 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:17.145 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:17.212 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:17.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:17.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:17.257 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T19:54:17.278 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1972181709' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:54:17.279 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:17 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:54:17.279 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1972181709' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:54:17.279 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:17 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:54:17.327 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:17.374 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:17.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1972181709' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:54:17.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:17 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:54:17.946 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:17.946 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:18.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:18 smithi144 ceph-mon[194833]: pgmap v312: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-09-23T19:54:18.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:54:18.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:18 smithi144 ceph-mon[194833]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T19:54:18.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:18 smithi144 ceph-mon[194833]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:18 smithi144 ceph-mon[194833]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:18 smithi144 ceph-mon[194833]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:18 smithi144 ceph-mon[194833]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.250 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:18 smithi144 ceph-mon[194833]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[221113]: pgmap v312: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-09-23T19:54:18.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:54:18.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[221113]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T19:54:18.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[221113]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[221113]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[221113]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[221113]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[221113]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[213871]: pgmap v312: 99 pgs: 99 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-09-23T19:54:18.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:54:18.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[213871]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T19:54:18.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[213871]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[213871]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[213871]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[213871]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:18 smithi135 ceph-mon[213871]: failed to encode map e140 with expected crc 2023-09-23T19:54:18.841 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:18.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:18.871 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T19:54:18.872 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:54:18.894 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:54:18.945 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:19.008 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:19.109 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:19.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:19.214 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:19.268 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:19.351 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:19.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:19.464 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:19.505 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:19.535 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:19.646 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:19.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:19.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:19.683 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:19.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:19.850 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:20.034 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:20.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:20.140 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:20.209 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:20.310 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:54:20.352 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:20.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:20.458 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:20.515 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:20 smithi135 ceph-mon[221113]: pgmap v314: 99 pgs: 99 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-09-23T19:54:20.515 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3320578592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:20.516 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:20 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:20.516 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:20 smithi135 ceph-mon[213871]: pgmap v314: 99 pgs: 99 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-09-23T19:54:20.517 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3320578592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:20.517 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:20 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:20.519 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:20.571 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:54:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:20 smithi144 ceph-mon[194833]: pgmap v314: 99 pgs: 99 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-09-23T19:54:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3320578592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:20 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:21.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:21 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:54:21.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:21 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T19:54:21.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:21 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:21.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:21 smithi144 ceph-mon[194833]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T19:54:21.387 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:21 smithi144 ceph-mon[194833]: failed to encode map e141 with expected crc 2023-09-23T19:54:21.387 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:21 smithi144 ceph-mon[194833]: failed to encode map e141 with expected crc 2023-09-23T19:54:21.687 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:21.688 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:21.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:54:21.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T19:54:21.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[221113]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[221113]: failed to encode map e141 with expected crc 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[221113]: failed to encode map e141 with expected crc 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[213871]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[213871]: failed to encode map e141 with expected crc 2023-09-23T19:54:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:21 smithi135 ceph-mon[213871]: failed to encode map e141 with expected crc 2023-09-23T19:54:21.752 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:21.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:21.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:21.857 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:21.944 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:22.054 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:22.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:22.065 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:22.164 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:22.189 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:22.230 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:22.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:22.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:22.333 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:22 smithi144 ceph-mon[194833]: pgmap v316: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:54:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:22.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:22.420 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:22 smithi135 ceph-mon[213871]: pgmap v316: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:54:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:22 smithi135 ceph-mon[221113]: pgmap v316: 99 pgs: 99 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T19:54:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:22.551 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:22.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:22.656 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:22.670 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:54:22.737 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:22.831 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:54:22.845 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:54:22.859 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:54:22.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:22.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:22.963 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:23.016 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:23.403 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:23 smithi144 ceph-mon[194833]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:54:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:23 smithi144 ceph-mon[194833]: pgmap v317: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-09-23T19:54:23.692 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:23 smithi135 ceph-mon[221113]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:54:23.693 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:23 smithi135 ceph-mon[221113]: pgmap v317: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-09-23T19:54:23.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:23 smithi135 ceph-mon[213871]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:54:23.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:23 smithi135 ceph-mon[213871]: pgmap v317: 99 pgs: 99 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-09-23T19:54:23.894 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:54:23.977 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:54:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:54:23] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T19:54:24.427 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:24.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:24.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:24.440 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":316,"stamp":"2023-09-23T19:54:21.156097+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6588078027,"num_objects":100482,"num_object_clones":0,"num_object_copies":301446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100482,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":132017,"num_write_kb":7422710,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129213,"ondisk_log_size":129213,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19909344,"kb_used_data":19334240,"kb_used_omap":32,"kb_used_meta":574623,"kb_avail":730051872,"statfs":{"total":767960285184,"available":747573116928,"internally_reserved":0,"allocated":19798261760,"data_stored":19793564654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":588414470},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,12,0,3,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-290586624,"num_objects":-4434,"num_object_clones":0,"num_object_copies":-13302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4434,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001896"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":119,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780174+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:54:11.780174+0000","last_peered":"2023-09-23T19:54:11.780174+0000","last_clean":"2023-09-23T19:54:11.780174+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:54:11.780174+0000","last_undegraded":"2023-09-23T19:54:11.780174+0000","last_fullsized":"2023-09-23T19:54:11.780174+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240493+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:54:17.240493+0000","last_peered":"2023-09-23T19:54:17.240493+0000","last_clean":"2023-09-23T19:54:17.240493+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:54:17.240493+0000","last_undegraded":"2023-09-23T19:54:17.240493+0000","last_fullsized":"2023-09-23T19:54:17.240493+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":77,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.793554+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:54:11.793554+0000","last_peered":"2023-09-23T19:54:11.793554+0000","last_clean":"2023-09-23T19:54:11.793554+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:54:11.793554+0000","last_undegraded":"2023-09-23T19:54:11.793554+0000","last_fullsized":"2023-09-23T19:54:11.793554+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":84,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.248933+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:54:17.248933+0000","last_peered":"2023-09-23T19:54:17.248933+0000","last_clean":"2023-09-23T19:54:17.248933+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:54:17.248933+0000","last_undegraded":"2023-09-23T19:54:17.248933+0000","last_fullsized":"2023-09-23T19:54:17.248933+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":121,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240153+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:54:17.240153+0000","last_peered":"2023-09-23T19:54:17.240153+0000","last_clean":"2023-09-23T19:54:17.240153+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:54:17.240153+0000","last_undegraded":"2023-09-23T19:54:17.240153+0000","last_fullsized":"2023-09-23T19:54:17.240153+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.790356+0000","last_change":"2023-09-23T19:49:51.374353+0000","last_active":"2023-09-23T19:54:11.790356+0000","last_peered":"2023-09-23T19:54:11.790356+0000","last_clean":"2023-09-23T19:54:11.790356+0000","last_became_active":"2023-09-23T19:49:51.373992+0000","last_became_peered":"2023-09-23T19:49:51.373992+0000","last_unstale":"2023-09-23T19:54:11.790356+0000","last_undegraded":"2023-09-23T19:54:11.790356+0000","last_fullsized":"2023-09-23T19:54:11.790356+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.344350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":83,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249399+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:54:17.249399+0000","last_peered":"2023-09-23T19:54:17.249399+0000","last_clean":"2023-09-23T19:54:17.249399+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:54:17.249399+0000","last_undegraded":"2023-09-23T19:54:17.249399+0000","last_fullsized":"2023-09-23T19:54:17.249399+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240032+0000","last_change":"2023-09-23T19:50:57.789294+0000","last_active":"2023-09-23T19:54:17.240032+0000","last_peered":"2023-09-23T19:54:17.240032+0000","last_clean":"2023-09-23T19:54:17.240032+0000","last_became_active":"2023-09-23T19:50:57.788922+0000","last_became_peered":"2023-09-23T19:50:57.788922+0000","last_unstale":"2023-09-23T19:54:17.240032+0000","last_undegraded":"2023-09-23T19:54:17.240032+0000","last_fullsized":"2023-09-23T19:54:17.240032+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:11.829374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.783448+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:54:11.783448+0000","last_peered":"2023-09-23T19:54:11.783448+0000","last_clean":"2023-09-23T19:54:11.783448+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:54:11.783448+0000","last_undegraded":"2023-09-23T19:54:11.783448+0000","last_fullsized":"2023-09-23T19:54:11.783448+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":54,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.241840+0000","last_change":"2023-09-23T19:50:57.789044+0000","last_active":"2023-09-23T19:54:17.241840+0000","last_peered":"2023-09-23T19:54:17.241840+0000","last_clean":"2023-09-23T19:54:17.241840+0000","last_became_active":"2023-09-23T19:50:57.788730+0000","last_became_peered":"2023-09-23T19:50:57.788730+0000","last_unstale":"2023-09-23T19:54:17.241840+0000","last_undegraded":"2023-09-23T19:54:17.241840+0000","last_fullsized":"2023-09-23T19:54:17.241840+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:23.585694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794824+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:54:11.794824+0000","last_peered":"2023-09-23T19:54:11.794824+0000","last_clean":"2023-09-23T19:54:11.794824+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:54:11.794824+0000","last_undegraded":"2023-09-23T19:54:11.794824+0000","last_fullsized":"2023-09-23T19:54:11.794824+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":78,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.793974+0000","last_change":"2023-09-23T19:50:23.998186+0000","last_active":"2023-09-23T19:54:11.793974+0000","last_peered":"2023-09-23T19:54:11.793974+0000","last_clean":"2023-09-23T19:54:11.793974+0000","last_became_active":"2023-09-23T19:50:23.997897+0000","last_became_peered":"2023-09-23T19:50:23.997897+0000","last_unstale":"2023-09-23T19:54:11.793974+0000","last_undegraded":"2023-09-23T19:54:11.793974+0000","last_fullsized":"2023-09-23T19:54:11.793974+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:57.792861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":106,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780899+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:54:11.780899+0000","last_peered":"2023-09-23T19:54:11.780899+0000","last_clean":"2023-09-23T19:54:11.780899+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:54:11.780899+0000","last_undegraded":"2023-09-23T19:54:11.780899+0000","last_fullsized":"2023-09-23T19:54:11.780899+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780443+0000","last_change":"2023-09-23T19:49:51.379282+0000","last_active":"2023-09-23T19:54:11.780443+0000","last_peered":"2023-09-23T19:54:11.780443+0000","last_clean":"2023-09-23T19:54:11.780443+0000","last_became_active":"2023-09-23T19:49:51.378849+0000","last_became_peered":"2023-09-23T19:49:51.378849+0000","last_unstale":"2023-09-23T19:54:11.780443+0000","last_undegraded":"2023-09-23T19:54:11.780443+0000","last_fullsized":"2023-09-23T19:54:11.780443+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:00.093034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":102,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.797668+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:54:11.797668+0000","last_peered":"2023-09-23T19:54:11.797668+0000","last_clean":"2023-09-23T19:54:11.797668+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:54:11.797668+0000","last_undegraded":"2023-09-23T19:54:11.797668+0000","last_fullsized":"2023-09-23T19:54:11.797668+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240634+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:54:17.240634+0000","last_peered":"2023-09-23T19:54:17.240634+0000","last_clean":"2023-09-23T19:54:17.240634+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:54:17.240634+0000","last_undegraded":"2023-09-23T19:54:17.240634+0000","last_fullsized":"2023-09-23T19:54:17.240634+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":83,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.244309+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:54:17.244309+0000","last_peered":"2023-09-23T19:54:17.244309+0000","last_clean":"2023-09-23T19:54:17.244309+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:54:17.244309+0000","last_undegraded":"2023-09-23T19:54:17.244309+0000","last_fullsized":"2023-09-23T19:54:17.244309+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.782754+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:54:11.782754+0000","last_peered":"2023-09-23T19:54:11.782754+0000","last_clean":"2023-09-23T19:54:11.782754+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:54:11.782754+0000","last_undegraded":"2023-09-23T19:54:11.782754+0000","last_fullsized":"2023-09-23T19:54:11.782754+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":83,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.795476+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:54:11.795476+0000","last_peered":"2023-09-23T19:54:11.795476+0000","last_clean":"2023-09-23T19:54:11.795476+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:54:11.795476+0000","last_undegraded":"2023-09-23T19:54:11.795476+0000","last_fullsized":"2023-09-23T19:54:11.795476+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":84,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.248836+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:54:17.248836+0000","last_peered":"2023-09-23T19:54:17.248836+0000","last_clean":"2023-09-23T19:54:17.248836+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:54:17.248836+0000","last_undegraded":"2023-09-23T19:54:17.248836+0000","last_fullsized":"2023-09-23T19:54:17.248836+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.789777+0000","last_change":"2023-09-23T19:48:37.567607+0000","last_active":"2023-09-23T19:54:11.789777+0000","last_peered":"2023-09-23T19:54:11.789777+0000","last_clean":"2023-09-23T19:54:11.789777+0000","last_became_active":"2023-09-23T19:48:37.567300+0000","last_became_peered":"2023-09-23T19:48:37.567300+0000","last_unstale":"2023-09-23T19:54:11.789777+0000","last_undegraded":"2023-09-23T19:54:11.789777+0000","last_fullsized":"2023-09-23T19:54:11.789777+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.088452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240814+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:54:17.240814+0000","last_peered":"2023-09-23T19:54:17.240814+0000","last_clean":"2023-09-23T19:54:17.240814+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:54:17.240814+0000","last_undegraded":"2023-09-23T19:54:17.240814+0000","last_fullsized":"2023-09-23T19:54:17.240814+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"140'8060","reported_seq":8096,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:18.092351+0000","last_change":"2023-09-23T19:52:21.187755+0000","last_active":"2023-09-23T19:54:18.092351+0000","last_peered":"2023-09-23T19:54:18.092351+0000","last_clean":"2023-09-23T19:54:18.092351+0000","last_became_active":"2023-09-23T19:52:21.187425+0000","last_became_peered":"2023-09-23T19:52:21.187425+0000","last_unstale":"2023-09-23T19:54:18.092351+0000","last_undegraded":"2023-09-23T19:54:18.092351+0000","last_fullsized":"2023-09-23T19:54:18.092351+0000","mapping_epoch":112,"log_start":"115'300","ondisk_log_start":"115'300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7760,"log_dups_size":0,"ondisk_log_size":7760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:24.504315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407502848,"num_objects":6218,"num_object_clones":0,"num_object_copies":18654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8060,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794777+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:54:11.794777+0000","last_peered":"2023-09-23T19:54:11.794777+0000","last_clean":"2023-09-23T19:54:11.794777+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:54:11.794777+0000","last_undegraded":"2023-09-23T19:54:11.794777+0000","last_fullsized":"2023-09-23T19:54:11.794777+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.785942+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:54:11.785942+0000","last_peered":"2023-09-23T19:54:11.785942+0000","last_clean":"2023-09-23T19:54:11.785942+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:54:11.785942+0000","last_undegraded":"2023-09-23T19:54:11.785942+0000","last_fullsized":"2023-09-23T19:54:11.785942+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"139'7951","reported_seq":7988,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.229283+0000","last_change":"2023-09-23T19:52:21.181571+0000","last_active":"2023-09-23T19:54:17.229283+0000","last_peered":"2023-09-23T19:54:17.229283+0000","last_clean":"2023-09-23T19:54:17.229283+0000","last_became_active":"2023-09-23T19:52:21.181249+0000","last_became_peered":"2023-09-23T19:52:21.181249+0000","last_unstale":"2023-09-23T19:54:17.229283+0000","last_undegraded":"2023-09-23T19:54:17.229283+0000","last_fullsized":"2023-09-23T19:54:17.229283+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:02.562956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407306240,"num_objects":6215,"num_object_clones":0,"num_object_copies":18645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7951,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":102,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.795213+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:54:11.795213+0000","last_peered":"2023-09-23T19:54:11.795213+0000","last_clean":"2023-09-23T19:54:11.795213+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:54:11.795213+0000","last_undegraded":"2023-09-23T19:54:11.795213+0000","last_fullsized":"2023-09-23T19:54:11.795213+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780387+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:54:11.780387+0000","last_peered":"2023-09-23T19:54:11.780387+0000","last_clean":"2023-09-23T19:54:11.780387+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:54:11.780387+0000","last_undegraded":"2023-09-23T19:54:11.780387+0000","last_fullsized":"2023-09-23T19:54:11.780387+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"140'8038","reported_seq":8075,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.230023+0000","last_change":"2023-09-23T19:52:21.183245+0000","last_active":"2023-09-23T19:54:17.230023+0000","last_peered":"2023-09-23T19:54:17.230023+0000","last_clean":"2023-09-23T19:54:17.230023+0000","last_became_active":"2023-09-23T19:52:21.182965+0000","last_became_peered":"2023-09-23T19:52:21.182965+0000","last_unstale":"2023-09-23T19:54:17.230023+0000","last_undegraded":"2023-09-23T19:54:17.230023+0000","last_fullsized":"2023-09-23T19:54:17.230023+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8038,"log_dups_size":0,"ondisk_log_size":8038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:14.836347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413007872,"num_objects":6302,"num_object_clones":0,"num_object_copies":18906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8038,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":84,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794660+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:54:11.794660+0000","last_peered":"2023-09-23T19:54:11.794660+0000","last_clean":"2023-09-23T19:54:11.794660+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:54:11.794660+0000","last_undegraded":"2023-09-23T19:54:11.794660+0000","last_fullsized":"2023-09-23T19:54:11.794660+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.790914+0000","last_change":"2023-09-23T19:49:51.380726+0000","last_active":"2023-09-23T19:54:11.790914+0000","last_peered":"2023-09-23T19:54:11.790914+0000","last_clean":"2023-09-23T19:54:11.790914+0000","last_became_active":"2023-09-23T19:49:51.380021+0000","last_became_peered":"2023-09-23T19:49:51.380021+0000","last_unstale":"2023-09-23T19:54:11.790914+0000","last_undegraded":"2023-09-23T19:54:11.790914+0000","last_fullsized":"2023-09-23T19:54:11.790914+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:16.892029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"139'8004","reported_seq":8039,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.212611+0000","last_change":"2023-09-23T19:52:21.183448+0000","last_active":"2023-09-23T19:54:17.212611+0000","last_peered":"2023-09-23T19:54:17.212611+0000","last_clean":"2023-09-23T19:54:17.212611+0000","last_became_active":"2023-09-23T19:52:21.183139+0000","last_became_peered":"2023-09-23T19:52:21.183139+0000","last_unstale":"2023-09-23T19:54:17.212611+0000","last_undegraded":"2023-09-23T19:54:17.212611+0000","last_fullsized":"2023-09-23T19:54:17.212611+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8004,"log_dups_size":0,"ondisk_log_size":8004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:28.123472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412090368,"num_objects":6288,"num_object_clones":0,"num_object_copies":18864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8004,"num_write_kb":457344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.782503+0000","last_change":"2023-09-23T19:50:57.788389+0000","last_active":"2023-09-23T19:54:11.782503+0000","last_peered":"2023-09-23T19:54:11.782503+0000","last_clean":"2023-09-23T19:54:11.782503+0000","last_became_active":"2023-09-23T19:50:57.788075+0000","last_became_peered":"2023-09-23T19:50:57.788075+0000","last_unstale":"2023-09-23T19:54:11.782503+0000","last_undegraded":"2023-09-23T19:54:11.782503+0000","last_fullsized":"2023-09-23T19:54:11.782503+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:06.180629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":84,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239246+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:54:17.239246+0000","last_peered":"2023-09-23T19:54:17.239246+0000","last_clean":"2023-09-23T19:54:17.239246+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:54:17.239246+0000","last_undegraded":"2023-09-23T19:54:17.239246+0000","last_fullsized":"2023-09-23T19:54:17.239246+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":520,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.795353+0000","last_change":"2023-09-23T19:52:57.349564+0000","last_active":"2023-09-23T19:54:11.795353+0000","last_peered":"2023-09-23T19:54:11.795353+0000","last_clean":"2023-09-23T19:54:11.795353+0000","last_became_active":"2023-09-23T19:52:57.349182+0000","last_became_peered":"2023-09-23T19:52:57.349182+0000","last_unstale":"2023-09-23T19:54:11.795353+0000","last_undegraded":"2023-09-23T19:54:11.795353+0000","last_fullsized":"2023-09-23T19:54:11.795353+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:25.262721+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"139'8194","reported_seq":8232,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.222959+0000","last_change":"2023-09-23T19:52:21.186399+0000","last_active":"2023-09-23T19:54:17.222959+0000","last_peered":"2023-09-23T19:54:17.222959+0000","last_clean":"2023-09-23T19:54:17.222959+0000","last_became_active":"2023-09-23T19:52:21.186119+0000","last_became_peered":"2023-09-23T19:52:21.186119+0000","last_unstale":"2023-09-23T19:54:17.222959+0000","last_undegraded":"2023-09-23T19:54:17.222959+0000","last_fullsized":"2023-09-23T19:54:17.222959+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:05.981956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422510616,"num_objects":6448,"num_object_clones":0,"num_object_copies":19344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6448,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8194,"num_write_kb":468481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240335+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:54:17.240335+0000","last_peered":"2023-09-23T19:54:17.240335+0000","last_clean":"2023-09-23T19:54:17.240335+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:54:17.240335+0000","last_undegraded":"2023-09-23T19:54:17.240335+0000","last_fullsized":"2023-09-23T19:54:17.240335+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780876+0000","last_change":"2023-09-23T19:50:57.790231+0000","last_active":"2023-09-23T19:54:11.780876+0000","last_peered":"2023-09-23T19:54:11.780876+0000","last_clean":"2023-09-23T19:54:11.780876+0000","last_became_active":"2023-09-23T19:50:57.789446+0000","last_became_peered":"2023-09-23T19:50:57.789446+0000","last_unstale":"2023-09-23T19:54:11.780876+0000","last_undegraded":"2023-09-23T19:54:11.780876+0000","last_fullsized":"2023-09-23T19:54:11.780876+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:41.313276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":452,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.785867+0000","last_change":"2023-09-23T19:52:58.803028+0000","last_active":"2023-09-23T19:54:11.785867+0000","last_peered":"2023-09-23T19:54:11.785867+0000","last_clean":"2023-09-23T19:54:11.785867+0000","last_became_active":"2023-09-23T19:52:57.353261+0000","last_became_peered":"2023-09-23T19:52:57.353261+0000","last_unstale":"2023-09-23T19:54:11.785867+0000","last_undegraded":"2023-09-23T19:54:11.785867+0000","last_fullsized":"2023-09-23T19:54:11.785867+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:53.333663+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.c","version":"140'8203","reported_seq":8240,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:19.395482+0000","last_change":"2023-09-23T19:52:21.179841+0000","last_active":"2023-09-23T19:54:19.395482+0000","last_peered":"2023-09-23T19:54:19.395482+0000","last_clean":"2023-09-23T19:54:19.395482+0000","last_became_active":"2023-09-23T19:52:21.179598+0000","last_became_peered":"2023-09-23T19:52:21.179598+0000","last_unstale":"2023-09-23T19:54:19.395482+0000","last_undegraded":"2023-09-23T19:54:19.395482+0000","last_fullsized":"2023-09-23T19:54:19.395482+0000","mapping_epoch":112,"log_start":"115'300","ondisk_log_start":"115'300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7903,"log_dups_size":0,"ondisk_log_size":7903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:39.463431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404815872,"num_objects":6177,"num_object_clones":0,"num_object_copies":18531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8203,"num_write_kb":460160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.790777+0000","last_change":"2023-09-23T19:49:51.380925+0000","last_active":"2023-09-23T19:54:11.790777+0000","last_peered":"2023-09-23T19:54:11.790777+0000","last_clean":"2023-09-23T19:54:11.790777+0000","last_became_active":"2023-09-23T19:49:51.380427+0000","last_became_peered":"2023-09-23T19:49:51.380427+0000","last_unstale":"2023-09-23T19:54:11.790777+0000","last_undegraded":"2023-09-23T19:54:11.790777+0000","last_fullsized":"2023-09-23T19:54:11.790777+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:49.879887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780947+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:54:11.780947+0000","last_peered":"2023-09-23T19:54:11.780947+0000","last_clean":"2023-09-23T19:54:11.780947+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:54:11.780947+0000","last_undegraded":"2023-09-23T19:54:11.780947+0000","last_fullsized":"2023-09-23T19:54:11.780947+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"140'8168","reported_seq":8204,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:18.079121+0000","last_change":"2023-09-23T19:52:21.187234+0000","last_active":"2023-09-23T19:54:18.079121+0000","last_peered":"2023-09-23T19:54:18.079121+0000","last_clean":"2023-09-23T19:54:18.079121+0000","last_became_active":"2023-09-23T19:52:21.186449+0000","last_became_peered":"2023-09-23T19:52:21.186449+0000","last_unstale":"2023-09-23T19:54:18.079121+0000","last_undegraded":"2023-09-23T19:54:18.079121+0000","last_fullsized":"2023-09-23T19:54:18.079121+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8168,"log_dups_size":0,"ondisk_log_size":8168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:15.473024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413532160,"num_objects":6310,"num_object_clones":0,"num_object_copies":18930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8168,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780360+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:54:11.780360+0000","last_peered":"2023-09-23T19:54:11.780360+0000","last_clean":"2023-09-23T19:54:11.780360+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:54:11.780360+0000","last_undegraded":"2023-09-23T19:54:11.780360+0000","last_fullsized":"2023-09-23T19:54:11.780360+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.781029+0000","last_change":"2023-09-23T19:50:23.998753+0000","last_active":"2023-09-23T19:54:11.781029+0000","last_peered":"2023-09-23T19:54:11.781029+0000","last_clean":"2023-09-23T19:54:11.781029+0000","last_became_active":"2023-09-23T19:50:23.998417+0000","last_became_peered":"2023-09-23T19:50:23.998417+0000","last_unstale":"2023-09-23T19:54:11.781029+0000","last_undegraded":"2023-09-23T19:54:11.781029+0000","last_fullsized":"2023-09-23T19:54:11.781029+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:11.002531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":452,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.782973+0000","last_change":"2023-09-23T19:52:59.809395+0000","last_active":"2023-09-23T19:54:11.782973+0000","last_peered":"2023-09-23T19:54:11.782973+0000","last_clean":"2023-09-23T19:54:11.782973+0000","last_became_active":"2023-09-23T19:52:57.359009+0000","last_became_peered":"2023-09-23T19:52:57.359009+0000","last_unstale":"2023-09-23T19:54:11.782973+0000","last_undegraded":"2023-09-23T19:54:11.782973+0000","last_fullsized":"2023-09-23T19:54:11.782973+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:56.921631+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.e","version":"140'8111","reported_seq":8147,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:18.106952+0000","last_change":"2023-09-23T19:52:21.184411+0000","last_active":"2023-09-23T19:54:18.106952+0000","last_peered":"2023-09-23T19:54:18.106952+0000","last_clean":"2023-09-23T19:54:18.106952+0000","last_became_active":"2023-09-23T19:52:21.184090+0000","last_became_peered":"2023-09-23T19:52:21.184090+0000","last_unstale":"2023-09-23T19:54:18.106952+0000","last_undegraded":"2023-09-23T19:54:18.106952+0000","last_fullsized":"2023-09-23T19:54:18.106952+0000","mapping_epoch":112,"log_start":"116'1200","ondisk_log_start":"116'1200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":6911,"log_dups_size":0,"ondisk_log_size":6911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:46.571567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407175168,"num_objects":6213,"num_object_clones":0,"num_object_copies":18639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8111,"num_write_kb":458368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":104,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240307+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:54:17.240307+0000","last_peered":"2023-09-23T19:54:17.240307+0000","last_clean":"2023-09-23T19:54:17.240307+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:54:17.240307+0000","last_undegraded":"2023-09-23T19:54:17.240307+0000","last_fullsized":"2023-09-23T19:54:17.240307+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":79,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246495+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:54:17.246495+0000","last_peered":"2023-09-23T19:54:17.246495+0000","last_clean":"2023-09-23T19:54:17.246495+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:54:17.246495+0000","last_undegraded":"2023-09-23T19:54:17.246495+0000","last_fullsized":"2023-09-23T19:54:17.246495+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":462,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.790298+0000","last_change":"2023-09-23T19:52:58.592695+0000","last_active":"2023-09-23T19:54:11.790298+0000","last_peered":"2023-09-23T19:54:11.790298+0000","last_clean":"2023-09-23T19:54:11.790298+0000","last_became_active":"2023-09-23T19:52:57.349277+0000","last_became_peered":"2023-09-23T19:52:57.349277+0000","last_unstale":"2023-09-23T19:54:11.790298+0000","last_undegraded":"2023-09-23T19:54:11.790298+0000","last_fullsized":"2023-09-23T19:54:11.790298+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.592597+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:25.665486+0000","scrub_duration":0.0050461000000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":475,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794247+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:54:11.794247+0000","last_peered":"2023-09-23T19:54:11.794247+0000","last_clean":"2023-09-23T19:54:11.794247+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:54:11.794247+0000","last_undegraded":"2023-09-23T19:54:11.794247+0000","last_fullsized":"2023-09-23T19:54:11.794247+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.782817+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:54:11.782817+0000","last_peered":"2023-09-23T19:54:11.782817+0000","last_clean":"2023-09-23T19:54:11.782817+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:54:11.782817+0000","last_undegraded":"2023-09-23T19:54:11.782817+0000","last_fullsized":"2023-09-23T19:54:11.782817+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":64,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.793885+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:54:11.793885+0000","last_peered":"2023-09-23T19:54:11.793885+0000","last_clean":"2023-09-23T19:54:11.793885+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:54:11.793885+0000","last_undegraded":"2023-09-23T19:54:11.793885+0000","last_fullsized":"2023-09-23T19:54:11.793885+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"141'8450","reported_seq":8488,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:20.657840+0000","last_change":"2023-09-23T19:52:21.185632+0000","last_active":"2023-09-23T19:54:20.657840+0000","last_peered":"2023-09-23T19:54:20.657840+0000","last_clean":"2023-09-23T19:54:20.657840+0000","last_became_active":"2023-09-23T19:52:21.185348+0000","last_became_peered":"2023-09-23T19:52:21.185348+0000","last_unstale":"2023-09-23T19:54:20.657840+0000","last_undegraded":"2023-09-23T19:54:20.657840+0000","last_fullsized":"2023-09-23T19:54:20.657840+0000","mapping_epoch":112,"log_start":"115'300","ondisk_log_start":"115'300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8150,"log_dups_size":0,"ondisk_log_size":8150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:30.337130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406847488,"num_objects":6208,"num_object_clones":0,"num_object_copies":18624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8450,"num_write_kb":469056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":452,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.786091+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T19:54:11.786091+0000","last_peered":"2023-09-23T19:54:11.786091+0000","last_clean":"2023-09-23T19:54:11.786091+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T19:54:11.786091+0000","last_undegraded":"2023-09-23T19:54:11.786091+0000","last_fullsized":"2023-09-23T19:54:11.786091+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.5","version":"141'8284","reported_seq":8322,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:20.751387+0000","last_change":"2023-09-23T19:52:21.181440+0000","last_active":"2023-09-23T19:54:20.751387+0000","last_peered":"2023-09-23T19:54:20.751387+0000","last_clean":"2023-09-23T19:54:20.751387+0000","last_became_active":"2023-09-23T19:52:21.181162+0000","last_became_peered":"2023-09-23T19:52:21.181162+0000","last_unstale":"2023-09-23T19:54:20.751387+0000","last_undegraded":"2023-09-23T19:54:20.751387+0000","last_fullsized":"2023-09-23T19:54:20.751387+0000","mapping_epoch":112,"log_start":"115'100","ondisk_log_start":"115'100","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8184,"log_dups_size":0,"ondisk_log_size":8184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:18.844043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407240704,"num_objects":6214,"num_object_clones":0,"num_object_copies":18642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8284,"num_write_kb":463936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":664,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.783196+0000","last_change":"2023-09-23T19:49:51.382830+0000","last_active":"2023-09-23T19:54:11.783196+0000","last_peered":"2023-09-23T19:54:11.783196+0000","last_clean":"2023-09-23T19:54:11.783196+0000","last_became_active":"2023-09-23T19:49:51.382367+0000","last_became_peered":"2023-09-23T19:49:51.382367+0000","last_unstale":"2023-09-23T19:54:11.783196+0000","last_undegraded":"2023-09-23T19:54:11.783196+0000","last_fullsized":"2023-09-23T19:54:11.783196+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:48.082643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780563+0000","last_change":"2023-09-23T19:48:37.573541+0000","last_active":"2023-09-23T19:54:11.780563+0000","last_peered":"2023-09-23T19:54:11.780563+0000","last_clean":"2023-09-23T19:54:11.780563+0000","last_became_active":"2023-09-23T19:48:37.573213+0000","last_became_peered":"2023-09-23T19:48:37.573213+0000","last_unstale":"2023-09-23T19:54:11.780563+0000","last_undegraded":"2023-09-23T19:54:11.780563+0000","last_fullsized":"2023-09-23T19:54:11.780563+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:16.981072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":94,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794178+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:54:11.794178+0000","last_peered":"2023-09-23T19:54:11.794178+0000","last_clean":"2023-09-23T19:54:11.794178+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:54:11.794178+0000","last_undegraded":"2023-09-23T19:54:11.794178+0000","last_fullsized":"2023-09-23T19:54:11.794178+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":520,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.795968+0000","last_change":"2023-09-23T19:52:57.745972+0000","last_active":"2023-09-23T19:54:11.795968+0000","last_peered":"2023-09-23T19:54:11.795968+0000","last_clean":"2023-09-23T19:54:11.795968+0000","last_became_active":"2023-09-23T19:52:57.348142+0000","last_became_peered":"2023-09-23T19:52:57.348142+0000","last_unstale":"2023-09-23T19:54:11.795968+0000","last_undegraded":"2023-09-23T19:54:11.795968+0000","last_fullsized":"2023-09-23T19:54:11.795968+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.745894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.745894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.660195+0000","scrub_duration":0.0079262489999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"140'8279","reported_seq":8316,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.230168+0000","last_change":"2023-09-23T19:52:21.183302+0000","last_active":"2023-09-23T19:54:17.230168+0000","last_peered":"2023-09-23T19:54:17.230168+0000","last_clean":"2023-09-23T19:54:17.230168+0000","last_became_active":"2023-09-23T19:52:21.183010+0000","last_became_peered":"2023-09-23T19:52:21.183010+0000","last_unstale":"2023-09-23T19:54:17.230168+0000","last_undegraded":"2023-09-23T19:54:17.230168+0000","last_fullsized":"2023-09-23T19:54:17.230168+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8279,"log_dups_size":0,"ondisk_log_size":8279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:01.323672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415825920,"num_objects":6345,"num_object_clones":0,"num_object_copies":19035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8279,"num_write_kb":467968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":624,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.789593+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:54:11.789593+0000","last_peered":"2023-09-23T19:54:11.789593+0000","last_clean":"2023-09-23T19:54:11.789593+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:54:11.789593+0000","last_undegraded":"2023-09-23T19:54:11.789593+0000","last_fullsized":"2023-09-23T19:54:11.789593+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":78,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.797844+0000","last_change":"2023-09-23T19:50:23.997623+0000","last_active":"2023-09-23T19:54:11.797844+0000","last_peered":"2023-09-23T19:54:11.797844+0000","last_clean":"2023-09-23T19:54:11.797844+0000","last_became_active":"2023-09-23T19:50:23.997428+0000","last_became_peered":"2023-09-23T19:50:23.997428+0000","last_unstale":"2023-09-23T19:54:11.797844+0000","last_undegraded":"2023-09-23T19:54:11.797844+0000","last_fullsized":"2023-09-23T19:54:11.797844+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:50.529980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794727+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:54:11.794727+0000","last_peered":"2023-09-23T19:54:11.794727+0000","last_clean":"2023-09-23T19:54:11.794727+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:54:11.794727+0000","last_undegraded":"2023-09-23T19:54:11.794727+0000","last_fullsized":"2023-09-23T19:54:11.794727+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":949,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.785674+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:54:11.785674+0000","last_peered":"2023-09-23T19:54:11.785674+0000","last_clean":"2023-09-23T19:54:11.785674+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:54:11.785674+0000","last_undegraded":"2023-09-23T19:54:11.785674+0000","last_fullsized":"2023-09-23T19:54:11.785674+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":501,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794520+0000","last_change":"2023-09-23T19:52:50.668453+0000","last_active":"2023-09-23T19:54:11.794520+0000","last_peered":"2023-09-23T19:54:11.794520+0000","last_clean":"2023-09-23T19:54:11.794520+0000","last_became_active":"2023-09-23T19:52:50.263397+0000","last_became_peered":"2023-09-23T19:52:50.263397+0000","last_unstale":"2023-09-23T19:54:11.794520+0000","last_undegraded":"2023-09-23T19:54:11.794520+0000","last_fullsized":"2023-09-23T19:54:11.794520+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:48.550472+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":53,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.244459+0000","last_change":"2023-09-23T19:50:57.787513+0000","last_active":"2023-09-23T19:54:17.244459+0000","last_peered":"2023-09-23T19:54:17.244459+0000","last_clean":"2023-09-23T19:54:17.244459+0000","last_became_active":"2023-09-23T19:50:57.787175+0000","last_became_peered":"2023-09-23T19:50:57.787175+0000","last_unstale":"2023-09-23T19:54:17.244459+0000","last_undegraded":"2023-09-23T19:54:17.244459+0000","last_fullsized":"2023-09-23T19:54:17.244459+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:19.081986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.790146+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:54:11.790146+0000","last_peered":"2023-09-23T19:54:11.790146+0000","last_clean":"2023-09-23T19:54:11.790146+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:54:11.790146+0000","last_undegraded":"2023-09-23T19:54:11.790146+0000","last_fullsized":"2023-09-23T19:54:11.790146+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"140'8191","reported_seq":8227,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:18.074622+0000","last_change":"2023-09-23T19:52:21.183523+0000","last_active":"2023-09-23T19:54:18.074622+0000","last_peered":"2023-09-23T19:54:18.074622+0000","last_clean":"2023-09-23T19:54:18.074622+0000","last_became_active":"2023-09-23T19:52:21.183198+0000","last_became_peered":"2023-09-23T19:52:21.183198+0000","last_unstale":"2023-09-23T19:54:18.074622+0000","last_undegraded":"2023-09-23T19:54:18.074622+0000","last_fullsized":"2023-09-23T19:54:18.074622+0000","mapping_epoch":112,"log_start":"115'400","ondisk_log_start":"115'400","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7791,"log_dups_size":0,"ondisk_log_size":7791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:07.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414515200,"num_objects":6325,"num_object_clones":0,"num_object_copies":18975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8191,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":453,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.793573+0000","last_change":"2023-09-23T19:53:33.852740+0000","last_active":"2023-09-23T19:54:11.793573+0000","last_peered":"2023-09-23T19:54:11.793573+0000","last_clean":"2023-09-23T19:54:11.793573+0000","last_became_active":"2023-09-23T19:52:57.350232+0000","last_became_peered":"2023-09-23T19:52:57.350232+0000","last_unstale":"2023-09-23T19:54:11.793573+0000","last_undegraded":"2023-09-23T19:54:11.793573+0000","last_fullsized":"2023-09-23T19:54:11.793573+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:18.009589+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"140'8190","reported_seq":8226,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:18.059452+0000","last_change":"2023-09-23T19:52:21.181927+0000","last_active":"2023-09-23T19:54:18.059452+0000","last_peered":"2023-09-23T19:54:18.059452+0000","last_clean":"2023-09-23T19:54:18.059452+0000","last_became_active":"2023-09-23T19:52:21.181527+0000","last_became_peered":"2023-09-23T19:52:21.181527+0000","last_unstale":"2023-09-23T19:54:18.059452+0000","last_undegraded":"2023-09-23T19:54:18.059452+0000","last_fullsized":"2023-09-23T19:54:18.059452+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8190,"log_dups_size":0,"ondisk_log_size":8190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.841652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415105024,"num_objects":6334,"num_object_clones":0,"num_object_copies":19002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8190,"num_write_kb":464768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":858,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794556+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:54:11.794556+0000","last_peered":"2023-09-23T19:54:11.794556+0000","last_clean":"2023-09-23T19:54:11.794556+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:54:11.794556+0000","last_undegraded":"2023-09-23T19:54:11.794556+0000","last_fullsized":"2023-09-23T19:54:11.794556+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":83,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.238751+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:54:17.238751+0000","last_peered":"2023-09-23T19:54:17.238751+0000","last_clean":"2023-09-23T19:54:17.238751+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:54:17.238751+0000","last_undegraded":"2023-09-23T19:54:17.238751+0000","last_fullsized":"2023-09-23T19:54:17.238751+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":97,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249104+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:54:17.249104+0000","last_peered":"2023-09-23T19:54:17.249104+0000","last_clean":"2023-09-23T19:54:17.249104+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:54:17.249104+0000","last_undegraded":"2023-09-23T19:54:17.249104+0000","last_fullsized":"2023-09-23T19:54:17.249104+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":462,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.790247+0000","last_change":"2023-09-23T19:53:06.516604+0000","last_active":"2023-09-23T19:54:11.790247+0000","last_peered":"2023-09-23T19:54:11.790247+0000","last_clean":"2023-09-23T19:54:11.790247+0000","last_became_active":"2023-09-23T19:52:57.344832+0000","last_became_peered":"2023-09-23T19:52:57.344832+0000","last_unstale":"2023-09-23T19:54:11.790247+0000","last_undegraded":"2023-09-23T19:54:11.790247+0000","last_fullsized":"2023-09-23T19:54:11.790247+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:06.516504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:06.516504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:30.290875+0000","scrub_duration":0.0084342629999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":218,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794744+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:54:11.794744+0000","last_peered":"2023-09-23T19:54:11.794744+0000","last_clean":"2023-09-23T19:54:11.794744+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:54:11.794744+0000","last_undegraded":"2023-09-23T19:54:11.794744+0000","last_fullsized":"2023-09-23T19:54:11.794744+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":78,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.794314+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:54:11.794314+0000","last_peered":"2023-09-23T19:54:11.794314+0000","last_clean":"2023-09-23T19:54:11.794314+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:54:11.794314+0000","last_undegraded":"2023-09-23T19:54:11.794314+0000","last_fullsized":"2023-09-23T19:54:11.794314+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":69,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239376+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:54:17.239376+0000","last_peered":"2023-09-23T19:54:17.239376+0000","last_clean":"2023-09-23T19:54:17.239376+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:54:17.239376+0000","last_undegraded":"2023-09-23T19:54:17.239376+0000","last_fullsized":"2023-09-23T19:54:17.239376+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"140'8272","reported_seq":8308,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:18.028665+0000","last_change":"2023-09-23T19:52:21.188633+0000","last_active":"2023-09-23T19:54:18.028665+0000","last_peered":"2023-09-23T19:54:18.028665+0000","last_clean":"2023-09-23T19:54:18.028665+0000","last_became_active":"2023-09-23T19:52:21.188310+0000","last_became_peered":"2023-09-23T19:52:21.188310+0000","last_unstale":"2023-09-23T19:54:18.028665+0000","last_undegraded":"2023-09-23T19:54:18.028665+0000","last_fullsized":"2023-09-23T19:54:18.028665+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8272,"log_dups_size":0,"ondisk_log_size":8272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:56.470945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417071104,"num_objects":6364,"num_object_clones":0,"num_object_copies":19092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8272,"num_write_kb":468352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":452,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780802+0000","last_change":"2023-09-23T19:52:59.792472+0000","last_active":"2023-09-23T19:54:11.780802+0000","last_peered":"2023-09-23T19:54:11.780802+0000","last_clean":"2023-09-23T19:54:11.780802+0000","last_became_active":"2023-09-23T19:52:57.356135+0000","last_became_peered":"2023-09-23T19:52:57.356135+0000","last_unstale":"2023-09-23T19:54:11.780802+0000","last_undegraded":"2023-09-23T19:54:11.780802+0000","last_fullsized":"2023-09-23T19:54:11.780802+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:49.843853+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240864+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:54:17.240864+0000","last_peered":"2023-09-23T19:54:17.240864+0000","last_clean":"2023-09-23T19:54:17.240864+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:54:17.240864+0000","last_undegraded":"2023-09-23T19:54:17.240864+0000","last_fullsized":"2023-09-23T19:54:17.240864+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":401,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.796115+0000","last_change":"2023-09-23T19:49:51.380375+0000","last_active":"2023-09-23T19:54:11.796115+0000","last_peered":"2023-09-23T19:54:11.796115+0000","last_clean":"2023-09-23T19:54:11.796115+0000","last_became_active":"2023-09-23T19:49:51.379793+0000","last_became_peered":"2023-09-23T19:49:51.379793+0000","last_unstale":"2023-09-23T19:54:11.796115+0000","last_undegraded":"2023-09-23T19:54:11.796115+0000","last_fullsized":"2023-09-23T19:54:11.796115+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:26.355152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":98,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240229+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:54:17.240229+0000","last_peered":"2023-09-23T19:54:17.240229+0000","last_clean":"2023-09-23T19:54:17.240229+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:54:17.240229+0000","last_undegraded":"2023-09-23T19:54:17.240229+0000","last_fullsized":"2023-09-23T19:54:17.240229+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"140'8279","reported_seq":8315,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:18.054659+0000","last_change":"2023-09-23T19:52:21.183759+0000","last_active":"2023-09-23T19:54:18.054659+0000","last_peered":"2023-09-23T19:54:18.054659+0000","last_clean":"2023-09-23T19:54:18.054659+0000","last_became_active":"2023-09-23T19:52:21.183451+0000","last_became_peered":"2023-09-23T19:52:21.183451+0000","last_unstale":"2023-09-23T19:54:18.054659+0000","last_undegraded":"2023-09-23T19:54:18.054659+0000","last_fullsized":"2023-09-23T19:54:18.054659+0000","mapping_epoch":112,"log_start":"116'1400","ondisk_log_start":"116'1400","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":6879,"log_dups_size":0,"ondisk_log_size":6879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:43.359001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411107328,"num_objects":6273,"num_object_clones":0,"num_object_copies":18819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8279,"num_write_kb":465664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":455,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240329+0000","last_change":"2023-09-23T19:53:00.705074+0000","last_active":"2023-09-23T19:54:17.240329+0000","last_peered":"2023-09-23T19:54:17.240329+0000","last_clean":"2023-09-23T19:54:17.240329+0000","last_became_active":"2023-09-23T19:52:57.353333+0000","last_became_peered":"2023-09-23T19:52:57.353333+0000","last_unstale":"2023-09-23T19:54:17.240329+0000","last_undegraded":"2023-09-23T19:54:17.240329+0000","last_fullsized":"2023-09-23T19:54:17.240329+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T19:53:00.704900+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:24.642289+0000","scrub_duration":0.0090978159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":371,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.790228+0000","last_change":"2023-09-23T19:50:23.999651+0000","last_active":"2023-09-23T19:54:11.790228+0000","last_peered":"2023-09-23T19:54:11.790228+0000","last_clean":"2023-09-23T19:54:11.790228+0000","last_became_active":"2023-09-23T19:50:23.999184+0000","last_became_peered":"2023-09-23T19:50:23.999184+0000","last_unstale":"2023-09-23T19:54:11.790228+0000","last_undegraded":"2023-09-23T19:54:11.790228+0000","last_fullsized":"2023-09-23T19:54:11.790228+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:41.010259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":87,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249027+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:54:17.249027+0000","last_peered":"2023-09-23T19:54:17.249027+0000","last_clean":"2023-09-23T19:54:17.249027+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:54:17.249027+0000","last_undegraded":"2023-09-23T19:54:17.249027+0000","last_fullsized":"2023-09-23T19:54:17.249027+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.779945+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:54:11.779945+0000","last_peered":"2023-09-23T19:54:11.779945+0000","last_clean":"2023-09-23T19:54:11.779945+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:54:11.779945+0000","last_undegraded":"2023-09-23T19:54:11.779945+0000","last_fullsized":"2023-09-23T19:54:11.779945+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"141'8436","reported_seq":8475,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:20.799058+0000","last_change":"2023-09-23T19:52:21.180582+0000","last_active":"2023-09-23T19:54:20.799058+0000","last_peered":"2023-09-23T19:54:20.799058+0000","last_clean":"2023-09-23T19:54:20.799058+0000","last_became_active":"2023-09-23T19:52:21.180255+0000","last_became_peered":"2023-09-23T19:52:21.180255+0000","last_unstale":"2023-09-23T19:54:20.799058+0000","last_undegraded":"2023-09-23T19:54:20.799058+0000","last_fullsized":"2023-09-23T19:54:20.799058+0000","mapping_epoch":112,"log_start":"115'700","ondisk_log_start":"115'700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7736,"log_dups_size":0,"ondisk_log_size":7736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:51.310686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407633920,"num_objects":6220,"num_object_clones":0,"num_object_copies":18660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8436,"num_write_kb":468992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":72,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239330+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:54:17.239330+0000","last_peered":"2023-09-23T19:54:17.239330+0000","last_clean":"2023-09-23T19:54:17.239330+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:54:17.239330+0000","last_undegraded":"2023-09-23T19:54:17.239330+0000","last_fullsized":"2023-09-23T19:54:17.239330+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":122,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.789920+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:54:11.789920+0000","last_peered":"2023-09-23T19:54:11.789920+0000","last_clean":"2023-09-23T19:54:11.789920+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:54:11.789920+0000","last_undegraded":"2023-09-23T19:54:11.789920+0000","last_fullsized":"2023-09-23T19:54:11.789920+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":102,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.782576+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:54:11.782576+0000","last_peered":"2023-09-23T19:54:11.782576+0000","last_clean":"2023-09-23T19:54:11.782576+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:54:11.782576+0000","last_undegraded":"2023-09-23T19:54:11.782576+0000","last_fullsized":"2023-09-23T19:54:11.782576+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780626+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:54:11.780626+0000","last_peered":"2023-09-23T19:54:11.780626+0000","last_clean":"2023-09-23T19:54:11.780626+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:54:11.780626+0000","last_undegraded":"2023-09-23T19:54:11.780626+0000","last_fullsized":"2023-09-23T19:54:11.780626+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.782893+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:54:11.782893+0000","last_peered":"2023-09-23T19:54:11.782893+0000","last_clean":"2023-09-23T19:54:11.782893+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:54:11.782893+0000","last_undegraded":"2023-09-23T19:54:11.782893+0000","last_fullsized":"2023-09-23T19:54:11.782893+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.780285+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:54:11.780285+0000","last_peered":"2023-09-23T19:54:11.780285+0000","last_clean":"2023-09-23T19:54:11.780285+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:54:11.780285+0000","last_undegraded":"2023-09-23T19:54:11.780285+0000","last_fullsized":"2023-09-23T19:54:11.780285+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240680+0000","last_change":"2023-09-23T19:50:57.786561+0000","last_active":"2023-09-23T19:54:17.240680+0000","last_peered":"2023-09-23T19:54:17.240680+0000","last_clean":"2023-09-23T19:54:17.240680+0000","last_became_active":"2023-09-23T19:50:57.786231+0000","last_became_peered":"2023-09-23T19:50:57.786231+0000","last_unstale":"2023-09-23T19:54:17.240680+0000","last_undegraded":"2023-09-23T19:54:17.240680+0000","last_fullsized":"2023-09-23T19:54:17.240680+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:27.327075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240851+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:54:17.240851+0000","last_peered":"2023-09-23T19:54:17.240851+0000","last_clean":"2023-09-23T19:54:17.240851+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:54:17.240851+0000","last_undegraded":"2023-09-23T19:54:17.240851+0000","last_fullsized":"2023-09-23T19:54:17.240851+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":87,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249199+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:54:17.249199+0000","last_peered":"2023-09-23T19:54:17.249199+0000","last_clean":"2023-09-23T19:54:17.249199+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:54:17.249199+0000","last_undegraded":"2023-09-23T19:54:17.249199+0000","last_fullsized":"2023-09-23T19:54:17.249199+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T19:54:11.779672+0000","last_change":"2023-09-23T19:49:51.378856+0000","last_active":"2023-09-23T19:54:11.779672+0000","last_peered":"2023-09-23T19:54:11.779672+0000","last_clean":"2023-09-23T19:54:11.779672+0000","last_became_active":"2023-09-23T19:49:51.378451+0000","last_became_peered":"2023-09-23T19:49:51.378451+0000","last_unstale":"2023-09-23T19:54:11.779672+0000","last_undegraded":"2023-09-23T19:54:11.779672+0000","last_fullsized":"2023-09-23T19:54:11.779672+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:03.276740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6583287832,"num_objects":100454,"num_object_clones":0,"num_object_copies":301362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100454,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131110,"num_write_kb":7409985,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19700789248,"data_stored":19700777032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126410,"ondisk_log_size":126410,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234158,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2095716,"kb_used_data":2037212,"kb_used_omap":0,"kb_used_meta":58496,"kb_avail":91649436,"statfs":{"total":95995035648,"available":93849022464,"internally_reserved":0,"allocated":2086105088,"data_stored":2085563127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":59899904},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":0.78600000000000003,"15min":0.65100000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.5890000000000004,"5min":5.5890000000000004,"15min":5.5890000000000004},"last":2.3849999999999998},{"interface":"front","average":{"1min":1.589,"5min":0.83899999999999997,"15min":0.69899999999999995},"min":{"1min":0.308,"5min":0.254,"15min":0.254},"max":{"1min":5.5750000000000002,"5min":5.5750000000000002,"15min":5.5750000000000002},"last":3.1019999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:53:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":0.88800000000000001,"15min":0.68000000000000005},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":11.093999999999999,"5min":11.093999999999999,"15min":11.093999999999999},"last":2.875},{"interface":"front","average":{"1min":1.722,"5min":0.86699999999999999,"15min":0.66500000000000004},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3529999999999998,"5min":5.1200000000000001,"15min":5.1200000000000001},"last":3.044}]},{"osd":2,"last update":"Sat Sep 23 19:53:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":1.2210000000000001,"15min":0.97499999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":17.556000000000001,"5min":17.556000000000001,"15min":17.556000000000001},"last":7.4390000000000001},{"interface":"front","average":{"1min":2.6539999999999999,"5min":1.0009999999999999,"15min":0.72399999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":17.300000000000001,"5min":17.300000000000001,"15min":17.300000000000001},"last":0.82199999999999995}]},{"osd":3,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.1240000000000001,"15min":0.93200000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3330000000000002,"5min":8.3330000000000002,"15min":8.3330000000000002},"last":2.0409999999999999},{"interface":"front","average":{"1min":1.597,"5min":0.90800000000000003,"15min":0.73199999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.0380000000000003,"5min":7.0380000000000003,"15min":7.0380000000000003},"last":2.1669999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":0.67600000000000005,"15min":0.53600000000000003},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.5979999999999999,"5min":3.5979999999999999,"15min":3.5979999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.585,"5min":0.78100000000000003,"15min":0.60199999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4550000000000001,"5min":6.4550000000000001,"15min":6.4550000000000001},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":0.74399999999999999,"15min":0.58799999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.2859999999999996,"5min":5.2859999999999996,"15min":5.2859999999999996},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.5629999999999999,"5min":0.76700000000000002,"15min":0.60999999999999999},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":5.0270000000000001,"5min":5.0270000000000001,"15min":5.0270000000000001},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":0.77900000000000003,"15min":0.63},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.0700000000000003,"5min":5.0700000000000003,"15min":5.0700000000000003},"last":0.312},{"interface":"front","average":{"1min":1.538,"5min":0.83999999999999997,"15min":0.66700000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.4580000000000002,"5min":5.4580000000000002,"15min":5.4580000000000002},"last":0.434}]}]},{"osd":6,"up_from":46,"seq":197568495795,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1669964,"kb_used_data":1628036,"kb_used_omap":0,"kb_used_meta":41920,"kb_avail":92075188,"statfs":{"total":95995035648,"available":94284992512,"internally_reserved":0,"allocated":1667108864,"data_stored":1666583051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42926080},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:52 2023","interfaces":[{"interface":"back","average":{"1min":4.415,"5min":1.3500000000000001,"15min":0.85199999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":60.738,"5min":60.738,"15min":60.738},"last":5.6749999999999998},{"interface":"front","average":{"1min":4.202,"5min":1.333,"15min":0.88700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":60.722999999999999,"5min":60.722999999999999,"15min":60.722999999999999},"last":5.6260000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.85799999999999998,"15min":0.69999999999999996},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6520000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":4.8019999999999996},{"interface":"front","average":{"1min":2.395,"5min":0.94299999999999995,"15min":0.69099999999999995},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":2.0289999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":7.8159999999999998,"5min":2.0390000000000001,"15min":1.0669999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":61.155000000000001,"5min":61.155000000000001,"15min":61.155000000000001},"last":4.6420000000000003},{"interface":"front","average":{"1min":7.9900000000000002,"5min":2.1400000000000001,"15min":1.1970000000000001},"min":{"1min":0.38700000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":60.506,"5min":60.506,"15min":60.506},"last":2.8559999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":0.97899999999999998,"15min":0.76700000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":5.7619999999999996},"last":4.3979999999999997},{"interface":"front","average":{"1min":2.1379999999999999,"5min":0.94299999999999995,"15min":0.69299999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":1.127}]},{"osd":4,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":1.0740000000000001,"15min":0.68600000000000005},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.4859999999999998,"5min":24.003,"15min":24.003},"last":2.9199999999999999},{"interface":"front","average":{"1min":1.607,"5min":1.0920000000000001,"15min":0.70199999999999996},"min":{"1min":0.378,"5min":0.313,"15min":0.26500000000000001},"max":{"1min":6.6020000000000003,"5min":24.802,"15min":24.802},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.147,"15min":0.71099999999999997},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":14.378,"5min":24.065999999999999,"15min":24.065999999999999},"last":4.4409999999999998},{"interface":"front","average":{"1min":1.641,"5min":1.1359999999999999,"15min":0.74399999999999999},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":6.0529999999999999,"5min":24.841000000000001,"15min":24.841000000000001},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 19:53:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.69399999999999995,"15min":0.60899999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.9589999999999996,"5min":4.9589999999999996,"15min":4.9589999999999996},"last":2.0640000000000001},{"interface":"front","average":{"1min":1.631,"5min":0.83499999999999996,"15min":0.67700000000000005},"min":{"1min":0.39400000000000002,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":7.6029999999999998,"5min":7.6029999999999998,"15min":7.6029999999999998},"last":0.58799999999999997}]}]},{"osd":1,"up_from":99,"seq":425201762360,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2075796,"kb_used_data":1999820,"kb_used_omap":7,"kb_used_meta":75960,"kb_avail":91669356,"statfs":{"total":95995035648,"available":93869420544,"internally_reserved":0,"allocated":2047815680,"data_stored":2047173098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":77783866},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":16.609999999999999,"5min":3.6970000000000001,"15min":1.472},"min":{"1min":0.34799999999999998,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":202.917,"5min":202.917,"15min":202.917},"last":12.708},{"interface":"front","average":{"1min":17.138999999999999,"5min":3.859,"15min":1.5289999999999999},"min":{"1min":0.372,"5min":0.25,"15min":0.25},"max":{"1min":203.34899999999999,"5min":203.34899999999999,"15min":203.34899999999999},"last":0.55500000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:53:32 2023","interfaces":[{"interface":"back","average":{"1min":15.391999999999999,"5min":3.3660000000000001,"15min":1.3540000000000001},"min":{"1min":0.214,"5min":0.192,"15min":0.192},"max":{"1min":231.57499999999999,"5min":231.57499999999999,"15min":231.57499999999999},"last":1.456},{"interface":"front","average":{"1min":15.177,"5min":3.323,"15min":1.3420000000000001},"min":{"1min":0.245,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":236.077,"5min":236.077,"15min":236.077},"last":12.664999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":0.97899999999999998,"15min":0.57299999999999995},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.8330000000000002,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.6559999999999999,"5min":0.76300000000000001,"15min":0.52100000000000002},"min":{"1min":0.25800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":5.218,"5min":5.5590000000000002,"15min":5.5590000000000002},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.024,"15min":0.77600000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.5369999999999999,"5min":7.5369999999999999,"15min":7.5369999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.5429999999999999,"5min":0.84999999999999998,"15min":0.69199999999999995},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.0499999999999998,"5min":5.0499999999999998,"15min":5.0499999999999998},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.93700000000000006,"15min":0.73899999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":6.3150000000000004,"5min":6.3150000000000004,"15min":6.3150000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.371,"5min":1.0940000000000001,"15min":0.79600000000000004},"min":{"1min":0.40999999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.7779999999999996,"5min":7.7779999999999996,"15min":7.7779999999999996},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.097,"15min":0.80500000000000005},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.9649999999999999,"5min":7.9649999999999999,"15min":7.9649999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.0800000000000001,"15min":0.82399999999999995},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":0.81399999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.179,"15min":0.82599999999999996},"min":{"1min":0.44500000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.2149999999999999,"5min":9.2149999999999999,"15min":9.2149999999999999},"last":0.504},{"interface":"front","average":{"1min":2.6099999999999999,"5min":1.1439999999999999,"15min":0.82899999999999996},"min":{"1min":0.40600000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":7.7309999999999999,"5min":7.7309999999999999,"15min":7.7309999999999999},"last":0.85799999999999998}]}]},{"osd":0,"up_from":94,"seq":403726925895,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3283024,"kb_used_data":3170952,"kb_used_omap":8,"kb_used_meta":112055,"kb_avail":90462128,"statfs":{"total":95995035648,"available":92633219072,"internally_reserved":0,"allocated":3247054848,"data_stored":3246416135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":114744929},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.45700000000000002,"15min":0.36799999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.5470000000000002,"5min":3.0950000000000002,"15min":3.0950000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.55500000000000005,"15min":0.41099999999999998},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.3010000000000002,"5min":8.3010000000000002,"15min":8.3010000000000002},"last":2.9950000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":4.2939999999999996,"5min":1.181,"15min":0.67700000000000005},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":63.677,"5min":63.677,"15min":63.677},"last":4.6870000000000003},{"interface":"front","average":{"1min":5.0819999999999999,"5min":1.3089999999999999,"15min":0.68700000000000006},"min":{"1min":0.34799999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":62.564,"5min":62.564,"15min":62.564},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":6.8789999999999996,"5min":1.71,"15min":0.82199999999999995},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":104.084,"5min":104.084,"15min":104.084},"last":3.0369999999999999},{"interface":"front","average":{"1min":7.5659999999999998,"5min":1.8360000000000001,"15min":0.86099999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":104.143,"5min":104.143,"15min":104.143},"last":4.5499999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.03,"15min":0.77800000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":8.5790000000000006,"5min":8.5790000000000006,"15min":8.5790000000000006},"last":4.8090000000000002},{"interface":"front","average":{"1min":1.508,"5min":0.81599999999999995,"15min":0.73399999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.4820000000000002,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":3.4060000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":0.84699999999999998,"15min":0.75900000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.6829999999999998,"5min":4.6829999999999998,"15min":4.6829999999999998},"last":4.7229999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":0.96799999999999997,"15min":0.80200000000000005},"min":{"1min":0.441,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":8.7279999999999998,"5min":8.7279999999999998,"15min":8.7279999999999998},"last":4.8620000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":0.89700000000000002,"15min":0.78300000000000003},"min":{"1min":0.40899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.9139999999999997,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":4.7469999999999999},{"interface":"front","average":{"1min":2.3410000000000002,"5min":0.996,"15min":0.81000000000000005},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.6920000000000002,"5min":8.6920000000000002,"15min":8.6920000000000002},"last":4.7919999999999998}]},{"osd":7,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":0.93000000000000005,"15min":0.80600000000000005},"min":{"1min":0.38200000000000001,"5min":0.372,"15min":0.372},"max":{"1min":5.9379999999999997,"5min":5.9379999999999997,"15min":5.9379999999999997},"last":11.214},{"interface":"front","average":{"1min":2.5059999999999998,"5min":1.075,"15min":0.84899999999999998},"min":{"1min":0.41099999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.6560000000000006,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":4.7759999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598833,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2504968,"kb_used_data":2433152,"kb_used_omap":12,"kb_used_meta":71795,"kb_avail":91240184,"statfs":{"total":95995035648,"available":93429948416,"internally_reserved":0,"allocated":2491547648,"data_stored":2490907503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":73519102},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":0.82199999999999995,"15min":0.50800000000000001},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":41.887999999999998,"5min":41.887999999999998,"15min":41.887999999999998},"last":6.8399999999999999},{"interface":"front","average":{"1min":2.968,"5min":0.92000000000000004,"15min":0.55500000000000005},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":41.838000000000001,"5min":41.838000000000001,"15min":41.838000000000001},"last":2.669}]},{"osd":1,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":0.68200000000000005,"15min":0.48199999999999998},"min":{"1min":0.215,"5min":0.214,"15min":0.214},"max":{"1min":6.5540000000000003,"5min":6.5540000000000003,"15min":6.5540000000000003},"last":7.3499999999999996},{"interface":"front","average":{"1min":1.0860000000000001,"5min":0.52400000000000002,"15min":0.433},"min":{"1min":0.27300000000000002,"5min":0.246,"15min":0.246},"max":{"1min":5.7590000000000003,"5min":5.7590000000000003,"15min":5.7590000000000003},"last":5.4199999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":0.88300000000000001,"15min":0.55400000000000005},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.4409999999999998,"5min":6.4409999999999998,"15min":6.4409999999999998},"last":7.327},{"interface":"front","average":{"1min":2.3889999999999998,"5min":0.86799999999999999,"15min":0.55400000000000005},"min":{"1min":0.22900000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.5039999999999996,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":5.1710000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":0.71799999999999997,"15min":0.66200000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.5039999999999996,"5min":4.5039999999999996,"15min":4.5039999999999996},"last":5.0720000000000001},{"interface":"front","average":{"1min":1.621,"5min":0.82199999999999995,"15min":0.68799999999999994},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0229999999999997,"5min":7.0229999999999997,"15min":7.0229999999999997},"last":7.2569999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":0.79300000000000004,"15min":0.68700000000000006},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.0259999999999998,"5min":4.0259999999999998,"15min":4.0259999999999998},"last":12.449999999999999},{"interface":"front","average":{"1min":1.728,"5min":0.84499999999999997,"15min":0.70099999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.2539999999999996,"5min":6.2539999999999996,"15min":6.2539999999999996},"last":2.641}]},{"osd":6,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":0.89900000000000002,"15min":0.70199999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.218,"5min":6.218,"15min":6.218},"last":7.2830000000000004},{"interface":"front","average":{"1min":1.7749999999999999,"5min":0.85899999999999999,"15min":0.70599999999999996},"min":{"1min":0.434,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.4130000000000003,"5min":6.4130000000000003,"15min":6.4130000000000003},"last":7.3029999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":0.84199999999999997,"15min":0.72199999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.6219999999999999,"5min":6.6219999999999999,"15min":6.6219999999999999},"last":7.375},{"interface":"front","average":{"1min":1.5549999999999999,"5min":0.83699999999999997,"15min":0.72599999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":5.6299999999999999,"5min":5.6299999999999999,"15min":5.6299999999999999},"last":12.406000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435307,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1240792,"kb_used_data":1187024,"kb_used_omap":4,"kb_used_meta":53755,"kb_avail":92504360,"statfs":{"total":95995035648,"available":94724464640,"internally_reserved":0,"allocated":1215512576,"data_stored":1214871803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":55045229},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.9449999999999998,"15min":1.2170000000000001},"min":{"1min":0.185,"5min":0.183,"15min":0.183},"max":{"1min":5.8719999999999999,"5min":168.85400000000001,"15min":168.85400000000001},"last":0.22},{"interface":"front","average":{"1min":1.7709999999999999,"5min":3.0419999999999998,"15min":1.256},"min":{"1min":0.23699999999999999,"5min":0.217,"15min":0.217},"max":{"1min":3.8700000000000001,"5min":168.608,"15min":168.608},"last":2.8900000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":0.72399999999999998,"15min":0.496},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":8.5289999999999999,"5min":8.5289999999999999,"15min":8.5289999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.506,"5min":0.65600000000000003,"15min":0.47199999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.1660000000000004,"5min":6.1660000000000004,"15min":6.1660000000000004},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":4.3929999999999998,"5min":4.6180000000000003,"15min":1.798},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":44.091999999999999,"5min":238.52000000000001,"15min":238.52000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":4.2789999999999999,"5min":4.5229999999999997,"15min":1.782},"min":{"1min":0.34899999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":43.929000000000002,"5min":238.18100000000001,"15min":238.18100000000001},"last":2.992}]},{"osd":4,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":0.78000000000000003,"15min":0.64900000000000002},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.4660000000000002,"5min":4.4660000000000002,"15min":4.4660000000000002},"last":69.236000000000004},{"interface":"front","average":{"1min":1.8620000000000001,"5min":0.94499999999999995,"15min":0.70499999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.3970000000000002,"5min":5.3970000000000002,"15min":5.3970000000000002},"last":69.373000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":0.88,"15min":0.69399999999999995},"min":{"1min":0.38700000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.7189999999999999,"5min":3.8370000000000002,"15min":3.8370000000000002},"last":2.9279999999999999},{"interface":"front","average":{"1min":2.202,"5min":0.96699999999999997,"15min":0.74099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":10.977,"5min":10.977,"15min":10.977},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":3.4809999999999999,"5min":1.2849999999999999,"15min":0.82099999999999995},"min":{"1min":0.38600000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.4399999999999995,"5min":9.4399999999999995,"15min":9.4399999999999995},"last":2.843},{"interface":"front","average":{"1min":2.1179999999999999,"5min":0.94599999999999995,"15min":0.72099999999999997},"min":{"1min":0.44400000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":10.932,"5min":10.932,"15min":10.932},"last":1.5069999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.018,"15min":0.752},"min":{"1min":0.44500000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":12.720000000000001,"5min":12.720000000000001,"15min":12.720000000000001},"last":2.375},{"interface":"front","average":{"1min":2.0139999999999998,"5min":0.95599999999999996,"15min":0.73799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.3170000000000002,"5min":6.3170000000000002,"15min":6.3170000000000002},"last":0.499}]}]},{"osd":4,"up_from":30,"seq":128849019068,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3321428,"kb_used_data":3236812,"kb_used_omap":0,"kb_used_meta":84608,"kb_avail":90423724,"statfs":{"total":95995035648,"available":92593893376,"internally_reserved":0,"allocated":3314495488,"data_stored":3313961805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":86638592},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":10.989000000000001,"5min":2.6459999999999999,"15min":1.2869999999999999},"min":{"1min":0.22900000000000001,"5min":0.22,"15min":0.22},"max":{"1min":86.069999999999993,"5min":86.069999999999993,"15min":86.069999999999993},"last":4.899},{"interface":"front","average":{"1min":10.878,"5min":2.6499999999999999,"15min":1.3129999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":86.051000000000002,"5min":86.051000000000002,"15min":86.051000000000002},"last":4.9770000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1869999999999998,"5min":1.2010000000000001,"15min":0.77100000000000002},"min":{"1min":0.27800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":10.021000000000001,"5min":10.021000000000001,"15min":10.021000000000001},"last":4.8769999999999998},{"interface":"front","average":{"1min":2.758,"5min":1.0780000000000001,"15min":0.73599999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":4.9390000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:53:30 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":1.359,"15min":0.82699999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":35.968000000000004,"5min":35.968000000000004,"15min":35.968000000000004},"last":0.25},{"interface":"front","average":{"1min":3.8010000000000002,"5min":1.2030000000000001,"15min":0.76500000000000001},"min":{"1min":0.35399999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":35.488999999999997,"5min":35.488999999999997,"15min":35.488999999999997},"last":5}]},{"osd":3,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":0.77300000000000002,"15min":0.61199999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":11.496,"5min":11.496,"15min":11.496},"last":9.5820000000000007},{"interface":"front","average":{"1min":2.4279999999999999,"5min":0.98299999999999998,"15min":0.72199999999999998},"min":{"1min":0.373,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":11.462,"5min":11.462,"15min":11.462},"last":9.5489999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":0.81899999999999995,"15min":0.56799999999999995},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.8220000000000001,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":5.0449999999999999},{"interface":"front","average":{"1min":1.492,"5min":0.73599999999999999,"15min":0.56100000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.9130000000000003,"5min":8.9130000000000003,"15min":8.9130000000000003},"last":9.6709999999999994}]},{"osd":6,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":0.84699999999999998,"15min":0.60199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.867,"5min":5.867,"15min":5.867},"last":4.8449999999999998},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.80300000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":5.0209999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.63600000000000001,"15min":0.57899999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29599999999999999},"max":{"1min":3.1259999999999999,"5min":3.1259999999999999,"15min":3.1259999999999999},"last":9.6129999999999995},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.70799999999999996,"15min":0.61399999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.9660000000000002,"5min":3.9660000000000002,"15min":3.9660000000000002},"last":0.61099999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757432,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3717656,"kb_used_data":3641232,"kb_used_omap":0,"kb_used_meta":76032,"kb_avail":90027496,"statfs":{"total":95995035648,"available":92188155904,"internally_reserved":0,"allocated":3728621568,"data_stored":3728088132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":77856768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,8,0,0,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":0.77000000000000002,"15min":0.64300000000000002},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.843,"5min":4.843,"15min":4.843},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.4199999999999999,"5min":0.73599999999999999,"15min":0.61699999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.9399999999999995,"5min":8.9399999999999995,"15min":8.9399999999999995},"last":0.76100000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":0.97399999999999998,"15min":0.67400000000000004},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":9.8460000000000001,"5min":9.8460000000000001,"15min":9.8460000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.202,"5min":0.86499999999999999,"15min":0.63800000000000001},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":8.9120000000000008,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":0.71899999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:53:30 2023","interfaces":[{"interface":"back","average":{"1min":8.0830000000000002,"5min":2.0470000000000002,"15min":1.0409999999999999},"min":{"1min":0.498,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":98.638000000000005,"5min":98.638000000000005,"15min":98.638000000000005},"last":0.69499999999999995},{"interface":"front","average":{"1min":8.1020000000000003,"5min":2.0910000000000002,"15min":1.089},"min":{"1min":0.54400000000000004,"5min":0.317,"15min":0.317},"max":{"1min":98.668000000000006,"5min":98.668000000000006,"15min":98.668000000000006},"last":0.74099999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":0.93400000000000005,"15min":0.67600000000000005},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.6470000000000002,"5min":4.6470000000000002,"15min":4.6470000000000002},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.853,"5min":0.95399999999999996,"15min":0.72199999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.0199999999999996,"5min":5.0199999999999996,"15min":5.0199999999999996},"last":0.83299999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":0.64600000000000002,"15min":0.51300000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.23699999999999999},"max":{"1min":4.1420000000000003,"5min":4.1420000000000003,"15min":4.1420000000000003},"last":1.363},{"interface":"front","average":{"1min":1.5660000000000001,"5min":0.76500000000000001,"15min":0.57299999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.3780000000000001,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":2.1190000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:54:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":0.91600000000000004,"15min":0.63400000000000001},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":10.180999999999999,"5min":10.180999999999999,"15min":10.180999999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.046,"15min":0.71299999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":10.406000000000001},"last":1.27}]},{"osd":7,"last update":"Sat Sep 23 19:53:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":0.68999999999999995,"15min":0.59199999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.29999999999999999},"max":{"1min":3.0750000000000002,"5min":3.0750000000000002,"15min":3.0750000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.4019999999999999,"5min":0.745,"15min":0.61599999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":2.8690000000000002,"5min":2.8690000000000002,"15min":2.8690000000000002},"last":2.036}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3229351936,"data_stored":3229351936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2038300672,"data_stored":2038300672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2487881728,"data_stored":2487877656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1212547072,"data_stored":1212547072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3288268800,"data_stored":3288268800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3725266944,"data_stored":3725262872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1653997568,"data_stored":1653997568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2065174528,"data_stored":2065170456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:54:24.448 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:54:24.532 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:24.565 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:24.858 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:24.919 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:24.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:24.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:25.014 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:54:25.025 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:25.060 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:54:25.091 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:25.108 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:54:25.141 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:54:25.182 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:54:25.241 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:54:25.267 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1138014552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:25.286 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:25.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:25.296 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:54:25.335 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:54:25.362 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:54:25.362 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:54:25.362 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:54:25.362 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:54:25.362 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:54:25.362 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:54:25.362 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:54:25.362 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:54:25.388 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1138014552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:25.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1138014552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:25.392 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:25.434 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:25.582 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:54:25.582 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T19:54:25.582 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T19:54:25.582 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T19:54:25.582 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.582 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:92986 5.7 5:edef64e1:::benchmark_data_smithi135_284570_object92985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T19:54:25.582 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:53:30.836278+0000", 2023-09-23T19:54:25.582 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 54.742859914, 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.169406051, 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 92986 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.583 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836278+0000", 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836278+0000", 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836277+0000", 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:25.584 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836319+0000", 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2228000000000003e-05 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836332+0000", 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2799000000000001e-05 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836338+0000", 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.767e-06 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836393+0000", 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4660000000000002e-05 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836543+0000", 2023-09-23T19:54:25.586 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000150476 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.836709+0000", 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016635200000000001 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.839900+0000", 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031905969999999999 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.587 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:30.839941+0000", 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0806999999999997e-05 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:31.005598+0000", 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16565706599999999 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.588 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:31.005639+0000", 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1026000000000003e-05 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:31.005684+0000", 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5559000000000002e-05 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:110143 5.7 5:e73eabe1:::benchmark_data_smithi135_284570_object110142:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T19:54:25.589 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:53:50.590209+0000", 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 34.988929685999999, 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.165731036, 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 110143 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.590 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.590209+0000", 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.590209+0000", 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.590207+0000", 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:25.591 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.590250+0000", 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3344999999999999e-05 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.590263+0000", 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2510000000000001e-05 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.592 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.590269+0000", 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2090000000000002e-06 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.591405+0000", 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0011362340000000001 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.591529+0000", 2023-09-23T19:54:25.593 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000124151 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.591655+0000", 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000125876 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.595217+0000", 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035612719999999999 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.594 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.599301+0000", 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040846179999999999 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.755840+0000", 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15653866599999999 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.755899+0000", 2023-09-23T19:54:25.595 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9283999999999997e-05 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:50.755940+0000", 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0447000000000001e-05 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:114615 5.7 5:e0433cb2:::benchmark_data_smithi135_284570_object114614:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:53:56.073574+0000", 2023-09-23T19:54:25.596 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 29.505564557, 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21698904199999999, 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 114615 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.597 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073574+0000", 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073574+0000", 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073572+0000", 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:25.598 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073616+0000", 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4610000000000001e-05 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073634+0000", 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7997000000000001e-05 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.599 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073642+0000", 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4460000000000004e-06 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073694+0000", 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2132999999999997e-05 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073843+0000", 2023-09-23T19:54:25.600 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014933000000000001 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.073975+0000", 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013124299999999999 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.077746+0000", 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037715180000000002 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.601 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.077834+0000", 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7645999999999994e-05 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.290473+0000", 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21263981300000001 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.290516+0000", 2023-09-23T19:54:25.602 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2959e-05 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.290563+0000", 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6282000000000002e-05 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:114623 5.7 5:eb72d67e:::benchmark_data_smithi135_284570_object114622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T19:54:25.603 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:53:56.139486+0000", 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 29.439651908999998, 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.161929503, 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 114623 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.604 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.139486+0000", 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.139486+0000", 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.139484+0000", 2023-09-23T19:54:25.605 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999976 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.139557+0000", 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3392000000000005e-05 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.139676+0000", 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000118758 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.606 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.139696+0000", 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0146999999999999e-05 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.139782+0000", 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5550999999999995e-05 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.139955+0000", 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017333800000000001 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.140151+0000", 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019609599999999999 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.143921+0000", 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037701240000000001 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.146694+0000", 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027728119999999999 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.301321+0000", 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15462710099999999 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.301375+0000", 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3501000000000002e-05 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:56.301416+0000", 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1162999999999999e-05 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:124375 5.f 5:fa220f90:::benchmark_data_smithi135_284570_object8591:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:25.610 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.841096+0000", 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.738042118999999, 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17780073399999999, 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 124375 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.611 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841096+0000", 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841096+0000", 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841095+0000", 2023-09-23T19:54:25.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841102+0000", 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9430000000000003e-06 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841114+0000", 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2064e-05 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.613 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841120+0000", 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4609999999999996e-06 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841173+0000", 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2917999999999997e-05 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841283+0000", 2023-09-23T19:54:25.614 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000110208 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.841388+0000", 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000104224 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.844552+0000", 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003164947 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.615 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.851973+0000", 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074202870000000002 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018779+0000", 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16680616400000001 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018841+0000", 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2323000000000006e-05 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018897+0000", 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5597999999999999e-05 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.617 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:124384 5.f 5:f8f237e4:::benchmark_data_smithi135_284570_object8600:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:08.848496+0000", 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.730642458999998, 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17051327299999999, 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 124384 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.618 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848496+0000", 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848496+0000", 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848494+0000", 2023-09-23T19:54:25.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848502+0000", 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1410000000000005e-06 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848521+0000", 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9055e-05 2023-09-23T19:54:25.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848528+0000", 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3030000000000002e-06 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848610+0000", 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1236000000000006e-05 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.622 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848726+0000", 2023-09-23T19:54:25.622 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000116128 2023-09-23T19:54:25.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.622 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.622 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:54:25.622 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.848831+0000", 2023-09-23T19:54:25.622 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105572 2023-09-23T19:54:25.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.625 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.625 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.625 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.855912+0000", 2023-09-23T19:54:25.625 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070812000000000002 2023-09-23T19:54:25.626 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.626 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.626 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.626 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:08.859671+0000", 2023-09-23T19:54:25.626 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037586730000000001 2023-09-23T19:54:25.626 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.626 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.626 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.627 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018935+0000", 2023-09-23T19:54:25.627 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15926378799999999 2023-09-23T19:54:25.627 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.627 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.627 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.627 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.018971+0000", 2023-09-23T19:54:25.627 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5649999999999999e-05 2023-09-23T19:54:25.627 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.628 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.628 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.628 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:09.019009+0000", 2023-09-23T19:54:25.628 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8483999999999999e-05 2023-09-23T19:54:25.628 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.628 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.628 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.628 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.629 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.629 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:126102 5.7 5:e90cf013:::benchmark_data_smithi135_284570_object10318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:54:25.629 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:11.209776+0000", 2023-09-23T19:54:25.629 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 14.369362397, 2023-09-23T19:54:25.629 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27505987300000001, 2023-09-23T19:54:25.629 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.629 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.629 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.630 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.630 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.630 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 126102 2023-09-23T19:54:25.630 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.630 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.630 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.630 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.631 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.209776+0000", 2023-09-23T19:54:25.631 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.631 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.631 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.631 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.631 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.209776+0000", 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.209775+0000", 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.209780+0000", 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9830000000000004e-06 2023-09-23T19:54:25.632 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.209791+0000", 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0954e-05 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.209797+0000", 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9349999999999999e-06 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.633 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.217662+0000", 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0078648269999999996 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.217764+0000", 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010273 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.634 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.217859+0000", 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4895000000000005e-05 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.222983+0000", 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0051236090000000003 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.241191+0000", 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.018208235 2023-09-23T19:54:25.635 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.484762+0000", 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.24357105700000001 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.484804+0000", 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1987000000000003e-05 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.636 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.484836+0000", 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1451000000000001e-05 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:126106 5.7 5:e7f69d01:::benchmark_data_smithi135_284570_object10322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:11.218174+0000", 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 14.360964406000001, 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27877995100000003, 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.637 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 126106 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.218174+0000", 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.638 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.218174+0000", 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.218172+0000", 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.639 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.218182+0000", 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0179e-05 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.218193+0000", 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0784999999999999e-05 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.218199+0000", 2023-09-23T19:54:25.640 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1600000000000003e-06 2023-09-23T19:54:25.641 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.641 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.641 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.641 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.223010+0000", 2023-09-23T19:54:25.641 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0048114630000000002 2023-09-23T19:54:25.641 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.641 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.641 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.223152+0000", 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014178500000000001 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.223284+0000", 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013174399999999999 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.642 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.235851+0000", 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012567241999999999 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.241349+0000", 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054976950000000004 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.496882+0000", 2023-09-23T19:54:25.643 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25553303700000002 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.496927+0000", 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4620999999999999e-05 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.496954+0000", 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7223e-05 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.644 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:126110 5.7 5:e4ece909:::benchmark_data_smithi135_284570_object10326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:11.226272+0000", 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 14.352865904, 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.270800859, 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.645 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.646 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.646 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 126110 2023-09-23T19:54:25.646 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.646 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.646 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.646 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.646 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226272+0000", 2023-09-23T19:54:25.646 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226272+0000", 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226271+0000", 2023-09-23T19:54:25.647 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:25.648 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.648 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.648 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.648 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226282+0000", 2023-09-23T19:54:25.648 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0547000000000001e-05 2023-09-23T19:54:25.648 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.648 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.648 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.649 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226293+0000", 2023-09-23T19:54:25.649 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1036000000000001e-05 2023-09-23T19:54:25.649 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.649 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.649 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.649 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226299+0000", 2023-09-23T19:54:25.649 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2659999999999998e-06 2023-09-23T19:54:25.649 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.650 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.650 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.650 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.231853+0000", 2023-09-23T19:54:25.650 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055545680000000002 2023-09-23T19:54:25.650 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.650 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.650 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.650 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.231993+0000", 2023-09-23T19:54:25.651 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000139801 2023-09-23T19:54:25.651 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.651 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.651 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.651 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.232120+0000", 2023-09-23T19:54:25.651 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012662400000000001 2023-09-23T19:54:25.652 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.652 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.652 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.652 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.241259+0000", 2023-09-23T19:54:25.652 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0091390320000000001 2023-09-23T19:54:25.652 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.652 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.652 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.241360+0000", 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000100858 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.497002+0000", 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25564257899999998 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.653 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.497042+0000", 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9320000000000003e-05 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.497073+0000", 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1537e-05 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.654 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:126113 5.7 5:e6ec8b0d:::benchmark_data_smithi135_284570_object10329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:11.231312+0000", 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 14.347825764, 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.265877541, 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.655 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 126113 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.231312+0000", 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.231312+0000", 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.656 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.231311+0000", 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.231322+0000", 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0869e-05 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.657 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.231337+0000", 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5653000000000001e-05 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.231345+0000", 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9480000000000008e-06 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.658 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.236949+0000", 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0056035490000000002 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.237067+0000", 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000118468 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.237178+0000", 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000110715 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.241342+0000", 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0041640189999999997 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.241409+0000", 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6940999999999995e-05 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.660 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.497110+0000", 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25570089200000001 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.497154+0000", 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3785999999999997e-05 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.661 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.497190+0000", 2023-09-23T19:54:25.662 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6507999999999997e-05 2023-09-23T19:54:25.662 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.662 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.662 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.662 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.662 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.663 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:126119 5.7 5:e6a72cf4:::benchmark_data_smithi135_284570_object10335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:54:25.663 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:11.244514+0000", 2023-09-23T19:54:25.663 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 14.33462469, 2023-09-23T19:54:25.663 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25545031099999999, 2023-09-23T19:54:25.663 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.663 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.663 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.663 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 126119 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244514+0000", 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.664 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244514+0000", 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244512+0000", 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244519+0000", 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7390000000000002e-06 2023-09-23T19:54:25.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244532+0000", 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3236e-05 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244540+0000", 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7069999999999992e-06 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.666 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244631+0000", 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1772000000000002e-05 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244801+0000", 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016931199999999999 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.244940+0000", 2023-09-23T19:54:25.667 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013963699999999999 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.248314+0000", 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033735570000000001 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.248425+0000", 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011069999999999999 2023-09-23T19:54:25.668 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.499853+0000", 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25142814299999999 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.499921+0000", 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8639000000000005e-05 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.669 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.499964+0000", 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2417999999999999e-05 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:127281 5.f 5:fa555460:::benchmark_data_smithi135_284570_object11497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.888122+0000", 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.691015822000001, 2023-09-23T19:54:25.670 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.194897876, 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127281 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888122+0000", 2023-09-23T19:54:25.671 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888122+0000", 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.672 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888121+0000", 2023-09-23T19:54:25.673 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:25.673 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.673 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.673 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.673 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888128+0000", 2023-09-23T19:54:25.673 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0149999999999998e-06 2023-09-23T19:54:25.673 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.674 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.674 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.674 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888141+0000", 2023-09-23T19:54:25.674 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3222e-05 2023-09-23T19:54:25.674 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.674 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.674 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888148+0000", 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5479999999999998e-06 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888194+0000", 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6511999999999998e-05 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888298+0000", 2023-09-23T19:54:25.675 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103847 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.888398+0000", 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9747999999999994e-05 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.896333+0000", 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0079357260000000006 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.676 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082641+0000", 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.186307317 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082949+0000", 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00030857399999999999 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.677 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082986+0000", 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7126000000000003e-05 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.083020+0000", 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3926e-05 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.678 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:127289 5.7 5:ecdf17c7:::benchmark_data_smithi135_284570_object11505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.891156+0000", 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.687982239, 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19165286100000001, 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127289 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.679 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891156+0000", 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891156+0000", 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.680 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891154+0000", 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891163+0000", 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9590000000000001e-06 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891175+0000", 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1851999999999999e-05 2023-09-23T19:54:25.681 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891181+0000", 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8239999999999996e-06 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891238+0000", 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7238000000000003e-05 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.682 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.683 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891339+0000", 2023-09-23T19:54:25.683 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010097 2023-09-23T19:54:25.683 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.683 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.683 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.683 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891459+0000", 2023-09-23T19:54:25.683 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000120178 2023-09-23T19:54:25.683 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082633+0000", 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.191173443 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082645+0000", 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2595000000000001e-05 2023-09-23T19:54:25.684 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082710+0000", 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5007999999999998e-05 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082772+0000", 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2181000000000007e-05 2023-09-23T19:54:25.685 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082809+0000", 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6387999999999998e-05 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:127290 5.7 5:e3f5d807:::benchmark_data_smithi135_284570_object11506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.891194+0000", 2023-09-23T19:54:25.686 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.687944428, 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19172078000000001, 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127290 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.687 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891194+0000", 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891194+0000", 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891193+0000", 2023-09-23T19:54:25.688 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891196+0000", 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7709999999999998e-06 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891202+0000", 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.575e-06 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.689 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.891205+0000", 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4319999999999999e-06 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895343+0000", 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0041377970000000003 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.690 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895451+0000", 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010758499999999999 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895548+0000", 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7697999999999999e-05 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082650+0000", 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18710146 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.691 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082685+0000", 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5675e-05 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082839+0000", 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015326700000000001 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.692 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082879+0000", 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0006000000000003e-05 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082915+0000", 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5849999999999997e-05 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.694 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.694 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:127278 5.6 e139/112 5:65b5482e:::benchmark_data_smithi135_284570_object11494:head v 139'8083, mlcod=139'8079)", 2023-09-23T19:54:25.694 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.894830+0000", 2023-09-23T19:54:25.694 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.684307937, 2023-09-23T19:54:25.694 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18412039999999999, 2023-09-23T19:54:25.694 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.694 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.695 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.695 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.894830+0000", 2023-09-23T19:54:25.695 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.695 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.695 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.696 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.894830+0000", 2023-09-23T19:54:25.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.696 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.696 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.696 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.696 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.894830+0000", 2023-09-23T19:54:25.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:25.697 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.697 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.697 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.894837+0000", 2023-09-23T19:54:25.697 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9770000000000003e-06 2023-09-23T19:54:25.697 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.894847+0000", 2023-09-23T19:54:25.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0349e-05 2023-09-23T19:54:25.698 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.698 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.894853+0000", 2023-09-23T19:54:25.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6489999999999996e-06 2023-09-23T19:54:25.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.699 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.699 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.894910+0000", 2023-09-23T19:54:25.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7144e-05 2023-09-23T19:54:25.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.894927+0000", 2023-09-23T19:54:25.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7255e-05 2023-09-23T19:54:25.700 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.700 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898390+0000", 2023-09-23T19:54:25.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003463458 2023-09-23T19:54:25.701 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.701 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.701 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.701 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078951+0000", 2023-09-23T19:54:25.701 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18056028099999999 2023-09-23T19:54:25.701 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.701 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.702 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.702 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.702 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.702 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:127285 5.4 e139/112 5:29355026:::benchmark_data_smithi135_284570_object11501:head v 139'7976, mlcod=139'7974)", 2023-09-23T19:54:25.702 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.895382+0000", 2023-09-23T19:54:25.702 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.683756354, 2023-09-23T19:54:25.702 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18688871500000001, 2023-09-23T19:54:25.703 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.703 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.703 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.703 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.703 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.703 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895382+0000", 2023-09-23T19:54:25.703 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.703 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.704 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895382+0000", 2023-09-23T19:54:25.704 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.704 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.705 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895381+0000", 2023-09-23T19:54:25.705 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:25.705 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.705 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.705 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.705 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895387+0000", 2023-09-23T19:54:25.705 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9370000000000001e-06 2023-09-23T19:54:25.705 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.706 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.706 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.706 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895397+0000", 2023-09-23T19:54:25.706 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6020000000000006e-06 2023-09-23T19:54:25.706 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.706 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.706 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.706 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895402+0000", 2023-09-23T19:54:25.707 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7169999999999999e-06 2023-09-23T19:54:25.707 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.707 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.707 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.707 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895500+0000", 2023-09-23T19:54:25.707 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8672000000000007e-05 2023-09-23T19:54:25.707 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.707 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.708 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895527+0000", 2023-09-23T19:54:25.708 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6356e-05 2023-09-23T19:54:25.708 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.708 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082230+0000", 2023-09-23T19:54:25.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18670365899999999 2023-09-23T19:54:25.709 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.709 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.709 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082271+0000", 2023-09-23T19:54:25.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0380000000000003e-05 2023-09-23T19:54:25.709 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.709 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.710 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.710 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.710 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.710 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:127291 5.4 e139/112 5:2d897107:::benchmark_data_smithi135_284570_object11507:head v 139'7977, mlcod=139'7974)", 2023-09-23T19:54:25.710 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.895415+0000", 2023-09-23T19:54:25.710 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.683722996, 2023-09-23T19:54:25.710 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.187055267, 2023-09-23T19:54:25.711 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.711 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.711 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.711 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.711 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.711 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895415+0000", 2023-09-23T19:54:25.711 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.711 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895415+0000", 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895415+0000", 2023-09-23T19:54:25.712 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895418+0000", 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.252e-06 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895424+0000", 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4299999999999997e-06 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.714 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.714 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.895427+0000", 2023-09-23T19:54:25.714 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.58e-06 2023-09-23T19:54:25.714 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.714 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.714 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.714 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078518+0000", 2023-09-23T19:54:25.715 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18309119700000001 2023-09-23T19:54:25.715 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.715 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.715 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.715 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078554+0000", 2023-09-23T19:54:25.715 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5113000000000001e-05 2023-09-23T19:54:25.715 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.716 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.716 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.716 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082443+0000", 2023-09-23T19:54:25.716 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038893920000000002 2023-09-23T19:54:25.716 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.716 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.716 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.716 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082471+0000", 2023-09-23T19:54:25.717 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7548000000000001e-05 2023-09-23T19:54:25.717 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.717 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.717 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.717 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.717 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.717 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:127287 5.e e139/112 5:7f91015f:::benchmark_data_smithi135_284570_object11503:head v 139'7871, mlcod=139'7869)", 2023-09-23T19:54:25.717 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.897241+0000", 2023-09-23T19:54:25.718 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.681897231000001, 2023-09-23T19:54:25.718 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.189588073, 2023-09-23T19:54:25.718 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.718 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.718 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.718 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.718 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897241+0000", 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897241+0000", 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.719 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.720 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.720 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897240+0000", 2023-09-23T19:54:25.720 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:25.720 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.720 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.720 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.720 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897246+0000", 2023-09-23T19:54:25.721 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8409999999999996e-06 2023-09-23T19:54:25.721 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.721 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.721 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.721 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897256+0000", 2023-09-23T19:54:25.721 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.927e-06 2023-09-23T19:54:25.721 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.721 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.722 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.722 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.897261+0000", 2023-09-23T19:54:25.722 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.702e-06 2023-09-23T19:54:25.722 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.722 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.722 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.722 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898207+0000", 2023-09-23T19:54:25.722 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000945717 2023-09-23T19:54:25.723 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.723 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.723 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.723 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898224+0000", 2023-09-23T19:54:25.723 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7206000000000001e-05 2023-09-23T19:54:25.723 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.723 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.723 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.086785+0000", 2023-09-23T19:54:25.724 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18856139299999999 2023-09-23T19:54:25.724 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.086829+0000", 2023-09-23T19:54:25.724 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3857999999999999e-05 2023-09-23T19:54:25.725 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.725 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.725 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.725 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.725 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.725 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop_reply(client.44591.0:127281 5.f e139/112 ondisk, result = 0)", 2023-09-23T19:54:25.725 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.898687+0000", 2023-09-23T19:54:25.726 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.680451423000001, 2023-09-23T19:54:25.726 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18433587100000001, 2023-09-23T19:54:25.726 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.726 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T19:54:25.726 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.726 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.726 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.726 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898687+0000", 2023-09-23T19:54:25.727 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.727 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.727 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.727 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898687+0000", 2023-09-23T19:54:25.727 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.727 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898686+0000", 2023-09-23T19:54:25.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:54:25.728 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.728 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898691+0000", 2023-09-23T19:54:25.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.322e-06 2023-09-23T19:54:25.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.729 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078746+0000", 2023-09-23T19:54:25.729 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.180054679 2023-09-23T19:54:25.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078751+0000", 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5199999999999999e-06 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082938+0000", 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0041871449999999998 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.730 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.731 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.731 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.082944+0000", 2023-09-23T19:54:25.731 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0460000000000002e-06 2023-09-23T19:54:25.731 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.731 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.731 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.731 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.083023+0000", 2023-09-23T19:54:25.731 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8909e-05 2023-09-23T19:54:25.732 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.732 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.732 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.732 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.732 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.732 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:127282 5.6 e139/112 5:6bb35daa:::benchmark_data_smithi135_284570_object11498:head v 139'8084, mlcod=139'8080)", 2023-09-23T19:54:25.732 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.899345+0000", 2023-09-23T19:54:25.733 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 12.679793104, 2023-09-23T19:54:25.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19142562099999999, 2023-09-23T19:54:25.733 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:25.733 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:54:25.733 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:25.733 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.733 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:25.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899345+0000", 2023-09-23T19:54:25.734 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.734 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:25.734 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899345+0000", 2023-09-23T19:54:25.734 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:25.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.735 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.735 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:25.735 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899345+0000", 2023-09-23T19:54:25.735 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:54:25.735 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.735 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.735 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:25.736 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.899350+0000", 2023-09-23T19:54:25.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3789999999999997e-06 2023-09-23T19:54:25.736 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.736 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.736 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:25.736 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078544+0000", 2023-09-23T19:54:25.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17919416099999999 2023-09-23T19:54:25.737 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.737 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.737 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:25.737 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078552+0000", 2023-09-23T19:54:25.737 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5939999999999996e-06 2023-09-23T19:54:25.737 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.737 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.737 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:25.738 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078977+0000", 2023-09-23T19:54:25.738 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00042523600000000001 2023-09-23T19:54:25.738 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.738 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.738 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:25.738 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.078995+0000", 2023-09-23T19:54:25.738 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7883000000000001e-05 2023-09-23T19:54:25.739 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.739 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.739 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:54:25.739 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.090724+0000", 2023-09-23T19:54:25.739 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011729646 2023-09-23T19:54:25.739 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:25.739 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:25.739 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.090771+0000", 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6334000000000001e-05 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:25.740 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:54:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:25.742 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:25.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:25.843 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:25.870 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:26.338 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:26 smithi135 ceph-mon[221113]: pgmap v318: 99 pgs: 99 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T19:54:26.338 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:26 smithi135 ceph-mon[213871]: pgmap v318: 99 pgs: 99 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T19:54:26.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:26 smithi144 ceph-mon[194833]: pgmap v318: 99 pgs: 99 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T19:54:26.624 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T19:54:26.770 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:26.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:26.846 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:26.874 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:26.924 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:27.076 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:27.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:27.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:27.181 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:27.254 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:27.288 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:27.383 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:27.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:27.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:27.491 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:27.568 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:27.682 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:27.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:27.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:27.792 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:54:27.861 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:28.012 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T19:54:28.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:28.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:54:28.115 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:28.145 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:28.223 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:28.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:28.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:28.238 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:54:28.330 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:28.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:28 smithi135 ceph-mon[213871]: pgmap v319: 99 pgs: 99 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T19:54:28.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1851054086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:28.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:28 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:28.396 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:28 smithi135 ceph-mon[221113]: pgmap v319: 99 pgs: 99 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T19:54:28.397 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1851054086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:28.397 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:28 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:28.397 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:28 smithi144 ceph-mon[194833]: pgmap v319: 99 pgs: 99 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T19:54:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1851054086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:28 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:54:28.890 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:54:29.160 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:29.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:29.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:29.264 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:29.326 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:138936 5.2 5:4c095038:::benchmark_data_smithi135_284570_object23152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:29.331189+0000", 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.010765999, 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010810929, 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:29.342 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 138936 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331189+0000", 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.343 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331189+0000", 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331188+0000", 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:29.344 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331193+0000", 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7439999999999998e-06 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331210+0000", 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7544e-05 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331215+0000", 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8319999999999996e-06 2023-09-23T19:54:29.345 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331281+0000", 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6037999999999994e-05 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331418+0000", 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000136694 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.346 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.331539+0000", 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000121017 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:138939 5.3 e142/112 5:c39d1a5e:::benchmark_data_smithi135_284570_object23155:head v 142'8846, mlcod=142'8844)", 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:29.337545+0000", 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0044103090000000003, 2023-09-23T19:54:29.347 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045064509999999999, 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.337545+0000", 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.348 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.337545+0000", 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.337544+0000", 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:54:29.349 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.337558+0000", 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3631999999999999e-05 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.337572+0000", 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.398e-05 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.337579+0000", 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9399999999999996e-06 2023-09-23T19:54:29.350 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.337681+0000", 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101964 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:29.337705+0000", 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4496000000000001e-05 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:29.351 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:54:29.352 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2023-09-23T19:54:29.352 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 2 2023-09-23T19:54:29.352 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:54:29.372 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:29 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:29.372 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:29 smithi144 ceph-mon[194833]: failed to encode map e142 with expected crc 2023-09-23T19:54:29.372 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:29 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:54:29.372 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:29 smithi144 ceph-mon[194833]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T19:54:29.373 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:29 smithi144 ceph-mon[194833]: failed to encode map e142 with expected crc 2023-09-23T19:54:29.474 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:29.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:29.495 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:29.495 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[221113]: failed to encode map e142 with expected crc 2023-09-23T19:54:29.496 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:54:29.496 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[221113]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T19:54:29.496 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[221113]: failed to encode map e142 with expected crc 2023-09-23T19:54:29.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:29.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[213871]: failed to encode map e142 with expected crc 2023-09-23T19:54:29.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:54:29.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[213871]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T19:54:29.496 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:29 smithi135 ceph-mon[213871]: failed to encode map e142 with expected crc 2023-09-23T19:54:29.580 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:29.681 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:29.760 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:29.771 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:29.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:29.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:29.878 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:29.917 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:29.998 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:30.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:30.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:30.102 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:30.142 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:54:30.214 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:54:30.215 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:30.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:30 smithi144 ceph-mon[194833]: pgmap v321: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-09-23T19:54:30.307 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T19:54:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:30 smithi135 ceph-mon[221113]: pgmap v321: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-09-23T19:54:30.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:30 smithi135 ceph-mon[213871]: pgmap v321: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-09-23T19:54:30.855 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:31.284 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:31.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:31.303 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:54:31.368 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":321,"stamp":"2023-09-23T19:54:29.158150+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6172317643,"num_objects":94138,"num_object_clones":0,"num_object_copies":282414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94138,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":138361,"num_write_kb":7422710,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132457,"ondisk_log_size":132457,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18722356,"kb_used_data":18164596,"kb_used_omap":32,"kb_used_meta":557535,"kb_avail":731238860,"statfs":{"total":767960285184,"available":748788592640,"internally_reserved":0,"allocated":18600546304,"data_stored":18595844221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":570916358},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3,1,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-459341824,"num_objects":-7009,"num_object_clones":0,"num_object_copies":-21027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7009,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.891947"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":120,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.252008+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:54:17.252008+0000","last_peered":"2023-09-23T19:54:17.252008+0000","last_clean":"2023-09-23T19:54:17.252008+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:54:17.252008+0000","last_undegraded":"2023-09-23T19:54:17.252008+0000","last_fullsized":"2023-09-23T19:54:17.252008+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240493+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:54:17.240493+0000","last_peered":"2023-09-23T19:54:17.240493+0000","last_clean":"2023-09-23T19:54:17.240493+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:54:17.240493+0000","last_undegraded":"2023-09-23T19:54:17.240493+0000","last_fullsized":"2023-09-23T19:54:17.240493+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":78,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.245386+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:54:17.245386+0000","last_peered":"2023-09-23T19:54:17.245386+0000","last_clean":"2023-09-23T19:54:17.245386+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:54:17.245386+0000","last_undegraded":"2023-09-23T19:54:17.245386+0000","last_fullsized":"2023-09-23T19:54:17.245386+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":84,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.248933+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:54:17.248933+0000","last_peered":"2023-09-23T19:54:17.248933+0000","last_clean":"2023-09-23T19:54:17.248933+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:54:17.248933+0000","last_undegraded":"2023-09-23T19:54:17.248933+0000","last_fullsized":"2023-09-23T19:54:17.248933+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":121,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240153+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:54:17.240153+0000","last_peered":"2023-09-23T19:54:17.240153+0000","last_clean":"2023-09-23T19:54:17.240153+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:54:17.240153+0000","last_undegraded":"2023-09-23T19:54:17.240153+0000","last_fullsized":"2023-09-23T19:54:17.240153+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.258885+0000","last_change":"2023-09-23T19:49:51.374353+0000","last_active":"2023-09-23T19:54:17.258885+0000","last_peered":"2023-09-23T19:54:17.258885+0000","last_clean":"2023-09-23T19:54:17.258885+0000","last_became_active":"2023-09-23T19:49:51.373992+0000","last_became_peered":"2023-09-23T19:49:51.373992+0000","last_unstale":"2023-09-23T19:54:17.258885+0000","last_undegraded":"2023-09-23T19:54:17.258885+0000","last_fullsized":"2023-09-23T19:54:17.258885+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.344350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":83,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249399+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:54:17.249399+0000","last_peered":"2023-09-23T19:54:17.249399+0000","last_clean":"2023-09-23T19:54:17.249399+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:54:17.249399+0000","last_undegraded":"2023-09-23T19:54:17.249399+0000","last_fullsized":"2023-09-23T19:54:17.249399+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240032+0000","last_change":"2023-09-23T19:50:57.789294+0000","last_active":"2023-09-23T19:54:17.240032+0000","last_peered":"2023-09-23T19:54:17.240032+0000","last_clean":"2023-09-23T19:54:17.240032+0000","last_became_active":"2023-09-23T19:50:57.788922+0000","last_became_peered":"2023-09-23T19:50:57.788922+0000","last_unstale":"2023-09-23T19:54:17.240032+0000","last_undegraded":"2023-09-23T19:54:17.240032+0000","last_fullsized":"2023-09-23T19:54:17.240032+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:11.829374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.244003+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:54:17.244003+0000","last_peered":"2023-09-23T19:54:17.244003+0000","last_clean":"2023-09-23T19:54:17.244003+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:54:17.244003+0000","last_undegraded":"2023-09-23T19:54:17.244003+0000","last_fullsized":"2023-09-23T19:54:17.244003+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":54,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.241840+0000","last_change":"2023-09-23T19:50:57.789044+0000","last_active":"2023-09-23T19:54:17.241840+0000","last_peered":"2023-09-23T19:54:17.241840+0000","last_clean":"2023-09-23T19:54:17.241840+0000","last_became_active":"2023-09-23T19:50:57.788730+0000","last_became_peered":"2023-09-23T19:50:57.788730+0000","last_unstale":"2023-09-23T19:54:17.241840+0000","last_undegraded":"2023-09-23T19:54:17.241840+0000","last_fullsized":"2023-09-23T19:54:17.241840+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:23.585694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":104,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.247425+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:54:17.247425+0000","last_peered":"2023-09-23T19:54:17.247425+0000","last_clean":"2023-09-23T19:54:17.247425+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:54:17.247425+0000","last_undegraded":"2023-09-23T19:54:17.247425+0000","last_fullsized":"2023-09-23T19:54:17.247425+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":79,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246499+0000","last_change":"2023-09-23T19:50:23.998186+0000","last_active":"2023-09-23T19:54:17.246499+0000","last_peered":"2023-09-23T19:54:17.246499+0000","last_clean":"2023-09-23T19:54:17.246499+0000","last_became_active":"2023-09-23T19:50:23.997897+0000","last_became_peered":"2023-09-23T19:50:23.997897+0000","last_unstale":"2023-09-23T19:54:17.246499+0000","last_undegraded":"2023-09-23T19:54:17.246499+0000","last_fullsized":"2023-09-23T19:54:17.246499+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:57.792861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":107,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.251098+0000","last_change":"2023-09-23T19:41:39.540438+0000","last_active":"2023-09-23T19:54:17.251098+0000","last_peered":"2023-09-23T19:54:17.251098+0000","last_clean":"2023-09-23T19:54:17.251098+0000","last_became_active":"2023-09-23T19:41:39.540175+0000","last_became_peered":"2023-09-23T19:41:39.540175+0000","last_unstale":"2023-09-23T19:54:17.251098+0000","last_undegraded":"2023-09-23T19:54:17.251098+0000","last_fullsized":"2023-09-23T19:54:17.251098+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:56.614514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.250761+0000","last_change":"2023-09-23T19:49:51.379282+0000","last_active":"2023-09-23T19:54:17.250761+0000","last_peered":"2023-09-23T19:54:17.250761+0000","last_clean":"2023-09-23T19:54:17.250761+0000","last_became_active":"2023-09-23T19:49:51.378849+0000","last_became_peered":"2023-09-23T19:49:51.378849+0000","last_unstale":"2023-09-23T19:54:17.250761+0000","last_undegraded":"2023-09-23T19:54:17.250761+0000","last_fullsized":"2023-09-23T19:54:17.250761+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:00.093034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246923+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:54:17.246923+0000","last_peered":"2023-09-23T19:54:17.246923+0000","last_clean":"2023-09-23T19:54:17.246923+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:54:17.246923+0000","last_undegraded":"2023-09-23T19:54:17.246923+0000","last_fullsized":"2023-09-23T19:54:17.246923+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240634+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:54:17.240634+0000","last_peered":"2023-09-23T19:54:17.240634+0000","last_clean":"2023-09-23T19:54:17.240634+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:54:17.240634+0000","last_undegraded":"2023-09-23T19:54:17.240634+0000","last_fullsized":"2023-09-23T19:54:17.240634+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":83,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.244309+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:54:17.244309+0000","last_peered":"2023-09-23T19:54:17.244309+0000","last_clean":"2023-09-23T19:54:17.244309+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:54:17.244309+0000","last_undegraded":"2023-09-23T19:54:17.244309+0000","last_fullsized":"2023-09-23T19:54:17.244309+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":100,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240036+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:54:17.240036+0000","last_peered":"2023-09-23T19:54:17.240036+0000","last_clean":"2023-09-23T19:54:17.240036+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:54:17.240036+0000","last_undegraded":"2023-09-23T19:54:17.240036+0000","last_fullsized":"2023-09-23T19:54:17.240036+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":84,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246914+0000","last_change":"2023-09-23T19:41:39.546298+0000","last_active":"2023-09-23T19:54:17.246914+0000","last_peered":"2023-09-23T19:54:17.246914+0000","last_clean":"2023-09-23T19:54:17.246914+0000","last_became_active":"2023-09-23T19:41:39.546037+0000","last_became_peered":"2023-09-23T19:41:39.546037+0000","last_unstale":"2023-09-23T19:54:17.246914+0000","last_undegraded":"2023-09-23T19:54:17.246914+0000","last_fullsized":"2023-09-23T19:54:17.246914+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:47.642075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":84,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.248836+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:54:17.248836+0000","last_peered":"2023-09-23T19:54:17.248836+0000","last_clean":"2023-09-23T19:54:17.248836+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:54:17.248836+0000","last_undegraded":"2023-09-23T19:54:17.248836+0000","last_fullsized":"2023-09-23T19:54:17.248836+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.252934+0000","last_change":"2023-09-23T19:48:37.567607+0000","last_active":"2023-09-23T19:54:17.252934+0000","last_peered":"2023-09-23T19:54:17.252934+0000","last_clean":"2023-09-23T19:54:17.252934+0000","last_became_active":"2023-09-23T19:48:37.567300+0000","last_became_peered":"2023-09-23T19:48:37.567300+0000","last_unstale":"2023-09-23T19:54:17.252934+0000","last_undegraded":"2023-09-23T19:54:17.252934+0000","last_fullsized":"2023-09-23T19:54:17.252934+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.088452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240814+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:54:17.240814+0000","last_peered":"2023-09-23T19:54:17.240814+0000","last_clean":"2023-09-23T19:54:17.240814+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:54:17.240814+0000","last_undegraded":"2023-09-23T19:54:17.240814+0000","last_fullsized":"2023-09-23T19:54:17.240814+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"141'8512","reported_seq":8549,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.977970+0000","last_change":"2023-09-23T19:52:21.187755+0000","last_active":"2023-09-23T19:54:27.977970+0000","last_peered":"2023-09-23T19:54:27.977970+0000","last_clean":"2023-09-23T19:54:27.977970+0000","last_became_active":"2023-09-23T19:52:21.187425+0000","last_became_peered":"2023-09-23T19:52:21.187425+0000","last_unstale":"2023-09-23T19:54:27.977970+0000","last_undegraded":"2023-09-23T19:54:27.977970+0000","last_fullsized":"2023-09-23T19:54:27.977970+0000","mapping_epoch":112,"log_start":"115'800","ondisk_log_start":"115'800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7712,"log_dups_size":0,"ondisk_log_size":7712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:24.504315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377880576,"num_objects":5766,"num_object_clones":0,"num_object_copies":17298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8512,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":102,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.247194+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:54:17.247194+0000","last_peered":"2023-09-23T19:54:17.247194+0000","last_clean":"2023-09-23T19:54:17.247194+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:54:17.247194+0000","last_undegraded":"2023-09-23T19:54:17.247194+0000","last_fullsized":"2023-09-23T19:54:17.247194+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":83,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239747+0000","last_change":"2023-09-23T19:41:40.596172+0000","last_active":"2023-09-23T19:54:17.239747+0000","last_peered":"2023-09-23T19:54:17.239747+0000","last_clean":"2023-09-23T19:54:17.239747+0000","last_became_active":"2023-09-23T19:41:40.595876+0000","last_became_peered":"2023-09-23T19:41:40.595876+0000","last_unstale":"2023-09-23T19:54:17.239747+0000","last_undegraded":"2023-09-23T19:54:17.239747+0000","last_fullsized":"2023-09-23T19:54:17.239747+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:57.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"141'8399","reported_seq":8437,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.223186+0000","last_change":"2023-09-23T19:52:21.181571+0000","last_active":"2023-09-23T19:54:27.223186+0000","last_peered":"2023-09-23T19:54:27.223186+0000","last_clean":"2023-09-23T19:54:27.223186+0000","last_became_active":"2023-09-23T19:52:21.181249+0000","last_became_peered":"2023-09-23T19:52:21.181249+0000","last_unstale":"2023-09-23T19:54:27.223186+0000","last_undegraded":"2023-09-23T19:54:27.223186+0000","last_fullsized":"2023-09-23T19:54:27.223186+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8399,"log_dups_size":0,"ondisk_log_size":8399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:02.562956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377946112,"num_objects":5767,"num_object_clones":0,"num_object_copies":17301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8399,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.247700+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:54:17.247700+0000","last_peered":"2023-09-23T19:54:17.247700+0000","last_clean":"2023-09-23T19:54:17.247700+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:54:17.247700+0000","last_undegraded":"2023-09-23T19:54:17.247700+0000","last_fullsized":"2023-09-23T19:54:17.247700+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":118,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.252517+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:54:17.252517+0000","last_peered":"2023-09-23T19:54:17.252517+0000","last_clean":"2023-09-23T19:54:17.252517+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:54:17.252517+0000","last_undegraded":"2023-09-23T19:54:17.252517+0000","last_fullsized":"2023-09-23T19:54:17.252517+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"141'8465","reported_seq":8503,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.239043+0000","last_change":"2023-09-23T19:52:21.183245+0000","last_active":"2023-09-23T19:54:27.239043+0000","last_peered":"2023-09-23T19:54:27.239043+0000","last_clean":"2023-09-23T19:54:27.239043+0000","last_became_active":"2023-09-23T19:52:21.182965+0000","last_became_peered":"2023-09-23T19:52:21.182965+0000","last_unstale":"2023-09-23T19:54:27.239043+0000","last_undegraded":"2023-09-23T19:54:27.239043+0000","last_fullsized":"2023-09-23T19:54:27.239043+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8465,"log_dups_size":0,"ondisk_log_size":8465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:14.836347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385024000,"num_objects":5875,"num_object_clones":0,"num_object_copies":17625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8465,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":85,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.247229+0000","last_change":"2023-09-23T19:41:39.538481+0000","last_active":"2023-09-23T19:54:17.247229+0000","last_peered":"2023-09-23T19:54:17.247229+0000","last_clean":"2023-09-23T19:54:17.247229+0000","last_became_active":"2023-09-23T19:41:39.538210+0000","last_became_peered":"2023-09-23T19:41:39.538210+0000","last_unstale":"2023-09-23T19:54:17.247229+0000","last_undegraded":"2023-09-23T19:54:17.247229+0000","last_fullsized":"2023-09-23T19:54:17.247229+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:55.165778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":100,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.254077+0000","last_change":"2023-09-23T19:49:51.380726+0000","last_active":"2023-09-23T19:54:17.254077+0000","last_peered":"2023-09-23T19:54:17.254077+0000","last_clean":"2023-09-23T19:54:17.254077+0000","last_became_active":"2023-09-23T19:49:51.380021+0000","last_became_peered":"2023-09-23T19:49:51.380021+0000","last_unstale":"2023-09-23T19:54:17.254077+0000","last_undegraded":"2023-09-23T19:54:17.254077+0000","last_fullsized":"2023-09-23T19:54:17.254077+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:16.892029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"141'8447","reported_seq":8485,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.226568+0000","last_change":"2023-09-23T19:52:21.183448+0000","last_active":"2023-09-23T19:54:27.226568+0000","last_peered":"2023-09-23T19:54:27.226568+0000","last_clean":"2023-09-23T19:54:27.226568+0000","last_became_active":"2023-09-23T19:52:21.183139+0000","last_became_peered":"2023-09-23T19:52:21.183139+0000","last_unstale":"2023-09-23T19:54:27.226568+0000","last_undegraded":"2023-09-23T19:54:27.226568+0000","last_fullsized":"2023-09-23T19:54:27.226568+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8447,"log_dups_size":0,"ondisk_log_size":8447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:28.123472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383057920,"num_objects":5845,"num_object_clones":0,"num_object_copies":17535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8447,"num_write_kb":457344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.241140+0000","last_change":"2023-09-23T19:50:57.788389+0000","last_active":"2023-09-23T19:54:17.241140+0000","last_peered":"2023-09-23T19:54:17.241140+0000","last_clean":"2023-09-23T19:54:17.241140+0000","last_became_active":"2023-09-23T19:50:57.788075+0000","last_became_peered":"2023-09-23T19:50:57.788075+0000","last_unstale":"2023-09-23T19:54:17.241140+0000","last_undegraded":"2023-09-23T19:54:17.241140+0000","last_fullsized":"2023-09-23T19:54:17.241140+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:06.180629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":84,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239246+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:54:17.239246+0000","last_peered":"2023-09-23T19:54:17.239246+0000","last_clean":"2023-09-23T19:54:17.239246+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:54:17.239246+0000","last_undegraded":"2023-09-23T19:54:17.239246+0000","last_fullsized":"2023-09-23T19:54:17.239246+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":521,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246771+0000","last_change":"2023-09-23T19:52:57.349564+0000","last_active":"2023-09-23T19:54:17.246771+0000","last_peered":"2023-09-23T19:54:17.246771+0000","last_clean":"2023-09-23T19:54:17.246771+0000","last_became_active":"2023-09-23T19:52:57.349182+0000","last_became_peered":"2023-09-23T19:52:57.349182+0000","last_unstale":"2023-09-23T19:54:17.246771+0000","last_undegraded":"2023-09-23T19:54:17.246771+0000","last_fullsized":"2023-09-23T19:54:17.246771+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:25.262721+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"141'8667","reported_seq":8706,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.152303+0000","last_change":"2023-09-23T19:52:21.186399+0000","last_active":"2023-09-23T19:54:27.152303+0000","last_peered":"2023-09-23T19:54:27.152303+0000","last_clean":"2023-09-23T19:54:27.152303+0000","last_became_active":"2023-09-23T19:52:21.186119+0000","last_became_peered":"2023-09-23T19:52:21.186119+0000","last_unstale":"2023-09-23T19:54:27.152303+0000","last_undegraded":"2023-09-23T19:54:27.152303+0000","last_fullsized":"2023-09-23T19:54:27.152303+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8667,"log_dups_size":0,"ondisk_log_size":8667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:05.981956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391512088,"num_objects":5975,"num_object_clones":0,"num_object_copies":17925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5975,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8667,"num_write_kb":468481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240335+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:54:17.240335+0000","last_peered":"2023-09-23T19:54:17.240335+0000","last_clean":"2023-09-23T19:54:17.240335+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:54:17.240335+0000","last_undegraded":"2023-09-23T19:54:17.240335+0000","last_fullsized":"2023-09-23T19:54:17.240335+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.251019+0000","last_change":"2023-09-23T19:50:57.790231+0000","last_active":"2023-09-23T19:54:17.251019+0000","last_peered":"2023-09-23T19:54:17.251019+0000","last_clean":"2023-09-23T19:54:17.251019+0000","last_became_active":"2023-09-23T19:50:57.789446+0000","last_became_peered":"2023-09-23T19:50:57.789446+0000","last_unstale":"2023-09-23T19:54:17.251019+0000","last_undegraded":"2023-09-23T19:54:17.251019+0000","last_fullsized":"2023-09-23T19:54:17.251019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:41.313276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":453,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239749+0000","last_change":"2023-09-23T19:52:58.803028+0000","last_active":"2023-09-23T19:54:17.239749+0000","last_peered":"2023-09-23T19:54:17.239749+0000","last_clean":"2023-09-23T19:54:17.239749+0000","last_became_active":"2023-09-23T19:52:57.353261+0000","last_became_peered":"2023-09-23T19:52:57.353261+0000","last_unstale":"2023-09-23T19:54:17.239749+0000","last_undegraded":"2023-09-23T19:54:17.239749+0000","last_fullsized":"2023-09-23T19:54:17.239749+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:53.333663+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.c","version":"141'8420","reported_seq":8458,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:24.341183+0000","last_change":"2023-09-23T19:52:21.179841+0000","last_active":"2023-09-23T19:54:24.341183+0000","last_peered":"2023-09-23T19:54:24.341183+0000","last_clean":"2023-09-23T19:54:24.341183+0000","last_became_active":"2023-09-23T19:52:21.179598+0000","last_became_peered":"2023-09-23T19:52:21.179598+0000","last_unstale":"2023-09-23T19:54:24.341183+0000","last_undegraded":"2023-09-23T19:54:24.341183+0000","last_fullsized":"2023-09-23T19:54:24.341183+0000","mapping_epoch":112,"log_start":"115'500","ondisk_log_start":"115'500","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7920,"log_dups_size":0,"ondisk_log_size":7920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:39.463431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390594560,"num_objects":5960,"num_object_clones":0,"num_object_copies":17880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8420,"num_write_kb":460160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.253950+0000","last_change":"2023-09-23T19:49:51.380925+0000","last_active":"2023-09-23T19:54:17.253950+0000","last_peered":"2023-09-23T19:54:17.253950+0000","last_clean":"2023-09-23T19:54:17.253950+0000","last_became_active":"2023-09-23T19:49:51.380427+0000","last_became_peered":"2023-09-23T19:49:51.380427+0000","last_unstale":"2023-09-23T19:54:17.253950+0000","last_undegraded":"2023-09-23T19:54:17.253950+0000","last_fullsized":"2023-09-23T19:54:17.253950+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:49.879887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":118,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.251372+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:54:17.251372+0000","last_peered":"2023-09-23T19:54:17.251372+0000","last_clean":"2023-09-23T19:54:17.251372+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:54:17.251372+0000","last_undegraded":"2023-09-23T19:54:17.251372+0000","last_fullsized":"2023-09-23T19:54:17.251372+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"141'8658","reported_seq":8695,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.999232+0000","last_change":"2023-09-23T19:52:21.187234+0000","last_active":"2023-09-23T19:54:27.999232+0000","last_peered":"2023-09-23T19:54:27.999232+0000","last_clean":"2023-09-23T19:54:27.999232+0000","last_became_active":"2023-09-23T19:52:21.186449+0000","last_became_peered":"2023-09-23T19:52:21.186449+0000","last_unstale":"2023-09-23T19:54:27.999232+0000","last_undegraded":"2023-09-23T19:54:27.999232+0000","last_fullsized":"2023-09-23T19:54:27.999232+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8658,"log_dups_size":0,"ondisk_log_size":8658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:15.473024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381419520,"num_objects":5820,"num_object_clones":0,"num_object_copies":17460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8658,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.252338+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:54:17.252338+0000","last_peered":"2023-09-23T19:54:17.252338+0000","last_clean":"2023-09-23T19:54:17.252338+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:54:17.252338+0000","last_undegraded":"2023-09-23T19:54:17.252338+0000","last_fullsized":"2023-09-23T19:54:17.252338+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":111,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.250058+0000","last_change":"2023-09-23T19:50:23.998753+0000","last_active":"2023-09-23T19:54:17.250058+0000","last_peered":"2023-09-23T19:54:17.250058+0000","last_clean":"2023-09-23T19:54:17.250058+0000","last_became_active":"2023-09-23T19:50:23.998417+0000","last_became_peered":"2023-09-23T19:50:23.998417+0000","last_unstale":"2023-09-23T19:54:17.250058+0000","last_undegraded":"2023-09-23T19:54:17.250058+0000","last_fullsized":"2023-09-23T19:54:17.250058+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:11.002531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":453,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240327+0000","last_change":"2023-09-23T19:52:59.809395+0000","last_active":"2023-09-23T19:54:17.240327+0000","last_peered":"2023-09-23T19:54:17.240327+0000","last_clean":"2023-09-23T19:54:17.240327+0000","last_became_active":"2023-09-23T19:52:57.359009+0000","last_became_peered":"2023-09-23T19:52:57.359009+0000","last_unstale":"2023-09-23T19:54:17.240327+0000","last_undegraded":"2023-09-23T19:54:17.240327+0000","last_fullsized":"2023-09-23T19:54:17.240327+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:56.921631+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.e","version":"141'8532","reported_seq":8569,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.979845+0000","last_change":"2023-09-23T19:52:21.184411+0000","last_active":"2023-09-23T19:54:27.979845+0000","last_peered":"2023-09-23T19:54:27.979845+0000","last_clean":"2023-09-23T19:54:27.979845+0000","last_became_active":"2023-09-23T19:52:21.184090+0000","last_became_peered":"2023-09-23T19:52:21.184090+0000","last_unstale":"2023-09-23T19:54:27.979845+0000","last_undegraded":"2023-09-23T19:54:27.979845+0000","last_fullsized":"2023-09-23T19:54:27.979845+0000","mapping_epoch":112,"log_start":"116'1700","ondisk_log_start":"116'1700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":6832,"log_dups_size":0,"ondisk_log_size":6832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:46.571567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379584512,"num_objects":5792,"num_object_clones":0,"num_object_copies":17376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8532,"num_write_kb":458368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":104,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240307+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:54:17.240307+0000","last_peered":"2023-09-23T19:54:17.240307+0000","last_clean":"2023-09-23T19:54:17.240307+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:54:17.240307+0000","last_undegraded":"2023-09-23T19:54:17.240307+0000","last_fullsized":"2023-09-23T19:54:17.240307+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":80,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:19.435702+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:54:19.435702+0000","last_peered":"2023-09-23T19:54:19.435702+0000","last_clean":"2023-09-23T19:54:19.435702+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:54:19.435702+0000","last_undegraded":"2023-09-23T19:54:19.435702+0000","last_fullsized":"2023-09-23T19:54:19.435702+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":463,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.258784+0000","last_change":"2023-09-23T19:52:58.592695+0000","last_active":"2023-09-23T19:54:17.258784+0000","last_peered":"2023-09-23T19:54:17.258784+0000","last_clean":"2023-09-23T19:54:17.258784+0000","last_became_active":"2023-09-23T19:52:57.349277+0000","last_became_peered":"2023-09-23T19:52:57.349277+0000","last_unstale":"2023-09-23T19:54:17.258784+0000","last_undegraded":"2023-09-23T19:54:17.258784+0000","last_fullsized":"2023-09-23T19:54:17.258784+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.592597+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:25.665486+0000","scrub_duration":0.0050461000000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":476,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246228+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:54:17.246228+0000","last_peered":"2023-09-23T19:54:17.246228+0000","last_clean":"2023-09-23T19:54:17.246228+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:54:17.246228+0000","last_undegraded":"2023-09-23T19:54:17.246228+0000","last_fullsized":"2023-09-23T19:54:17.246228+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":118,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240108+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:54:17.240108+0000","last_peered":"2023-09-23T19:54:17.240108+0000","last_clean":"2023-09-23T19:54:17.240108+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:54:17.240108+0000","last_undegraded":"2023-09-23T19:54:17.240108+0000","last_fullsized":"2023-09-23T19:54:17.240108+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246446+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:54:17.246446+0000","last_peered":"2023-09-23T19:54:17.246446+0000","last_clean":"2023-09-23T19:54:17.246446+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:54:17.246446+0000","last_undegraded":"2023-09-23T19:54:17.246446+0000","last_fullsized":"2023-09-23T19:54:17.246446+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"141'8694","reported_seq":8731,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:25.767990+0000","last_change":"2023-09-23T19:52:21.185632+0000","last_active":"2023-09-23T19:54:25.767990+0000","last_peered":"2023-09-23T19:54:25.767990+0000","last_clean":"2023-09-23T19:54:25.767990+0000","last_became_active":"2023-09-23T19:52:21.185348+0000","last_became_peered":"2023-09-23T19:52:21.185348+0000","last_unstale":"2023-09-23T19:54:25.767990+0000","last_undegraded":"2023-09-23T19:54:25.767990+0000","last_fullsized":"2023-09-23T19:54:25.767990+0000","mapping_epoch":112,"log_start":"115'500","ondisk_log_start":"115'500","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:30.337130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390856704,"num_objects":5964,"num_object_clones":0,"num_object_copies":17892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8694,"num_write_kb":469056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":453,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239952+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T19:54:17.239952+0000","last_peered":"2023-09-23T19:54:17.239952+0000","last_clean":"2023-09-23T19:54:17.239952+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T19:54:17.239952+0000","last_undegraded":"2023-09-23T19:54:17.239952+0000","last_fullsized":"2023-09-23T19:54:17.239952+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.5","version":"141'8492","reported_seq":8530,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:25.782339+0000","last_change":"2023-09-23T19:52:21.181440+0000","last_active":"2023-09-23T19:54:25.782339+0000","last_peered":"2023-09-23T19:54:25.782339+0000","last_clean":"2023-09-23T19:54:25.782339+0000","last_became_active":"2023-09-23T19:52:21.181162+0000","last_became_peered":"2023-09-23T19:52:21.181162+0000","last_unstale":"2023-09-23T19:54:25.782339+0000","last_undegraded":"2023-09-23T19:54:25.782339+0000","last_fullsized":"2023-09-23T19:54:25.782339+0000","mapping_epoch":112,"log_start":"115'300","ondisk_log_start":"115'300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:18.844043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393609216,"num_objects":6006,"num_object_clones":0,"num_object_copies":18018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8492,"num_write_kb":463936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":665,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240603+0000","last_change":"2023-09-23T19:49:51.382830+0000","last_active":"2023-09-23T19:54:17.240603+0000","last_peered":"2023-09-23T19:54:17.240603+0000","last_clean":"2023-09-23T19:54:17.240603+0000","last_became_active":"2023-09-23T19:49:51.382367+0000","last_became_peered":"2023-09-23T19:49:51.382367+0000","last_unstale":"2023-09-23T19:54:17.240603+0000","last_undegraded":"2023-09-23T19:54:17.240603+0000","last_fullsized":"2023-09-23T19:54:17.240603+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:48.082643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":102,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.250834+0000","last_change":"2023-09-23T19:48:37.573541+0000","last_active":"2023-09-23T19:54:17.250834+0000","last_peered":"2023-09-23T19:54:17.250834+0000","last_clean":"2023-09-23T19:54:17.250834+0000","last_became_active":"2023-09-23T19:48:37.573213+0000","last_became_peered":"2023-09-23T19:48:37.573213+0000","last_unstale":"2023-09-23T19:54:17.250834+0000","last_undegraded":"2023-09-23T19:54:17.250834+0000","last_fullsized":"2023-09-23T19:54:17.250834+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:16.981072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":95,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246150+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:54:17.246150+0000","last_peered":"2023-09-23T19:54:17.246150+0000","last_clean":"2023-09-23T19:54:17.246150+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:54:17.246150+0000","last_undegraded":"2023-09-23T19:54:17.246150+0000","last_fullsized":"2023-09-23T19:54:17.246150+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":521,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246549+0000","last_change":"2023-09-23T19:52:57.745972+0000","last_active":"2023-09-23T19:54:17.246549+0000","last_peered":"2023-09-23T19:54:17.246549+0000","last_clean":"2023-09-23T19:54:17.246549+0000","last_became_active":"2023-09-23T19:52:57.348142+0000","last_became_peered":"2023-09-23T19:52:57.348142+0000","last_unstale":"2023-09-23T19:54:17.246549+0000","last_undegraded":"2023-09-23T19:54:17.246549+0000","last_fullsized":"2023-09-23T19:54:17.246549+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.745894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.745894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.660195+0000","scrub_duration":0.0079262489999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"141'8729","reported_seq":8767,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.138575+0000","last_change":"2023-09-23T19:52:21.183302+0000","last_active":"2023-09-23T19:54:27.138575+0000","last_peered":"2023-09-23T19:54:27.138575+0000","last_clean":"2023-09-23T19:54:27.138575+0000","last_became_active":"2023-09-23T19:52:21.183010+0000","last_became_peered":"2023-09-23T19:52:21.183010+0000","last_unstale":"2023-09-23T19:54:27.138575+0000","last_undegraded":"2023-09-23T19:54:27.138575+0000","last_fullsized":"2023-09-23T19:54:27.138575+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8729,"log_dups_size":0,"ondisk_log_size":8729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:01.323672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386334720,"num_objects":5895,"num_object_clones":0,"num_object_copies":17685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8729,"num_write_kb":467968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":625,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.229313+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:54:17.229313+0000","last_peered":"2023-09-23T19:54:17.229313+0000","last_clean":"2023-09-23T19:54:17.229313+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:54:17.229313+0000","last_undegraded":"2023-09-23T19:54:17.229313+0000","last_fullsized":"2023-09-23T19:54:17.229313+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":79,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.245668+0000","last_change":"2023-09-23T19:50:23.997623+0000","last_active":"2023-09-23T19:54:17.245668+0000","last_peered":"2023-09-23T19:54:17.245668+0000","last_clean":"2023-09-23T19:54:17.245668+0000","last_became_active":"2023-09-23T19:50:23.997428+0000","last_became_peered":"2023-09-23T19:50:23.997428+0000","last_unstale":"2023-09-23T19:54:17.245668+0000","last_undegraded":"2023-09-23T19:54:17.245668+0000","last_fullsized":"2023-09-23T19:54:17.245668+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:50.529980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246423+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:54:17.246423+0000","last_peered":"2023-09-23T19:54:17.246423+0000","last_clean":"2023-09-23T19:54:17.246423+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:54:17.246423+0000","last_undegraded":"2023-09-23T19:54:17.246423+0000","last_fullsized":"2023-09-23T19:54:17.246423+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":950,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240219+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:54:17.240219+0000","last_peered":"2023-09-23T19:54:17.240219+0000","last_clean":"2023-09-23T19:54:17.240219+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:54:17.240219+0000","last_undegraded":"2023-09-23T19:54:17.240219+0000","last_fullsized":"2023-09-23T19:54:17.240219+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":502,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246510+0000","last_change":"2023-09-23T19:52:50.668453+0000","last_active":"2023-09-23T19:54:17.246510+0000","last_peered":"2023-09-23T19:54:17.246510+0000","last_clean":"2023-09-23T19:54:17.246510+0000","last_became_active":"2023-09-23T19:52:50.263397+0000","last_became_peered":"2023-09-23T19:52:50.263397+0000","last_unstale":"2023-09-23T19:54:17.246510+0000","last_undegraded":"2023-09-23T19:54:17.246510+0000","last_fullsized":"2023-09-23T19:54:17.246510+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:48.550472+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":53,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.244459+0000","last_change":"2023-09-23T19:50:57.787513+0000","last_active":"2023-09-23T19:54:17.244459+0000","last_peered":"2023-09-23T19:54:17.244459+0000","last_clean":"2023-09-23T19:54:17.244459+0000","last_became_active":"2023-09-23T19:50:57.787175+0000","last_became_peered":"2023-09-23T19:50:57.787175+0000","last_unstale":"2023-09-23T19:54:17.244459+0000","last_undegraded":"2023-09-23T19:54:17.244459+0000","last_fullsized":"2023-09-23T19:54:17.244459+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:19.081986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":83,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.253498+0000","last_change":"2023-09-23T19:41:40.598113+0000","last_active":"2023-09-23T19:54:17.253498+0000","last_peered":"2023-09-23T19:54:17.253498+0000","last_clean":"2023-09-23T19:54:17.253498+0000","last_became_active":"2023-09-23T19:41:40.597808+0000","last_became_peered":"2023-09-23T19:41:40.597808+0000","last_unstale":"2023-09-23T19:54:17.253498+0000","last_undegraded":"2023-09-23T19:54:17.253498+0000","last_fullsized":"2023-09-23T19:54:17.253498+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:19.373510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"141'8692","reported_seq":8729,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.941220+0000","last_change":"2023-09-23T19:52:21.183523+0000","last_active":"2023-09-23T19:54:27.941220+0000","last_peered":"2023-09-23T19:54:27.941220+0000","last_clean":"2023-09-23T19:54:27.941220+0000","last_became_active":"2023-09-23T19:52:21.183198+0000","last_became_peered":"2023-09-23T19:52:21.183198+0000","last_unstale":"2023-09-23T19:54:27.941220+0000","last_undegraded":"2023-09-23T19:54:27.941220+0000","last_fullsized":"2023-09-23T19:54:27.941220+0000","mapping_epoch":112,"log_start":"115'900","ondisk_log_start":"115'900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7792,"log_dups_size":0,"ondisk_log_size":7792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:07.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381681664,"num_objects":5824,"num_object_clones":0,"num_object_copies":17472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8692,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":454,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.245738+0000","last_change":"2023-09-23T19:53:33.852740+0000","last_active":"2023-09-23T19:54:17.245738+0000","last_peered":"2023-09-23T19:54:17.245738+0000","last_clean":"2023-09-23T19:54:17.245738+0000","last_became_active":"2023-09-23T19:52:57.350232+0000","last_became_peered":"2023-09-23T19:52:57.350232+0000","last_unstale":"2023-09-23T19:54:17.245738+0000","last_undegraded":"2023-09-23T19:54:17.245738+0000","last_fullsized":"2023-09-23T19:54:17.245738+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:18.009589+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"141'8626","reported_seq":8663,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.968024+0000","last_change":"2023-09-23T19:52:21.181927+0000","last_active":"2023-09-23T19:54:27.968024+0000","last_peered":"2023-09-23T19:54:27.968024+0000","last_clean":"2023-09-23T19:54:27.968024+0000","last_became_active":"2023-09-23T19:52:21.181527+0000","last_became_peered":"2023-09-23T19:52:21.181527+0000","last_unstale":"2023-09-23T19:54:27.968024+0000","last_undegraded":"2023-09-23T19:54:27.968024+0000","last_fullsized":"2023-09-23T19:54:27.968024+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8626,"log_dups_size":0,"ondisk_log_size":8626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.841652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386531328,"num_objects":5898,"num_object_clones":0,"num_object_copies":17694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8626,"num_write_kb":464768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":859,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246606+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:54:17.246606+0000","last_peered":"2023-09-23T19:54:17.246606+0000","last_clean":"2023-09-23T19:54:17.246606+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:54:17.246606+0000","last_undegraded":"2023-09-23T19:54:17.246606+0000","last_fullsized":"2023-09-23T19:54:17.246606+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":83,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.238751+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:54:17.238751+0000","last_peered":"2023-09-23T19:54:17.238751+0000","last_clean":"2023-09-23T19:54:17.238751+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:54:17.238751+0000","last_undegraded":"2023-09-23T19:54:17.238751+0000","last_fullsized":"2023-09-23T19:54:17.238751+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":97,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249104+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:54:17.249104+0000","last_peered":"2023-09-23T19:54:17.249104+0000","last_clean":"2023-09-23T19:54:17.249104+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:54:17.249104+0000","last_undegraded":"2023-09-23T19:54:17.249104+0000","last_fullsized":"2023-09-23T19:54:17.249104+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":463,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.253595+0000","last_change":"2023-09-23T19:53:06.516604+0000","last_active":"2023-09-23T19:54:17.253595+0000","last_peered":"2023-09-23T19:54:17.253595+0000","last_clean":"2023-09-23T19:54:17.253595+0000","last_became_active":"2023-09-23T19:52:57.344832+0000","last_became_peered":"2023-09-23T19:52:57.344832+0000","last_unstale":"2023-09-23T19:54:17.253595+0000","last_undegraded":"2023-09-23T19:54:17.253595+0000","last_fullsized":"2023-09-23T19:54:17.253595+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:06.516504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:06.516504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:30.290875+0000","scrub_duration":0.0084342629999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":219,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.247340+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:54:17.247340+0000","last_peered":"2023-09-23T19:54:17.247340+0000","last_clean":"2023-09-23T19:54:17.247340+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:54:17.247340+0000","last_undegraded":"2023-09-23T19:54:17.247340+0000","last_fullsized":"2023-09-23T19:54:17.247340+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":79,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246315+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:54:17.246315+0000","last_peered":"2023-09-23T19:54:17.246315+0000","last_clean":"2023-09-23T19:54:17.246315+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:54:17.246315+0000","last_undegraded":"2023-09-23T19:54:17.246315+0000","last_fullsized":"2023-09-23T19:54:17.246315+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":69,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239376+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:54:17.239376+0000","last_peered":"2023-09-23T19:54:17.239376+0000","last_clean":"2023-09-23T19:54:17.239376+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:54:17.239376+0000","last_undegraded":"2023-09-23T19:54:17.239376+0000","last_fullsized":"2023-09-23T19:54:17.239376+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"141'8722","reported_seq":8759,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.984114+0000","last_change":"2023-09-23T19:52:21.188633+0000","last_active":"2023-09-23T19:54:27.984114+0000","last_peered":"2023-09-23T19:54:27.984114+0000","last_clean":"2023-09-23T19:54:27.984114+0000","last_became_active":"2023-09-23T19:52:21.188310+0000","last_became_peered":"2023-09-23T19:52:21.188310+0000","last_unstale":"2023-09-23T19:54:27.984114+0000","last_undegraded":"2023-09-23T19:54:27.984114+0000","last_fullsized":"2023-09-23T19:54:27.984114+0000","mapping_epoch":112,"log_start":"115'300","ondisk_log_start":"115'300","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8422,"log_dups_size":0,"ondisk_log_size":8422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:56.470945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387579904,"num_objects":5914,"num_object_clones":0,"num_object_copies":17742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8722,"num_write_kb":468352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":453,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.251272+0000","last_change":"2023-09-23T19:52:59.792472+0000","last_active":"2023-09-23T19:54:17.251272+0000","last_peered":"2023-09-23T19:54:17.251272+0000","last_clean":"2023-09-23T19:54:17.251272+0000","last_became_active":"2023-09-23T19:52:57.356135+0000","last_became_peered":"2023-09-23T19:52:57.356135+0000","last_unstale":"2023-09-23T19:54:17.251272+0000","last_undegraded":"2023-09-23T19:54:17.251272+0000","last_fullsized":"2023-09-23T19:54:17.251272+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:49.843853+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240864+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:54:17.240864+0000","last_peered":"2023-09-23T19:54:17.240864+0000","last_clean":"2023-09-23T19:54:17.240864+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:54:17.240864+0000","last_undegraded":"2023-09-23T19:54:17.240864+0000","last_fullsized":"2023-09-23T19:54:17.240864+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":402,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.246203+0000","last_change":"2023-09-23T19:49:51.380375+0000","last_active":"2023-09-23T19:54:17.246203+0000","last_peered":"2023-09-23T19:54:17.246203+0000","last_clean":"2023-09-23T19:54:17.246203+0000","last_became_active":"2023-09-23T19:49:51.379793+0000","last_became_peered":"2023-09-23T19:49:51.379793+0000","last_unstale":"2023-09-23T19:54:17.246203+0000","last_undegraded":"2023-09-23T19:54:17.246203+0000","last_fullsized":"2023-09-23T19:54:17.246203+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:26.355152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":98,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240229+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:54:17.240229+0000","last_peered":"2023-09-23T19:54:17.240229+0000","last_clean":"2023-09-23T19:54:17.240229+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:54:17.240229+0000","last_undegraded":"2023-09-23T19:54:17.240229+0000","last_fullsized":"2023-09-23T19:54:17.240229+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"141'8730","reported_seq":8767,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:27.934240+0000","last_change":"2023-09-23T19:52:21.183759+0000","last_active":"2023-09-23T19:54:27.934240+0000","last_peered":"2023-09-23T19:54:27.934240+0000","last_clean":"2023-09-23T19:54:27.934240+0000","last_became_active":"2023-09-23T19:52:21.183451+0000","last_became_peered":"2023-09-23T19:52:21.183451+0000","last_unstale":"2023-09-23T19:54:27.934240+0000","last_undegraded":"2023-09-23T19:54:27.934240+0000","last_fullsized":"2023-09-23T19:54:27.934240+0000","mapping_epoch":112,"log_start":"116'1900","ondisk_log_start":"116'1900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:43.359001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381550592,"num_objects":5822,"num_object_clones":0,"num_object_copies":17466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8730,"num_write_kb":465664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":455,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240329+0000","last_change":"2023-09-23T19:53:00.705074+0000","last_active":"2023-09-23T19:54:17.240329+0000","last_peered":"2023-09-23T19:54:17.240329+0000","last_clean":"2023-09-23T19:54:17.240329+0000","last_became_active":"2023-09-23T19:52:57.353333+0000","last_became_peered":"2023-09-23T19:52:57.353333+0000","last_unstale":"2023-09-23T19:54:17.240329+0000","last_undegraded":"2023-09-23T19:54:17.240329+0000","last_fullsized":"2023-09-23T19:54:17.240329+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T19:53:00.704900+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:24.642289+0000","scrub_duration":0.0090978159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":372,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.254520+0000","last_change":"2023-09-23T19:50:23.999651+0000","last_active":"2023-09-23T19:54:17.254520+0000","last_peered":"2023-09-23T19:54:17.254520+0000","last_clean":"2023-09-23T19:54:17.254520+0000","last_became_active":"2023-09-23T19:50:23.999184+0000","last_became_peered":"2023-09-23T19:50:23.999184+0000","last_unstale":"2023-09-23T19:54:17.254520+0000","last_undegraded":"2023-09-23T19:54:17.254520+0000","last_fullsized":"2023-09-23T19:54:17.254520+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:41.010259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":87,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249027+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:54:17.249027+0000","last_peered":"2023-09-23T19:54:17.249027+0000","last_clean":"2023-09-23T19:54:17.249027+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:54:17.249027+0000","last_undegraded":"2023-09-23T19:54:17.249027+0000","last_fullsized":"2023-09-23T19:54:17.249027+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":84,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.252400+0000","last_change":"2023-09-23T19:41:40.350897+0000","last_active":"2023-09-23T19:54:17.252400+0000","last_peered":"2023-09-23T19:54:17.252400+0000","last_clean":"2023-09-23T19:54:17.252400+0000","last_became_active":"2023-09-23T19:41:40.350643+0000","last_became_peered":"2023-09-23T19:41:40.350643+0000","last_unstale":"2023-09-23T19:54:17.252400+0000","last_undegraded":"2023-09-23T19:54:17.252400+0000","last_fullsized":"2023-09-23T19:54:17.252400+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:35.769972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"141'8669","reported_seq":8708,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-09-23T19:54:25.738248+0000","last_change":"2023-09-23T19:52:21.180582+0000","last_active":"2023-09-23T19:54:25.738248+0000","last_peered":"2023-09-23T19:54:25.738248+0000","last_clean":"2023-09-23T19:54:25.738248+0000","last_became_active":"2023-09-23T19:52:21.180255+0000","last_became_peered":"2023-09-23T19:52:21.180255+0000","last_unstale":"2023-09-23T19:54:25.738248+0000","last_undegraded":"2023-09-23T19:54:25.738248+0000","last_fullsized":"2023-09-23T19:54:25.738248+0000","mapping_epoch":112,"log_start":"115'900","ondisk_log_start":"115'900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7769,"log_dups_size":0,"ondisk_log_size":7769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:51.310686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392364032,"num_objects":5987,"num_object_clones":0,"num_object_copies":17961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8669,"num_write_kb":468992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":72,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.239330+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:54:17.239330+0000","last_peered":"2023-09-23T19:54:17.239330+0000","last_clean":"2023-09-23T19:54:17.239330+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:54:17.239330+0000","last_undegraded":"2023-09-23T19:54:17.239330+0000","last_fullsized":"2023-09-23T19:54:17.239330+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":123,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.229805+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:54:17.229805+0000","last_peered":"2023-09-23T19:54:17.229805+0000","last_clean":"2023-09-23T19:54:17.229805+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:54:17.229805+0000","last_undegraded":"2023-09-23T19:54:17.229805+0000","last_fullsized":"2023-09-23T19:54:17.229805+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.241216+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:54:17.241216+0000","last_peered":"2023-09-23T19:54:17.241216+0000","last_clean":"2023-09-23T19:54:17.241216+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:54:17.241216+0000","last_undegraded":"2023-09-23T19:54:17.241216+0000","last_fullsized":"2023-09-23T19:54:17.241216+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":118,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.250903+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:54:17.250903+0000","last_peered":"2023-09-23T19:54:17.250903+0000","last_clean":"2023-09-23T19:54:17.250903+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:54:17.250903+0000","last_undegraded":"2023-09-23T19:54:17.250903+0000","last_fullsized":"2023-09-23T19:54:17.250903+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":86,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240194+0000","last_change":"2023-09-23T19:41:39.543343+0000","last_active":"2023-09-23T19:54:17.240194+0000","last_peered":"2023-09-23T19:54:17.240194+0000","last_clean":"2023-09-23T19:54:17.240194+0000","last_became_active":"2023-09-23T19:41:39.543097+0000","last_became_peered":"2023-09-23T19:41:39.543097+0000","last_unstale":"2023-09-23T19:54:17.240194+0000","last_undegraded":"2023-09-23T19:54:17.240194+0000","last_fullsized":"2023-09-23T19:54:17.240194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:39.527773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":118,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.252123+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:54:17.252123+0000","last_peered":"2023-09-23T19:54:17.252123+0000","last_clean":"2023-09-23T19:54:17.252123+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:54:17.252123+0000","last_undegraded":"2023-09-23T19:54:17.252123+0000","last_fullsized":"2023-09-23T19:54:17.252123+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240680+0000","last_change":"2023-09-23T19:50:57.786561+0000","last_active":"2023-09-23T19:54:17.240680+0000","last_peered":"2023-09-23T19:54:17.240680+0000","last_clean":"2023-09-23T19:54:17.240680+0000","last_became_active":"2023-09-23T19:50:57.786231+0000","last_became_peered":"2023-09-23T19:50:57.786231+0000","last_unstale":"2023-09-23T19:54:17.240680+0000","last_undegraded":"2023-09-23T19:54:17.240680+0000","last_fullsized":"2023-09-23T19:54:17.240680+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:27.327075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.240851+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:54:17.240851+0000","last_peered":"2023-09-23T19:54:17.240851+0000","last_clean":"2023-09-23T19:54:17.240851+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:54:17.240851+0000","last_undegraded":"2023-09-23T19:54:17.240851+0000","last_fullsized":"2023-09-23T19:54:17.240851+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":87,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249199+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:54:17.249199+0000","last_peered":"2023-09-23T19:54:17.249199+0000","last_clean":"2023-09-23T19:54:17.249199+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:54:17.249199+0000","last_undegraded":"2023-09-23T19:54:17.249199+0000","last_fullsized":"2023-09-23T19:54:17.249199+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T19:54:17.249277+0000","last_change":"2023-09-23T19:49:51.378856+0000","last_active":"2023-09-23T19:54:17.249277+0000","last_peered":"2023-09-23T19:54:17.249277+0000","last_clean":"2023-09-23T19:54:17.249277+0000","last_became_active":"2023-09-23T19:49:51.378451+0000","last_became_peered":"2023-09-23T19:49:51.378451+0000","last_unstale":"2023-09-23T19:54:17.249277+0000","last_undegraded":"2023-09-23T19:54:17.249277+0000","last_fullsized":"2023-09-23T19:54:17.249277+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:03.276740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6167527448,"num_objects":94110,"num_object_clones":0,"num_object_copies":282330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94110,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137454,"num_write_kb":7409985,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18508361728,"data_stored":18508349512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129654,"ondisk_log_size":129654,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234160,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1937964,"kb_used_data":1886116,"kb_used_omap":0,"kb_used_meta":51776,"kb_avail":91807188,"statfs":{"total":95995035648,"available":94010560512,"internally_reserved":0,"allocated":1931382784,"data_stored":1930840701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":53018624},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":0.78600000000000003,"15min":0.65100000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.5890000000000004,"5min":5.5890000000000004,"15min":5.5890000000000004},"last":1.4590000000000001},{"interface":"front","average":{"1min":1.589,"5min":0.83899999999999997,"15min":0.69899999999999995},"min":{"1min":0.308,"5min":0.254,"15min":0.254},"max":{"1min":5.5750000000000002,"5min":5.5750000000000002,"15min":5.5750000000000002},"last":1.2170000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:53:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":0.88800000000000001,"15min":0.68000000000000005},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":11.093999999999999,"5min":11.093999999999999,"15min":11.093999999999999},"last":0.501},{"interface":"front","average":{"1min":1.722,"5min":0.86699999999999999,"15min":0.66500000000000004},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3529999999999998,"5min":5.1200000000000001,"15min":5.1200000000000001},"last":1.2809999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:54:26 2023","interfaces":[{"interface":"back","average":{"1min":2.488,"5min":1.548,"15min":1.0840000000000001},"min":{"1min":0.34799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.3599999999999994,"5min":17.556000000000001,"15min":17.556000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.254,"5min":1.534,"15min":0.90200000000000002},"min":{"1min":0.34499999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.465,"5min":17.300000000000001,"15min":17.300000000000001},"last":1.2470000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.1240000000000001,"15min":0.93200000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3330000000000002,"5min":8.3330000000000002,"15min":8.3330000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.597,"5min":0.90800000000000003,"15min":0.73199999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.0380000000000003,"5min":7.0380000000000003,"15min":7.0380000000000003},"last":1.359}]},{"osd":4,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":0.67600000000000005,"15min":0.53600000000000003},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.5979999999999999,"5min":3.5979999999999999,"15min":3.5979999999999999},"last":1.171},{"interface":"front","average":{"1min":1.585,"5min":0.78100000000000003,"15min":0.60199999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4550000000000001,"5min":6.4550000000000001,"15min":6.4550000000000001},"last":1.423}]},{"osd":5,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":0.74399999999999999,"15min":0.58799999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.2859999999999996,"5min":5.2859999999999996,"15min":5.2859999999999996},"last":1.3300000000000001},{"interface":"front","average":{"1min":1.5629999999999999,"5min":0.76700000000000002,"15min":0.60999999999999999},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":5.0270000000000001,"5min":5.0270000000000001,"15min":5.0270000000000001},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":0.77900000000000003,"15min":0.63},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.0700000000000003,"5min":5.0700000000000003,"15min":5.0700000000000003},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.538,"5min":0.83999999999999997,"15min":0.66700000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.4580000000000002,"5min":5.4580000000000002,"15min":5.4580000000000002},"last":1.488}]}]},{"osd":6,"up_from":46,"seq":197568495797,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1554780,"kb_used_data":1507604,"kb_used_omap":0,"kb_used_meta":47168,"kb_avail":92190372,"statfs":{"total":95995035648,"available":94402940928,"internally_reserved":0,"allocated":1543786496,"data_stored":1543260439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":48300032},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:52 2023","interfaces":[{"interface":"back","average":{"1min":4.415,"5min":1.3500000000000001,"15min":0.85199999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":60.738,"5min":60.738,"15min":60.738},"last":0.29299999999999998},{"interface":"front","average":{"1min":4.202,"5min":1.333,"15min":0.88700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":60.722999999999999,"5min":60.722999999999999,"15min":60.722999999999999},"last":1.8580000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.85799999999999998,"15min":0.69999999999999996},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6520000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":2.9929999999999999},{"interface":"front","average":{"1min":2.395,"5min":0.94299999999999995,"15min":0.69099999999999995},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":7.8159999999999998,"5min":2.0390000000000001,"15min":1.0669999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":61.155000000000001,"5min":61.155000000000001,"15min":61.155000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":7.9900000000000002,"5min":2.1400000000000001,"15min":1.1970000000000001},"min":{"1min":0.38700000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":60.506,"5min":60.506,"15min":60.506},"last":5.7380000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":0.97899999999999998,"15min":0.76700000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":5.7619999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.1379999999999999,"5min":0.94299999999999995,"15min":0.69299999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":0.66800000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":1.0740000000000001,"15min":0.68600000000000005},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.4859999999999998,"5min":24.003,"15min":24.003},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.607,"5min":1.0920000000000001,"15min":0.70199999999999996},"min":{"1min":0.378,"5min":0.313,"15min":0.26500000000000001},"max":{"1min":6.6020000000000003,"5min":24.802,"15min":24.802},"last":1.8}]},{"osd":5,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.147,"15min":0.71099999999999997},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":14.378,"5min":24.065999999999999,"15min":24.065999999999999},"last":0.63},{"interface":"front","average":{"1min":1.641,"5min":1.1359999999999999,"15min":0.74399999999999999},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":6.0529999999999999,"5min":24.841000000000001,"15min":24.841000000000001},"last":1.835}]},{"osd":7,"last update":"Sat Sep 23 19:53:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.69399999999999995,"15min":0.60899999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.9589999999999996,"5min":4.9589999999999996,"15min":4.9589999999999996},"last":0.309},{"interface":"front","average":{"1min":1.631,"5min":0.83499999999999996,"15min":0.67700000000000005},"min":{"1min":0.39400000000000002,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":7.6029999999999998,"5min":7.6029999999999998,"15min":7.6029999999999998},"last":3.1960000000000002}]}]},{"osd":1,"up_from":99,"seq":425201762361,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989532,"kb_used_data":1925844,"kb_used_omap":7,"kb_used_meta":63672,"kb_avail":91755620,"statfs":{"total":95995035648,"available":93957754880,"internally_reserved":0,"allocated":1972064256,"data_stored":1971421552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":65200954},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":16.609999999999999,"5min":3.6970000000000001,"15min":1.472},"min":{"1min":0.34799999999999998,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":202.917,"5min":202.917,"15min":202.917},"last":4.923},{"interface":"front","average":{"1min":17.138999999999999,"5min":3.859,"15min":1.5289999999999999},"min":{"1min":0.372,"5min":0.25,"15min":0.25},"max":{"1min":203.34899999999999,"5min":203.34899999999999,"15min":203.34899999999999},"last":4.7089999999999996}]},{"osd":2,"last update":"Sat Sep 23 19:53:32 2023","interfaces":[{"interface":"back","average":{"1min":15.391999999999999,"5min":3.3660000000000001,"15min":1.3540000000000001},"min":{"1min":0.214,"5min":0.192,"15min":0.192},"max":{"1min":231.57499999999999,"5min":231.57499999999999,"15min":231.57499999999999},"last":4.0540000000000003},{"interface":"front","average":{"1min":15.177,"5min":3.323,"15min":1.3420000000000001},"min":{"1min":0.245,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":236.077,"5min":236.077,"15min":236.077},"last":9.9600000000000009}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":0.97899999999999998,"15min":0.57299999999999995},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.8330000000000002,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":4.0300000000000002},{"interface":"front","average":{"1min":1.6559999999999999,"5min":0.76300000000000001,"15min":0.52100000000000002},"min":{"1min":0.25800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":5.218,"5min":5.5590000000000002,"15min":5.5590000000000002},"last":4.7409999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.024,"15min":0.77600000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.5369999999999999,"5min":7.5369999999999999,"15min":7.5369999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.5429999999999999,"5min":0.84999999999999998,"15min":0.69199999999999995},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.0499999999999998,"5min":5.0499999999999998,"15min":5.0499999999999998},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.93700000000000006,"15min":0.73899999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":6.3150000000000004,"5min":6.3150000000000004,"15min":6.3150000000000004},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.371,"5min":1.0940000000000001,"15min":0.79600000000000004},"min":{"1min":0.40999999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.7779999999999996,"5min":7.7779999999999996,"15min":7.7779999999999996},"last":3.98}]},{"osd":6,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.097,"15min":0.80500000000000005},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.9649999999999999,"5min":7.9649999999999999,"15min":7.9649999999999999},"last":5.3659999999999997},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.0800000000000001,"15min":0.82399999999999995},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":4.6550000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.179,"15min":0.82599999999999996},"min":{"1min":0.44500000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.2149999999999999,"5min":9.2149999999999999,"15min":9.2149999999999999},"last":5.29},{"interface":"front","average":{"1min":2.6099999999999999,"5min":1.1439999999999999,"15min":0.82899999999999996},"min":{"1min":0.40600000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":7.7309999999999999,"5min":7.7309999999999999,"15min":7.7309999999999999},"last":0.53500000000000003}]}]},{"osd":0,"up_from":94,"seq":403726925896,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3163928,"kb_used_data":3060048,"kb_used_omap":8,"kb_used_meta":103863,"kb_avail":90581224,"statfs":{"total":95995035648,"available":92755173376,"internally_reserved":0,"allocated":3133489152,"data_stored":3132850317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":106356321},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.45700000000000002,"15min":0.36799999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.5470000000000002,"5min":3.0950000000000002,"15min":3.0950000000000002},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.55500000000000005,"15min":0.41099999999999998},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.3010000000000002,"5min":8.3010000000000002,"15min":8.3010000000000002},"last":0.38}]},{"osd":2,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":4.2939999999999996,"5min":1.181,"15min":0.67700000000000005},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":63.677,"5min":63.677,"15min":63.677},"last":0.39500000000000002},{"interface":"front","average":{"1min":5.0819999999999999,"5min":1.3089999999999999,"15min":0.68700000000000006},"min":{"1min":0.34799999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":62.564,"5min":62.564,"15min":62.564},"last":0.34100000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":6.8789999999999996,"5min":1.71,"15min":0.82199999999999995},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":104.084,"5min":104.084,"15min":104.084},"last":0.41299999999999998},{"interface":"front","average":{"1min":7.5659999999999998,"5min":1.8360000000000001,"15min":0.86099999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":104.143,"5min":104.143,"15min":104.143},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.03,"15min":0.77800000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":8.5790000000000006,"5min":8.5790000000000006,"15min":8.5790000000000006},"last":110.312},{"interface":"front","average":{"1min":1.508,"5min":0.81599999999999995,"15min":0.73399999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.4820000000000002,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":110.256}]},{"osd":5,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":0.84699999999999998,"15min":0.75900000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.6829999999999998,"5min":4.6829999999999998,"15min":4.6829999999999998},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.1360000000000001,"5min":0.96799999999999997,"15min":0.80200000000000005},"min":{"1min":0.441,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":8.7279999999999998,"5min":8.7279999999999998,"15min":8.7279999999999998},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":0.89700000000000002,"15min":0.78300000000000003},"min":{"1min":0.40899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.9139999999999997,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":109.044},{"interface":"front","average":{"1min":2.3410000000000002,"5min":0.996,"15min":0.81000000000000005},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.6920000000000002,"5min":8.6920000000000002,"15min":8.6920000000000002},"last":109.06399999999999}]},{"osd":7,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":0.93000000000000005,"15min":0.80600000000000005},"min":{"1min":0.38200000000000001,"5min":0.372,"15min":0.372},"max":{"1min":5.9379999999999997,"5min":5.9379999999999997,"15min":5.9379999999999997},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.5059999999999998,"5min":1.075,"15min":0.84899999999999998},"min":{"1min":0.41099999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.6560000000000006,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":0.52100000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598835,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2321436,"kb_used_data":2257748,"kb_used_omap":12,"kb_used_meta":63603,"kb_avail":91423716,"statfs":{"total":95995035648,"available":93617885184,"internally_reserved":0,"allocated":2311933952,"data_stored":2311289806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":65130494},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":0.82199999999999995,"15min":0.50800000000000001},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":41.887999999999998,"5min":41.887999999999998,"15min":41.887999999999998},"last":1.099},{"interface":"front","average":{"1min":2.968,"5min":0.92000000000000004,"15min":0.55500000000000005},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":41.838000000000001,"5min":41.838000000000001,"15min":41.838000000000001},"last":1.782}]},{"osd":1,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":0.68200000000000005,"15min":0.48199999999999998},"min":{"1min":0.215,"5min":0.214,"15min":0.214},"max":{"1min":6.5540000000000003,"5min":6.5540000000000003,"15min":6.5540000000000003},"last":5.1959999999999997},{"interface":"front","average":{"1min":1.0860000000000001,"5min":0.52400000000000002,"15min":0.433},"min":{"1min":0.27300000000000002,"5min":0.246,"15min":0.246},"max":{"1min":5.7590000000000003,"5min":5.7590000000000003,"15min":5.7590000000000003},"last":1.0229999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":0.88300000000000001,"15min":0.55400000000000005},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.4409999999999998,"5min":6.4409999999999998,"15min":6.4409999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.3889999999999998,"5min":0.86799999999999999,"15min":0.55400000000000005},"min":{"1min":0.22900000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.5039999999999996,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":1.071}]},{"osd":4,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":0.71799999999999997,"15min":0.66200000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.5039999999999996,"5min":4.5039999999999996,"15min":4.5039999999999996},"last":1.21},{"interface":"front","average":{"1min":1.621,"5min":0.82199999999999995,"15min":0.68799999999999994},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0229999999999997,"5min":7.0229999999999997,"15min":7.0229999999999997},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":0.79300000000000004,"15min":0.68700000000000006},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.0259999999999998,"5min":4.0259999999999998,"15min":4.0259999999999998},"last":1.05},{"interface":"front","average":{"1min":1.728,"5min":0.84499999999999997,"15min":0.70099999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.2539999999999996,"5min":6.2539999999999996,"15min":6.2539999999999996},"last":1.8140000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":0.89900000000000002,"15min":0.70199999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.218,"5min":6.218,"15min":6.218},"last":1.837},{"interface":"front","average":{"1min":1.7749999999999999,"5min":0.85899999999999999,"15min":0.70599999999999996},"min":{"1min":0.434,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.4130000000000003,"5min":6.4130000000000003,"15min":6.4130000000000003},"last":5.2800000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":0.84199999999999997,"15min":0.72199999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.6219999999999999,"5min":6.6219999999999999,"15min":6.6219999999999999},"last":5.2450000000000001},{"interface":"front","average":{"1min":1.5549999999999999,"5min":0.83699999999999997,"15min":0.72599999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":5.6299999999999999,"5min":5.6299999999999999,"15min":5.6299999999999999},"last":2.4940000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435308,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1202136,"kb_used_data":1144272,"kb_used_omap":4,"kb_used_meta":57851,"kb_avail":92543016,"statfs":{"total":95995035648,"available":94764048384,"internally_reserved":0,"allocated":1171734528,"data_stored":1171093755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":59239533},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.9449999999999998,"15min":1.2170000000000001},"min":{"1min":0.185,"5min":0.183,"15min":0.183},"max":{"1min":5.8719999999999999,"5min":168.85400000000001,"15min":168.85400000000001},"last":0.20200000000000001},{"interface":"front","average":{"1min":1.7709999999999999,"5min":3.0419999999999998,"15min":1.256},"min":{"1min":0.23699999999999999,"5min":0.217,"15min":0.217},"max":{"1min":3.8700000000000001,"5min":168.608,"15min":168.608},"last":2.605}]},{"osd":1,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":0.72399999999999998,"15min":0.496},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":8.5289999999999999,"5min":8.5289999999999999,"15min":8.5289999999999999},"last":2.5299999999999998},{"interface":"front","average":{"1min":1.506,"5min":0.65600000000000003,"15min":0.47199999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.1660000000000004,"5min":6.1660000000000004,"15min":6.1660000000000004},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":4.3929999999999998,"5min":4.6180000000000003,"15min":1.798},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":44.091999999999999,"5min":238.52000000000001,"15min":238.52000000000001},"last":2.5619999999999998},{"interface":"front","average":{"1min":4.2789999999999999,"5min":4.5229999999999997,"15min":1.782},"min":{"1min":0.34899999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":43.929000000000002,"5min":238.18100000000001,"15min":238.18100000000001},"last":0.42399999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":0.78000000000000003,"15min":0.64900000000000002},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.4660000000000002,"5min":4.4660000000000002,"15min":4.4660000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.8620000000000001,"5min":0.94499999999999995,"15min":0.70499999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.3970000000000002,"5min":5.3970000000000002,"15min":5.3970000000000002},"last":5.4800000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":0.88,"15min":0.69399999999999995},"min":{"1min":0.38700000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.7189999999999999,"5min":3.8370000000000002,"15min":3.8370000000000002},"last":2.9319999999999999},{"interface":"front","average":{"1min":2.202,"5min":0.96699999999999997,"15min":0.74099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":10.977,"5min":10.977,"15min":10.977},"last":2.6899999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":3.4809999999999999,"5min":1.2849999999999999,"15min":0.82099999999999995},"min":{"1min":0.38600000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.4399999999999995,"5min":9.4399999999999995,"15min":9.4399999999999995},"last":4.399},{"interface":"front","average":{"1min":2.1179999999999999,"5min":0.94599999999999995,"15min":0.72099999999999997},"min":{"1min":0.44400000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":10.932,"5min":10.932,"15min":10.932},"last":2.6160000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.018,"15min":0.752},"min":{"1min":0.44500000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":12.720000000000001,"5min":12.720000000000001,"15min":12.720000000000001},"last":3.5489999999999999},{"interface":"front","average":{"1min":2.0139999999999998,"5min":0.95599999999999996,"15min":0.73799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.3170000000000002,"5min":6.3170000000000002,"15min":6.3170000000000002},"last":0.78900000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019070,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3084836,"kb_used_data":3006108,"kb_used_omap":0,"kb_used_meta":78720,"kb_avail":90660316,"statfs":{"total":95995035648,"available":92836163584,"internally_reserved":0,"allocated":3078254592,"data_stored":3077720665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":80609280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":10.989000000000001,"5min":2.6459999999999999,"15min":1.2869999999999999},"min":{"1min":0.22900000000000001,"5min":0.22,"15min":0.22},"max":{"1min":86.069999999999993,"5min":86.069999999999993,"15min":86.069999999999993},"last":0.33000000000000002},{"interface":"front","average":{"1min":10.878,"5min":2.6499999999999999,"15min":1.3129999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":86.051000000000002,"5min":86.051000000000002,"15min":86.051000000000002},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1869999999999998,"5min":1.2010000000000001,"15min":0.77100000000000002},"min":{"1min":0.27800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":10.021000000000001,"5min":10.021000000000001,"15min":10.021000000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":2.758,"5min":1.0780000000000001,"15min":0.73599999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":0.255}]},{"osd":2,"last update":"Sat Sep 23 19:53:30 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":1.359,"15min":0.82699999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":35.968000000000004,"5min":35.968000000000004,"15min":35.968000000000004},"last":2.109},{"interface":"front","average":{"1min":3.8010000000000002,"5min":1.2030000000000001,"15min":0.76500000000000001},"min":{"1min":0.35399999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":35.488999999999997,"5min":35.488999999999997,"15min":35.488999999999997},"last":0.61699999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":0.77300000000000002,"15min":0.61199999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":11.496,"5min":11.496,"15min":11.496},"last":4.8680000000000003},{"interface":"front","average":{"1min":2.4279999999999999,"5min":0.98299999999999998,"15min":0.72199999999999998},"min":{"1min":0.373,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":11.462,"5min":11.462,"15min":11.462},"last":4.9450000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":0.81899999999999995,"15min":0.56799999999999995},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.8220000000000001,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.492,"5min":0.73599999999999999,"15min":0.56100000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.9130000000000003,"5min":8.9130000000000003,"15min":8.9130000000000003},"last":12.675000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":0.84699999999999998,"15min":0.60199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.867,"5min":5.867,"15min":5.867},"last":1.8660000000000001},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.80300000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 19:54:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6200000000000001,"5min":1.2410000000000001,"15min":0.78300000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.29599999999999999},"max":{"1min":11.59,"5min":11.59,"15min":11.59},"last":4.915},{"interface":"front","average":{"1min":2.7669999999999999,"5min":1.1339999999999999,"15min":0.76000000000000001},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.3089999999999993,"5min":9.3089999999999993,"15min":9.3089999999999993},"last":0.35499999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757434,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3467744,"kb_used_data":3376856,"kb_used_omap":0,"kb_used_meta":90880,"kb_avail":90277408,"statfs":{"total":95995035648,"available":92444065792,"internally_reserved":0,"allocated":3457900544,"data_stored":3457366986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":93061120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":0.77000000000000002,"15min":0.64300000000000002},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.843,"5min":4.843,"15min":4.843},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.4199999999999999,"5min":0.73599999999999999,"15min":0.61699999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.9399999999999995,"5min":8.9399999999999995,"15min":8.9399999999999995},"last":0.31900000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":0.97399999999999998,"15min":0.67400000000000004},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":9.8460000000000001,"5min":9.8460000000000001,"15min":9.8460000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.202,"5min":0.86499999999999999,"15min":0.63800000000000001},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":8.9120000000000008,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":1.7090000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:53:30 2023","interfaces":[{"interface":"back","average":{"1min":8.0830000000000002,"5min":2.0470000000000002,"15min":1.0409999999999999},"min":{"1min":0.498,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":98.638000000000005,"5min":98.638000000000005,"15min":98.638000000000005},"last":1.01},{"interface":"front","average":{"1min":8.1020000000000003,"5min":2.0910000000000002,"15min":1.089},"min":{"1min":0.54400000000000004,"5min":0.317,"15min":0.317},"max":{"1min":98.668000000000006,"5min":98.668000000000006,"15min":98.668000000000006},"last":1.0309999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":0.93400000000000005,"15min":0.67600000000000005},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.6470000000000002,"5min":4.6470000000000002,"15min":4.6470000000000002},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.853,"5min":0.95399999999999996,"15min":0.72199999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.0199999999999996,"5min":5.0199999999999996,"15min":5.0199999999999996},"last":1.349}]},{"osd":4,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":0.64600000000000002,"15min":0.51300000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.23699999999999999},"max":{"1min":4.1420000000000003,"5min":4.1420000000000003,"15min":4.1420000000000003},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.5660000000000001,"5min":0.76500000000000001,"15min":0.57299999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.3780000000000001,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 19:54:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":0.91600000000000004,"15min":0.63400000000000001},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":10.180999999999999,"5min":10.180999999999999,"15min":10.180999999999999},"last":0.435},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.046,"15min":0.71299999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":10.406000000000001},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:53:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":0.68999999999999995,"15min":0.59199999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.29999999999999999},"max":{"1min":3.0750000000000002,"5min":3.0750000000000002,"15min":3.0750000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.4019999999999999,"5min":0.745,"15min":0.61599999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":2.8690000000000002,"5min":2.8690000000000002,"15min":2.8690000000000002},"last":0.45000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3113091072,"data_stored":3113091072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1964310528,"data_stored":1964310528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2307919872,"data_stored":2307915800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1167065088,"data_stored":1167065088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3055484928,"data_stored":3055484928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3455782912,"data_stored":3455778840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1531314176,"data_stored":1531314176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1913393152,"data_stored":1913389080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:54:31.374 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T19:54:31.375 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:54:31.375 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T19:54:31.375 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:54:31.375 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:54:31.375 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T19:54:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:31 smithi144 ceph-mon[194833]: from='client.44714 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:54:31.389 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:31.480 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:31.513 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:31 smithi135 ceph-mon[213871]: from='client.44714 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:54:31.513 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:31 smithi135 ceph-mon[221113]: from='client.44714 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:54:31.623 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:31.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:31.750 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:31.778 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:31.820 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:31.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:31.923 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:31.968 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:32.125 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:32.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:32.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:32.243 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:32.267 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:32.324 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:32.363 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:32.538 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:32 smithi135 ceph-mon[213871]: pgmap v322: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:54:32.540 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:32 smithi135 ceph-mon[221113]: pgmap v322: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:54:32.612 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:32 smithi144 ceph-mon[194833]: pgmap v322: 99 pgs: 99 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-09-23T19:54:32.654 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:32.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:32.773 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:32.871 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:32.891 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:54:32.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:54:32.948 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:54:32.948 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T19:54:32.948 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T19:54:32.948 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T19:54:32.948 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:54:32.953 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:54:32.980 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:54:33.026 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:54:33.071 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:54:33.110 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:54:33.146 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:54:33.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:54:33.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:54:33.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:54:33.162 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:54:33.163 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:54:33.163 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:54:33.163 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:54:33.163 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:54:33.180 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:33.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:33.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:33.284 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:33.317 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:33 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4264061748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:33 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4264061748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:33 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4264061748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:54:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:54:33] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T19:54:33.994 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T19:54:34.270 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:34 smithi144 ceph-mon[194833]: pgmap v323: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:54:34.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:34 smithi135 ceph-mon[213871]: pgmap v323: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:54:34.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:34 smithi135 ceph-mon[221113]: pgmap v323: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T19:54:34.393 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:34.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:34.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:34.500 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:34.577 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:34.716 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:34.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:34.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:34.772 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:34.830 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:34.895 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:35.011 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:35.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:35.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:35.117 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:35.144 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:35.323 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:35.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:35.432 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:35.469 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:35.749 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:35.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:35.854 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:35.928 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:36.128 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:36.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:36.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:36.236 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:36.308 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:36 smithi135 ceph-mon[213871]: pgmap v324: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-23T19:54:36.312 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:36.314 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:36 smithi135 ceph-mon[221113]: pgmap v324: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-23T19:54:36.398 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:54:36.398 INFO:teuthology.orchestra.run.smithi135.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:54:36.399 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:54:36.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:36.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:36.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:36.529 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:36.547 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:36 smithi144 ceph-mon[194833]: pgmap v324: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-09-23T19:54:36.586 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:36.648 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:36.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:36.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:36.751 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:36.787 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:36.813 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:36.813 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:36.830 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:36.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:36.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:36.859 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:54:36.934 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:36.964 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:37.067 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:37.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:37.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:37.180 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:37.223 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:37.323 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:54:37.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:37 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1276425245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:37.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:37 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:37 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1276425245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:37 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:37 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1276425245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:37 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:37.654 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:54:37.689 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:37.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:37.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:37.793 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:37.820 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:37.949 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:37.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:37.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:38.079 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:38.151 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:38.279 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:38.282 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:38.284 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:38.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:38.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:38.399 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:38.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[221113]: pgmap v325: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:54:38.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:38.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:38.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[221113]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T19:54:38.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[221113]: failed to encode map e143 with expected crc 2023-09-23T19:54:38.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[221113]: failed to encode map e143 with expected crc 2023-09-23T19:54:38.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[213871]: pgmap v325: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:54:38.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:38.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:38.456 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[213871]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T19:54:38.456 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[213871]: failed to encode map e143 with expected crc 2023-09-23T19:54:38.456 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:38 smithi135 ceph-mon[213871]: failed to encode map e143 with expected crc 2023-09-23T19:54:38.456 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:38.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:38 smithi144 ceph-mon[194833]: pgmap v325: 99 pgs: 99 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-09-23T19:54:38.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:38 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:38.592 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:38 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:38.592 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:38 smithi144 ceph-mon[194833]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T19:54:38.592 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:38 smithi144 ceph-mon[194833]: failed to encode map e143 with expected crc 2023-09-23T19:54:38.592 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:38 smithi144 ceph-mon[194833]: failed to encode map e143 with expected crc 2023-09-23T19:54:38.595 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:38.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:38.701 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:38.748 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:38.896 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:38.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:38.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:39.003 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:39.076 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:39.120 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:39.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:39.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:39.223 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:39.249 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:39.290 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:39.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:39 smithi144 ceph-mon[194833]: pgmap v327: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T19:54:39.395 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:39.420 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:39.461 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:39.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:39.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:39.564 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:39.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:39 smithi135 ceph-mon[213871]: pgmap v327: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T19:54:39.616 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:39 smithi135 ceph-mon[221113]: pgmap v327: 99 pgs: 99 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T19:54:39.625 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:39.755 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:39.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:39.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:39.862 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:39.934 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:40.131 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:40.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:40.189 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:54:40.240 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:40.368 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:40.465 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:40.466 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:40.472 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:54:40.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:40 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3718532151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:40.495 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:40.509 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:40 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3718532151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:40.509 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:40 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3718532151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:40.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:40.510 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:54:40.543 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:54:40.583 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:54:40.611 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:40.642 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:54:40.685 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:40.696 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:54:40.740 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T19:54:40.789 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:54:40.819 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:40.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:54:40.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:54:40.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:54:40.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:54:40.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:54:40.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:54:40.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:54:40.823 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:54:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:40.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:40.848 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:40.928 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:40.956 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:41.078 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:41.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:41.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:41.185 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:41.231 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:41.424 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T19:54:41.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:41 smithi144 ceph-mon[194833]: pgmap v328: 99 pgs: 99 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T19:54:41.789 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:41.798 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:41 smithi135 ceph-mon[213871]: pgmap v328: 99 pgs: 99 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T19:54:41.799 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:41 smithi135 ceph-mon[221113]: pgmap v328: 99 pgs: 99 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T19:54:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:41.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:41.901 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:41.961 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:42.071 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:42.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:42.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:42.091 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:42.190 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:42.249 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:42.493 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:54:42.493 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:54:42.493 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:54:42.493 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T19:54:42.493 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T19:54:42.574 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:42.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:42.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:42.682 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:42.707 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:42.749 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:42.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:42.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:42.785 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:54:42 smithi144 systemd[1]: Stopping Ceph osd.6 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T19:54:42.852 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:42.884 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:43.211 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:43 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:54:43.448 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:43.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:43.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:43.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:43 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:54:43.553 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:43.623 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:54:43 smithi144 conmon[173765]: logger=sqlstore.transactions t=2023-09-23T19:54:43.438177868Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T19:54:43.623 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:54:43 smithi144 conmon[173765]: logger=cleanup t=2023-09-23T19:54:43.623981275Z level=info msg="Completed cleanup jobs" duration=197.057831ms 2023-09-23T19:54:43.624 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:43 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:54:43.637 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:43.718 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:43.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:43.822 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:43.876 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:54:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:54:43] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T19:54:43.879 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:43.889 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:54:43 smithi144 conmon[131356]: 2023-09-23T19:54:43.767+0000 7f87f74f1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T19:54:43.889 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:54:43 smithi144 conmon[131356]: 2023-09-23T19:54:43.767+0000 7f87f74f1700 -1 osd.6 143 *** Got signal Terminated *** 2023-09-23T19:54:43.889 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:54:43 smithi144 conmon[131356]: 2023-09-23T19:54:43.767+0000 7f87f74f1700 -1 osd.6 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:54:43.966 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:54:43.989 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:44.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:44.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:44.102 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:44.197 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:44.435 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:54:44.435 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T19:54:44.435 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T19:54:44.436 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T19:54:44.436 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:54:44.457 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:44 smithi144 ceph-mon[194833]: pgmap v329: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:54:44.457 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:44 smithi144 ceph-mon[194833]: osd.6 marked itself down and dead 2023-09-23T19:54:44.482 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:44.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:44.585 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:44.614 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:44 smithi135 ceph-mon[221113]: pgmap v329: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:54:44.614 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:44 smithi135 ceph-mon[221113]: osd.6 marked itself down and dead 2023-09-23T19:54:44.614 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:44 smithi135 ceph-mon[213871]: pgmap v329: 99 pgs: 99 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T19:54:44.615 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:44 smithi135 ceph-mon[213871]: osd.6 marked itself down and dead 2023-09-23T19:54:44.617 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:44.647 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:44.667 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:44.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:44.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:44.771 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:44.802 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:45.262 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:54:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:45 smithi144 ceph-mon[194833]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:54:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:45 smithi144 ceph-mon[194833]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T19:54:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:45 smithi144 ceph-mon[194833]: failed to encode map e144 with expected crc 2023-09-23T19:54:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3770467545' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:45 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:45.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[221113]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:54:45.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[221113]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T19:54:45.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[221113]: failed to encode map e144 with expected crc 2023-09-23T19:54:45.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3770467545' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:45.501 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:45.503 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[213871]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T19:54:45.503 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[213871]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T19:54:45.503 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[213871]: failed to encode map e144 with expected crc 2023-09-23T19:54:45.504 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3770467545' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:45.504 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:45 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:54:45.900 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T19:54:45.983 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:45.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:46.087 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:46.153 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:54:45 smithi144 bash[220605]: d04b0ed25c0ffcf48d3e97838091a753e8b262112a3a098c6649da28f2524e05 2023-09-23T19:54:46.162 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:46.396 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:46.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:46.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:46.404 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: pgmap v331: 99 pgs: 5 peering, 8 stale+active+clean, 86 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-09-23T19:54:46.404 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:54:46.404 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:46.404 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.404 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.404 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.405 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.405 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:46.405 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T19:54:46.405 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:46 smithi144 ceph-mon[194833]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: pgmap v331: 99 pgs: 5 peering, 8 stale+active+clean, 86 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-09-23T19:54:46.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:54:46.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:46.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:46.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T19:54:46.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[221113]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: pgmap v331: 99 pgs: 5 peering, 8 stale+active+clean, 86 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T19:54:46.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:46 smithi135 ceph-mon[213871]: failed to encode map e145 with expected crc 2023-09-23T19:54:46.502 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:46.589 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:46.738 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:46.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:46.779 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:54:46.846 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:46.930 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:47.069 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:47.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:47.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:47.181 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:47.238 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:47.359 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:47.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:47.467 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:47.529 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:47.623 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:54:47.656 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:54:47.689 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:54:47.732 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:54:47.757 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:47.765 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:54:47.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:47.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:47.870 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:47.896 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:47.989 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:47.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:48.064 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:54:48.093 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:48.101 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:54:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:54:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:54:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:54:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:54:48.102 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:54:48.102 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:54:48.102 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:54:48.102 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:54:48.157 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:48 smithi144 ceph-mon[194833]: pgmap v333: 99 pgs: 3 active+undersized+degraded, 5 peering, 4 stale+active+clean, 6 active+undersized, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 545 op/s; 10343/248559 objects degraded (4.161%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:48 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 10343/248559 objects degraded (4.161%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:54:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:48 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1664102641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:48.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:48 smithi135 ceph-mon[213871]: pgmap v333: 99 pgs: 3 active+undersized+degraded, 5 peering, 4 stale+active+clean, 6 active+undersized, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 545 op/s; 10343/248559 objects degraded (4.161%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:48.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:48 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 10343/248559 objects degraded (4.161%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:54:48.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:48 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1664102641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:48.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:48 smithi135 ceph-mon[221113]: pgmap v333: 99 pgs: 3 active+undersized+degraded, 5 peering, 4 stale+active+clean, 6 active+undersized, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 545 op/s; 10343/248559 objects degraded (4.161%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:48.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:48 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 10343/248559 objects degraded (4.161%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T19:54:48.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:48 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1664102641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:48.694 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:48.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:48.797 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:48.866 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:54:48.885 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:49.031 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T19:54:49.094 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:54:49 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6.service: Succeeded. 2023-09-23T19:54:49.095 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 19:54:49 smithi144 systemd[1]: Stopped Ceph osd.6 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T19:54:49.124 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:54:49.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:49.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:49.162 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:54:49.162 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T19:54:49.228 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:54:49.297 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:49.476 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:54:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:54:49.479 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:54:49.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:49.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:54:49.553 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:49.554 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:49.581 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:49.668 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:49.780 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:49.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:49.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:49.855 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:49.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:49.988 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:50.097 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:50.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:50.201 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:50.234 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:50 smithi135 ceph-mon[221113]: pgmap v334: 99 pgs: 9 active+undersized+degraded, 5 peering, 22 active+undersized, 63 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 668 op/s; 20341/243129 objects degraded (8.366%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:50.235 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:50 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:50.235 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:50 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:50.235 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:50 smithi135 ceph-mon[213871]: pgmap v334: 99 pgs: 9 active+undersized+degraded, 5 peering, 22 active+undersized, 63 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 668 op/s; 20341/243129 objects degraded (8.366%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:50.235 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:50 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:50.235 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:50 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:50.254 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:50.405 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:54:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:50.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:50.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:50 smithi144 ceph-mon[194833]: pgmap v334: 99 pgs: 9 active+undersized+degraded, 5 peering, 22 active+undersized, 63 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 668 op/s; 20341/243129 objects degraded (8.366%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:50.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:50 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:50.482 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:50 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:50.483 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:54:50.510 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:50.654 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:50.802 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:50.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:50.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:50.924 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:50.955 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:51.442 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:51.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:51.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:51.480 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:54:51.546 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:51.554 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:51 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:54:51.556 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:51 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:54:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:51 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:54:51.663 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:51.771 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:51.779 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:51.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:51.901 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:51.954 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:52.108 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:52.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:52.213 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:52.310 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:52.494 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:52.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:52.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:52.611 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:52 smithi144 ceph-mon[194833]: pgmap v335: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 629 op/s; 19936/240072 objects degraded (8.304%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:52.665 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[221113]: pgmap v335: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 629 op/s; 19936/240072 objects degraded (8.304%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:52.666 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.666 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.666 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.666 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:52.666 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[213871]: pgmap v335: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 629 op/s; 19936/240072 objects degraded (8.304%); 39 KiB/s, 0 objects/s recovering 2023-09-23T19:54:52.666 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.666 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.667 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:52.667 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:54:52.690 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:52.796 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:52.816 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:52.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:52.928 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:52.985 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:53.085 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:53.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:53.195 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:53.222 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:53.263 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:54:53.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:53.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:53.370 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:53.396 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:53.497 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:53.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:53.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:53.538 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:54:53.567 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:54:53.567 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:54:53.567 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:53.567 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3042506356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:53.567 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:53.567 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:54:53.567 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:54:53.568 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:53.568 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3042506356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:53.568 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:53.605 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:53.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:54:53.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:54:53.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:53.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3042506356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:53.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:54:53.664 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:53.915 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:54:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:54:53] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.43.0" 2023-09-23T19:54:54.163 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:54:54.163 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:54:54.164 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T19:54:54.164 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:54:54.255 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:54.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:54.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:54.363 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:54.390 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:54.469 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[213871]: pgmap v336: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 584 op/s; 19936/236916 objects degraded (8.415%); 38 KiB/s, 0 objects/s recovering 2023-09-23T19:54:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[213871]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T19:54:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 19936/236916 objects degraded (8.415%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:54:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:54:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[213871]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T19:54:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[213871]: failed to encode map e146 with expected crc 2023-09-23T19:54:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[213871]: failed to encode map e146 with expected crc 2023-09-23T19:54:54.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[221113]: pgmap v336: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 584 op/s; 19936/236916 objects degraded (8.415%); 38 KiB/s, 0 objects/s recovering 2023-09-23T19:54:54.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[221113]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T19:54:54.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 19936/236916 objects degraded (8.415%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:54:54.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:54:54.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:54.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[221113]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T19:54:54.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[221113]: failed to encode map e146 with expected crc 2023-09-23T19:54:54.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:54 smithi135 ceph-mon[221113]: failed to encode map e146 with expected crc 2023-09-23T19:54:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:54.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:54.584 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:54.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:54 smithi144 ceph-mon[194833]: pgmap v336: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 584 op/s; 19936/236916 objects degraded (8.415%); 38 KiB/s, 0 objects/s recovering 2023-09-23T19:54:54.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:54 smithi144 ceph-mon[194833]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T19:54:54.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:54 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 19936/236916 objects degraded (8.415%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:54:54.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:54 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:54:54.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:54 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:54:54.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:54 smithi144 ceph-mon[194833]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T19:54:54.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:54 smithi144 ceph-mon[194833]: failed to encode map e146 with expected crc 2023-09-23T19:54:54.639 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:54 smithi144 ceph-mon[194833]: failed to encode map e146 with expected crc 2023-09-23T19:54:54.657 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:54.732 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:54.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:54.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:54.842 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:54.899 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:55.192 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:55.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:55.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:55.255 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:54:55.256 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:54:55.301 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:55.394 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:55.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:55 smithi135 ceph-mon[221113]: failed to encode map e146 with expected crc 2023-09-23T19:54:55.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:55 smithi135 ceph-mon[213871]: failed to encode map e146 with expected crc 2023-09-23T19:54:55.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:55 smithi144 ceph-mon[194833]: failed to encode map e146 with expected crc 2023-09-23T19:54:55.890 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:55.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:55.992 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:56.026 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:56.135 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:56.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:56.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:56.244 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:56.270 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:56 smithi135 ceph-mon[213871]: pgmap v338: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 667 op/s; 19497/232218 objects degraded (8.396%); 34 KiB/s, 0 objects/s recovering 2023-09-23T19:54:56.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:56 smithi135 ceph-mon[221113]: pgmap v338: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 667 op/s; 19497/232218 objects degraded (8.396%); 34 KiB/s, 0 objects/s recovering 2023-09-23T19:54:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:56 smithi144 ceph-mon[194833]: pgmap v338: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 667 op/s; 19497/232218 objects degraded (8.396%); 34 KiB/s, 0 objects/s recovering 2023-09-23T19:54:56.739 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:54:56.877 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:54:56.887 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:54:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:56.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:54:56.989 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:57.071 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:57.170 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:57.446 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:57.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:57.555 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:54:57.588 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:57.673 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:54:57.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:57.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:54:57.783 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:54:57.817 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:57.867 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:54:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:57.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:54:57.972 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:54:57.991 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:54:58.048 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:54:58.063 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:58.084 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:54:58.103 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:54:58.143 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:54:58.183 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:54:58.231 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:54:58.265 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:54:58.266 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:54:58.266 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:54:58.266 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:54:58.266 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:54:58.266 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:54:58.266 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:54:58.266 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:54:58.266 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:54:58.267 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:54:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:58.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:54:58.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:58 smithi135 ceph-mon[213871]: pgmap v339: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 19061/229506 objects degraded (8.305%) 2023-09-23T19:54:58.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:58.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/32634294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:58.297 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:58 smithi135 ceph-mon[221113]: pgmap v339: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 19061/229506 objects degraded (8.305%) 2023-09-23T19:54:58.297 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:58.297 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/32634294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:58.373 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:54:58.438 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:58 smithi144 ceph-mon[194833]: pgmap v339: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 19061/229506 objects degraded (8.305%) 2023-09-23T19:54:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:54:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/32634294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:54:59.074 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:54:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:59.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:54:59.179 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:54:59.225 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:59.306 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:54:59.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:54:59.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:54:59.413 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:54:59.445 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:54:59 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 18637/222207 objects degraded (8.387%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:54:59.445 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:54:59 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 18637/222207 objects degraded (8.387%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:54:59.447 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:54:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:54:59 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 18637/222207 objects degraded (8.387%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:00.114 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:55:00.135 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:55:00.137 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:00.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:00.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:00.246 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:00.332 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:00.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:00 smithi144 ceph-mon[194833]: pgmap v340: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 18637/222207 objects degraded (8.387%) 2023-09-23T19:55:00.670 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:00 smithi135 ceph-mon[213871]: pgmap v340: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 18637/222207 objects degraded (8.387%) 2023-09-23T19:55:00.672 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:00 smithi135 ceph-mon[221113]: pgmap v340: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 18637/222207 objects degraded (8.387%) 2023-09-23T19:55:01.431 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:01.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:01.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:01.544 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:01.585 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:01.664 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:01.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:01.776 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:01.818 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:01.891 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:01.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:02.003 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:02.099 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:02.136 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:55:02.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:02 smithi144 ceph-mon[194833]: pgmap v341: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 18637/221427 objects degraded (8.417%) 2023-09-23T19:55:02.309 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:02 smithi135 ceph-mon[213871]: pgmap v341: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 18637/221427 objects degraded (8.417%) 2023-09-23T19:55:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:55:02.392 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:55:02.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:02 smithi135 ceph-mon[221113]: pgmap v341: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 18637/221427 objects degraded (8.417%) 2023-09-23T19:55:02.887 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:02.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:02.999 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:03.052 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:03.549 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:03.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:03.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:03.664 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:03.707 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:03.762 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":341,"stamp":"2023-09-23T19:55:00.478762+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4840143639,"num_objects":73809,"num_object_clones":0,"num_object_copies":221427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18637,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73809,"num_whiteouts":0,"num_read":6084,"num_read_kb":13379,"num_write":158734,"num_write_kb":7423150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":322018,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138871,"ondisk_log_size":138871,"up":261,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14999240,"kb_used_data":14377096,"kb_used_omap":32,"kb_used_meta":621343,"kb_avail":734961976,"statfs":{"total":767960285184,"available":752601063424,"internally_reserved":0,"allocated":14722146304,"data_stored":14717399298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":636255750},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-407306240,"num_objects":-6215,"num_object_clones":0,"num_object_copies":-18645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6215,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.315148"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":125,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.253804+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:54:45.253804+0000","last_peered":"2023-09-23T19:54:45.253804+0000","last_clean":"2023-09-23T19:54:45.253804+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:54:45.253804+0000","last_undegraded":"2023-09-23T19:54:45.253804+0000","last_fullsized":"2023-09-23T19:54:45.253804+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":122,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.253237+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:54:45.253237+0000","last_peered":"2023-09-23T19:54:45.253237+0000","last_clean":"2023-09-23T19:54:45.253237+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:54:45.253237+0000","last_undegraded":"2023-09-23T19:54:45.253237+0000","last_fullsized":"2023-09-23T19:54:45.253237+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.257635+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:54:45.257635+0000","last_peered":"2023-09-23T19:54:45.257635+0000","last_clean":"2023-09-23T19:54:45.257635+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:54:45.257635+0000","last_undegraded":"2023-09-23T19:54:45.257635+0000","last_fullsized":"2023-09-23T19:54:45.257635+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":90,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.256063+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:54:45.256063+0000","last_peered":"2023-09-23T19:54:45.256063+0000","last_clean":"2023-09-23T19:54:45.256063+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:54:45.256063+0000","last_undegraded":"2023-09-23T19:54:45.256063+0000","last_fullsized":"2023-09-23T19:54:45.256063+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.249550+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:54:45.249550+0000","last_peered":"2023-09-23T19:54:45.249550+0000","last_clean":"2023-09-23T19:54:45.249550+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:54:45.249550+0000","last_undegraded":"2023-09-23T19:54:45.249550+0000","last_fullsized":"2023-09-23T19:54:45.249550+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.330109+0000","last_change":"2023-09-23T19:54:45.329703+0000","last_active":"2023-09-23T19:54:45.330109+0000","last_peered":"2023-09-23T19:54:45.330109+0000","last_clean":"2023-09-23T19:48:37.564795+0000","last_became_active":"2023-09-23T19:54:45.329703+0000","last_became_peered":"2023-09-23T19:54:45.329703+0000","last_unstale":"2023-09-23T19:54:45.330109+0000","last_undegraded":"2023-09-23T19:54:45.330109+0000","last_fullsized":"2023-09-23T19:54:45.326339+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:32.384765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":88,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.257345+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:54:45.257345+0000","last_peered":"2023-09-23T19:54:45.257345+0000","last_clean":"2023-09-23T19:54:45.257345+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:54:45.257345+0000","last_undegraded":"2023-09-23T19:54:45.257345+0000","last_fullsized":"2023-09-23T19:54:45.257345+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":114,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.257050+0000","last_change":"2023-09-23T19:54:45.256818+0000","last_active":"2023-09-23T19:54:45.257050+0000","last_peered":"2023-09-23T19:54:45.257050+0000","last_clean":"2023-09-23T19:54:37.301254+0000","last_became_active":"2023-09-23T19:54:45.256818+0000","last_became_peered":"2023-09-23T19:54:45.256818+0000","last_unstale":"2023-09-23T19:54:45.257050+0000","last_undegraded":"2023-09-23T19:54:45.257050+0000","last_fullsized":"2023-09-23T19:54:45.249522+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:38.432198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.325316+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:54:45.325316+0000","last_peered":"2023-09-23T19:54:45.325316+0000","last_clean":"2023-09-23T19:54:45.325316+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:54:45.325316+0000","last_undegraded":"2023-09-23T19:54:45.325316+0000","last_fullsized":"2023-09-23T19:54:45.325316+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.326715+0000","last_change":"2023-09-23T19:54:45.326468+0000","last_active":"2023-09-23T19:54:45.326715+0000","last_peered":"2023-09-23T19:54:45.326715+0000","last_clean":"2023-09-23T19:54:37.301193+0000","last_became_active":"2023-09-23T19:54:45.326468+0000","last_became_peered":"2023-09-23T19:54:45.326468+0000","last_unstale":"2023-09-23T19:54:45.326715+0000","last_undegraded":"2023-09-23T19:54:45.326715+0000","last_fullsized":"2023-09-23T19:54:45.249759+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:55.164985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":109,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.254243+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:54:45.254243+0000","last_peered":"2023-09-23T19:54:45.254243+0000","last_clean":"2023-09-23T19:54:45.254243+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:54:45.254243+0000","last_undegraded":"2023-09-23T19:54:45.254243+0000","last_fullsized":"2023-09-23T19:54:45.254243+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":91,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.260566+0000","last_change":"2023-09-23T19:54:45.260238+0000","last_active":"2023-09-23T19:54:45.260566+0000","last_peered":"2023-09-23T19:54:45.260566+0000","last_clean":"2023-09-23T19:54:37.321829+0000","last_became_active":"2023-09-23T19:54:45.260238+0000","last_became_peered":"2023-09-23T19:54:45.260238+0000","last_unstale":"2023-09-23T19:54:45.260566+0000","last_undegraded":"2023-09-23T19:54:45.260566+0000","last_fullsized":"2023-09-23T19:54:45.256657+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:53.941330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":119,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:45.331521+0000","last_change":"2023-09-23T19:54:45.330589+0000","last_active":"2023-09-23T19:54:45.331521+0000","last_peered":"2023-09-23T19:54:45.331521+0000","last_clean":"2023-09-23T19:54:37.309094+0000","last_became_active":"2023-09-23T19:54:45.330589+0000","last_became_peered":"2023-09-23T19:54:45.330589+0000","last_unstale":"2023-09-23T19:54:45.331521+0000","last_undegraded":"2023-09-23T19:54:45.326907+0000","last_fullsized":"2023-09-23T19:54:45.326790+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:44.510855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":113,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.327532+0000","last_change":"2023-09-23T19:54:45.327308+0000","last_active":"2023-09-23T19:54:45.327532+0000","last_peered":"2023-09-23T19:54:45.327532+0000","last_clean":"2023-09-23T19:54:37.307838+0000","last_became_active":"2023-09-23T19:54:45.327308+0000","last_became_peered":"2023-09-23T19:54:45.327308+0000","last_unstale":"2023-09-23T19:54:45.327532+0000","last_undegraded":"2023-09-23T19:54:45.327532+0000","last_fullsized":"2023-09-23T19:54:45.254218+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:34.491557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":108,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.325578+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:54:45.325578+0000","last_peered":"2023-09-23T19:54:45.325578+0000","last_clean":"2023-09-23T19:54:45.325578+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:54:45.325578+0000","last_undegraded":"2023-09-23T19:54:45.325578+0000","last_fullsized":"2023-09-23T19:54:45.325578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.250662+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:54:45.250662+0000","last_peered":"2023-09-23T19:54:45.250662+0000","last_clean":"2023-09-23T19:54:45.250662+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:54:45.250662+0000","last_undegraded":"2023-09-23T19:54:45.250662+0000","last_fullsized":"2023-09-23T19:54:45.250662+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":88,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.256093+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:54:45.256093+0000","last_peered":"2023-09-23T19:54:45.256093+0000","last_clean":"2023-09-23T19:54:45.256093+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:54:45.256093+0000","last_undegraded":"2023-09-23T19:54:45.256093+0000","last_fullsized":"2023-09-23T19:54:45.256093+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.255266+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:54:45.255266+0000","last_peered":"2023-09-23T19:54:45.255266+0000","last_clean":"2023-09-23T19:54:45.255266+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:54:45.255266+0000","last_undegraded":"2023-09-23T19:54:45.255266+0000","last_fullsized":"2023-09-23T19:54:45.255266+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":96,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.331111+0000","last_change":"2023-09-23T19:54:45.330750+0000","last_active":"2023-09-23T19:54:45.331111+0000","last_peered":"2023-09-23T19:54:45.331111+0000","last_clean":"2023-09-23T19:54:37.324687+0000","last_became_active":"2023-09-23T19:54:45.330750+0000","last_became_peered":"2023-09-23T19:54:45.330750+0000","last_unstale":"2023-09-23T19:54:45.331111+0000","last_undegraded":"2023-09-23T19:54:45.331111+0000","last_fullsized":"2023-09-23T19:54:45.253851+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:18.890717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.255986+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:54:45.255986+0000","last_peered":"2023-09-23T19:54:45.255986+0000","last_clean":"2023-09-23T19:54:45.255986+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:54:45.255986+0000","last_undegraded":"2023-09-23T19:54:45.255986+0000","last_fullsized":"2023-09-23T19:54:45.255986+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:45.915990+0000","last_change":"2023-09-23T19:54:45.331318+0000","last_active":"2023-09-23T19:54:45.915990+0000","last_peered":"2023-09-23T19:54:45.915990+0000","last_clean":"2023-09-23T19:41:41.886282+0000","last_became_active":"2023-09-23T19:54:45.331318+0000","last_became_peered":"2023-09-23T19:54:45.331318+0000","last_unstale":"2023-09-23T19:54:45.915990+0000","last_undegraded":"2023-09-23T19:54:45.255936+0000","last_fullsized":"2023-09-23T19:54:45.255813+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:16.760756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":108,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.250541+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:54:45.250541+0000","last_peered":"2023-09-23T19:54:45.250541+0000","last_clean":"2023-09-23T19:54:45.250541+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:54:45.250541+0000","last_undegraded":"2023-09-23T19:54:45.250541+0000","last_fullsized":"2023-09-23T19:54:45.250541+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"146'9737","reported_seq":9785,"reported_epoch":146,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:58.118497+0000","last_change":"2023-09-23T19:54:45.331117+0000","last_active":"2023-09-23T19:54:58.118497+0000","last_peered":"2023-09-23T19:54:58.118497+0000","last_clean":"2023-09-23T19:54:44.215053+0000","last_became_active":"2023-09-23T19:54:45.331117+0000","last_became_peered":"2023-09-23T19:54:45.331117+0000","last_unstale":"2023-09-23T19:54:58.118497+0000","last_undegraded":"2023-09-23T19:54:45.254419+0000","last_fullsized":"2023-09-23T19:54:45.254311+0000","mapping_epoch":144,"log_start":"116'2000","ondisk_log_start":"116'2000","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7737,"log_dups_size":0,"ondisk_log_size":7737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:20.895307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297598976,"num_objects":4541,"num_object_clones":0,"num_object_copies":13623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4541,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9737,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":4541}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":107,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.325717+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:54:45.325717+0000","last_peered":"2023-09-23T19:54:45.325717+0000","last_clean":"2023-09-23T19:54:45.325717+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:54:45.325717+0000","last_undegraded":"2023-09-23T19:54:45.325717+0000","last_fullsized":"2023-09-23T19:54:45.325717+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":95,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.329301+0000","last_change":"2023-09-23T19:54:45.329085+0000","last_active":"2023-09-23T19:54:45.329301+0000","last_peered":"2023-09-23T19:54:45.329301+0000","last_clean":"2023-09-23T19:54:37.309463+0000","last_became_active":"2023-09-23T19:54:45.329085+0000","last_became_peered":"2023-09-23T19:54:45.329085+0000","last_unstale":"2023-09-23T19:54:45.329301+0000","last_undegraded":"2023-09-23T19:54:45.329301+0000","last_fullsized":"2023-09-23T19:54:45.254841+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:07.066246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"146'9602","reported_seq":9645,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:57.251777+0000","last_change":"2023-09-23T19:52:21.181571+0000","last_active":"2023-09-23T19:54:57.251777+0000","last_peered":"2023-09-23T19:54:57.251777+0000","last_clean":"2023-09-23T19:54:57.251777+0000","last_became_active":"2023-09-23T19:52:21.181249+0000","last_became_peered":"2023-09-23T19:52:21.181249+0000","last_unstale":"2023-09-23T19:54:57.251777+0000","last_undegraded":"2023-09-23T19:54:57.251777+0000","last_fullsized":"2023-09-23T19:54:57.251777+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":9602,"log_dups_size":0,"ondisk_log_size":9602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:02.562956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299106304,"num_objects":4564,"num_object_clones":0,"num_object_copies":13692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9602,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":108,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.325484+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:54:45.325484+0000","last_peered":"2023-09-23T19:54:45.325484+0000","last_clean":"2023-09-23T19:54:45.325484+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:54:45.325484+0000","last_undegraded":"2023-09-23T19:54:45.325484+0000","last_fullsized":"2023-09-23T19:54:45.325484+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":123,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.253809+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:54:45.253809+0000","last_peered":"2023-09-23T19:54:45.253809+0000","last_clean":"2023-09-23T19:54:45.253809+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:54:45.253809+0000","last_undegraded":"2023-09-23T19:54:45.253809+0000","last_fullsized":"2023-09-23T19:54:45.253809+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"146'9722","reported_seq":9766,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:57.261209+0000","last_change":"2023-09-23T19:52:21.183245+0000","last_active":"2023-09-23T19:54:57.261209+0000","last_peered":"2023-09-23T19:54:57.261209+0000","last_clean":"2023-09-23T19:54:57.261209+0000","last_became_active":"2023-09-23T19:52:21.182965+0000","last_became_peered":"2023-09-23T19:52:21.182965+0000","last_unstale":"2023-09-23T19:54:57.261209+0000","last_undegraded":"2023-09-23T19:54:57.261209+0000","last_fullsized":"2023-09-23T19:54:57.261209+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":9722,"log_dups_size":0,"ondisk_log_size":9722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:14.836347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302645248,"num_objects":4618,"num_object_clones":0,"num_object_copies":13854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9722,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":97,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:45.328639+0000","last_change":"2023-09-23T19:54:45.328405+0000","last_active":"2023-09-23T19:54:45.328639+0000","last_peered":"2023-09-23T19:54:45.328639+0000","last_clean":"2023-09-23T19:54:37.326665+0000","last_became_active":"2023-09-23T19:54:45.328405+0000","last_became_peered":"2023-09-23T19:54:45.328405+0000","last_unstale":"2023-09-23T19:54:45.328639+0000","last_undegraded":"2023-09-23T19:54:45.254169+0000","last_fullsized":"2023-09-23T19:54:45.254056+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:10.168687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":61,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.331673+0000","last_change":"2023-09-23T19:54:45.331478+0000","last_active":"2023-09-23T19:54:45.331673+0000","last_peered":"2023-09-23T19:54:45.331673+0000","last_clean":"2023-09-23T19:48:37.563757+0000","last_became_active":"2023-09-23T19:54:45.331478+0000","last_became_peered":"2023-09-23T19:54:45.331478+0000","last_unstale":"2023-09-23T19:54:45.331673+0000","last_undegraded":"2023-09-23T19:54:45.331673+0000","last_fullsized":"2023-09-23T19:54:45.256149+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.555906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"146'9721","reported_seq":9765,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:57.251403+0000","last_change":"2023-09-23T19:52:21.183448+0000","last_active":"2023-09-23T19:54:57.251403+0000","last_peered":"2023-09-23T19:54:57.251403+0000","last_clean":"2023-09-23T19:54:57.251403+0000","last_became_active":"2023-09-23T19:52:21.183139+0000","last_became_peered":"2023-09-23T19:52:21.183139+0000","last_unstale":"2023-09-23T19:54:57.251403+0000","last_undegraded":"2023-09-23T19:54:57.251403+0000","last_fullsized":"2023-09-23T19:54:57.251403+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":9721,"log_dups_size":0,"ondisk_log_size":9721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:28.123472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299565056,"num_objects":4571,"num_object_clones":0,"num_object_copies":13713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9721,"num_write_kb":457344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":113,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.328656+0000","last_change":"2023-09-23T19:54:45.328431+0000","last_active":"2023-09-23T19:54:45.328656+0000","last_peered":"2023-09-23T19:54:45.328656+0000","last_clean":"2023-09-23T19:54:37.309760+0000","last_became_active":"2023-09-23T19:54:45.328431+0000","last_became_peered":"2023-09-23T19:54:45.328431+0000","last_unstale":"2023-09-23T19:54:45.328656+0000","last_undegraded":"2023-09-23T19:54:45.328656+0000","last_fullsized":"2023-09-23T19:54:45.254881+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.882212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":89,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.255201+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:54:45.255201+0000","last_peered":"2023-09-23T19:54:45.255201+0000","last_clean":"2023-09-23T19:54:45.255201+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:54:45.255201+0000","last_undegraded":"2023-09-23T19:54:45.255201+0000","last_fullsized":"2023-09-23T19:54:45.255201+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":526,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.253708+0000","last_change":"2023-09-23T19:52:57.349564+0000","last_active":"2023-09-23T19:54:45.253708+0000","last_peered":"2023-09-23T19:54:45.253708+0000","last_clean":"2023-09-23T19:54:45.253708+0000","last_became_active":"2023-09-23T19:52:57.349182+0000","last_became_peered":"2023-09-23T19:52:57.349182+0000","last_unstale":"2023-09-23T19:54:45.253708+0000","last_undegraded":"2023-09-23T19:54:45.253708+0000","last_fullsized":"2023-09-23T19:54:45.253708+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:25.262721+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"146'9972","reported_seq":10017,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:57.276090+0000","last_change":"2023-09-23T19:52:21.186399+0000","last_active":"2023-09-23T19:54:57.276090+0000","last_peered":"2023-09-23T19:54:57.276090+0000","last_clean":"2023-09-23T19:54:57.276090+0000","last_became_active":"2023-09-23T19:52:21.186119+0000","last_became_peered":"2023-09-23T19:52:21.186119+0000","last_unstale":"2023-09-23T19:54:57.276090+0000","last_undegraded":"2023-09-23T19:54:57.276090+0000","last_fullsized":"2023-09-23T19:54:57.276090+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":9972,"log_dups_size":0,"ondisk_log_size":9972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:05.981956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305987608,"num_objects":4670,"num_object_clones":0,"num_object_copies":14010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4670,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9972,"num_write_kb":468481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":106,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.252810+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:54:45.252810+0000","last_peered":"2023-09-23T19:54:45.252810+0000","last_clean":"2023-09-23T19:54:45.252810+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:54:45.252810+0000","last_undegraded":"2023-09-23T19:54:45.252810+0000","last_fullsized":"2023-09-23T19:54:45.252810+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":113,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.330083+0000","last_change":"2023-09-23T19:54:45.329835+0000","last_active":"2023-09-23T19:54:45.330083+0000","last_peered":"2023-09-23T19:54:45.330083+0000","last_clean":"2023-09-23T19:54:37.307789+0000","last_became_active":"2023-09-23T19:54:45.329835+0000","last_became_peered":"2023-09-23T19:54:45.329835+0000","last_unstale":"2023-09-23T19:54:45.330083+0000","last_undegraded":"2023-09-23T19:54:45.330083+0000","last_fullsized":"2023-09-23T19:54:45.253447+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:43.945453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":458,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.254922+0000","last_change":"2023-09-23T19:52:58.803028+0000","last_active":"2023-09-23T19:54:45.254922+0000","last_peered":"2023-09-23T19:54:45.254922+0000","last_clean":"2023-09-23T19:54:45.254922+0000","last_became_active":"2023-09-23T19:52:57.353261+0000","last_became_peered":"2023-09-23T19:52:57.353261+0000","last_unstale":"2023-09-23T19:54:45.254922+0000","last_undegraded":"2023-09-23T19:54:45.254922+0000","last_fullsized":"2023-09-23T19:54:45.254922+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:53.333663+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.c","version":"146'9916","reported_seq":9960,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:59.420233+0000","last_change":"2023-09-23T19:52:21.179841+0000","last_active":"2023-09-23T19:54:59.420233+0000","last_peered":"2023-09-23T19:54:59.420233+0000","last_clean":"2023-09-23T19:54:59.420233+0000","last_became_active":"2023-09-23T19:52:21.179598+0000","last_became_peered":"2023-09-23T19:52:21.179598+0000","last_unstale":"2023-09-23T19:54:59.420233+0000","last_undegraded":"2023-09-23T19:54:59.420233+0000","last_fullsized":"2023-09-23T19:54:59.420233+0000","mapping_epoch":112,"log_start":"116'2000","ondisk_log_start":"116'2000","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7916,"log_dups_size":0,"ondisk_log_size":7916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:39.463431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292552704,"num_objects":4464,"num_object_clones":0,"num_object_copies":13392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9916,"num_write_kb":460160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":63,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.329304+0000","last_change":"2023-09-23T19:54:45.328988+0000","last_active":"2023-09-23T19:54:45.329304+0000","last_peered":"2023-09-23T19:54:45.329304+0000","last_clean":"2023-09-23T19:48:37.563883+0000","last_became_active":"2023-09-23T19:54:45.328988+0000","last_became_peered":"2023-09-23T19:54:45.328988+0000","last_unstale":"2023-09-23T19:54:45.329304+0000","last_undegraded":"2023-09-23T19:54:45.329304+0000","last_fullsized":"2023-09-23T19:54:45.254217+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:54.383663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":123,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.327018+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:54:45.327018+0000","last_peered":"2023-09-23T19:54:45.327018+0000","last_clean":"2023-09-23T19:54:45.327018+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:54:45.327018+0000","last_undegraded":"2023-09-23T19:54:45.327018+0000","last_fullsized":"2023-09-23T19:54:45.327018+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"146'9891","reported_seq":9940,"reported_epoch":146,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:58.162759+0000","last_change":"2023-09-23T19:54:45.332253+0000","last_active":"2023-09-23T19:54:58.162759+0000","last_peered":"2023-09-23T19:54:58.162759+0000","last_clean":"2023-09-23T19:54:44.237174+0000","last_became_active":"2023-09-23T19:54:45.332253+0000","last_became_peered":"2023-09-23T19:54:45.332253+0000","last_unstale":"2023-09-23T19:54:58.162759+0000","last_undegraded":"2023-09-23T19:54:45.325988+0000","last_fullsized":"2023-09-23T19:54:45.325859+0000","mapping_epoch":144,"log_start":"116'1000","ondisk_log_start":"116'1000","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8891,"log_dups_size":0,"ondisk_log_size":8891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:55.913331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300613632,"num_objects":4587,"num_object_clones":0,"num_object_copies":13761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4587,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9891,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":4587}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":107,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.253671+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:54:45.253671+0000","last_peered":"2023-09-23T19:54:45.253671+0000","last_clean":"2023-09-23T19:54:45.253671+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:54:45.253671+0000","last_undegraded":"2023-09-23T19:54:45.253671+0000","last_fullsized":"2023-09-23T19:54:45.253671+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.330226+0000","last_change":"2023-09-23T19:54:45.329999+0000","last_active":"2023-09-23T19:54:45.330226+0000","last_peered":"2023-09-23T19:54:45.330226+0000","last_clean":"2023-09-23T19:54:37.308104+0000","last_became_active":"2023-09-23T19:54:45.329999+0000","last_became_peered":"2023-09-23T19:54:45.329999+0000","last_unstale":"2023-09-23T19:54:45.330226+0000","last_undegraded":"2023-09-23T19:54:45.330226+0000","last_fullsized":"2023-09-23T19:54:45.325949+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:37.998933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":465,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.329266+0000","last_change":"2023-09-23T19:54:45.328719+0000","last_active":"2023-09-23T19:54:45.329266+0000","last_peered":"2023-09-23T19:54:45.329266+0000","last_clean":"2023-09-23T19:54:37.309745+0000","last_became_active":"2023-09-23T19:54:45.328719+0000","last_became_peered":"2023-09-23T19:54:45.328719+0000","last_unstale":"2023-09-23T19:54:45.329266+0000","last_undegraded":"2023-09-23T19:54:45.329266+0000","last_fullsized":"2023-09-23T19:54:45.254858+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:11.097517+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.e","version":"146'9754","reported_seq":9796,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:58.195002+0000","last_change":"2023-09-23T19:52:21.184411+0000","last_active":"2023-09-23T19:54:58.195002+0000","last_peered":"2023-09-23T19:54:58.195002+0000","last_clean":"2023-09-23T19:54:58.195002+0000","last_became_active":"2023-09-23T19:52:21.184090+0000","last_became_peered":"2023-09-23T19:52:21.184090+0000","last_unstale":"2023-09-23T19:54:58.195002+0000","last_undegraded":"2023-09-23T19:54:58.195002+0000","last_fullsized":"2023-09-23T19:54:58.195002+0000","mapping_epoch":112,"log_start":"119'2900","ondisk_log_start":"119'2900","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":6854,"log_dups_size":0,"ondisk_log_size":6854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:46.571567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299499520,"num_objects":4570,"num_object_clones":0,"num_object_copies":13710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9754,"num_write_kb":458368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":109,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.249703+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:54:45.249703+0000","last_peered":"2023-09-23T19:54:45.249703+0000","last_clean":"2023-09-23T19:54:45.249703+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:54:45.249703+0000","last_undegraded":"2023-09-23T19:54:45.249703+0000","last_fullsized":"2023-09-23T19:54:45.249703+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":85,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.258511+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:54:45.258511+0000","last_peered":"2023-09-23T19:54:45.258511+0000","last_clean":"2023-09-23T19:54:45.258511+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:54:45.258511+0000","last_undegraded":"2023-09-23T19:54:45.258511+0000","last_fullsized":"2023-09-23T19:54:45.258511+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":428,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.823724+0000","last_change":"2023-09-23T19:54:45.328153+0000","last_active":"2023-09-23T19:54:45.823724+0000","last_peered":"2023-09-23T19:54:45.823724+0000","last_clean":"2023-09-23T19:45:37.119152+0000","last_became_active":"2023-09-23T19:54:45.328153+0000","last_became_peered":"2023-09-23T19:54:45.328153+0000","last_unstale":"2023-09-23T19:54:45.823724+0000","last_undegraded":"2023-09-23T19:54:45.823724+0000","last_fullsized":"2023-09-23T19:54:45.250769+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.592597+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":481,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.258198+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:54:45.258198+0000","last_peered":"2023-09-23T19:54:45.258198+0000","last_clean":"2023-09-23T19:54:45.258198+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:54:45.258198+0000","last_undegraded":"2023-09-23T19:54:45.258198+0000","last_fullsized":"2023-09-23T19:54:45.258198+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":123,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.255289+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:54:45.255289+0000","last_peered":"2023-09-23T19:54:45.255289+0000","last_clean":"2023-09-23T19:54:45.255289+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:54:45.255289+0000","last_undegraded":"2023-09-23T19:54:45.255289+0000","last_fullsized":"2023-09-23T19:54:45.255289+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.256579+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:54:45.256579+0000","last_peered":"2023-09-23T19:54:45.256579+0000","last_clean":"2023-09-23T19:54:45.256579+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:54:45.256579+0000","last_undegraded":"2023-09-23T19:54:45.256579+0000","last_fullsized":"2023-09-23T19:54:45.256579+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"146'9922","reported_seq":9972,"reported_epoch":146,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:55.860757+0000","last_change":"2023-09-23T19:54:45.331362+0000","last_active":"2023-09-23T19:54:55.860757+0000","last_peered":"2023-09-23T19:54:55.860757+0000","last_clean":"2023-09-23T19:54:44.232037+0000","last_became_active":"2023-09-23T19:54:45.331362+0000","last_became_peered":"2023-09-23T19:54:45.331362+0000","last_unstale":"2023-09-23T19:54:55.860757+0000","last_undegraded":"2023-09-23T19:54:45.256798+0000","last_fullsized":"2023-09-23T19:54:45.256688+0000","mapping_epoch":144,"log_start":"116'1800","ondisk_log_start":"116'1800","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:07.483580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310378496,"num_objects":4736,"num_object_clones":0,"num_object_copies":14208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4736,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9922,"num_write_kb":469056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4736}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":458,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.254977+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T19:54:45.254977+0000","last_peered":"2023-09-23T19:54:45.254977+0000","last_clean":"2023-09-23T19:54:45.254977+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T19:54:45.254977+0000","last_undegraded":"2023-09-23T19:54:45.254977+0000","last_fullsized":"2023-09-23T19:54:45.254977+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.5","version":"146'9843","reported_seq":9886,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:56.015928+0000","last_change":"2023-09-23T19:52:21.181440+0000","last_active":"2023-09-23T19:54:56.015928+0000","last_peered":"2023-09-23T19:54:56.015928+0000","last_clean":"2023-09-23T19:54:56.015928+0000","last_became_active":"2023-09-23T19:52:21.181162+0000","last_became_peered":"2023-09-23T19:52:21.181162+0000","last_unstale":"2023-09-23T19:54:56.015928+0000","last_undegraded":"2023-09-23T19:54:56.015928+0000","last_fullsized":"2023-09-23T19:54:56.015928+0000","mapping_epoch":112,"log_start":"116'1700","ondisk_log_start":"116'1700","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8143,"log_dups_size":0,"ondisk_log_size":8143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:18.844043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305070080,"num_objects":4655,"num_object_clones":0,"num_object_copies":13965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9843,"num_write_kb":463936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":677,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:45.330660+0000","last_change":"2023-09-23T19:54:45.330327+0000","last_active":"2023-09-23T19:54:45.330660+0000","last_peered":"2023-09-23T19:54:45.330660+0000","last_clean":"2023-09-23T19:54:37.309968+0000","last_became_active":"2023-09-23T19:54:45.330327+0000","last_became_peered":"2023-09-23T19:54:45.330327+0000","last_unstale":"2023-09-23T19:54:45.330660+0000","last_undegraded":"2023-09-23T19:54:45.326583+0000","last_fullsized":"2023-09-23T19:54:45.255367+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:15.555675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":114,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.330202+0000","last_change":"2023-09-23T19:54:45.329968+0000","last_active":"2023-09-23T19:54:45.330202+0000","last_peered":"2023-09-23T19:54:45.330202+0000","last_clean":"2023-09-23T19:54:37.307758+0000","last_became_active":"2023-09-23T19:54:45.329968+0000","last_became_peered":"2023-09-23T19:54:45.329968+0000","last_unstale":"2023-09-23T19:54:45.330202+0000","last_undegraded":"2023-09-23T19:54:45.330202+0000","last_fullsized":"2023-09-23T19:54:45.254567+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:06.731053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.258168+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:54:45.258168+0000","last_peered":"2023-09-23T19:54:45.258168+0000","last_clean":"2023-09-23T19:54:45.258168+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:54:45.258168+0000","last_undegraded":"2023-09-23T19:54:45.258168+0000","last_fullsized":"2023-09-23T19:54:45.258168+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":526,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.253669+0000","last_change":"2023-09-23T19:52:57.745972+0000","last_active":"2023-09-23T19:54:45.253669+0000","last_peered":"2023-09-23T19:54:45.253669+0000","last_clean":"2023-09-23T19:54:45.253669+0000","last_became_active":"2023-09-23T19:52:57.348142+0000","last_became_peered":"2023-09-23T19:52:57.348142+0000","last_unstale":"2023-09-23T19:54:45.253669+0000","last_undegraded":"2023-09-23T19:54:45.253669+0000","last_fullsized":"2023-09-23T19:54:45.253669+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.745894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.745894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.660195+0000","scrub_duration":0.0079262489999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"146'10020","reported_seq":10064,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:57.278387+0000","last_change":"2023-09-23T19:52:21.183302+0000","last_active":"2023-09-23T19:54:57.278387+0000","last_peered":"2023-09-23T19:54:57.278387+0000","last_clean":"2023-09-23T19:54:57.278387+0000","last_became_active":"2023-09-23T19:52:21.183010+0000","last_became_peered":"2023-09-23T19:52:21.183010+0000","last_unstale":"2023-09-23T19:54:57.278387+0000","last_undegraded":"2023-09-23T19:54:57.278387+0000","last_fullsized":"2023-09-23T19:54:57.278387+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":10020,"log_dups_size":0,"ondisk_log_size":10020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:01.323672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301727744,"num_objects":4604,"num_object_clones":0,"num_object_copies":13812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10020,"num_write_kb":467968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":631,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.253688+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:54:45.253688+0000","last_peered":"2023-09-23T19:54:45.253688+0000","last_clean":"2023-09-23T19:54:45.253688+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:54:45.253688+0000","last_undegraded":"2023-09-23T19:54:45.253688+0000","last_fullsized":"2023-09-23T19:54:45.253688+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":90,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.260127+0000","last_change":"2023-09-23T19:54:45.259787+0000","last_active":"2023-09-23T19:54:45.260127+0000","last_peered":"2023-09-23T19:54:45.260127+0000","last_clean":"2023-09-23T19:54:37.324099+0000","last_became_active":"2023-09-23T19:54:45.259787+0000","last_became_peered":"2023-09-23T19:54:45.259787+0000","last_unstale":"2023-09-23T19:54:45.260127+0000","last_undegraded":"2023-09-23T19:54:45.260127+0000","last_fullsized":"2023-09-23T19:54:45.255700+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:30.823666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":114,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.258443+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:54:45.258443+0000","last_peered":"2023-09-23T19:54:45.258443+0000","last_clean":"2023-09-23T19:54:45.258443+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:54:45.258443+0000","last_undegraded":"2023-09-23T19:54:45.258443+0000","last_fullsized":"2023-09-23T19:54:45.258443+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":955,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.255322+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:54:45.255322+0000","last_peered":"2023-09-23T19:54:45.255322+0000","last_clean":"2023-09-23T19:54:45.255322+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:54:45.255322+0000","last_undegraded":"2023-09-23T19:54:45.255322+0000","last_fullsized":"2023-09-23T19:54:45.255322+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":514,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.328405+0000","last_change":"2023-09-23T19:54:45.328163+0000","last_active":"2023-09-23T19:54:45.328405+0000","last_peered":"2023-09-23T19:54:45.328405+0000","last_clean":"2023-09-23T19:54:37.325099+0000","last_became_active":"2023-09-23T19:54:45.328163+0000","last_became_peered":"2023-09-23T19:54:45.328163+0000","last_unstale":"2023-09-23T19:54:45.328405+0000","last_undegraded":"2023-09-23T19:54:45.328405+0000","last_fullsized":"2023-09-23T19:54:45.254108+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:35.290506+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.328635+0000","last_change":"2023-09-23T19:54:45.328321+0000","last_active":"2023-09-23T19:54:45.328635+0000","last_peered":"2023-09-23T19:54:45.328635+0000","last_clean":"2023-09-23T19:54:37.298552+0000","last_became_active":"2023-09-23T19:54:45.328321+0000","last_became_peered":"2023-09-23T19:54:45.328321+0000","last_unstale":"2023-09-23T19:54:45.328635+0000","last_undegraded":"2023-09-23T19:54:45.328635+0000","last_fullsized":"2023-09-23T19:54:45.250215+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:44.881597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.332399+0000","last_change":"2023-09-23T19:54:45.332190+0000","last_active":"2023-09-23T19:54:45.332399+0000","last_peered":"2023-09-23T19:54:45.332399+0000","last_clean":"2023-09-23T19:41:39.520007+0000","last_became_active":"2023-09-23T19:54:45.332190+0000","last_became_peered":"2023-09-23T19:54:45.332190+0000","last_unstale":"2023-09-23T19:54:45.332399+0000","last_undegraded":"2023-09-23T19:54:45.332399+0000","last_fullsized":"2023-09-23T19:54:45.327279+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:25.992565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"146'9950","reported_seq":9992,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:58.132044+0000","last_change":"2023-09-23T19:52:21.183523+0000","last_active":"2023-09-23T19:54:58.132044+0000","last_peered":"2023-09-23T19:54:58.132044+0000","last_clean":"2023-09-23T19:54:58.132044+0000","last_became_active":"2023-09-23T19:52:21.183198+0000","last_became_peered":"2023-09-23T19:52:21.183198+0000","last_unstale":"2023-09-23T19:54:58.132044+0000","last_undegraded":"2023-09-23T19:54:58.132044+0000","last_fullsized":"2023-09-23T19:54:58.132044+0000","mapping_epoch":112,"log_start":"116'2200","ondisk_log_start":"116'2200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7750,"log_dups_size":0,"ondisk_log_size":7750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:07.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299237376,"num_objects":4566,"num_object_clones":0,"num_object_copies":13698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9950,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":465,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.330767+0000","last_change":"2023-09-23T19:54:45.330514+0000","last_active":"2023-09-23T19:54:45.330767+0000","last_peered":"2023-09-23T19:54:45.330767+0000","last_clean":"2023-09-23T19:54:37.321801+0000","last_became_active":"2023-09-23T19:54:45.330514+0000","last_became_peered":"2023-09-23T19:54:45.330514+0000","last_unstale":"2023-09-23T19:54:45.330767+0000","last_undegraded":"2023-09-23T19:54:45.330767+0000","last_fullsized":"2023-09-23T19:54:45.257209+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:04.752509+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"146'9896","reported_seq":9938,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:58.137598+0000","last_change":"2023-09-23T19:52:21.181927+0000","last_active":"2023-09-23T19:54:58.137598+0000","last_peered":"2023-09-23T19:54:58.137598+0000","last_clean":"2023-09-23T19:54:58.137598+0000","last_became_active":"2023-09-23T19:52:21.181527+0000","last_became_peered":"2023-09-23T19:52:21.181527+0000","last_unstale":"2023-09-23T19:54:58.137598+0000","last_undegraded":"2023-09-23T19:54:58.137598+0000","last_fullsized":"2023-09-23T19:54:58.137598+0000","mapping_epoch":112,"log_start":"116'1000","ondisk_log_start":"116'1000","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8896,"log_dups_size":0,"ondisk_log_size":8896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.841652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303300608,"num_objects":4628,"num_object_clones":0,"num_object_copies":13884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9896,"num_write_kb":464768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":864,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.254368+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:54:45.254368+0000","last_peered":"2023-09-23T19:54:45.254368+0000","last_clean":"2023-09-23T19:54:45.254368+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:54:45.254368+0000","last_undegraded":"2023-09-23T19:54:45.254368+0000","last_fullsized":"2023-09-23T19:54:45.254368+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":89,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:50.916722+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:54:50.916722+0000","last_peered":"2023-09-23T19:54:50.916722+0000","last_clean":"2023-09-23T19:54:50.916722+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:54:50.916722+0000","last_undegraded":"2023-09-23T19:54:50.916722+0000","last_fullsized":"2023-09-23T19:54:50.916722+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":102,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.256173+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:54:45.256173+0000","last_peered":"2023-09-23T19:54:45.256173+0000","last_clean":"2023-09-23T19:54:45.256173+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:54:45.256173+0000","last_undegraded":"2023-09-23T19:54:45.256173+0000","last_fullsized":"2023-09-23T19:54:45.256173+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":421,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.916161+0000","last_change":"2023-09-23T19:54:45.260113+0000","last_active":"2023-09-23T19:54:45.916161+0000","last_peered":"2023-09-23T19:54:45.916161+0000","last_clean":"2023-09-23T19:45:37.119152+0000","last_became_active":"2023-09-23T19:54:45.260113+0000","last_became_peered":"2023-09-23T19:54:45.260113+0000","last_unstale":"2023-09-23T19:54:45.916161+0000","last_undegraded":"2023-09-23T19:54:45.916161+0000","last_fullsized":"2023-09-23T19:54:45.254638+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:06.516504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:06.516504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":224,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.254133+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:54:45.254133+0000","last_peered":"2023-09-23T19:54:45.254133+0000","last_clean":"2023-09-23T19:54:45.254133+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:54:45.254133+0000","last_undegraded":"2023-09-23T19:54:45.254133+0000","last_fullsized":"2023-09-23T19:54:45.254133+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.258251+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:54:45.258251+0000","last_peered":"2023-09-23T19:54:45.258251+0000","last_clean":"2023-09-23T19:54:45.258251+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:54:45.258251+0000","last_undegraded":"2023-09-23T19:54:45.258251+0000","last_fullsized":"2023-09-23T19:54:45.258251+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:50.916794+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:54:50.916794+0000","last_peered":"2023-09-23T19:54:50.916794+0000","last_clean":"2023-09-23T19:54:50.916794+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:54:50.916794+0000","last_undegraded":"2023-09-23T19:54:50.916794+0000","last_fullsized":"2023-09-23T19:54:50.916794+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"146'9874","reported_seq":9918,"reported_epoch":146,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:55.870896+0000","last_change":"2023-09-23T19:54:45.331245+0000","last_active":"2023-09-23T19:54:55.870896+0000","last_peered":"2023-09-23T19:54:55.870896+0000","last_clean":"2023-09-23T19:54:44.178479+0000","last_became_active":"2023-09-23T19:54:45.331245+0000","last_became_peered":"2023-09-23T19:54:45.331245+0000","last_unstale":"2023-09-23T19:54:55.870896+0000","last_undegraded":"2023-09-23T19:54:45.255724+0000","last_fullsized":"2023-09-23T19:54:45.255607+0000","mapping_epoch":144,"log_start":"116'1715","ondisk_log_start":"116'1715","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":8159,"log_dups_size":0,"ondisk_log_size":8159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:52.938092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312082432,"num_objects":4762,"num_object_clones":0,"num_object_copies":14286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4762,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9874,"num_write_kb":468352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4762}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":465,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.331547+0000","last_change":"2023-09-23T19:54:45.330408+0000","last_active":"2023-09-23T19:54:45.331547+0000","last_peered":"2023-09-23T19:54:45.331547+0000","last_clean":"2023-09-23T19:54:37.307730+0000","last_became_active":"2023-09-23T19:54:45.330408+0000","last_became_peered":"2023-09-23T19:54:45.330408+0000","last_unstale":"2023-09-23T19:54:45.331547+0000","last_undegraded":"2023-09-23T19:54:45.331547+0000","last_fullsized":"2023-09-23T19:54:45.325245+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:54.949323+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":106,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.250635+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:54:45.250635+0000","last_peered":"2023-09-23T19:54:45.250635+0000","last_clean":"2023-09-23T19:54:45.250635+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:54:45.250635+0000","last_undegraded":"2023-09-23T19:54:45.250635+0000","last_fullsized":"2023-09-23T19:54:45.250635+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":413,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.331073+0000","last_change":"2023-09-23T19:54:45.330659+0000","last_active":"2023-09-23T19:54:45.331073+0000","last_peered":"2023-09-23T19:54:45.331073+0000","last_clean":"2023-09-23T19:54:37.321990+0000","last_became_active":"2023-09-23T19:54:45.330659+0000","last_became_peered":"2023-09-23T19:54:45.330659+0000","last_unstale":"2023-09-23T19:54:45.331073+0000","last_undegraded":"2023-09-23T19:54:45.331073+0000","last_fullsized":"2023-09-23T19:54:45.255445+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:54.478074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:53.513011+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:54:53.513011+0000","last_peered":"2023-09-23T19:54:53.513011+0000","last_clean":"2023-09-23T19:54:53.513011+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:54:53.513011+0000","last_undegraded":"2023-09-23T19:54:53.513011+0000","last_fullsized":"2023-09-23T19:54:53.513011+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"146'10058","reported_seq":10100,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:58.161147+0000","last_change":"2023-09-23T19:52:21.183759+0000","last_active":"2023-09-23T19:54:58.161147+0000","last_peered":"2023-09-23T19:54:58.161147+0000","last_clean":"2023-09-23T19:54:58.161147+0000","last_became_active":"2023-09-23T19:52:21.183451+0000","last_became_peered":"2023-09-23T19:52:21.183451+0000","last_unstale":"2023-09-23T19:54:58.161147+0000","last_undegraded":"2023-09-23T19:54:58.161147+0000","last_fullsized":"2023-09-23T19:54:58.161147+0000","mapping_epoch":112,"log_start":"120'3200","ondisk_log_start":"120'3200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:43.359001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294518784,"num_objects":4494,"num_object_clones":0,"num_object_copies":13482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10058,"num_write_kb":465664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":460,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.250435+0000","last_change":"2023-09-23T19:53:00.705074+0000","last_active":"2023-09-23T19:54:45.250435+0000","last_peered":"2023-09-23T19:54:45.250435+0000","last_clean":"2023-09-23T19:54:45.250435+0000","last_became_active":"2023-09-23T19:52:57.353333+0000","last_became_peered":"2023-09-23T19:52:57.353333+0000","last_unstale":"2023-09-23T19:54:45.250435+0000","last_undegraded":"2023-09-23T19:54:45.250435+0000","last_fullsized":"2023-09-23T19:54:45.250435+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T19:53:00.704900+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:24.642289+0000","scrub_duration":0.0090978159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":308,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.328457+0000","last_change":"2023-09-23T19:54:45.328104+0000","last_active":"2023-09-23T19:54:45.328457+0000","last_peered":"2023-09-23T19:54:45.328457+0000","last_clean":"2023-09-23T19:43:08.441095+0000","last_became_active":"2023-09-23T19:54:45.328104+0000","last_became_peered":"2023-09-23T19:54:45.328104+0000","last_unstale":"2023-09-23T19:54:45.328457+0000","last_undegraded":"2023-09-23T19:54:45.328457+0000","last_fullsized":"2023-09-23T19:54:45.325453+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:40.212909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":92,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.256118+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:54:45.256118+0000","last_peered":"2023-09-23T19:54:45.256118+0000","last_clean":"2023-09-23T19:54:45.256118+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:54:45.256118+0000","last_undegraded":"2023-09-23T19:54:45.256118+0000","last_fullsized":"2023-09-23T19:54:45.256118+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":96,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.330553+0000","last_change":"2023-09-23T19:54:45.330341+0000","last_active":"2023-09-23T19:54:45.330553+0000","last_peered":"2023-09-23T19:54:45.330553+0000","last_clean":"2023-09-23T19:54:37.305426+0000","last_became_active":"2023-09-23T19:54:45.330341+0000","last_became_peered":"2023-09-23T19:54:45.330341+0000","last_unstale":"2023-09-23T19:54:45.330553+0000","last_undegraded":"2023-09-23T19:54:45.330553+0000","last_fullsized":"2023-09-23T19:54:45.253699+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:22.534625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"146'9905","reported_seq":9949,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T19:54:55.806838+0000","last_change":"2023-09-23T19:52:21.180582+0000","last_active":"2023-09-23T19:54:55.806838+0000","last_peered":"2023-09-23T19:54:55.806838+0000","last_clean":"2023-09-23T19:54:55.806838+0000","last_became_active":"2023-09-23T19:52:21.180255+0000","last_became_peered":"2023-09-23T19:52:21.180255+0000","last_unstale":"2023-09-23T19:54:55.806838+0000","last_undegraded":"2023-09-23T19:54:55.806838+0000","last_fullsized":"2023-09-23T19:54:55.806838+0000","mapping_epoch":112,"log_start":"116'2200","ondisk_log_start":"116'2200","created":112,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:52:20.162416+0000","last_clean_scrub_stamp":"2023-09-23T19:52:20.162416+0000","objects_scrubbed":0,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:51.310686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311361536,"num_objects":4751,"num_object_clones":0,"num_object_copies":14253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9905,"num_write_kb":468992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":77,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.255356+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:54:45.255356+0000","last_peered":"2023-09-23T19:54:45.255356+0000","last_clean":"2023-09-23T19:54:45.255356+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:54:45.255356+0000","last_undegraded":"2023-09-23T19:54:45.255356+0000","last_fullsized":"2023-09-23T19:54:45.255356+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":128,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.253845+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:54:45.253845+0000","last_peered":"2023-09-23T19:54:45.253845+0000","last_clean":"2023-09-23T19:54:45.253845+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:54:45.253845+0000","last_undegraded":"2023-09-23T19:54:45.253845+0000","last_fullsized":"2023-09-23T19:54:45.253845+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":108,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.255335+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:54:45.255335+0000","last_peered":"2023-09-23T19:54:45.255335+0000","last_clean":"2023-09-23T19:54:45.255335+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:54:45.255335+0000","last_undegraded":"2023-09-23T19:54:45.255335+0000","last_fullsized":"2023-09-23T19:54:45.255335+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.254585+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:54:45.254585+0000","last_peered":"2023-09-23T19:54:45.254585+0000","last_clean":"2023-09-23T19:54:45.254585+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:54:45.254585+0000","last_undegraded":"2023-09-23T19:54:45.254585+0000","last_fullsized":"2023-09-23T19:54:45.254585+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":98,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:54:45.333145+0000","last_change":"2023-09-23T19:54:45.332893+0000","last_active":"2023-09-23T19:54:45.333145+0000","last_peered":"2023-09-23T19:54:45.333145+0000","last_clean":"2023-09-23T19:54:37.310107+0000","last_became_active":"2023-09-23T19:54:45.332893+0000","last_became_peered":"2023-09-23T19:54:45.332893+0000","last_unstale":"2023-09-23T19:54:45.333145+0000","last_undegraded":"2023-09-23T19:54:45.327254+0000","last_fullsized":"2023-09-23T19:54:45.327139+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:12.007433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":123,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.254389+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:54:45.254389+0000","last_peered":"2023-09-23T19:54:45.254389+0000","last_clean":"2023-09-23T19:54:45.254389+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:54:45.254389+0000","last_undegraded":"2023-09-23T19:54:45.254389+0000","last_fullsized":"2023-09-23T19:54:45.254389+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":121,"reported_epoch":146,"state":"active+undersized","last_fresh":"2023-09-23T19:54:53.513976+0000","last_change":"2023-09-23T19:54:45.253972+0000","last_active":"2023-09-23T19:54:53.513976+0000","last_peered":"2023-09-23T19:54:53.513976+0000","last_clean":"2023-09-23T19:54:37.303960+0000","last_became_active":"2023-09-23T19:54:45.253972+0000","last_became_peered":"2023-09-23T19:54:45.253972+0000","last_unstale":"2023-09-23T19:54:53.513976+0000","last_undegraded":"2023-09-23T19:54:53.513976+0000","last_fullsized":"2023-09-23T19:54:45.250762+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:27.944201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":129,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:45.249631+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:54:45.249631+0000","last_peered":"2023-09-23T19:54:45.249631+0000","last_clean":"2023-09-23T19:54:45.249631+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:54:45.249631+0000","last_undegraded":"2023-09-23T19:54:45.249631+0000","last_fullsized":"2023-09-23T19:54:45.249631+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":93,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T19:54:50.916935+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:54:50.916935+0000","last_peered":"2023-09-23T19:54:50.916935+0000","last_clean":"2023-09-23T19:54:50.916935+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:54:50.916935+0000","last_undegraded":"2023-09-23T19:54:50.916935+0000","last_fullsized":"2023-09-23T19:54:50.916935+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":113,"reported_epoch":145,"state":"active+undersized","last_fresh":"2023-09-23T19:54:45.327062+0000","last_change":"2023-09-23T19:54:45.326835+0000","last_active":"2023-09-23T19:54:45.327062+0000","last_peered":"2023-09-23T19:54:45.327062+0000","last_clean":"2023-09-23T19:54:37.305270+0000","last_became_active":"2023-09-23T19:54:45.326835+0000","last_became_peered":"2023-09-23T19:54:45.326835+0000","last_unstale":"2023-09-23T19:54:45.327062+0000","last_undegraded":"2023-09-23T19:54:45.327062+0000","last_fullsized":"2023-09-23T19:54:45.253489+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:50.091323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":4835246104,"num_objects":73781,"num_object_clones":0,"num_object_copies":221343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73781,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157783,"num_write_kb":7409985,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14625681408,"data_stored":14625669192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136068,"ondisk_log_size":136068,"up":44,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":84,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697708,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":322018,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":27,"acting":27,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":21,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234166,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1538260,"kb_used_data":1473932,"kb_used_omap":0,"kb_used_meta":64320,"kb_avail":92206892,"statfs":{"total":95995035648,"available":94419857408,"internally_reserved":0,"allocated":1509306368,"data_stored":1508763675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":65863680},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:50 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.0249999999999999,"15min":0.73099999999999998},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":9.625,"5min":9.625,"15min":9.625},"last":1.508},{"interface":"front","average":{"1min":2.536,"5min":1.22,"15min":0.82599999999999996},"min":{"1min":0.26100000000000001,"5min":0.254,"15min":0.254},"max":{"1min":9.0820000000000007,"5min":9.0820000000000007,"15min":9.0820000000000007},"last":2.5960000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:53:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":0.88800000000000001,"15min":0.68000000000000005},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":11.093999999999999,"5min":11.093999999999999,"15min":11.093999999999999},"last":1.478},{"interface":"front","average":{"1min":1.722,"5min":0.86699999999999999,"15min":0.66500000000000004},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3529999999999998,"5min":5.1200000000000001,"15min":5.1200000000000001},"last":1.649}]},{"osd":2,"last update":"Sat Sep 23 19:54:26 2023","interfaces":[{"interface":"back","average":{"1min":2.488,"5min":1.548,"15min":1.0840000000000001},"min":{"1min":0.34799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.3599999999999994,"5min":17.556000000000001,"15min":17.556000000000001},"last":1.411},{"interface":"front","average":{"1min":3.254,"5min":1.534,"15min":0.90200000000000002},"min":{"1min":0.34499999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.465,"5min":17.300000000000001,"15min":17.300000000000001},"last":2.3399999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.1240000000000001,"15min":0.93200000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3330000000000002,"5min":8.3330000000000002,"15min":8.3330000000000002},"last":1.5409999999999999},{"interface":"front","average":{"1min":1.597,"5min":0.90800000000000003,"15min":0.73199999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.0380000000000003,"5min":7.0380000000000003,"15min":7.0380000000000003},"last":2.3999999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":2.734,"5min":1.125,"15min":0.68799999999999994},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.496,"5min":11.496,"15min":11.496},"last":3.948},{"interface":"front","average":{"1min":2.5990000000000002,"5min":1.1879999999999999,"15min":0.74299999999999999},"min":{"1min":0.41199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.4930000000000003,"5min":7.4930000000000003,"15min":7.4930000000000003},"last":1.4490000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8759999999999999,"5min":1.0109999999999999,"15min":0.68100000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.327,"5min":7.327,"15min":7.327},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.5649999999999999,"5min":1.161,"15min":0.748},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":13.148999999999999,"5min":13.148999999999999,"15min":13.148999999999999},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.0189999999999999,"15min":0.71499999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.6449999999999996,"5min":5.6449999999999996,"15min":5.6449999999999996},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.121,"15min":0.76500000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.7720000000000002,"5min":5.7720000000000002,"15min":5.7720000000000002},"last":0.38900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495800,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379500,"kb_used_data":1334692,"kb_used_omap":0,"kb_used_meta":44800,"kb_avail":92365652,"statfs":{"total":95995035648,"available":94582427648,"internally_reserved":0,"allocated":1366724608,"data_stored":1366198307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":45875200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:52 2023","interfaces":[{"interface":"back","average":{"1min":4.415,"5min":1.3500000000000001,"15min":0.85199999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":60.738,"5min":60.738,"15min":60.738},"last":1.476},{"interface":"front","average":{"1min":4.202,"5min":1.333,"15min":0.88700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":60.722999999999999,"5min":60.722999999999999,"15min":60.722999999999999},"last":1.377}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.85799999999999998,"15min":0.69999999999999996},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6520000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.395,"5min":0.94299999999999995,"15min":0.69099999999999995},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":4.3029999999999999,"5min":2.7829999999999999,"15min":1.3149999999999999},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.245},"max":{"1min":53.667999999999999,"5min":61.155000000000001,"15min":61.155000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":4.024,"5min":2.7989999999999999,"15min":1.417},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":53.628,"5min":60.506,"15min":60.506},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":0.97899999999999998,"15min":0.76700000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":5.7619999999999996},"last":1.361},{"interface":"front","average":{"1min":2.1379999999999999,"5min":0.94299999999999995,"15min":0.69299999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":1.387}]},{"osd":4,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":1.0740000000000001,"15min":0.68600000000000005},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.4859999999999998,"5min":24.003,"15min":24.003},"last":0.377},{"interface":"front","average":{"1min":1.607,"5min":1.0920000000000001,"15min":0.70199999999999996},"min":{"1min":0.378,"5min":0.313,"15min":0.26500000000000001},"max":{"1min":6.6020000000000003,"5min":24.802,"15min":24.802},"last":1.3149999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.147,"15min":0.71099999999999997},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":14.378,"5min":24.065999999999999,"15min":24.065999999999999},"last":1.4079999999999999},{"interface":"front","average":{"1min":1.641,"5min":1.1359999999999999,"15min":0.74399999999999999},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":6.0529999999999999,"5min":24.841000000000001,"15min":24.841000000000001},"last":1.2769999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:54:39 2023","interfaces":[{"interface":"back","average":{"1min":6.1509999999999998,"5min":1.8149999999999999,"15min":0.98099999999999998},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":105.71299999999999,"5min":105.71299999999999,"15min":105.71299999999999},"last":2.1579999999999999},{"interface":"front","average":{"1min":6.5620000000000003,"5min":2.0259999999999998,"15min":1.075},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":103.206,"5min":103.206,"15min":103.206},"last":0.46100000000000002}]}]},{"osd":1,"up_from":99,"seq":425201762368,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1531660,"kb_used_data":1455620,"kb_used_omap":7,"kb_used_meta":75960,"kb_avail":92213492,"statfs":{"total":95995035648,"available":94426615808,"internally_reserved":0,"allocated":1490554880,"data_stored":1489904084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":77783866},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":16.609999999999999,"5min":3.6970000000000001,"15min":1.472},"min":{"1min":0.34799999999999998,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":202.917,"5min":202.917,"15min":202.917},"last":0.55000000000000004},{"interface":"front","average":{"1min":17.138999999999999,"5min":3.859,"15min":1.5289999999999999},"min":{"1min":0.372,"5min":0.25,"15min":0.25},"max":{"1min":203.34899999999999,"5min":203.34899999999999,"15min":203.34899999999999},"last":4.0570000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8559999999999999,"5min":4.0670000000000002,"15min":1.5880000000000001},"min":{"1min":0.221,"5min":0.192,"15min":0.192},"max":{"1min":10.303000000000001,"5min":231.57499999999999,"15min":231.57499999999999},"last":8.5679999999999996},{"interface":"front","average":{"1min":3.004,"5min":3.8530000000000002,"15min":1.5189999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":12.664999999999999,"5min":236.077,"15min":236.077},"last":0.57099999999999995}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":0.97899999999999998,"15min":0.57299999999999995},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.8330000000000002,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.6559999999999999,"5min":0.76300000000000001,"15min":0.52100000000000002},"min":{"1min":0.25800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":5.218,"5min":5.5590000000000002,"15min":5.5590000000000002},"last":4.093}]},{"osd":4,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.024,"15min":0.77600000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.5369999999999999,"5min":7.5369999999999999,"15min":7.5369999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.5429999999999999,"5min":0.84999999999999998,"15min":0.69199999999999995},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.0499999999999998,"5min":5.0499999999999998,"15min":5.0499999999999998},"last":4.1159999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.93700000000000006,"15min":0.73899999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":6.3150000000000004,"5min":6.3150000000000004,"15min":6.3150000000000004},"last":4.1369999999999996},{"interface":"front","average":{"1min":2.371,"5min":1.0940000000000001,"15min":0.79600000000000004},"min":{"1min":0.40999999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.7779999999999996,"5min":7.7779999999999996,"15min":7.7779999999999996},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.097,"15min":0.80500000000000005},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.9649999999999999,"5min":7.9649999999999999,"15min":7.9649999999999999},"last":14.897},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.0800000000000001,"15min":0.82399999999999995},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":14.904999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.179,"15min":0.82599999999999996},"min":{"1min":0.44500000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.2149999999999999,"5min":9.2149999999999999,"15min":9.2149999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.6099999999999999,"5min":1.1439999999999999,"15min":0.82899999999999996},"min":{"1min":0.40600000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":7.7309999999999999,"5min":7.7309999999999999,"15min":7.7309999999999999},"last":2.1970000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925902,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2531720,"kb_used_data":2427456,"kb_used_omap":8,"kb_used_meta":103863,"kb_avail":91213432,"statfs":{"total":95995035648,"available":93402554368,"internally_reserved":0,"allocated":2485714944,"data_stored":2485068017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":106356321},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.45700000000000002,"15min":0.36799999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.5470000000000002,"5min":3.0950000000000002,"15min":3.0950000000000002},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.55500000000000005,"15min":0.41099999999999998},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.3010000000000002,"5min":8.3010000000000002,"15min":8.3010000000000002},"last":0.63200000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:54:36 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.518,"15min":0.78900000000000003},"min":{"1min":0.21099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.7010000000000005,"5min":63.677,"15min":63.677},"last":2.8140000000000001},{"interface":"front","average":{"1min":2.8759999999999999,"5min":1.8089999999999999,"15min":0.85399999999999998},"min":{"1min":0.34100000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":10.805999999999999,"5min":62.564,"15min":62.564},"last":1.9930000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":6.8789999999999996,"5min":1.71,"15min":0.82199999999999995},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":104.084,"5min":104.084,"15min":104.084},"last":0.61799999999999999},{"interface":"front","average":{"1min":7.5659999999999998,"5min":1.8360000000000001,"15min":0.86099999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":104.143,"5min":104.143,"15min":104.143},"last":3.431}]},{"osd":4,"last update":"Sat Sep 23 19:54:50 2023","interfaces":[{"interface":"back","average":{"1min":8.3040000000000003,"5min":2.5600000000000001,"15min":1.288},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":110.312,"5min":110.312,"15min":110.312},"last":0.29099999999999998},{"interface":"front","average":{"1min":7.2859999999999996,"5min":2.1339999999999999,"15min":1.173},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":110.256,"5min":110.256,"15min":110.256},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":1.135,"15min":0.85499999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.4979999999999993,"5min":8.4979999999999993,"15min":8.4979999999999993},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.8929999999999998,"5min":1.403,"15min":0.94699999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":0.89700000000000002,"15min":0.78300000000000003},"min":{"1min":0.40899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.9139999999999997,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":2.4100000000000001},{"interface":"front","average":{"1min":2.3410000000000002,"5min":0.996,"15min":0.81000000000000005},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.6920000000000002,"5min":8.6920000000000002,"15min":8.6920000000000002},"last":1.1679999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9660000000000002,"5min":1.375,"15min":0.95399999999999996},"min":{"1min":0.433,"5min":0.372,"15min":0.372},"max":{"1min":11.214,"5min":11.214,"15min":11.214},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.9079999999999999,"5min":1.51,"15min":0.99399999999999999},"min":{"1min":0.437,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.0020000000000007,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":0.41799999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598841,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1879116,"kb_used_data":1785156,"kb_used_omap":12,"kb_used_meta":93939,"kb_avail":91866036,"statfs":{"total":95995035648,"available":94070820864,"internally_reserved":0,"allocated":1827999744,"data_stored":1827347506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":96194558},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.2529999999999999,"15min":0.65200000000000002},"min":{"1min":0.224,"5min":0.216,"15min":0.216},"max":{"1min":9.1690000000000005,"5min":41.887999999999998,"15min":41.887999999999998},"last":5.2199999999999998},{"interface":"front","average":{"1min":2.367,"5min":1.319,"15min":0.68799999999999994},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":5.7720000000000002,"5min":41.838000000000001,"15min":41.838000000000001},"last":0.66100000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.577,"5min":1.121,"15min":0.629},"min":{"1min":0.245,"5min":0.214,"15min":0.214},"max":{"1min":10.667999999999999,"5min":10.667999999999999,"15min":10.667999999999999},"last":0.374},{"interface":"front","average":{"1min":2.3660000000000001,"5min":0.91900000000000004,"15min":0.56499999999999995},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":12.031000000000001,"5min":12.031000000000001,"15min":12.031000000000001},"last":0.32900000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":0.88300000000000001,"15min":0.55400000000000005},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.4409999999999998,"5min":6.4409999999999998,"15min":6.4409999999999998},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.3889999999999998,"5min":0.86799999999999999,"15min":0.55400000000000005},"min":{"1min":0.22900000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.5039999999999996,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":0.48699999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":1.0640000000000001,"15min":0.77800000000000002},"min":{"1min":0.30499999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.2609999999999992,"5min":9.2609999999999992,"15min":9.2609999999999992},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.4780000000000002,"5min":1.194,"15min":0.81200000000000006},"min":{"1min":0.34200000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.2569999999999997,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":14.478999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5299999999999998,"5min":1.3720000000000001,"15min":0.88},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":13.035,"5min":13.035,"15min":13.035},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.359,"5min":1.1910000000000001,"15min":0.81599999999999995},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.9179999999999993,"5min":8.9179999999999993,"15min":8.9179999999999993},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.827,"5min":1.343,"15min":0.84999999999999998},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4740000000000002,"5min":7.4740000000000002,"15min":7.4740000000000002},"last":1.0660000000000001},{"interface":"front","average":{"1min":2.899,"5min":1.3120000000000001,"15min":0.85699999999999998},"min":{"1min":0.42399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.3029999999999999,"5min":7.3029999999999999,"15min":7.3029999999999999},"last":1.008}]},{"osd":7,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":3.7400000000000002,"5min":1.4570000000000001,"15min":0.92700000000000005},"min":{"1min":0.48399999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":8.7469999999999999,"5min":8.7469999999999999,"15min":8.7469999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":4.2789999999999999,"5min":1.5589999999999999,"15min":0.96699999999999997},"min":{"1min":0.41199999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":12.406000000000001,"5min":12.406000000000001,"15min":12.406000000000001},"last":0.42199999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435314,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":957652,"kb_used_data":899532,"kb_used_omap":4,"kb_used_meta":57851,"kb_avail":92787500,"statfs":{"total":95995035648,"available":95014400000,"internally_reserved":0,"allocated":921120768,"data_stored":920461422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":59239533},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.9449999999999998,"15min":1.2170000000000001},"min":{"1min":0.185,"5min":0.183,"15min":0.183},"max":{"1min":5.8719999999999999,"5min":168.85400000000001,"15min":168.85400000000001},"last":1.1890000000000001},{"interface":"front","average":{"1min":1.7709999999999999,"5min":3.0419999999999998,"15min":1.256},"min":{"1min":0.23699999999999999,"5min":0.217,"15min":0.217},"max":{"1min":3.8700000000000001,"5min":168.608,"15min":168.608},"last":2.6829999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":0.72399999999999998,"15min":0.496},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":8.5289999999999999,"5min":8.5289999999999999,"15min":8.5289999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.506,"5min":0.65600000000000003,"15min":0.47199999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.1660000000000004,"5min":6.1660000000000004,"15min":6.1660000000000004},"last":0.52400000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":4.3929999999999998,"5min":4.6180000000000003,"15min":1.798},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":44.091999999999999,"5min":238.52000000000001,"15min":238.52000000000001},"last":1.861},{"interface":"front","average":{"1min":4.2789999999999999,"5min":4.5229999999999997,"15min":1.782},"min":{"1min":0.34899999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":43.929000000000002,"5min":238.18100000000001,"15min":238.18100000000001},"last":1.8120000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":0.78000000000000003,"15min":0.64900000000000002},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.4660000000000002,"5min":4.4660000000000002,"15min":4.4660000000000002},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.8620000000000001,"5min":0.94499999999999995,"15min":0.70499999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.3970000000000002,"5min":5.3970000000000002,"15min":5.3970000000000002},"last":2.7149999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":0.88,"15min":0.69399999999999995},"min":{"1min":0.38700000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.7189999999999999,"5min":3.8370000000000002,"15min":3.8370000000000002},"last":5.6120000000000001},{"interface":"front","average":{"1min":2.202,"5min":0.96699999999999997,"15min":0.74099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":10.977,"5min":10.977,"15min":10.977},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":3.4809999999999999,"5min":1.2849999999999999,"15min":0.82099999999999995},"min":{"1min":0.38600000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.4399999999999995,"5min":9.4399999999999995,"15min":9.4399999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.1179999999999999,"5min":0.94599999999999995,"15min":0.72099999999999997},"min":{"1min":0.44400000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":10.932,"5min":10.932,"15min":10.932},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.018,"15min":0.752},"min":{"1min":0.44500000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":12.720000000000001,"5min":12.720000000000001,"15min":12.720000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.0139999999999998,"5min":0.95599999999999996,"15min":0.73799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.3170000000000002,"5min":6.3170000000000002,"15min":6.3170000000000002},"last":0.49399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019076,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2432780,"kb_used_data":2342724,"kb_used_omap":0,"kb_used_meta":90048,"kb_avail":91312372,"statfs":{"total":95995035648,"available":93503868928,"internally_reserved":0,"allocated":2398949376,"data_stored":2398414839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":92209152},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":2.9239999999999999,"15min":1.3799999999999999},"min":{"1min":0.224,"5min":0.22,"15min":0.22},"max":{"1min":4.899,"5min":86.069999999999993,"15min":86.069999999999993},"last":4.1459999999999999},{"interface":"front","average":{"1min":1.573,"5min":2.835,"15min":1.375},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.9770000000000003,"5min":86.051000000000002,"15min":86.051000000000002},"last":4.1609999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1869999999999998,"5min":1.2010000000000001,"15min":0.77100000000000002},"min":{"1min":0.27800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":10.021000000000001,"5min":10.021000000000001,"15min":10.021000000000001},"last":3.1040000000000001},{"interface":"front","average":{"1min":2.758,"5min":1.0780000000000001,"15min":0.73599999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":0.61599999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.7390000000000001,"15min":0.95399999999999996},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":9.6630000000000003,"5min":35.968000000000004,"15min":35.968000000000004},"last":3.915},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.6060000000000001,"15min":0.89900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.199999999999999,"5min":35.488999999999997,"15min":35.488999999999997},"last":0.64500000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":0.77300000000000002,"15min":0.61199999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":11.496,"5min":11.496,"15min":11.496},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.4279999999999999,"5min":0.98299999999999998,"15min":0.72199999999999998},"min":{"1min":0.373,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":11.462,"5min":11.462,"15min":11.462},"last":3.9670000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:54:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":1.2410000000000001,"15min":0.71699999999999997},"min":{"1min":0.315,"5min":0.246,"15min":0.246},"max":{"1min":10.246,"5min":10.246,"15min":10.246},"last":3.3380000000000001},{"interface":"front","average":{"1min":4.2290000000000001,"5min":1.466,"15min":0.81799999999999995},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.675000000000001,"5min":12.675000000000001,"15min":12.675000000000001},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":0.84699999999999998,"15min":0.60199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.867,"5min":5.867,"15min":5.867},"last":8.0150000000000006},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.80300000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":3.931}]},{"osd":7,"last update":"Sat Sep 23 19:54:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6200000000000001,"5min":1.2410000000000001,"15min":0.78300000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.29599999999999999},"max":{"1min":11.59,"5min":11.59,"15min":11.59},"last":0.433},{"interface":"front","average":{"1min":2.7669999999999999,"5min":1.1339999999999999,"15min":0.76000000000000001},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.3089999999999993,"5min":9.3089999999999993,"15min":9.3089999999999993},"last":3.161}]}]},{"osd":5,"up_from":38,"seq":163208757440,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2748552,"kb_used_data":2657984,"kb_used_omap":0,"kb_used_meta":90560,"kb_avail":90996600,"statfs":{"total":95995035648,"available":93180518400,"internally_reserved":0,"allocated":2721775616,"data_stored":2721241448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":92733440},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:54:49 2023","interfaces":[{"interface":"back","average":{"1min":2.766,"5min":1.2070000000000001,"15min":0.78900000000000003},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":24.852,"5min":24.852,"15min":24.852},"last":2.891},{"interface":"front","average":{"1min":3.2829999999999999,"5min":1.2809999999999999,"15min":0.79900000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":24.638999999999999,"5min":24.638999999999999,"15min":24.638999999999999},"last":1.2470000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":0.97399999999999998,"15min":0.67400000000000004},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":9.8460000000000001,"5min":9.8460000000000001,"15min":9.8460000000000001},"last":2.9260000000000002},{"interface":"front","average":{"1min":2.202,"5min":0.86499999999999999,"15min":0.63800000000000001},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":8.9120000000000008,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":1.6120000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:54:33 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":2.3929999999999998,"15min":1.157},"min":{"1min":0.29599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.9039999999999999,"5min":98.638000000000005,"15min":98.638000000000005},"last":2.9540000000000002},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.4329999999999998,"15min":1.2030000000000001},"min":{"1min":0.31900000000000001,"5min":0.317,"15min":0.317},"max":{"1min":6.8700000000000001,"5min":98.668000000000006,"15min":98.668000000000006},"last":2.9809999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":0.93400000000000005,"15min":0.67600000000000005},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.6470000000000002,"5min":4.6470000000000002,"15min":4.6470000000000002},"last":0.439},{"interface":"front","average":{"1min":1.853,"5min":0.95399999999999996,"15min":0.72199999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.0199999999999996,"5min":5.0199999999999996,"15min":5.0199999999999996},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:54:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":0.96699999999999997,"15min":0.624},"min":{"1min":0.27100000000000002,"5min":0.249,"15min":0.23699999999999999},"max":{"1min":10.077,"5min":10.077,"15min":10.077},"last":2.117},{"interface":"front","average":{"1min":2.8780000000000001,"5min":1.236,"15min":0.73899999999999999},"min":{"1min":0.33700000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.226000000000001,"5min":10.226000000000001,"15min":10.226000000000001},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:54:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":0.91600000000000004,"15min":0.63400000000000001},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":10.180999999999999,"5min":10.180999999999999,"15min":10.180999999999999},"last":4.2949999999999999},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.046,"15min":0.71299999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":10.406000000000001},"last":3.879}]},{"osd":7,"last update":"Sat Sep 23 19:54:33 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.0169999999999999,"15min":0.70099999999999996},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":13.734999999999999,"5min":13.734999999999999,"15min":13.734999999999999},"last":3.1509999999999998},{"interface":"front","average":{"1min":2.395,"5min":1.115,"15min":0.73899999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.30099999999999999},"max":{"1min":10.041,"5min":10.041,"15min":10.041},"last":3.282}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2457010176,"data_stored":2457010176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1478295552,"data_stored":1478295552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1810173952,"data_stored":1810169880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":916389888,"data_stored":916389888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2391080960,"data_stored":2391080960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2715160576,"data_stored":2715156504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1364852736,"data_stored":1364852736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1492717568,"data_stored":1492713496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:55:03.768 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-09-23T19:55:03.770 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:03.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:03.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:03.819 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:55:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:55:03] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.43.0" 2023-09-23T19:55:03.878 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:03.986 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:04.312 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:04.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:04.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:04.331 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:04.405 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:55:04.406 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T19:55:04.406 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T19:55:04.406 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T19:55:04.406 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:55:04.421 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:04.445 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:04 smithi135 ceph-mon[221113]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:55:04.445 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:04 smithi135 ceph-mon[221113]: pgmap v342: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 18204/215127 objects degraded (8.462%) 2023-09-23T19:55:04.445 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:04 smithi135 ceph-mon[213871]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:55:04.445 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:04 smithi135 ceph-mon[213871]: pgmap v342: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 18204/215127 objects degraded (8.462%) 2023-09-23T19:55:04.454 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:04.501 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:04.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:04.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:04.610 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:04 smithi144 ceph-mon[194833]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:55:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:04 smithi144 ceph-mon[194833]: pgmap v342: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 18204/215127 objects degraded (8.462%) 2023-09-23T19:55:04.670 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:04.718 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:04.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:04.823 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:04.873 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:04.972 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:04.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:04.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:05.083 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:05.142 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:05.189 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:05.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:05.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:05.292 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:05.353 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:05.354 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:55:05.387 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3601663489' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:05.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3601663489' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:05.430 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:05.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:05.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:05.538 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:05.585 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:05.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3601663489' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:05.918 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:55:06.066 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:06.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:06.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:06.176 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:06.240 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:06 smithi144 ceph-mon[194833]: pgmap v343: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 17731/210132 objects degraded (8.438%) 2023-09-23T19:55:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:06 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 17731/210132 objects degraded (8.438%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:06 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3601663489' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:06 smithi144 ceph-mon[194833]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T19:55:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:06 smithi144 ceph-mon[194833]: failed to encode map e147 with expected crc 2023-09-23T19:55:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[221113]: pgmap v343: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 17731/210132 objects degraded (8.438%) 2023-09-23T19:55:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 17731/210132 objects degraded (8.438%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3601663489' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[221113]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T19:55:06.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[221113]: failed to encode map e147 with expected crc 2023-09-23T19:55:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[213871]: pgmap v343: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 17731/210132 objects degraded (8.438%) 2023-09-23T19:55:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 17731/210132 objects degraded (8.438%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3601663489' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[213871]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T19:55:06.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:06 smithi135 ceph-mon[213871]: failed to encode map e147 with expected crc 2023-09-23T19:55:06.947 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:06.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:06.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:07.054 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:07.087 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:07.363 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:07.616 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:07.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:07.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:07.735 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:07.771 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:07.827 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:07.849 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:07.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:07.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:07.958 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:08.037 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:08.105 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:08 smithi144 ceph-mon[194833]: pgmap v345: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 17245/207210 objects degraded (8.322%) 2023-09-23T19:55:08.106 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:08.106 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:08.108 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:08 smithi135 ceph-mon[213871]: pgmap v345: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 17245/207210 objects degraded (8.322%) 2023-09-23T19:55:08.108 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:08.108 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:08.239 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:08.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:08.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:08.354 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:08.398 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:08.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:08 smithi135 ceph-mon[221113]: pgmap v345: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 17245/207210 objects degraded (8.322%) 2023-09-23T19:55:08.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:08.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:08.476 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:08.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:08.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:08.583 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:08.659 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:08.729 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:55:08.729 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:55:08.729 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:55:08.729 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:55:08.730 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:55:08.730 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:55:08.730 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:55:08.730 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:55:08.730 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:55:08.730 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:55:08.730 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:55:08.730 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:55:08.731 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:55:08.731 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:55:08.731 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:55:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2261722136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:09.406 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:55:09.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2261722136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:09.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2261722136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:55:10.314 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:55:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:10 smithi135 ceph-mon[221113]: pgmap v346: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 16800/199695 objects degraded (8.413%) 2023-09-23T19:55:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:10 smithi135 ceph-mon[213871]: pgmap v346: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 16800/199695 objects degraded (8.413%) 2023-09-23T19:55:10.553 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:10.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:10.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:10 smithi144 ceph-mon[194833]: pgmap v346: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 16800/199695 objects degraded (8.413%) 2023-09-23T19:55:10.660 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:10.707 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:11.128 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:11.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:11.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:11.233 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:11.334 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:11.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:11 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 16800/198999 objects degraded (8.442%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:11.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:11 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 16800/198999 objects degraded (8.442%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:11.475 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:11.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:11.592 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:11 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 16800/198999 objects degraded (8.442%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:11.642 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:11.654 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:11.701 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:11.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:11.807 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:11.857 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:11.931 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:11.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:11.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:12.036 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:12.075 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:12.157 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:12.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:12.262 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:12.284 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 1 pgp_num to 11 2023-09-23T19:55:12.300 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:12.314 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:55:12.322 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:55:12.322 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:55:12.374 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:12.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:12.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:12.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:12 smithi135 ceph-mon[213871]: pgmap v347: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 16800/198999 objects degraded (8.442%) 2023-09-23T19:55:12.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4141138652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:55:12.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:12 smithi135 ceph-mon[221113]: pgmap v347: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 16800/198999 objects degraded (8.442%) 2023-09-23T19:55:12.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4141138652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:55:12.484 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:12.556 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:12.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:12 smithi144 ceph-mon[194833]: pgmap v347: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 16800/198999 objects degraded (8.442%) 2023-09-23T19:55:12.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4141138652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:55:13.601 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4141138652' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T19:55:13.601 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:13 smithi135 ceph-mon[213871]: osdmap e148: 8 total, 7 up, 8 in 2023-09-23T19:55:13.602 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:13 smithi135 ceph-mon[213871]: failed to encode map e148 with expected crc 2023-09-23T19:55:13.602 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4141138652' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T19:55:13.602 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:13 smithi135 ceph-mon[221113]: osdmap e148: 8 total, 7 up, 8 in 2023-09-23T19:55:13.602 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:13 smithi135 ceph-mon[221113]: failed to encode map e148 with expected crc 2023-09-23T19:55:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4141138652' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T19:55:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:13 smithi144 ceph-mon[194833]: osdmap e148: 8 total, 7 up, 8 in 2023-09-23T19:55:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:13 smithi144 ceph-mon[194833]: failed to encode map e148 with expected crc 2023-09-23T19:55:13.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:55:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:55:13] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2023-09-23T19:55:14.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:14 smithi135 ceph-mon[213871]: failed to encode map e148 with expected crc 2023-09-23T19:55:14.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:14 smithi135 ceph-mon[213871]: pgmap v349: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 16333/192579 objects degraded (8.481%) 2023-09-23T19:55:14.391 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:14 smithi135 ceph-mon[221113]: failed to encode map e148 with expected crc 2023-09-23T19:55:14.391 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:14 smithi135 ceph-mon[221113]: pgmap v349: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 16333/192579 objects degraded (8.481%) 2023-09-23T19:55:14.455 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:14.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:14.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:14.566 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:14 smithi144 ceph-mon[194833]: failed to encode map e148 with expected crc 2023-09-23T19:55:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:14 smithi144 ceph-mon[194833]: pgmap v349: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 16333/192579 objects degraded (8.481%) 2023-09-23T19:55:14.644 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:14.773 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:55:14.802 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:14.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:14.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:14.915 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:14.930 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:15.004 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:15.175 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:15.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:15.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:15.280 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:15.315 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:15.405 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:15.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:15.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:15.513 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:15.591 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:15.894 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:15.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:15.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:15.999 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:16.070 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:16.111 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:16.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:16.216 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:16.241 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:16.273 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:55:16.284 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:16.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:16 smithi144 ceph-mon[194833]: pgmap v350: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 15905/188031 objects degraded (8.459%) 2023-09-23T19:55:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2664047116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:16 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:16.392 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:16.443 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:16.542 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:16.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:16.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:16.649 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:16.705 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:16 smithi135 ceph-mon[221113]: pgmap v350: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 15905/188031 objects degraded (8.459%) 2023-09-23T19:55:16.706 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2664047116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:16.706 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:16 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:16.707 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:16 smithi135 ceph-mon[213871]: pgmap v350: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 15905/188031 objects degraded (8.459%) 2023-09-23T19:55:16.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2664047116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:16.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:16 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:16.711 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:16.875 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:16.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:16.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:17.044 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:17.295 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:17.295 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[221113]: failed to encode map e149 with expected crc 2023-09-23T19:55:17.295 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[221113]: failed to encode map e149 with expected crc 2023-09-23T19:55:17.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:17.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[213871]: failed to encode map e149 with expected crc 2023-09-23T19:55:17.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[213871]: failed to encode map e149 with expected crc 2023-09-23T19:55:17.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:17.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[213871]: osdmap e149: 8 total, 7 up, 8 in 2023-09-23T19:55:17.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 15905/188031 objects degraded (8.459%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:17.383 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:17.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:17.492 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:17.537 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:17.546 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:17.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:17.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[221113]: osdmap e149: 8 total, 7 up, 8 in 2023-09-23T19:55:17.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 15905/188031 objects degraded (8.459%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:17.617 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:17.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:17.622 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:17 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:17.623 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:17 smithi144 ceph-mon[194833]: failed to encode map e149 with expected crc 2023-09-23T19:55:17.623 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:17 smithi144 ceph-mon[194833]: failed to encode map e149 with expected crc 2023-09-23T19:55:17.623 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:17 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:17.623 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:17 smithi144 ceph-mon[194833]: osdmap e149: 8 total, 7 up, 8 in 2023-09-23T19:55:17.623 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 15905/188031 objects degraded (8.459%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:17.723 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:17.784 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:17.878 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:17.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:17.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:17.987 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:18.033 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:18.040 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:55:18.072 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:55:18.102 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:18.106 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:55:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:18.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:18.147 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:55:18.193 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:55:18.212 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:18.232 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:55:18.280 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:55:18.280 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:55:18.280 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:55:18.280 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:55:18.280 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:55:18.281 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:55:18.281 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:55:18.281 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:55:18.281 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:55:18.303 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:18.305 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:18 smithi144 ceph-mon[194833]: pgmap v352: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 15432/185274 objects degraded (8.329%) 2023-09-23T19:55:18.305 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/874312713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:18.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:18 smithi135 ceph-mon[213871]: pgmap v352: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 15432/185274 objects degraded (8.329%) 2023-09-23T19:55:18.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/874312713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:18.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:18 smithi135 ceph-mon[221113]: pgmap v352: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 15432/185274 objects degraded (8.329%) 2023-09-23T19:55:18.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/874312713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:19.094 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:55:19.095 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:55:19.095 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:55:19.095 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T19:55:19.520 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:55:19.649 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:19.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:19.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:19.754 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:19.828 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:20.074 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:20.075 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:55:20.075 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:55:20.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:20.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:20.188 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:20.265 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:20.363 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:20 smithi135 ceph-mon[213871]: pgmap v353: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 14961/177738 objects degraded (8.417%) 2023-09-23T19:55:20.363 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:20 smithi135 ceph-mon[221113]: pgmap v353: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 14961/177738 objects degraded (8.417%) 2023-09-23T19:55:20.365 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:20.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:20.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:20.483 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:20.535 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:20.591 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:20.604 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:20.608 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:20 smithi144 ceph-mon[194833]: pgmap v353: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 14961/177738 objects degraded (8.417%) 2023-09-23T19:55:20.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:20.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:20.642 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:20.709 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:20.762 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:55:20.762 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:55:20.762 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:175036 5.c 5:3f3e0833:::benchmark_data_smithi135_284570_object59252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:20.731009+0000", 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.025040828000000001, 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.025076036999999999, 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:20.763 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 175036 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731009+0000", 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731009+0000", 2023-09-23T19:55:20.764 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731007+0000", 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731017+0000", 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0441e-05 2023-09-23T19:55:20.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731038+0000", 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1460000000000001e-05 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731050+0000", 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1127999999999999e-05 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731105+0000", 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5318999999999997e-05 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731270+0000", 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000165565 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731413+0000", 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014208800000000001 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.743775+0000", 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012362167 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:20.768 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.743967+0000", 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019191100000000001 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:175038 5.c 5:327c4745:::benchmark_data_smithi135_284570_object59254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:20.731077+0000", 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.024971854000000002, 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.025113716000000001, 2023-09-23T19:55:20.769 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 175038 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731077+0000", 2023-09-23T19:55:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731077+0000", 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731077+0000", 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731081+0000", 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2089999999999999e-06 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731089+0000", 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3690000000000007e-06 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:20.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731105+0000", 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5716999999999999e-05 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.735490+0000", 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043843739999999999 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.735620+0000", 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013057300000000001 2023-09-23T19:55:20.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.735747+0000", 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012695600000000001 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.743780+0000", 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080330160000000005 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754486+0000", 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010706373999999999 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:175046 5.c 5:34852b21:::benchmark_data_smithi135_284570_object59262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:20.731114+0000", 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.024935386, 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.025171307, 2023-09-23T19:55:20.775 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 175046 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731114+0000", 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731114+0000", 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731113+0000", 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731117+0000", 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5860000000000001e-06 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731124+0000", 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4540000000000001e-06 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.731130+0000", 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7259999999999999e-06 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.739634+0000", 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085034020000000002 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.739770+0000", 2023-09-23T19:55:20.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013686699999999999 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.739891+0000", 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000120172 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.751306+0000", 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011415312 2023-09-23T19:55:20.780 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754521+0000", 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032155619999999999 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:175054 5.5 e149/112 5:ac30ae52:::benchmark_data_smithi135_284570_object59270:head v 149'10980, mlcod=149'10976)", 2023-09-23T19:55:20.781 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:20.751942+0000", 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0041075900000000004, 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0044309520000000002, 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.751942+0000", 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.751942+0000", 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.751941+0000", 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:20.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.751950+0000", 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9830000000000002e-06 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.751962+0000", 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2277e-05 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.751970+0000", 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.875e-06 2023-09-23T19:55:20.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.752036+0000", 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5654999999999994e-05 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.752054+0000", 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8318999999999999e-05 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.785 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:175058 5.0 e149/112 5:054159ec:::benchmark_data_smithi135_284570_object59274:head v 149'11072, mlcod=149'11070)", 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:20.754647+0000", 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.001401972, 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017991909999999999, 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754647+0000", 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754647+0000", 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754647+0000", 2023-09-23T19:55:20.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754653+0000", 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9750000000000004e-06 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754661+0000", 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3550000000000005e-06 2023-09-23T19:55:20.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754667+0000", 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5199999999999997e-06 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754708+0000", 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0958999999999999e-05 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:20.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:20.754727+0000", 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8921999999999999e-05 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 5 2023-09-23T19:55:20.790 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:55:20.796 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:21.125 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:21.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:21.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:21.234 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:21.315 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:22.076 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:55:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:22 smithi135 ceph-mon[221113]: pgmap v354: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 14961/177117 objects degraded (8.447%) 2023-09-23T19:55:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 14961/177117 objects degraded (8.447%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:22.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:22 smithi135 ceph-mon[213871]: pgmap v354: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 14961/177117 objects degraded (8.447%) 2023-09-23T19:55:22.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:22.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 14961/177117 objects degraded (8.447%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:22 smithi144 ceph-mon[194833]: pgmap v354: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 14961/177117 objects degraded (8.447%) 2023-09-23T19:55:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:22.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 14961/177117 objects degraded (8.447%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:22.834 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:22.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:22.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:22.941 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:22.978 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:23.052 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:23.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:23.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:23.095 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:55:23.165 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:23.265 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:23.398 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:23 smithi135 ceph-mon[213871]: pgmap v355: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 14467/170346 objects degraded (8.493%) 2023-09-23T19:55:23.529 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:23.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:23.634 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:23.661 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:23 smithi144 ceph-mon[194833]: pgmap v355: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 14467/170346 objects degraded (8.493%) 2023-09-23T19:55:23.664 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:23.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:23 smithi135 ceph-mon[221113]: pgmap v355: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 14467/170346 objects degraded (8.493%) 2023-09-23T19:55:23.726 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:23.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:23.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:23.832 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:23.857 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:23.939 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:23.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:55:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:55:23] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.43.0" 2023-09-23T19:55:24.051 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:24.112 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:24.211 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:24.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:24.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:24.323 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:24.362 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:24.421 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:24.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:24.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:24.522 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:24.531 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:24.623 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:24.886 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:24.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:24.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:24.999 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:25.054 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:25.086 INFO:teuthology.orchestra.run.smithi135.stdout:moved 30 / 297 (10.101%) 2023-09-23T19:55:25.086 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.55277 -> 9.19294 (expected baseline 6.03053) 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 30 -> 36 pgs (0.707071 -> 0.848485 * mean) 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.4 with 44 -> 37 pgs (1.03704 -> 0.872054 * mean) 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 1 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0157 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0172 2023-09-23T19:55:25.087 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 weight 1.0000 -> 0.7042 2023-09-23T19:55:25.088 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 weight 1.0000 -> 0.7546 2023-09-23T19:55:25.088 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 weight 1.0000 -> 0.7933 2023-09-23T19:55:25.101 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:55:25.134 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:25.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:25.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:25.158 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:55:25 smithi135 conmon[201366]: 2023-09-23T19:55:25.083+0000 7fcf49f9e700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T19:55:25.209 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:25.239 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:25.331 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:25.778 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:25.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:25.854 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:25 smithi135 ceph-mon[221113]: pgmap v356: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 14000/165477 objects degraded (8.460%) 2023-09-23T19:55:25.854 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:25 smithi135 ceph-mon[221113]: from='client.44771 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:25.855 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:25 smithi135 ceph-mon[213871]: pgmap v356: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 14000/165477 objects degraded (8.460%) 2023-09-23T19:55:25.856 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:25 smithi135 ceph-mon[213871]: from='client.44771 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:25 smithi144 ceph-mon[194833]: pgmap v356: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 14000/165477 objects degraded (8.460%) 2023-09-23T19:55:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:25 smithi144 ceph-mon[194833]: from='client.44771 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:25.888 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:25.914 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:26.601 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:55:26.602 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:26.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:26.710 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:26.737 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:26.885 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1259506862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:55:26.886 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:55:26.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1259506862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:55:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:55:26.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1259506862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:55:26.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:55:27.279 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:27.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:27.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:27.388 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:27.415 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:27.590 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: pgmap v357: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 13535/162717 objects degraded (8.318%) 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: osdmap e150: 8 total, 7 up, 8 in 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.591 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.592 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.592 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 13535/162717 objects degraded (8.318%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:27.859 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: pgmap v357: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 13535/162717 objects degraded (8.318%) 2023-09-23T19:55:27.859 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:27.859 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.859 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.859 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.860 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.860 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:55:27.860 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: osdmap e150: 8 total, 7 up, 8 in 2023-09-23T19:55:27.860 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.860 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.860 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.860 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 13535/162717 objects degraded (8.318%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: pgmap v357: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 13535/162717 objects degraded (8.318%) 2023-09-23T19:55:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:55:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: osdmap e150: 8 total, 7 up, 8 in 2023-09-23T19:55:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: failed to encode map e150 with expected crc 2023-09-23T19:55:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 13535/162717 objects degraded (8.318%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:28.092 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:28.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:28.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:28.188 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:55:28.189 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T19:55:28.209 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:28.251 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:28.329 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:28.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:28.437 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:28.476 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:28.527 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:28.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:28.634 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:28.705 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:29.273 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:55:29.273 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:55:29.296 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:55:29.347 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:55:29.390 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:55:29.428 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:55:29.460 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:55:29.461 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:55:29.461 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:55:29.461 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:55:29.461 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:55:29.461 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:55:29.461 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:55:29.461 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:55:29.461 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:55:29.477 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:29.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:29.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:29.588 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:29.678 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:29.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:29 smithi135 ceph-mon[213871]: pgmap v359: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 13072/155046 objects degraded (8.431%) 2023-09-23T19:55:29.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1178263820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:29.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:29 smithi135 ceph-mon[221113]: pgmap v359: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 13072/155046 objects degraded (8.431%) 2023-09-23T19:55:29.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1178263820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:29.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:29 smithi144 ceph-mon[194833]: pgmap v359: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 13072/155046 objects degraded (8.431%) 2023-09-23T19:55:29.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1178263820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:30.132 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:55:30.317 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:30.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:30.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:30.404 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:55:30.405 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:55:30.421 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:30.457 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:30.550 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:30.553 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:30.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:30.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:30.667 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:30.766 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:31.073 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:55:31.073 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:55:31.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:31.092 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:31.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:31.193 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:31.224 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:31.307 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:31.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:31.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:31.420 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:31.467 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:31.551 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:31.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:31.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:31 smithi135 ceph-mon[213871]: pgmap v360: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 13072/154359 objects degraded (8.469%) 2023-09-23T19:55:31.658 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:31.688 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:31.772 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:31.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:31.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:31.880 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:31 smithi144 ceph-mon[194833]: pgmap v360: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 13072/154359 objects degraded (8.469%) 2023-09-23T19:55:31.908 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:31 smithi135 ceph-mon[221113]: pgmap v360: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 13072/154359 objects degraded (8.469%) 2023-09-23T19:55:31.912 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:32.406 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:55:32.414 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:32.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:32.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:32.525 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:32.553 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:32.595 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:32.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:32.701 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:32.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 13072/154359 objects degraded (8.469%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:32.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 13072/154359 objects degraded (8.469%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:32.747 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:32.818 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:32.818 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:32.824 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 13072/154359 objects degraded (8.469%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:32.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:32.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:32.870 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:55:32.926 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:32.970 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:33.041 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:33.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:33.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:33.156 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:33.277 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:33.409 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:33.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:33.515 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:33.577 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:33.678 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:33.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:33.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:33.698 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:33.789 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:33.858 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:33.960 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:55:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:55:33] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.43.0" 2023-09-23T19:55:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:34 smithi135 ceph-mon[213871]: pgmap v361: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 12606/148029 objects degraded (8.516%) 2023-09-23T19:55:34.393 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:34.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:34.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:34.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:34 smithi135 ceph-mon[221113]: pgmap v361: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 12606/148029 objects degraded (8.516%) 2023-09-23T19:55:34.513 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:34.591 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.55277 -> 6.55277 (expected baseline 6.03053) 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 30 -> 30 pgs (0.707071 -> 0.707071 * mean) 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.4 with 44 -> 44 pgs (1.03704 -> 1.03704 * mean) 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:55:34.623 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0140 2023-09-23T19:55:34.624 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.1396 2023-09-23T19:55:34.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:34 smithi144 ceph-mon[194833]: pgmap v361: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 12606/148029 objects degraded (8.516%) 2023-09-23T19:55:34.644 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:55:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:36 smithi144 ceph-mon[194833]: pgmap v362: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 12123/143109 objects degraded (8.471%) 2023-09-23T19:55:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:36 smithi144 ceph-mon[194833]: from='client.34899 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:36.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:36 smithi135 ceph-mon[221113]: pgmap v362: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 12123/143109 objects degraded (8.471%) 2023-09-23T19:55:36.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:36 smithi135 ceph-mon[221113]: from='client.34899 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:36.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:36 smithi135 ceph-mon[213871]: pgmap v362: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 12123/143109 objects degraded (8.471%) 2023-09-23T19:55:36.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:36 smithi135 ceph-mon[213871]: from='client.34899 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:36.784 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:36.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:36.846 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:55:36.846 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-23T19:55:36.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:36.959 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:37.150 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:37.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:37.251 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:37.297 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:37.387 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:37.392 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:37.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:37 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12123/143109 objects degraded (8.471%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:37.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:37.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:37 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12123/143109 objects degraded (8.471%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:37.495 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:37.554 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:37.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:37.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:37 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12123/143109 objects degraded (8.471%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:37.922 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:37.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:37.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:38.030 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:38.097 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:55:38.114 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:38.164 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:55:38.217 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:55:38.257 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:55:38.317 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:55:38.327 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:55:38.365 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:55:38.406 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:55:38.406 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:55:38.406 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:55:38.406 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:55:38.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:55:38.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:55:38.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:55:38.407 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:55:38.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:55:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:38 smithi135 ceph-mon[221113]: pgmap v363: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 11655/140280 objects degraded (8.308%) 2023-09-23T19:55:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4143443705' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:38 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3702454645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:38 smithi135 ceph-mon[213871]: pgmap v363: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 11655/140280 objects degraded (8.308%) 2023-09-23T19:55:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4143443705' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:38 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3702454645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:38.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:38 smithi144 ceph-mon[194833]: pgmap v363: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 11655/140280 objects degraded (8.308%) 2023-09-23T19:55:38.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4143443705' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:38.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:38 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:55:38.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3702454645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:39.188 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:39.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:39.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:39.297 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:39.323 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:55:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[213871]: osdmap e151: 8 total, 7 up, 8 in 2023-09-23T19:55:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[213871]: failed to encode map e151 with expected crc 2023-09-23T19:55:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[213871]: failed to encode map e151 with expected crc 2023-09-23T19:55:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[213871]: failed to encode map e151 with expected crc 2023-09-23T19:55:39.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:55:39.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:39.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[221113]: osdmap e151: 8 total, 7 up, 8 in 2023-09-23T19:55:39.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[221113]: failed to encode map e151 with expected crc 2023-09-23T19:55:39.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[221113]: failed to encode map e151 with expected crc 2023-09-23T19:55:39.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:39 smithi135 ceph-mon[221113]: failed to encode map e151 with expected crc 2023-09-23T19:55:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:39 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:55:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:39 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:55:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:39 smithi144 ceph-mon[194833]: osdmap e151: 8 total, 7 up, 8 in 2023-09-23T19:55:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:39 smithi144 ceph-mon[194833]: failed to encode map e151 with expected crc 2023-09-23T19:55:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:39 smithi144 ceph-mon[194833]: failed to encode map e151 with expected crc 2023-09-23T19:55:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:39 smithi144 ceph-mon[194833]: failed to encode map e151 with expected crc 2023-09-23T19:55:40.617 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:40.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:40.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:40.621 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:40 smithi135 ceph-mon[221113]: pgmap v365: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 11203/132360 objects degraded (8.464%) 2023-09-23T19:55:40.621 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:40 smithi135 ceph-mon[213871]: pgmap v365: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 11203/132360 objects degraded (8.464%) 2023-09-23T19:55:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:40 smithi144 ceph-mon[194833]: pgmap v365: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 11203/132360 objects degraded (8.464%) 2023-09-23T19:55:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:55:40.690 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:55:40.722 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:40.774 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:40.837 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:40.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:40.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:40.943 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:40.990 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:41.034 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:41.078 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:41.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:41.193 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:41.230 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:41.323 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:41.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:41.431 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:41.491 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:41.557 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:55:41.557 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:55:41.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:55:41.557 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:55:41.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.557 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:67877 5.c 5:3302d86b:::benchmark_data_smithi135_284570_object67876:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T19:55:41.557 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:05.885148+0000", 2023-09-23T19:55:41.557 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 155.669025898, 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.15899455400000001, 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 67877 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885148+0000", 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885148+0000", 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885146+0000", 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885171+0000", 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4445999999999999e-05 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885189+0000", 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8189000000000002e-05 2023-09-23T19:55:41.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885197+0000", 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3839999999999997e-06 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885305+0000", 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107932 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885434+0000", 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012834500000000001 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885556+0000", 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012217599999999999 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.890202+0000", 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046460700000000004 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.891487+0000", 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012849999999999999 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:06.044044+0000", 2023-09-23T19:55:41.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.152556777 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:06.044096+0000", 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2159e-05 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:06.044143+0000", 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6931000000000002e-05 2023-09-23T19:55:41.564 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:67878 5.c 5:3b08d2a0:::benchmark_data_smithi135_284570_object67877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:05.885214+0000", 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 155.668959484, 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.15902464899999999, 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.565 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 67878 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885214+0000", 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885214+0000", 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885214+0000", 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885234+0000", 2023-09-23T19:55:41.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0140999999999999e-05 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885241+0000", 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3690000000000001e-06 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.885247+0000", 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0000000000000002e-06 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.887674+0000", 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002426124 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.887798+0000", 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123934 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.887932+0000", 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000134509 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.890208+0000", 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022760200000000001 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:05.891519+0000", 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0013113409999999999 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:06.044176+0000", 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.152656872 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:06.044210+0000", 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3764999999999999e-05 2023-09-23T19:55:41.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:06.044239+0000", 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9102e-05 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:110150 5.c 5:34506594:::benchmark_data_smithi135_284570_object110149:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:50.593665+0000", 2023-09-23T19:55:41.572 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 110.960508984, 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.156236504, 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 110150 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.573 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593665+0000", 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593665+0000", 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593663+0000", 2023-09-23T19:55:41.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593688+0000", 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4805999999999999e-05 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593701+0000", 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2858e-05 2023-09-23T19:55:41.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593709+0000", 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8820000000000001e-06 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593751+0000", 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2400000000000001e-05 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593839+0000", 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7408000000000001e-05 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.593930+0000", 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1475000000000001e-05 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.596484+0000", 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025540150000000002 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.600048+0000", 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003563849 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.749813+0000", 2023-09-23T19:55:41.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14976497599999999 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.749863+0000", 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9554e-05 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:50.749901+0000", 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8915000000000003e-05 2023-09-23T19:55:41.579 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:113458 5.c 5:365467a7:::benchmark_data_smithi135_284570_object113457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:54.635849+0000", 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 106.91832482300001, 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.15539958000000001, 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.580 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.581 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.581 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 113458 2023-09-23T19:55:41.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.581 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.635849+0000", 2023-09-23T19:55:41.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.635849+0000", 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.635848+0000", 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.635872+0000", 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4508e-05 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.635885+0000", 2023-09-23T19:55:41.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3239e-05 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.635894+0000", 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6309999999999992e-06 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.635960+0000", 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5999000000000003e-05 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.636080+0000", 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000120129 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.636213+0000", 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013284399999999999 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.640027+0000", 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038137240000000001 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.643468+0000", 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034410650000000001 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.791165+0000", 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14769760900000001 2023-09-23T19:55:41.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.791213+0000", 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8089e-05 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:54.791249+0000", 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5323e-05 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.587 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:114614 5.c 5:3bdfca03:::benchmark_data_smithi135_284570_object114613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:56.071208+0000", 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.48296626600001, 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22332307600000001, 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.588 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 114614 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071208+0000", 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071208+0000", 2023-09-23T19:55:41.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071206+0000", 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071227+0000", 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0344999999999999e-05 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071236+0000", 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2450000000000004e-06 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071241+0000", 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5199999999999997e-06 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071281+0000", 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9801000000000001e-05 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071370+0000", 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8949999999999994e-05 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071455+0000", 2023-09-23T19:55:41.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4414999999999997e-05 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.074979+0000", 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0035246019999999999 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.078084+0000", 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031047700000000002 2023-09-23T19:55:41.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.294440+0000", 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21635629100000001 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.294490+0000", 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9446000000000001e-05 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.294531+0000", 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1065000000000001e-05 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:114622 5.c 5:3d988ee4:::benchmark_data_smithi135_284570_object114621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:56.139334+0000", 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.414840356, 2023-09-23T19:55:41.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.158325888, 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 114622 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139334+0000", 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139334+0000", 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139332+0000", 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:55:41.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139356+0000", 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4430000000000002e-05 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139369+0000", 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3222e-05 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139378+0000", 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8400000000000001e-06 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139432+0000", 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3377000000000001e-05 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139561+0000", 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000128928 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.139715+0000", 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015446399999999999 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.143292+0000", 2023-09-23T19:55:41.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0035765910000000001 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.144047+0000", 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00075543699999999997 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.297561+0000", 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.15351377899999999 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.297617+0000", 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6616999999999997e-05 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.297659+0000", 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2024000000000003e-05 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:124388 5.c 5:33065abd:::benchmark_data_smithi135_284570_object8604:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:08.856332+0000", 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 92.697842012999999, 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17133683499999999, 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 124388 2023-09-23T19:55:41.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856332+0000", 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856332+0000", 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856330+0000", 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856338+0000", 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9209999999999992e-06 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856356+0000", 2023-09-23T19:55:41.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7846000000000002e-05 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856364+0000", 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5140000000000001e-06 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856418+0000", 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3306000000000001e-05 2023-09-23T19:55:41.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856549+0000", 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013117700000000001 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.856656+0000", 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000106958 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.861042+0000", 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043862900000000002 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:08.865830+0000", 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047881299999999998 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:09.027598+0000", 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16176744200000001 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:09.027635+0000", 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7592000000000002e-05 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:09.027669+0000", 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3417999999999998e-05 2023-09-23T19:55:41.609 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:125823 5.c 5:3be4aa05:::benchmark_data_smithi135_284570_object10039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:10.733379+0000", 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 90.820794844000005, 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.19459162999999999, 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.610 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 125823 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733379+0000", 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733379+0000", 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733378+0000", 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733385+0000", 2023-09-23T19:55:41.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4089999999999997e-06 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733402+0000", 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6799e-05 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733410+0000", 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0860000000000002e-06 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733464+0000", 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3875000000000002e-05 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733598+0000", 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013450499999999999 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.733718+0000", 2023-09-23T19:55:41.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000120028 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.737476+0000", 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0037575830000000001 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.740571+0000", 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030955079999999999 2023-09-23T19:55:41.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.927899+0000", 2023-09-23T19:55:41.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.18732748499999999 2023-09-23T19:55:41.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.927942+0000", 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.303e-05 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:10.927971+0000", 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8853e-05 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:126118 5.c 5:380143a9:::benchmark_data_smithi135_284570_object10334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.241628+0000", 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 90.312546072000004, 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24666907800000001, 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 126118 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241628+0000", 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241628+0000", 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241627+0000", 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241632+0000", 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.31e-06 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241641+0000", 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2660000000000007e-06 2023-09-23T19:55:41.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241647+0000", 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8579999999999996e-06 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241685+0000", 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7818999999999998e-05 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241766+0000", 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0822000000000006e-05 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241835+0000", 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8881999999999995e-05 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.244659+0000", 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028244749999999999 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.246015+0000", 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0013558470000000001 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.488191+0000", 2023-09-23T19:55:41.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24217628999999999 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.488251+0000", 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9348999999999997e-05 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.488297+0000", 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6428999999999999e-05 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.625 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:140733 5.c 5:3c10dae1:::benchmark_data_smithi135_284570_object24949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:31.742139+0000", 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 69.812035273000006, 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16339213499999999, 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.626 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 140733 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742139+0000", 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742139+0000", 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742137+0000", 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742144+0000", 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4409999999999998e-06 2023-09-23T19:55:41.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742159+0000", 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5296000000000001e-05 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742168+0000", 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2650000000000002e-06 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742210+0000", 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1304999999999998e-05 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742331+0000", 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000121397 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.742452+0000", 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000120878 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.745484+0000", 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003031672 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.746515+0000", 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010313080000000001 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.905432+0000", 2023-09-23T19:55:41.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.158917319 2023-09-23T19:55:41.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.905488+0000", 2023-09-23T19:55:41.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6038000000000001e-05 2023-09-23T19:55:41.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:31.905531+0000", 2023-09-23T19:55:41.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2586999999999997e-05 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:148373 5.c 5:3fc24fd7:::benchmark_data_smithi135_284570_object32589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.657866+0000", 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 58.896308032, 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.230255707, 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.634 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148373 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657866+0000", 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657866+0000", 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657865+0000", 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657870+0000", 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4219999999999999e-06 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657883+0000", 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2418999999999999e-05 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657888+0000", 2023-09-23T19:55:41.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7320000000000003e-06 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657948+0000", 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9524999999999997e-05 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658053+0000", 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000105282 2023-09-23T19:55:41.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658154+0000", 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101173 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887885+0000", 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.229730404 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887926+0000", 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1737999999999999e-05 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888044+0000", 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117816 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888083+0000", 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8380999999999997e-05 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888122+0000", 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9079000000000003e-05 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:148374 5.c 5:31a4f63e:::benchmark_data_smithi135_284570_object32590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.658148+0000", 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 58.896026436, 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.23127130900000001, 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148374 2023-09-23T19:55:41.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658148+0000", 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658148+0000", 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658147+0000", 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658151+0000", 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1200000000000004e-06 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658163+0000", 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.224e-05 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658169+0000", 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.015e-06 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661984+0000", 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038152279999999999 2023-09-23T19:55:41.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662082+0000", 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8055999999999998e-05 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662151+0000", 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8774000000000003e-05 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888037+0000", 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22588576199999999 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888126+0000", 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9195000000000002e-05 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889336+0000", 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012101569999999999 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889381+0000", 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4808000000000002e-05 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889419+0000", 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7645000000000003e-05 2023-09-23T19:55:41.648 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:148373 5.c e143/112 ondisk, result = 0)", 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.661352+0000", 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 58.892821505000001, 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22653495600000001, 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:55:41.649 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661352+0000", 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661352+0000", 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661352+0000", 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661357+0000", 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8050000000000003e-06 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661368+0000", 2023-09-23T19:55:41.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0984e-05 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661374+0000", 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8810000000000001e-06 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887872+0000", 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.226497741 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887879+0000", 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0419999999999997e-06 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887887+0000", 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4079999999999998e-06 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:148377 5.5 e143/112 5:a6d497bb:::benchmark_data_smithi135_284570_object32593:head v 143'9284, mlcod=143'9282)", 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.662002+0000", 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 58.892171572999999, 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.225817616, 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.654 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662002+0000", 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662002+0000", 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662002+0000", 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:55:41.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662008+0000", 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9850000000000003e-06 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662015+0000", 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5950000000000001e-06 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662020+0000", 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7219999999999999e-06 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662064+0000", 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3977000000000003e-05 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662082+0000", 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8164000000000001e-05 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664929+0000", 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028465999999999999 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887820+0000", 2023-09-23T19:55:41.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.222891216 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:148380 5.0 e143/112 5:05c78c8a:::benchmark_data_smithi135_284570_object32596:head v 143'9387, mlcod=143'9385)", 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.664519+0000", 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 58.889654505999999, 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.223721211, 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.659 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664519+0000", 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664519+0000", 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664519+0000", 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664526+0000", 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4739999999999999e-06 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664538+0000", 2023-09-23T19:55:41.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2330000000000001e-05 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664546+0000", 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3429999999999998e-06 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664589+0000", 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3686000000000002e-05 2023-09-23T19:55:41.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664610+0000", 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0885000000000001e-05 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888206+0000", 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22359541199999999 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888241+0000", 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4885000000000002e-05 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:148373 5.c e143/112 ondisk, result = 0)", 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.666374+0000", 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 58.887799872000002, 2023-09-23T19:55:41.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.221554634, 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.666374+0000", 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.666374+0000", 2023-09-23T19:55:41.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.666373+0000", 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.666381+0000", 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6020000000000002e-06 2023-09-23T19:55:41.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887631+0000", 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22124992099999999 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887647+0000", 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5733999999999999e-05 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887915+0000", 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00026879200000000001 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887921+0000", 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8359999999999997e-06 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887929+0000", 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4189999999999996e-06 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:160443 5.c 5:3015b93a:::benchmark_data_smithi135_284570_object44659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:00.934539+0000", 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 40.619634654999999, 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.174520904, 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.669 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 160443 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934539+0000", 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934539+0000", 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934537+0000", 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934547+0000", 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1209999999999996e-06 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934562+0000", 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5299e-05 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934571+0000", 2023-09-23T19:55:41.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8529999999999998e-06 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934620+0000", 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9276000000000001e-05 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934703+0000", 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2618000000000001e-05 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.934797+0000", 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4059000000000002e-05 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.937740+0000", 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029431629999999999 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.939088+0000", 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0013482850000000001 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:01.108962+0000", 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16987422699999999 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:01.109016+0000", 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3287e-05 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:01.109060+0000", 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4610000000000001e-05 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.676 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:160445 5.c 5:3028e760:::benchmark_data_smithi135_284570_object44661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:00.939967+0000", 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 40.614207462000003, 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16958309999999999, 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 160445 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.677 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.939967+0000", 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.939967+0000", 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.939965+0000", 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.939971+0000", 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6960000000000002e-06 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.939980+0000", 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3430000000000002e-06 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.939986+0000", 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1680000000000001e-06 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.940020+0000", 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4180000000000001e-05 2023-09-23T19:55:41.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.940099+0000", 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8607000000000001e-05 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.940176+0000", 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7099000000000002e-05 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.942824+0000", 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026481529999999999 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:00.943004+0000", 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017955099999999999 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:01.109478+0000", 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16647363500000001 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:01.109517+0000", 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9848000000000002e-05 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:01.109550+0000", 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2237999999999999e-05 2023-09-23T19:55:41.683 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:178913 5.c 5:36ac370d:::benchmark_data_smithi135_284570_object63129:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:25.870577+0000", 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 15.683596705999999, 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.158067342, 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.684 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 178913 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870577+0000", 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870577+0000", 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870576+0000", 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870583+0000", 2023-09-23T19:55:41.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7170000000000008e-06 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870596+0000", 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2943e-05 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870605+0000", 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4460000000000001e-06 2023-09-23T19:55:41.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870659+0000", 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4438000000000003e-05 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870785+0000", 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000126124 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.870901+0000", 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011608300000000001 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.873848+0000", 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002947037 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:25.876788+0000", 2023-09-23T19:55:41.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029392120000000001 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:26.028555+0000", 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.151767918 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:26.028605+0000", 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9147999999999998e-05 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:26.028645+0000", 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0000999999999999e-05 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:189893 5.c 5:3cf87985:::benchmark_data_smithi135_284570_object74109:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:40.360813+0000", 2023-09-23T19:55:41.691 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 1.1933608929999999, 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20771811300000001, 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 189893 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:55:41.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360813+0000", 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360813+0000", 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360812+0000", 2023-09-23T19:55:41.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360817+0000", 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5049999999999998e-06 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360830+0000", 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2425999999999999e-05 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360835+0000", 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5949999999999998e-06 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360873+0000", 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7963000000000002e-05 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:55:41.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360983+0000", 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010943899999999999 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.361050+0000", 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6923999999999998e-05 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.363988+0000", 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029381730000000001 2023-09-23T19:55:41.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.365200+0000", 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012125459999999999 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.568446+0000", 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20324604499999999 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.568493+0000", 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6761000000000002e-05 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.568531+0000", 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8022999999999998e-05 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:55:41.698 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:55:41.699 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:55:41.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:41.700 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:41.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:41.801 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:41.875 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:42.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:42 smithi135 ceph-mon[213871]: pgmap v366: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11203/131709 objects degraded (8.506%) 2023-09-23T19:55:42.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11203/131709 objects degraded (8.506%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:42 smithi135 ceph-mon[221113]: pgmap v366: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11203/131709 objects degraded (8.506%) 2023-09-23T19:55:42.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11203/131709 objects degraded (8.506%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:42.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:42 smithi144 ceph-mon[194833]: pgmap v366: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11203/131709 objects degraded (8.506%) 2023-09-23T19:55:42.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11203/131709 objects degraded (8.506%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:42.691 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:55:42.699 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:42.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:42.807 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:42.847 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:42.925 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:42.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:42.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:42.988 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:55:43.039 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:43.114 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:43.314 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:55:43.344 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:43.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:43.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:43.458 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:43.612 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:43.763 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:43.886 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:55:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:55:43] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.43.0" 2023-09-23T19:55:43.895 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:43.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:44.014 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:44.095 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:44.136 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:44.442 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:44.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:44.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:44.555 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:55:44.592 INFO:teuthology.orchestra.run.smithi135.stdout:moved 24 / 297 (8.08081%) 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.55277 -> 8.08753 (expected baseline 6.03053) 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 30 -> 35 pgs (0.707071 -> 0.824916 * mean) 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.4 with 44 -> 40 pgs (1.03704 -> 0.942761 * mean) 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 3 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0122 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0134 2023-09-23T19:55:44.593 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 weight 1.0000 -> 0.7350 2023-09-23T19:55:44.594 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0 weight 1.0000 -> 0.7826 2023-09-23T19:55:44.594 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 1.0000 -> 0.8275 2023-09-23T19:55:44.594 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4 weight 1.0000 -> 0.8344 2023-09-23T19:55:44.594 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:44 smithi135 ceph-mon[221113]: pgmap v367: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10728/125391 objects degraded (8.556%) 2023-09-23T19:55:44.595 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:44 smithi144 ceph-mon[194833]: pgmap v367: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10728/125391 objects degraded (8.556%) 2023-09-23T19:55:44.595 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:44 smithi135 ceph-mon[213871]: pgmap v367: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10728/125391 objects degraded (8.556%) 2023-09-23T19:55:44.596 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:55:44 smithi135 conmon[201366]: 2023-09-23T19:55:44.592+0000 7fcf49f9e700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T19:55:44.599 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:44.623 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:55:44.656 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:55:44.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:44.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:55:44.764 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:44.775 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:44.852 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:46.387 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:46.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:46.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:46.492 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:46.562 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:46 smithi144 ceph-mon[194833]: pgmap v368: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 10241/120330 objects degraded (8.511%) 2023-09-23T19:55:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:46 smithi144 ceph-mon[194833]: from='client.34914 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:46.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:46 smithi135 ceph-mon[213871]: pgmap v368: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 10241/120330 objects degraded (8.511%) 2023-09-23T19:55:46.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:46 smithi135 ceph-mon[213871]: from='client.34914 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:46 smithi135 ceph-mon[221113]: pgmap v368: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 10241/120330 objects degraded (8.511%) 2023-09-23T19:55:46.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:46 smithi135 ceph-mon[221113]: from='client.34914 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:47.346 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:47.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:47.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:47.431 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:55:47.431 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T19:55:47.459 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:47.588 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:47.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 10241/120330 objects degraded (8.511%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:47.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 10241/120330 objects degraded (8.511%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:47.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 10241/120330 objects degraded (8.511%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T19:55:47.967 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:47.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:48.074 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:48.121 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:48.569 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:55:48.596 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:55:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:48 smithi144 ceph-mon[194833]: pgmap v369: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 9817/117561 objects degraded (8.351%) 2023-09-23T19:55:48.638 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:55:48.665 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:55:48.721 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:55:48.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:48 smithi135 ceph-mon[221113]: pgmap v369: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 9817/117561 objects degraded (8.351%) 2023-09-23T19:55:48.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:48 smithi135 ceph-mon[213871]: pgmap v369: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 9817/117561 objects degraded (8.351%) 2023-09-23T19:55:48.765 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:55:48.803 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:55:48.804 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:55:48.804 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:55:48.804 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:55:48.804 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:55:48.804 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:55:48.804 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:55:48.804 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:55:48.804 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:55:49.113 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:49.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:49.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:49.225 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:49.280 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:49.370 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:49.375 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:49 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/10065469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:49.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:49.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:49.483 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:49.548 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:49 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/10065469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:49.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:49 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/10065469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:49.744 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:49.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:49.796 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:49.830 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:55:49.831 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:55:49.858 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:49.914 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:49.997 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:50.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:50.101 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:50.210 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:50.468 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:55:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:50.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:55:50.482 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:55:50.482 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:55:50.574 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:50.619 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:50.698 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:50.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:50.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:50.703 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:50 smithi144 ceph-mon[194833]: pgmap v370: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9364/110097 objects degraded (8.505%) 2023-09-23T19:55:50.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:50 smithi135 ceph-mon[221113]: pgmap v370: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9364/110097 objects degraded (8.505%) 2023-09-23T19:55:50.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:50 smithi135 ceph-mon[213871]: pgmap v370: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9364/110097 objects degraded (8.505%) 2023-09-23T19:55:50.803 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:50.873 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:51.271 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:51.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:51.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:51.374 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:51.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:51 smithi135 ceph-mon[213871]: pgmap v371: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 9364/109407 objects degraded (8.559%) 2023-09-23T19:55:51.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/78996000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:51.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:51.397 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:55:51.405 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:51.446 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:51.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:51.551 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:51.603 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:51 smithi135 ceph-mon[221113]: pgmap v371: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 9364/109407 objects degraded (8.559%) 2023-09-23T19:55:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/78996000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:51.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:51.833 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:55:51.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:51 smithi144 ceph-mon[194833]: pgmap v371: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 9364/109407 objects degraded (8.559%) 2023-09-23T19:55:51.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/78996000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:51.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:55:52.022 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:55:52.054 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:52.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:52.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:52.174 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:52.268 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[213871]: failed to encode map e152 with expected crc 2023-09-23T19:55:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[213871]: osdmap e152: 8 total, 7 up, 8 in 2023-09-23T19:55:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[213871]: failed to encode map e152 with expected crc 2023-09-23T19:55:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9364/109407 objects degraded (8.559%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:55:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[221113]: failed to encode map e152 with expected crc 2023-09-23T19:55:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[221113]: osdmap e152: 8 total, 7 up, 8 in 2023-09-23T19:55:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[221113]: failed to encode map e152 with expected crc 2023-09-23T19:55:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:52 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9364/109407 objects degraded (8.559%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:55:52.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:52 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:55:52.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:52 smithi144 ceph-mon[194833]: failed to encode map e152 with expected crc 2023-09-23T19:55:52.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:55:52.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:52 smithi144 ceph-mon[194833]: osdmap e152: 8 total, 7 up, 8 in 2023-09-23T19:55:52.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:52 smithi144 ceph-mon[194833]: failed to encode map e152 with expected crc 2023-09-23T19:55:52.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:55:52.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:52 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9364/109407 objects degraded (8.559%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:55:53.426 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:53.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:53.532 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:53.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:53 smithi135 ceph-mon[221113]: failed to encode map e152 with expected crc 2023-09-23T19:55:53.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:53 smithi135 ceph-mon[221113]: pgmap v373: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 8881/103080 objects degraded (8.616%) 2023-09-23T19:55:53.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:55:53.601 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:53 smithi135 ceph-mon[213871]: failed to encode map e152 with expected crc 2023-09-23T19:55:53.601 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:53 smithi135 ceph-mon[213871]: pgmap v373: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 8881/103080 objects degraded (8.616%) 2023-09-23T19:55:53.601 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:55:53.602 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:53.710 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:53.788 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:53.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:53.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:53 smithi144 ceph-mon[194833]: failed to encode map e152 with expected crc 2023-09-23T19:55:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:53 smithi144 ceph-mon[194833]: pgmap v373: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 8881/103080 objects degraded (8.616%) 2023-09-23T19:55:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:55:53.899 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:55:53.937 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:53.977 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:55:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:55:53] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-09-23T19:55:53.996 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:54.315 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:55:54.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:54.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:55:54.428 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:54.498 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:54.879 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.55277 -> 6.55277 (expected baseline 6.03053) 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 30 -> 30 pgs (0.707071 -> 0.707071 * mean) 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.4 with 44 -> 44 pgs (1.03704 -> 1.03704 * mean) 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 3 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0105 2023-09-23T19:55:54.880 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.1049 2023-09-23T19:55:54.904 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:55:54.932 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:54.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:54.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:55.036 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:55:55.080 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:55.379 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:55:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:55 smithi135 ceph-mon[221113]: pgmap v374: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8419/98274 objects degraded (8.567%) 2023-09-23T19:55:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:55 smithi135 ceph-mon[221113]: from='client.44816 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:55.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:55 smithi135 ceph-mon[213871]: pgmap v374: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8419/98274 objects degraded (8.567%) 2023-09-23T19:55:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:55 smithi135 ceph-mon[213871]: from='client.44816 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:55.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:55 smithi144 ceph-mon[194833]: pgmap v374: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8419/98274 objects degraded (8.567%) 2023-09-23T19:55:55.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:55 smithi144 ceph-mon[194833]: from='client.44816 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:55:57.500 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:55:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:57.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:55:57.618 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:55:57.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:57 smithi135 ceph-mon[221113]: pgmap v375: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7999/95658 objects degraded (8.362%) 2023-09-23T19:55:57.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7999/95658 objects degraded (8.362%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:55:57.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:57.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:57.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:57 smithi135 ceph-mon[213871]: pgmap v375: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7999/95658 objects degraded (8.362%) 2023-09-23T19:55:57.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7999/95658 objects degraded (8.362%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:55:57.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:57.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:57.676 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:57.730 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T19:55:57.730 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T19:55:57.772 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:55:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:57.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:55:57.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:57 smithi144 ceph-mon[194833]: pgmap v375: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7999/95658 objects degraded (8.362%) 2023-09-23T19:55:57.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7999/95658 objects degraded (8.362%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:55:57.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:57.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:55:57.889 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:55:57.985 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:58.265 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:55:58.272 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:55:58.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:58.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:55:58.323 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:55:58.384 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:55:58.384 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:55:58.386 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:55:58.390 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:55:58.427 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:55:58.427 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:58.460 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:55:58.503 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:55:58.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:55:58.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:55:58.521 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:55:58.571 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:55:58.584 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2391889256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:58.584 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2391889256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:55:58.609 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:55:58.612 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:55:58.683 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:55:58.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2391889256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:55:59.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:55:59 smithi135 ceph-mon[221113]: pgmap v376: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 7550/88293 objects degraded (8.551%) 2023-09-23T19:55:59.589 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:55:59 smithi135 ceph-mon[213871]: pgmap v376: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 7550/88293 objects degraded (8.551%) 2023-09-23T19:55:59.885 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:55:59 smithi144 ceph-mon[194833]: pgmap v376: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 7550/88293 objects degraded (8.551%) 2023-09-23T19:56:00.422 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:56:00.886 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:00.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:00.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:56:00.938 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:56:01.001 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:01.042 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:01.163 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:01.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:01.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:01.244 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:01.271 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:01.363 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:01.855 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:01 smithi135 ceph-mon[213871]: pgmap v377: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7550/87675 objects degraded (8.611%) 2023-09-23T19:56:01.855 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:01 smithi135 ceph-mon[221113]: pgmap v377: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7550/87675 objects degraded (8.611%) 2023-09-23T19:56:01.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:01 smithi144 ceph-mon[194833]: pgmap v377: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7550/87675 objects degraded (8.611%) 2023-09-23T19:56:01.934 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:01.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:01.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:02.049 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:02.078 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:02.090 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:02.158 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:02.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:02.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:02.265 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:02.386 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:02.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:02 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7550/87675 objects degraded (8.611%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:02.939 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:56:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:02 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7550/87675 objects degraded (8.611%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:02.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:02 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7550/87675 objects degraded (8.611%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:03.177 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:03.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:03.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:03.282 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:03.339 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:03.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:03 smithi135 ceph-mon[213871]: pgmap v378: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 7049/81402 objects degraded (8.659%) 2023-09-23T19:56:03.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:03 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2768051545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:03.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:03 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:03.635 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:56:03.676 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:03.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:03.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:03.786 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:03.859 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:56:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:56:03] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-09-23T19:56:03.860 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:03 smithi135 ceph-mon[221113]: pgmap v378: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 7049/81402 objects degraded (8.659%) 2023-09-23T19:56:03.861 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:03 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2768051545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:03.861 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:03 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:03.872 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:03 smithi144 ceph-mon[194833]: pgmap v378: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 7049/81402 objects degraded (8.659%) 2023-09-23T19:56:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:03 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2768051545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:03 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:04.387 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:04.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:04.496 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:04.560 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:04.649 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:04 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:04.649 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:04 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:04.649 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:04 smithi135 ceph-mon[221113]: osdmap e153: 8 total, 7 up, 8 in 2023-09-23T19:56:04.649 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:04 smithi135 ceph-mon[221113]: failed to encode map e153 with expected crc 2023-09-23T19:56:04.650 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:04 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:04.650 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:04 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:04.651 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:04 smithi135 ceph-mon[213871]: osdmap e153: 8 total, 7 up, 8 in 2023-09-23T19:56:04.651 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:04 smithi135 ceph-mon[213871]: failed to encode map e153 with expected crc 2023-09-23T19:56:04.675 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:04.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:04.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:04.779 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:04.872 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:04 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:04 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:04 smithi144 ceph-mon[194833]: osdmap e153: 8 total, 7 up, 8 in 2023-09-23T19:56:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:04 smithi144 ceph-mon[194833]: failed to encode map e153 with expected crc 2023-09-23T19:56:05.376 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:05.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:05.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:05.488 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:05.522 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:05.603 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:05.610 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:05.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:05.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:05.712 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:05.809 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:05 smithi144 ceph-mon[194833]: pgmap v380: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 6594/76620 objects degraded (8.606%) 2023-09-23T19:56:05.980 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:05 smithi135 ceph-mon[213871]: pgmap v380: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 6594/76620 objects degraded (8.606%) 2023-09-23T19:56:05.980 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:05 smithi135 ceph-mon[221113]: pgmap v380: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 6594/76620 objects degraded (8.606%) 2023-09-23T19:56:05.996 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:06.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:06.049 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T19:56:06.049 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:56:06.049 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.55277 -> 6.55277 (expected baseline 6.03053) 2023-09-23T19:56:06.049 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 30 -> 30 pgs (0.707071 -> 0.707071 * mean) 2023-09-23T19:56:06.049 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.4 with 44 -> 44 pgs (1.03704 -> 1.03704 * mean) 2023-09-23T19:56:06.049 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:56:06.049 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T19:56:06.050 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 1 2023-09-23T19:56:06.050 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:56:06.050 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0088 2023-09-23T19:56:06.050 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0881 2023-09-23T19:56:06.071 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:56:06.105 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:06.153 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:06.227 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:06.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:06.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:06.335 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:06.365 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:06.460 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:06.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:06.567 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:06.634 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:06.668 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:56:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:06 smithi144 ceph-mon[194833]: from='client.54664 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:06.886 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:06 smithi135 ceph-mon[213871]: from='client.54664 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:06.889 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:06 smithi135 ceph-mon[221113]: from='client.54664 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:07.037 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:56:07.037 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:56:07.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:56:07.037 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:56:07.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.037 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:126099 5.b 5:d8dd17cf:::benchmark_data_smithi135_284570_object10315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.210526+0000", 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 115.621519445, 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29520065499999998, 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 126099 2023-09-23T19:56:07.038 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210526+0000", 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210526+0000", 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.039 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210525+0000", 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210528+0000", 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4719999999999998e-06 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.040 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210533+0000", 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4179999999999996e-06 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210536+0000", 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.959e-06 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.213754+0000", 2023-09-23T19:56:07.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003218051 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.213859+0000", 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000104831 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.213944+0000", 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4603000000000003e-05 2023-09-23T19:56:07.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.217831+0000", 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038871470000000001 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501313+0000", 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28348219099999999 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.505638+0000", 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043253129999999999 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.505685+0000", 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.702e-05 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.505726+0000", 2023-09-23T19:56:07.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1003e-05 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:126100 5.d 5:b4307950:::benchmark_data_smithi135_284570_object10316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.210554+0000", 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 115.62149137500001, 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.290823736, 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.045 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 126100 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210554+0000", 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.046 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210554+0000", 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210554+0000", 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210556+0000", 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1780000000000002e-06 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210562+0000", 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6329999999999999e-06 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210567+0000", 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2259999999999999e-06 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210605+0000", 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8293000000000001e-05 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210713+0000", 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107648 2023-09-23T19:56:07.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.210811+0000", 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8888999999999995e-05 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.213984+0000", 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031728250000000002 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222079+0000", 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080946560000000004 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222175+0000", 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6564999999999994e-05 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501332+0000", 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27915637500000001 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501378+0000", 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5760000000000002e-05 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.052 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:126099 5.b e138/112 ondisk, result = 0)", 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.222042+0000", 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 115.61000321100001, 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27927860300000001, 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.053 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222042+0000", 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222042+0000", 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222042+0000", 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967296 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222043+0000", 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4789999999999999e-06 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222047+0000", 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0670000000000002e-06 2023-09-23T19:56:07.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222050+0000", 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4169999999999999e-06 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222183+0000", 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013349299999999999 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501297+0000", 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.279113795 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501321+0000", 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3487999999999999e-05 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.057 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:126100 5.d e138/112 ondisk, result = 0)", 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.222119+0000", 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 115.609926051, 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27927829399999998, 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.058 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222119+0000", 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222119+0000", 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222119+0000", 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222121+0000", 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4849999999999999e-06 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222128+0000", 2023-09-23T19:56:07.060 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4119999999999998e-06 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222131+0000", 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.151e-06 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222166+0000", 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5571999999999998e-05 2023-09-23T19:56:07.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222172+0000", 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3589999999999999e-06 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501397+0000", 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.279225634 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.062 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:126112 5.6 5:63d106bd:::benchmark_data_smithi135_284570_object10328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.226372+0000", 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 115.605673145, 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29269265999999999, 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.063 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 126112 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.226372+0000", 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.226372+0000", 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.226371+0000", 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.226377+0000", 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0170000000000002e-06 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501280+0000", 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.274903177 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501300+0000", 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9942e-05 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.066 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.505672+0000", 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043715869999999997 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.505795+0000", 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012362300000000001 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.505875+0000", 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9752999999999993e-05 2023-09-23T19:56:07.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.517963+0000", 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012088141 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.518609+0000", 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000646125 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.519007+0000", 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00039779799999999999 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.519045+0000", 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7598000000000002e-05 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.069 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.519065+0000", 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9996999999999998e-05 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:126108 5.4 e138/112 5:21918726:::benchmark_data_smithi135_284570_object10324:head v 138'7901, mlcod=138'7899)", 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.226433+0000", 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 115.605612026, 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27955861500000001, 2023-09-23T19:56:07.070 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.226433+0000", 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.226433+0000", 2023-09-23T19:56:07.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.226433+0000", 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.226438+0000", 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2009999999999998e-06 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501568+0000", 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27512996000000001 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501576+0000", 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6459999999999993e-06 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501620+0000", 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4807e-05 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.501642+0000", 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2092000000000001e-05 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.074 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.505959+0000", 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043163259999999997 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.505992+0000", 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3073e-05 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.075 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149806 5.a 5:571c9a77:::benchmark_data_smithi135_284570_object34022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:45.837031+0000", 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 80.995014463999993, 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27810474200000002, 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.076 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149806 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.837031+0000", 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.837031+0000", 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.837030+0000", 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.837034+0000", 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6890000000000002e-06 2023-09-23T19:56:07.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.837041+0000", 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6309999999999997e-06 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.837046+0000", 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1200000000000001e-06 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.837085+0000", 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8999999999999999e-05 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.837208+0000", 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000122998 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:46.099558+0000", 2023-09-23T19:56:07.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26235056099999998 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:46.114003+0000", 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014445019999999999 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:46.114014+0000", 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0499999999999999e-05 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:46.115038+0000", 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010240169999999999 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:46.115090+0000", 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2420999999999998e-05 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:46.115136+0000", 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5371999999999999e-05 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195610 5.8 5:13ab778b:::benchmark_data_smithi135_284570_object79826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086316+0000", 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.745729656999998, 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32473328200000001, 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.083 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195610 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086316+0000", 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.084 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086316+0000", 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086315+0000", 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086318+0000", 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.616e-06 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086324+0000", 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0000000000000002e-06 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086329+0000", 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0629999999999996e-06 2023-09-23T19:56:07.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099057+0000", 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012727894999999999 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099206+0000", 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014897700000000001 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099330+0000", 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012414300000000001 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.105020+0000", 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056901069999999998 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410872+0000", 2023-09-23T19:56:07.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30585215500000001 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410959+0000", 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7231999999999994e-05 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411010+0000", 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.024e-05 2023-09-23T19:56:07.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411049+0000", 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9240999999999999e-05 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195612 5.a 5:51964534:::benchmark_data_smithi135_284570_object79828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:07.090 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086334+0000", 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.745710947999999, 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31885195999999999, 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195612 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.091 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.092 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086336+0000", 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2479999999999999e-06 2023-09-23T19:56:07.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086341+0000", 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0610000000000003e-06 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086346+0000", 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5310000000000003e-06 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086419+0000", 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3072999999999996e-05 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086523+0000", 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000103767 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086633+0000", 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110221 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.095308+0000", 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0086747639999999997 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099671+0000", 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043637499999999996 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405096+0000", 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30542450199999999 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405140+0000", 2023-09-23T19:56:07.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3736000000000003e-05 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405186+0000", 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6621e-05 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.098 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195613 5.a 5:51e9d81c:::benchmark_data_smithi135_284570_object79829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086357+0000", 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.745687796999999, 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.321475978, 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195613 2023-09-23T19:56:07.099 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T19:56:07.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086360+0000", 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6060000000000001e-06 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086365+0000", 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8440000000000003e-06 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086371+0000", 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.096e-06 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.090840+0000", 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004469794 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.091046+0000", 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000205508 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.091154+0000", 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107816 2023-09-23T19:56:07.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099701+0000", 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085474149999999992 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099715+0000", 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4365999999999999e-05 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405243+0000", 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30552804300000003 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405285+0000", 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1223999999999997e-05 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.407833+0000", 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025487410000000002 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195623 5.8 5:1605c7cf:::benchmark_data_smithi135_284570_object79839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106079+0000", 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.725966093, 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30509230399999998, 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.106 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195623 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106079+0000", 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106079+0000", 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106078+0000", 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106087+0000", 2023-09-23T19:56:07.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8689999999999993e-06 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106101+0000", 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3971000000000001e-05 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106110+0000", 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9800000000000004e-06 2023-09-23T19:56:07.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106177+0000", 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7464000000000003e-05 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106310+0000", 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000132712 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106435+0000", 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000125499 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111791+0000", 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053554220000000003 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410920+0000", 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29912940799999999 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411095+0000", 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017488599999999999 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411136+0000", 2023-09-23T19:56:07.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1517999999999998e-05 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411171+0000", 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5009999999999999e-05 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.113 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195626 5.6 5:6edb6a83:::benchmark_data_smithi135_284570_object79842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106123+0000", 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.725922733000001, 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30425704100000001, 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195626 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106123+0000", 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106123+0000", 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106122+0000", 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106126+0000", 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1300000000000003e-06 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106133+0000", 2023-09-23T19:56:07.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8109999999999997e-06 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106138+0000", 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2569999999999998e-06 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106419+0000", 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028042200000000001 2023-09-23T19:56:07.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106516+0000", 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7001000000000002e-05 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106616+0000", 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100272 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405132+0000", 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.298516432 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405142+0000", 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2189999999999993e-06 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410291+0000", 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051497000000000001 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410341+0000", 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0207000000000002e-05 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410380+0000", 2023-09-23T19:56:07.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8109999999999999e-05 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:195612 5.a e151/112 ondisk, result = 0)", 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106148+0000", 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.725896975000001, 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29904286699999999, 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:56:07.121 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106152+0000", 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1230000000000002e-06 2023-09-23T19:56:07.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106163+0000", 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0951000000000001e-05 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106169+0000", 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8220000000000001e-06 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405078+0000", 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.298908224 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405088+0000", 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0686999999999999e-05 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405191+0000", 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010278 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195629 5.8 5:14b98469:::benchmark_data_smithi135_284570_object79845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.110999+0000", 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.721046447999999, 2023-09-23T19:56:07.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31090222000000001, 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195629 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110999+0000", 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110999+0000", 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110998+0000", 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111004+0000", 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8809999999999998e-06 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111022+0000", 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7354999999999999e-05 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111032+0000", 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0251000000000001e-05 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111900+0000", 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00086782700000000005 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112014+0000", 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000113835 2023-09-23T19:56:07.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112123+0000", 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010916000000000001 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411233+0000", 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.299110243 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411284+0000", 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0732999999999997e-05 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421815+0000", 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010530813999999999 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421859+0000", 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4240000000000003e-05 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421901+0000", 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2107000000000002e-05 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195630 5.6 5:65c744e5:::benchmark_data_smithi135_284570_object79846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.111039+0000", 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.721005945999998, 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.304778728, 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.134 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195630 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111042+0000", 2023-09-23T19:56:07.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.507e-06 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111049+0000", 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9399999999999996e-06 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111055+0000", 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4439999999999999e-06 2023-09-23T19:56:07.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111663+0000", 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060795100000000002 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111801+0000", 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013784600000000001 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111940+0000", 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000139574 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405147+0000", 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29320712999999998 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410463+0000", 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053154830000000002 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415730+0000", 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052668560000000003 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415777+0000", 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7123000000000003e-05 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415818+0000", 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1347999999999998e-05 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:195613 5.a e151/112 ondisk, result = 0)", 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.111802+0000", 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.720242832, 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29603929699999998, 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111802+0000", 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111802+0000", 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111802+0000", 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111807+0000", 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6210000000000001e-06 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111818+0000", 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0623e-05 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111824+0000", 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6540000000000002e-06 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405231+0000", 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.293406204 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405238+0000", 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.267e-06 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.407842+0000", 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026037289999999999 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:195610 5.8 e151/112 ondisk, result = 0)", 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.112245+0000", 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.719799791, 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29862952999999998, 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T19:56:07.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112250+0000", 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0350000000000001e-06 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405043+0000", 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29279347100000003 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405055+0000", 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1627000000000001e-05 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410857+0000", 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058025580000000002 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410866+0000", 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3450000000000006e-06 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410875+0000", 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.217e-06 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195633 5.b 5:d886e22d:::benchmark_data_smithi135_284570_object79849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.113988+0000", 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.718057727000001, 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32055229600000001, 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195633 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.152 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113988+0000", 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113988+0000", 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113986+0000", 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113993+0000", 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5889999999999999e-06 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405037+0000", 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29104433699999999 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405056+0000", 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8836999999999999e-05 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405147+0000", 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1032000000000005e-05 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405287+0000", 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014026500000000001 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:07.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405410+0000", 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000122636 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415431+0000", 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010021077 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.423733+0000", 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083021039999999994 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434462+0000", 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010729140999999999 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434502+0000", 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9712000000000002e-05 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434540+0000", 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7759999999999998e-05 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:195626 5.6 e151/112 ondisk, result = 0)", 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.121130+0000", 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 18.710915526000001, 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28401833300000001, 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.121130+0000", 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.121130+0000", 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.121128+0000", 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:07.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.121139+0000", 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.092e-05 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405038+0000", 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28389862900000001 2023-09-23T19:56:07.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405055+0000", 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6960999999999999e-05 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405117+0000", 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2071999999999998e-05 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405126+0000", 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6330000000000002e-06 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405148+0000", 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2466000000000001e-05 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203734 5.b 5:dd140fcc:::benchmark_data_smithi135_284570_object87950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.309502+0000", 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.5225436649999997, 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29768655599999999, 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:56:07.165 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203734 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309502+0000", 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:56:07.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309502+0000", 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309501+0000", 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309504+0000", 2023-09-23T19:56:07.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.225e-06 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309513+0000", 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4309999999999994e-06 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309517+0000", 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0199999999999996e-06 2023-09-23T19:56:07.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309557+0000", 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9762999999999999e-05 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309632+0000", 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5908999999999994e-05 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309704+0000", 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1668000000000006e-05 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.313325+0000", 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036211889999999999 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.313868+0000", 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00054303499999999998 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607103+0000", 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.293234781 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607154+0000", 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0914000000000001e-05 2023-09-23T19:56:07.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607188+0000", 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4112999999999997e-05 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:56:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:56:07.540 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:07.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:07.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:07.653 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:07.718 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:07 smithi135 ceph-mon[221113]: pgmap v381: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6123/73785 objects degraded (8.298%) 2023-09-23T19:56:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:07 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6123/73785 objects degraded (8.298%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:07.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:07 smithi135 ceph-mon[213871]: pgmap v381: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6123/73785 objects degraded (8.298%) 2023-09-23T19:56:07.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:07.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:07 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6123/73785 objects degraded (8.298%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:07 smithi144 ceph-mon[194833]: pgmap v381: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6123/73785 objects degraded (8.298%) 2023-09-23T19:56:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:07 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6123/73785 objects degraded (8.298%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:09.348 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:56:09.490 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:09.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:09.605 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:09.632 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:09.688 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:09.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:09.794 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:09.796 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:09.834 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:09.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:09 smithi144 ceph-mon[194833]: pgmap v382: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5662/66312 objects degraded (8.538%) 2023-09-23T19:56:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:09 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:09 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:09.892 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:09.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:09.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:09.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:09 smithi135 ceph-mon[213871]: pgmap v382: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5662/66312 objects degraded (8.538%) 2023-09-23T19:56:09.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:09 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:09.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:09 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:09.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:09 smithi135 ceph-mon[221113]: pgmap v382: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5662/66312 objects degraded (8.538%) 2023-09-23T19:56:09.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:09 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:09.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:09 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:09.999 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:10.046 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:10.128 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:10.135 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:10.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:10.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:10.240 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:10.311 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:10.714 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:10 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:56:10.714 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:10 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:56:10.714 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:10 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:10.835 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:56:10.862 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:56:10.937 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:56:10.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:10 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:56:10.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:10 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:56:10.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:10 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:10.974 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:56:11.010 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:56:11.059 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:56:11.092 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:56:11.092 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:56:11.092 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:56:11.092 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:56:11.093 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:56:11.093 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:56:11.093 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:56:11.093 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:56:11.093 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:56:11.107 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:11.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:11.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:11.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:10 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:56:11.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:10 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:56:11.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:10 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:11.219 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:11.294 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:11.395 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:11.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:11.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:11.508 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:11.559 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:11.911 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:11 smithi135 ceph-mon[221113]: pgmap v383: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 5662/65604 objects degraded (8.631%) 2023-09-23T19:56:11.911 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:11 smithi135 ceph-mon[221113]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T19:56:11.911 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:11 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3293345843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:11.912 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:11 smithi135 ceph-mon[213871]: pgmap v383: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 5662/65604 objects degraded (8.631%) 2023-09-23T19:56:11.912 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:11 smithi135 ceph-mon[213871]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T19:56:11.912 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:11 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3293345843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:12.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:11 smithi144 ceph-mon[194833]: pgmap v383: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 5662/65604 objects degraded (8.631%) 2023-09-23T19:56:12.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:11 smithi144 ceph-mon[194833]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T19:56:12.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:11 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3293345843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:12.585 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:56:12.585 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:56:12.585 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T19:56:12.585 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:56:12.586 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:56:12.586 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T19:56:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:56:12.650 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:56:12.687 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:12.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:12.798 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:12.853 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:12.867 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5662/65604 objects degraded (8.631%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:12.870 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5662/65604 objects degraded (8.631%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:12.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5662/65604 objects degraded (8.631%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:12.967 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:12.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:12.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:13.072 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:13.135 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:13.154 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:13.243 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:13.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:13.357 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:13.396 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:13.581 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:13.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:13.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:13.687 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:13.726 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:56:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:56:13] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-09-23T19:56:13.763 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:13.943 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:13.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:13.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:14.063 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:14.106 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:14.161 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:14.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:14.266 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:14.293 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:14.294 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:14.331 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:14 smithi144 ceph-mon[194833]: pgmap v384: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5173/62742 objects degraded (8.245%) 2023-09-23T19:56:14.536 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:14 smithi135 ceph-mon[213871]: pgmap v384: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5173/62742 objects degraded (8.245%) 2023-09-23T19:56:14.537 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:14 smithi135 ceph-mon[221113]: pgmap v384: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5173/62742 objects degraded (8.245%) 2023-09-23T19:56:14.557 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:14.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:14.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:14.650 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:56:14.669 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:14.776 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:14.940 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:14.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:14.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:15.044 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:15.137 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:15.286 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:56:15.545 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/895437680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:15.545 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:15 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:15.546 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/895437680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:15.546 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:15 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:15.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/895437680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:15.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:15 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:15.961 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:15.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:16.012 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:56:16.066 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:16.091 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:16.131 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:16.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:16.234 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:16.328 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:16.375 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:16 smithi144 ceph-mon[194833]: pgmap v385: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 4693/54690 objects degraded (8.581%) 2023-09-23T19:56:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:16 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:16 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:16 smithi144 ceph-mon[194833]: osdmap e154: 8 total, 7 up, 8 in 2023-09-23T19:56:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:16 smithi144 ceph-mon[194833]: failed to encode map e154 with expected crc 2023-09-23T19:56:16.435 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:16.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[221113]: pgmap v385: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 4693/54690 objects degraded (8.581%) 2023-09-23T19:56:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:16.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[221113]: osdmap e154: 8 total, 7 up, 8 in 2023-09-23T19:56:16.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[221113]: failed to encode map e154 with expected crc 2023-09-23T19:56:16.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[213871]: pgmap v385: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 4693/54690 objects degraded (8.581%) 2023-09-23T19:56:16.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:16.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:16.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[213871]: osdmap e154: 8 total, 7 up, 8 in 2023-09-23T19:56:16.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:16 smithi135 ceph-mon[213871]: failed to encode map e154 with expected crc 2023-09-23T19:56:16.559 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:16.600 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:16.741 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:16.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:16.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:16.852 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:16.913 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:17.008 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:17.169 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:17.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:17.250 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:56:17.250 INFO:teuthology.orchestra.run.smithi144.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:56:17.250 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:56:17.276 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:17.338 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:17.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:17 smithi144 ceph-mon[194833]: failed to encode map e154 with expected crc 2023-09-23T19:56:17.426 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4693/54690 objects degraded (8.581%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:17.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:17 smithi135 ceph-mon[213871]: failed to encode map e154 with expected crc 2023-09-23T19:56:17.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4693/54690 objects degraded (8.581%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:17.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:17 smithi135 ceph-mon[221113]: failed to encode map e154 with expected crc 2023-09-23T19:56:17.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4693/54690 objects degraded (8.581%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:17.940 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:17.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:17.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:18.052 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:18.115 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:18.220 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:56:18.221 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:18.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:18.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:18.330 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:18.436 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:18.510 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:18 smithi135 ceph-mon[221113]: pgmap v387: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 4693/53325 objects degraded (8.801%) 2023-09-23T19:56:18.510 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:18 smithi135 ceph-mon[213871]: pgmap v387: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 4693/53325 objects degraded (8.801%) 2023-09-23T19:56:18.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:18 smithi144 ceph-mon[194833]: pgmap v387: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 4693/53325 objects degraded (8.801%) 2023-09-23T19:56:18.684 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:18.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:18.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:18.803 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:18.848 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:18.936 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:18.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:19.037 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:56:19.037 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:56:19.038 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:56:19.038 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:56:19.040 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:56:19.043 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:19.077 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:56:19.080 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:19.118 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:56:19.119 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:56:19.119 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:56:19.119 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:56:19.119 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:56:19.119 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:56:19.119 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:56:19.119 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:56:19.119 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:56:19.155 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:19.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:19.264 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:19.331 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:19.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/799203583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:19.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/799203583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/799203583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:20.050 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:20.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:20.157 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:20.219 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:20 smithi135 ceph-mon[221113]: pgmap v388: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 908 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3732/43443 objects degraded (8.591%) 2023-09-23T19:56:20.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:20 smithi135 ceph-mon[213871]: pgmap v388: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 908 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3732/43443 objects degraded (8.591%) 2023-09-23T19:56:20.488 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:20.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:20.512 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:56:20.512 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:56:20.515 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:20.602 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:20 smithi144 ceph-mon[194833]: pgmap v388: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 908 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3732/43443 objects degraded (8.591%) 2023-09-23T19:56:20.662 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:20.765 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:20.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:20.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:20.887 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:20.991 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:21.081 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:56:21.081 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T19:56:21.081 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T19:56:21.081 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T19:56:21.082 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:56:21.146 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:21.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:21.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:21.257 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:21.345 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:22.340 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:22.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:22.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:22.448 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:22.512 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:22.513 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:56:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:22 smithi144 ceph-mon[194833]: pgmap v389: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 894 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3732/42765 objects degraded (8.727%) 2023-09-23T19:56:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 3732/42765 objects degraded (8.727%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:22 smithi135 ceph-mon[213871]: pgmap v389: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 894 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3732/42765 objects degraded (8.727%) 2023-09-23T19:56:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:22.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 3732/42765 objects degraded (8.727%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:22.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:22 smithi135 ceph-mon[221113]: pgmap v389: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 894 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3732/42765 objects degraded (8.727%) 2023-09-23T19:56:22.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:56:22.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:22.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 3732/42765 objects degraded (8.727%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:22.846 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:56:22.947 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:22.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:22.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:23.058 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:23.099 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:23.174 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:23.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:23.181 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:56:23.181 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:56:23.182 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:56:23.182 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T19:56:23.205 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:23.282 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:23.347 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:23.632 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:23.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:23.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:23.738 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:23.766 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:23.827 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:23.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:23.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:23.915 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:56:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:56:23] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T19:56:23.936 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:24.031 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:24.137 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:24.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:24.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:24.241 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:24.294 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:24.338 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:24.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:24.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:24.352 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:56:24.442 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:24.467 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:24.509 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:24.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:24.614 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:24 smithi144 ceph-mon[194833]: pgmap v390: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 834 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3234/39897 objects degraded (8.106%) 2023-09-23T19:56:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3595392631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:24.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:24 smithi135 ceph-mon[221113]: pgmap v390: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 834 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3234/39897 objects degraded (8.106%) 2023-09-23T19:56:24.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3595392631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:24.672 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:24 smithi135 ceph-mon[213871]: pgmap v390: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 834 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3234/39897 objects degraded (8.106%) 2023-09-23T19:56:24.673 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3595392631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:24.676 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:25.430 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:25.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:25.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:25.539 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:25.616 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:25.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:25 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:56:25.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3595392631' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:25.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:25 smithi135 ceph-mon[221113]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T19:56:25.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:25 smithi135 ceph-mon[221113]: failed to encode map e155 with expected crc 2023-09-23T19:56:25.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:25 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:56:25.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3595392631' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:25.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:25 smithi144 ceph-mon[194833]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T19:56:25.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:25 smithi144 ceph-mon[194833]: failed to encode map e155 with expected crc 2023-09-23T19:56:25.635 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:25 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:56:25.635 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3595392631' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:25.635 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:25 smithi135 ceph-mon[213871]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T19:56:25.635 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:25 smithi135 ceph-mon[213871]: failed to encode map e155 with expected crc 2023-09-23T19:56:25.636 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:25.636 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:25.766 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:25.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:25.887 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:25.966 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:26.376 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:26.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:26.489 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:26.514 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:56:26.549 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:56:26.600 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:26 smithi135 ceph-mon[213871]: pgmap v392: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 682 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 2758/32592 objects degraded (8.462%) 2023-09-23T19:56:26.600 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:26 smithi135 ceph-mon[221113]: pgmap v392: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 682 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 2758/32592 objects degraded (8.462%) 2023-09-23T19:56:26.601 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:26.604 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:56:26.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:26 smithi144 ceph-mon[194833]: pgmap v392: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 682 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 2758/32592 objects degraded (8.462%) 2023-09-23T19:56:26.640 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:56:26.691 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:56:26.723 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:56:26.769 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:56:26.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:56:26.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:56:26.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:56:26.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:56:26.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:56:26.770 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:56:26.770 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:56:26.771 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:56:27.659 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:56:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:27 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/746991791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 2758/32592 objects degraded (8.462%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:27.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:27 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/746991791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:27.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 2758/32592 objects degraded (8.462%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:27.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:27 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/746991791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:27.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 2758/32592 objects degraded (8.462%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:28 smithi135 ceph-mon[213871]: pgmap v393: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 653 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2758/31194 objects degraded (8.841%) 2023-09-23T19:56:28.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:28 smithi135 ceph-mon[221113]: pgmap v393: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 653 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2758/31194 objects degraded (8.841%) 2023-09-23T19:56:28.528 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:28.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:28.635 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:28.682 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:28.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:28 smithi144 ceph-mon[194833]: pgmap v393: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 653 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2758/31194 objects degraded (8.841%) 2023-09-23T19:56:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:56:29.205 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:56:29.237 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:29.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:29.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:29.349 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:29.464 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:29 smithi135 ceph-mon[213871]: pgmap v394: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 457 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 1869/21789 objects degraded (8.578%) 2023-09-23T19:56:29.468 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:29 smithi135 ceph-mon[221113]: pgmap v394: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 457 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 1869/21789 objects degraded (8.578%) 2023-09-23T19:56:29.481 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:29.742 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:29.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:29.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:29.860 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:29.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:29 smithi144 ceph-mon[194833]: pgmap v394: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 457 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 1869/21789 objects degraded (8.578%) 2023-09-23T19:56:29.927 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:29.930 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:29.931 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:29.971 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:30.279 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:30.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:30.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.55277 -> 6.55277 (expected baseline 6.03053) 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 30 -> 30 pgs (0.707071 -> 0.707071 * mean) 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.4 with 44 -> 44 pgs (1.03704 -> 1.03704 * mean) 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T19:56:30.386 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:56:30.389 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0046 2023-09-23T19:56:30.389 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0051 2023-09-23T19:56:30.389 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 1.0000 -> 0.9500 2023-09-23T19:56:30.390 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:30.406 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:56:30.439 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:30.453 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:30 smithi135 ceph-mon[213871]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:30.453 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:56:30 smithi135 conmon[201366]: 2023-09-23T19:56:30.376+0000 7fcf49f9e700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-23T19:56:30.453 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:30 smithi135 ceph-mon[221113]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:30.548 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:30.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:30.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:30.659 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:30.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:30 smithi144 ceph-mon[194833]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:30.728 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:30.791 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:30.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:30.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:30.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:30.954 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:31.206 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:56:31.399 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:56:31.508 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:31.519 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:31 smithi135 ceph-mon[213871]: pgmap v395: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 440 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1869/20967 objects degraded (8.914%) 2023-09-23T19:56:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:31.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:31.529 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:31 smithi135 ceph-mon[221113]: pgmap v395: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 440 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1869/20967 objects degraded (8.914%) 2023-09-23T19:56:31.621 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:31.697 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:31 smithi144 ceph-mon[194833]: pgmap v395: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 440 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1869/20967 objects degraded (8.914%) 2023-09-23T19:56:32.005 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:32.081 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:32.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:32.188 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:32.233 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:32.237 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T19:56:32.237 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T19:56:32.348 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:32.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:32.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:32.440 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1869/20967 objects degraded (8.914%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:32.457 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:32.508 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1869/20967 objects degraded (8.914%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:32.511 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:32.659 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:32.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:32.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:32.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1869/20967 objects degraded (8.914%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:32.763 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:32.830 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:33.018 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:33.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:33.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:33.152 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:33.175 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:33.193 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:33.250 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:33.518 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:33 smithi135 ceph-mon[213871]: pgmap v396: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 375 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 1340/17886 objects degraded (7.492%) 2023-09-23T19:56:33.518 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:33 smithi135 ceph-mon[221113]: pgmap v396: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 375 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 1340/17886 objects degraded (7.492%) 2023-09-23T19:56:33.708 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:33.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:33.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:33.822 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:33.848 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:33 smithi144 ceph-mon[194833]: pgmap v396: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 375 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 1340/17886 objects degraded (7.492%) 2023-09-23T19:56:33.852 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:33.943 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:33.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:33.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:56:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:56:33] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T19:56:34.050 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:34.075 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:34.116 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:34.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:34.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:34.219 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:34.250 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:34.475 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:56:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:34 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/94897973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:56:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:34 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:56:34.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:34 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/94897973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:56:34.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:34 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:56:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:34 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/94897973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:56:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:34 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:56:35.319 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:35.320 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:35 smithi144 ceph-mon[194833]: pgmap v397: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 861/10077 objects degraded (8.544%) 2023-09-23T19:56:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:35 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:35 smithi144 ceph-mon[194833]: failed to encode map e156 with expected crc 2023-09-23T19:56:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:35 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:56:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:35 smithi144 ceph-mon[194833]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T19:56:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:35 smithi144 ceph-mon[194833]: failed to encode map e156 with expected crc 2023-09-23T19:56:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[221113]: pgmap v397: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 861/10077 objects degraded (8.544%) 2023-09-23T19:56:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[221113]: failed to encode map e156 with expected crc 2023-09-23T19:56:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:56:35.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[221113]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T19:56:35.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[221113]: failed to encode map e156 with expected crc 2023-09-23T19:56:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[213871]: pgmap v397: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 213 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 861/10077 objects degraded (8.544%) 2023-09-23T19:56:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[213871]: failed to encode map e156 with expected crc 2023-09-23T19:56:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:56:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[213871]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T19:56:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:35 smithi135 ceph-mon[213871]: failed to encode map e156 with expected crc 2023-09-23T19:56:35.805 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:35.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:35.909 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:35.962 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:36.223 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:36.224 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:56:36.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:36.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:36.333 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:36.429 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:36.605 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:36.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:36.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:36.713 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:36.791 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:36.838 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:36.881 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:36.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:36.988 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:37.085 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:37.217 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:37.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:37.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:37.284 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:56:37.337 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:37.346 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:56:37.396 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:56:37.433 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:37.434 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:56:37.434 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Removed 115781 objects 2023-09-23T19:56:37.434 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Clean up completed and total clean up time :159.703 2023-09-23T19:56:37.453 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:56:37.473 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:56:37.477 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:37.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:37.493 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:56:37.493 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:56:37.493 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:56:37.494 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:56:37.494 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:56:37.494 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:56:37.494 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:56:37.494 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:56:37.494 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:56:37.581 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:37.610 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:37.652 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:37.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[221113]: pgmap v399: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 861/8757 objects degraded (9.832%) 2023-09-23T19:56:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1611921311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 861/8757 objects degraded (9.832%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'15937'}"}]: dispatch 2023-09-23T19:56:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[213871]: pgmap v399: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 861/8757 objects degraded (9.832%) 2023-09-23T19:56:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1611921311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 861/8757 objects degraded (9.832%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'15937'}"}]: dispatch 2023-09-23T19:56:37.755 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:37.779 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:37 smithi144 ceph-mon[194833]: pgmap v399: 99 pgs: 9 active+undersized+degraded, 27 active+undersized, 63 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 861/8757 objects degraded (9.832%) 2023-09-23T19:56:37.780 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:37.780 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:37 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1611921311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:37.780 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:37 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 861/8757 objects degraded (9.832%), 9 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:37.780 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'15937'}"}]: dispatch 2023-09-23T19:56:37.782 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:37.792 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T19:56:37.792 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T19:56:37.823 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:37.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:37.926 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:37.951 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:37.996 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:37.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:37.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:38.100 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:38.128 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:38.172 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:38.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:38.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.55277 -> 5.1369 (expected baseline 6.03053) 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 30 -> 32 pgs (0.707071 -> 0.754209 * mean) 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.4 with 44 -> 49 pgs (1.03704 -> 1.15488 * mean) 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 3 2023-09-23T19:56:38.252 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:56:38.253 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0036 2023-09-23T19:56:38.253 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0039 2023-09-23T19:56:38.253 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 1.0000 -> 0.2432 2023-09-23T19:56:38.278 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:38.305 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:38.401 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:38.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:38.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:38.479 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:56:38.504 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:38.560 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:38.575 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:56:38.575 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T19:56:38.575 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T19:56:38.576 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T19:56:38.576 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:56:38.655 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:56:38.655 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:56:38.705 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:38.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:38.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:38.808 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:38.835 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:39.058 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:39.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:39.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:39.162 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:39.169 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T19:56:39.169 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T19:56:39.190 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:39.207 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:39.253 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:39.287 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:39.288 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:56:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:39.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:39.392 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:39.418 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[221113]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:39.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[221113]: failed to encode map e157 with expected crc 2023-09-23T19:56:39.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[221113]: failed to encode map e157 with expected crc 2023-09-23T19:56:39.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'15937'}"}]': finished 2023-09-23T19:56:39.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[221113]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T19:56:39.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[213871]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:39.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[213871]: failed to encode map e157 with expected crc 2023-09-23T19:56:39.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[213871]: failed to encode map e157 with expected crc 2023-09-23T19:56:39.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'15937'}"}]': finished 2023-09-23T19:56:39.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:39 smithi135 ceph-mon[213871]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T19:56:39.421 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:39.519 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:39.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:39.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:39.623 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:39 smithi144 ceph-mon[194833]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:39 smithi144 ceph-mon[194833]: failed to encode map e157 with expected crc 2023-09-23T19:56:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:39 smithi144 ceph-mon[194833]: failed to encode map e157 with expected crc 2023-09-23T19:56:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'15937'}"}]': finished 2023-09-23T19:56:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:39 smithi144 ceph-mon[194833]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T19:56:39.651 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:39.669 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:40.451 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:40.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:40.556 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:40.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:40 smithi144 ceph-mon[194833]: pgmap v401: 99 pgs: 6 peering, 6 active+undersized+degraded, 24 active+undersized, 63 active+clean; 24 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 88/990 objects degraded (8.889%) 2023-09-23T19:56:40.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:40 smithi144 ceph-mon[194833]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T19:56:40.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:40 smithi144 ceph-mon[194833]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:40 smithi144 ceph-mon[194833]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:40 smithi144 ceph-mon[194833]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:40 smithi144 ceph-mon[194833]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:40 smithi144 ceph-mon[194833]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:40 smithi144 ceph-mon[194833]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.583 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:40.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[213871]: pgmap v401: 99 pgs: 6 peering, 6 active+undersized+degraded, 24 active+undersized, 63 active+clean; 24 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 88/990 objects degraded (8.889%) 2023-09-23T19:56:40.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[213871]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T19:56:40.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[213871]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[213871]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[213871]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[213871]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[213871]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[213871]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[221113]: pgmap v401: 99 pgs: 6 peering, 6 active+undersized+degraded, 24 active+undersized, 63 active+clean; 24 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 88/990 objects degraded (8.889%) 2023-09-23T19:56:40.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[221113]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T19:56:40.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[221113]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[221113]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[221113]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[221113]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[221113]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:40 smithi135 ceph-mon[221113]: failed to encode map e158 with expected crc 2023-09-23T19:56:40.657 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:56:40.681 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:40.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:40.788 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:40.814 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:40.870 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:40.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:40.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:40.899 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:40.973 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:41.002 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:41.096 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:41.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:41.101 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:56:41.101 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T19:56:41.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T19:56:41.101 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T19:56:41.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.101 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:125817 5.0 e137/112 5:007c7e62:::benchmark_data_smithi135_284570_object10033:head v 137'7963, mlcod=137'7959)", 2023-09-23T19:56:41.101 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:10.727040+0000", 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 150.37205605, 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.20207420300000001, 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727040+0000", 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727040+0000", 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727040+0000", 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727048+0000", 2023-09-23T19:56:41.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.653e-06 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727057+0000", 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1770000000000004e-06 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.727062+0000", 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7079999999999996e-06 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918255+0000", 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.191192998 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918282+0000", 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7053e-05 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.929082+0000", 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010800172 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.929115+0000", 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2199999999999997e-05 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.106 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:125823 5.c e137/112 5:3be4aa05:::benchmark_data_smithi135_284570_object10039:head v 137'7817, mlcod=137'7815)", 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:10.733899+0000", 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 150.365197112, 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19391745499999999, 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.733899+0000", 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.733899+0000", 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.733899+0000", 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.733906+0000", 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8900000000000001e-06 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918216+0000", 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.184310378 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918232+0000", 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5702999999999999e-05 2023-09-23T19:56:41.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918287+0000", 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5600999999999998e-05 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918309+0000", 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2082999999999998e-05 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.927716+0000", 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0094066219999999999 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.927817+0000", 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010086000000000001 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:125826 5.e 5:77df08cc:::benchmark_data_smithi135_284570_object10042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:10.736113+0000", 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 150.36298331099999, 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19501096800000001, 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 125826 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.112 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.736113+0000", 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.736113+0000", 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.736111+0000", 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.736123+0000", 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.226e-05 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918216+0000", 2023-09-23T19:56:41.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18209268300000001 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918236+0000", 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0194999999999999e-05 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918286+0000", 2023-09-23T19:56:41.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0318999999999997e-05 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918429+0000", 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014285799999999999 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.918555+0000", 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012537900000000001 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.925579+0000", 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070240140000000003 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.927711+0000", 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002131984 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.931038+0000", 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033274670000000002 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.931084+0000", 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5541000000000003e-05 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.931124+0000", 2023-09-23T19:56:41.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0367999999999997e-05 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:127287 5.e 5:7f91015f:::benchmark_data_smithi135_284570_object11503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.892771+0000", 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 148.206325658, 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19526686700000001, 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.119 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127287 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892771+0000", 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892771+0000", 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892769+0000", 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892777+0000", 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9500000000000001e-06 2023-09-23T19:56:41.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892791+0000", 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4039000000000001e-05 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892799+0000", 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6499999999999996e-06 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.122 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892845+0000", 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6211000000000002e-05 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.892958+0000", 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113736 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.893064+0000", 2023-09-23T19:56:41.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105321 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.898810+0000", 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0057458459999999998 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.084220+0000", 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.185410361 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.087948+0000", 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037277460000000001 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.087996+0000", 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8072000000000003e-05 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.088038+0000", 2023-09-23T19:56:41.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1841000000000001e-05 2023-09-23T19:56:41.126 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.126 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.126 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:127294 5.e 5:7c49cd13:::benchmark_data_smithi135_284570_object11510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:12.904248+0000", 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 148.19484855499999, 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.194697595, 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127294 2023-09-23T19:56:41.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.904248+0000", 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.904248+0000", 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.904246+0000", 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:12.904253+0000", 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7120000000000003e-06 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.130 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.130 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076794+0000", 2023-09-23T19:56:41.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17254071600000001 2023-09-23T19:56:41.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.130 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076807+0000", 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2687999999999999e-05 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076854+0000", 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7380999999999999e-05 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.076999+0000", 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000145432 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.077128+0000", 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012904100000000001 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.084542+0000", 2023-09-23T19:56:41.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074134409999999998 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.084552+0000", 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6919999999999997e-06 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.098879+0000", 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014327174999999999 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.098910+0000", 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1044000000000003e-05 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:13.098945+0000", 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5649999999999999e-05 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.134 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:147010 5.f e143/112 5:ffd43230:::benchmark_data_smithi135_284570_object31226:head v 143'9222, mlcod=143'9220)", 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:40.814452+0000", 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 120.28464466600001, 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18902136899999999, 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.814452+0000", 2023-09-23T19:56:41.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.814452+0000", 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.814451+0000", 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.814458+0000", 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2429999999999999e-06 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.814468+0000", 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.978e-06 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.137 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.814474+0000", 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6699999999999999e-06 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.814512+0000", 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8002e-05 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.814527+0000", 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5138e-05 2023-09-23T19:56:41.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817909+0000", 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033819259999999999 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.003473+0000", 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18556414199999999 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.139 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:147004 5.0 e143/112 5:0570f923:::benchmark_data_smithi135_284570_object31220:head v 143'9316, mlcod=143'9314)", 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:40.817631+0000", 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 120.281465068, 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.192543672, 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.140 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817631+0000", 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817631+0000", 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817631+0000", 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967296 2023-09-23T19:56:41.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817635+0000", 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9349999999999996e-06 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817641+0000", 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4809999999999997e-06 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817644+0000", 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.185e-06 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817685+0000", 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1368000000000001e-05 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.817704+0000", 2023-09-23T19:56:41.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9134999999999999e-05 2023-09-23T19:56:41.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.144 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.144 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.010137+0000", 2023-09-23T19:56:41.144 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19243279899999999 2023-09-23T19:56:41.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.010175+0000", 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7978999999999997e-05 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44591.0:147012 5.f e143/112 5:f458b56b:::benchmark_data_smithi135_284570_object31228:head v 143'9223, mlcod=143'9220)", 2023-09-23T19:56:41.145 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:40.818329+0000", 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 120.280766943, 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19524111199999999, 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.818329+0000", 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.146 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.818329+0000", 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.818329+0000", 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:40.818335+0000", 2023-09-23T19:56:41.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0109999999999997e-06 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.000015+0000", 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18168044799999999 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.000024+0000", 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6009999999999995e-06 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.004269+0000", 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004245006 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.004288+0000", 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8595999999999999e-05 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.013543+0000", 2023-09-23T19:56:41.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0092555469999999994 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:41.013571+0000", 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7438999999999999e-05 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:148378 5.e 5:74ea49b5:::benchmark_data_smithi135_284570_object32594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:56:41.150 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:42.665378+0000", 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 118.43371878799999, 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.232768735, 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148378 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.151 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665378+0000", 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665378+0000", 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665377+0000", 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:41.152 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665380+0000", 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0460000000000001e-06 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665386+0000", 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1909999999999997e-06 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.153 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665390+0000", 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9559999999999999e-06 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665442+0000", 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1412999999999997e-05 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665569+0000", 2023-09-23T19:56:41.154 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012707800000000001 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665690+0000", 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012113 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.675796+0000", 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010105866 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.684880+0000", 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090838169999999992 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898054+0000", 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21317402199999999 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898103+0000", 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9048999999999997e-05 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898146+0000", 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3585999999999999e-05 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.157 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:148379 5.e 5:772fcc4e:::benchmark_data_smithi135_284570_object32595:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:42.665408+0000", 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 118.433688875, 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.232853012, 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148379 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.158 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665408+0000", 2023-09-23T19:56:41.159 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665408+0000", 2023-09-23T19:56:41.160 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.160 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.160 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.162 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665407+0000", 2023-09-23T19:56:41.162 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:41.162 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.162 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.163 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665410+0000", 2023-09-23T19:56:41.163 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.65e-06 2023-09-23T19:56:41.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.164 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665415+0000", 2023-09-23T19:56:41.164 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2299999999999999e-06 2023-09-23T19:56:41.164 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.164 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.164 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.164 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665419+0000", 2023-09-23T19:56:41.164 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1540000000000004e-06 2023-09-23T19:56:41.164 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.165 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.165 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.165 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.669141+0000", 2023-09-23T19:56:41.165 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037221590000000001 2023-09-23T19:56:41.165 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.165 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.165 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.669250+0000", 2023-09-23T19:56:41.166 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000108668 2023-09-23T19:56:41.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.166 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.166 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.669358+0000", 2023-09-23T19:56:41.166 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107997 2023-09-23T19:56:41.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.167 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.167 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.675806+0000", 2023-09-23T19:56:41.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0064474700000000003 2023-09-23T19:56:41.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.684988+0000", 2023-09-23T19:56:41.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0091826259999999993 2023-09-23T19:56:41.168 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898190+0000", 2023-09-23T19:56:41.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.213201737 2023-09-23T19:56:41.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.169 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.169 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898226+0000", 2023-09-23T19:56:41.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6353000000000002e-05 2023-09-23T19:56:41.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.170 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898261+0000", 2023-09-23T19:56:41.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4272e-05 2023-09-23T19:56:41.170 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.170 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.170 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:148382 5.e 5:7ea26d6e:::benchmark_data_smithi135_284570_object32598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:42.668946+0000", 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 118.430150274, 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.229404887, 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.171 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148382 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.668946+0000", 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.668946+0000", 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.668946+0000", 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.668948+0000", 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8980000000000001e-06 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.173 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.668951+0000", 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2880000000000001e-06 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.668953+0000", 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2230000000000001e-06 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.672704+0000", 2023-09-23T19:56:41.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037508089999999999 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.672806+0000", 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102025 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.672926+0000", 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000119916 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.675943+0000", 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0030173880000000002 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.685023+0000", 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090799629999999999 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898291+0000", 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21326729 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898322+0000", 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1816999999999997e-05 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898351+0000", 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8547999999999998e-05 2023-09-23T19:56:41.177 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:193680 5.e 5:7b988700:::benchmark_data_smithi135_284570_object77896:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:45.440110+0000", 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 55.658986405999997, 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18856071899999999, 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.178 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 193680 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440110+0000", 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440110+0000", 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440109+0000", 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440114+0000", 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5669999999999996e-06 2023-09-23T19:56:41.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440126+0000", 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1919999999999999e-05 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440130+0000", 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8130000000000003e-06 2023-09-23T19:56:41.181 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440168+0000", 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7494999999999999e-05 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440262+0000", 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4091000000000004e-05 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440346+0000", 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3988999999999998e-05 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.446151+0000", 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0058051789999999997 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.446284+0000", 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000133189 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.628581+0000", 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18229709999999999 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.628630+0000", 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8145000000000001e-05 2023-09-23T19:56:41.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.628671+0000", 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1177999999999998e-05 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:193681 5.e 5:7021d368:::benchmark_data_smithi135_284570_object77897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:41.185 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:45.440140+0000", 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 55.658956879999998, 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.188640167, 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 193681 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440140+0000", 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440140+0000", 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440139+0000", 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440141+0000", 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.748e-06 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440147+0000", 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2040000000000002e-06 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.440150+0000", 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5299999999999999e-06 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.443090+0000", 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029398499999999999 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.443214+0000", 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000124797 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.443298+0000", 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3961000000000005e-05 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.447087+0000", 2023-09-23T19:56:41.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003788231 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.448112+0000", 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0010254870000000001 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.628703+0000", 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.180591375 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.628746+0000", 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2920000000000002e-05 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:45.628780+0000", 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3340999999999998e-05 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:203736 5.e 5:769acb00:::benchmark_data_smithi135_284570_object87952:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:59.317474+0000", 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 41.781622749999997, 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.270062107, 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 203736 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.193 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317474+0000", 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317474+0000", 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317473+0000", 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317480+0000", 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.593e-06 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317497+0000", 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6568000000000001e-05 2023-09-23T19:56:41.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317502+0000", 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7379999999999999e-06 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317537+0000", 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4981999999999998e-05 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317662+0000", 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012491800000000001 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317802+0000", 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000140109 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.320633+0000", 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028309450000000001 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.320859+0000", 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00022572599999999999 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.587439+0000", 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.26657951000000002 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.587492+0000", 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3653999999999998e-05 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.587536+0000", 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3483999999999999e-05 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.199 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:203739 5.e 5:78001ac3:::benchmark_data_smithi135_284570_object87955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:59.322087+0000", 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 41.777009225, 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.26868331000000001, 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.200 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 203739 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322087+0000", 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322087+0000", 2023-09-23T19:56:41.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322087+0000", 2023-09-23T19:56:41.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:41.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322092+0000", 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2299999999999999e-06 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322102+0000", 2023-09-23T19:56:41.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0463e-05 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322107+0000", 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8069999999999999e-06 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322142+0000", 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4669999999999998e-05 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322218+0000", 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5876000000000003e-05 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322284+0000", 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6440999999999996e-05 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.325790+0000", 2023-09-23T19:56:41.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035055339999999998 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.327103+0000", 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0013134450000000001 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.590673+0000", 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.26356996599999999 2023-09-23T19:56:41.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.590725+0000", 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1783000000000002e-05 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.590771+0000", 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5779999999999999e-05 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.207 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:203744 5.e 5:756207b1:::benchmark_data_smithi135_284570_object87960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:59.339381+0000", 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 41.759715188999998, 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25202122599999999, 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.208 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 203744 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339381+0000", 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339381+0000", 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339380+0000", 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339386+0000", 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6880000000000002e-06 2023-09-23T19:56:41.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339401+0000", 2023-09-23T19:56:41.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4447000000000001e-05 2023-09-23T19:56:41.213 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339408+0000", 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.109e-06 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339450+0000", 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2456999999999997e-05 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339539+0000", 2023-09-23T19:56:41.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9030000000000006e-05 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339615+0000", 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5193999999999999e-05 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.342421+0000", 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028068070000000001 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.342614+0000", 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000192652 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.591327+0000", 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.24871296400000001 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.591370+0000", 2023-09-23T19:56:41.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2577999999999998e-05 2023-09-23T19:56:41.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.591402+0000", 2023-09-23T19:56:41.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2910999999999997e-05 2023-09-23T19:56:41.217 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.218 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.218 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.218 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:222016 5.e 5:7c7c053d:::benchmark_data_smithi135_284570_object106232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:24.285082+0000", 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.814014738000001, 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.242105458, 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 222016 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.219 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285082+0000", 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285082+0000", 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.220 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285080+0000", 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285087+0000", 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7819999999999996e-06 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285099+0000", 2023-09-23T19:56:41.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2251e-05 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285105+0000", 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9390000000000002e-06 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285186+0000", 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0529000000000001e-05 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285291+0000", 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105424 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285396+0000", 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010489600000000001 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.292362+0000", 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0069656470000000002 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305916+0000", 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013554372 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527116+0000", 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.22120009299999999 2023-09-23T19:56:41.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527158+0000", 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1838999999999997e-05 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527187+0000", 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9204999999999999e-05 2023-09-23T19:56:41.225 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:222020 5.e 5:7ff5b4db:::benchmark_data_smithi135_284570_object106236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:24.285119+0000", 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.813977259000001, 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.24215035400000001, 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.226 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 222020 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285119+0000", 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285119+0000", 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285119+0000", 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285121+0000", 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3520000000000001e-06 2023-09-23T19:56:41.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285127+0000", 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5720000000000002e-06 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285130+0000", 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.614e-06 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.288912+0000", 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037813339999999999 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.289029+0000", 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000117748 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.289144+0000", 2023-09-23T19:56:41.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011466100000000001 2023-09-23T19:56:41.231 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.231 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.231 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.231 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.292514+0000", 2023-09-23T19:56:41.231 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033697620000000001 2023-09-23T19:56:41.231 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.231 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.231 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.232 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.310303+0000", 2023-09-23T19:56:41.232 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.017789503000000002 2023-09-23T19:56:41.232 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.232 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.232 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.232 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527222+0000", 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.216918746 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527248+0000", 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6203e-05 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.233 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.234 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527270+0000", 2023-09-23T19:56:41.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1217e-05 2023-09-23T19:56:41.234 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.234 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.234 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:222028 5.e 5:73df6521:::benchmark_data_smithi135_284570_object106244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:24.296871+0000", 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.802224979999998, 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.23046222299999999, 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.235 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 222028 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.296871+0000", 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.296871+0000", 2023-09-23T19:56:41.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.296870+0000", 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.296875+0000", 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9520000000000002e-06 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.237 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.296886+0000", 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0399e-05 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.296891+0000", 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5829999999999997e-06 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.238 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.296937+0000", 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6087000000000001e-05 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.297070+0000", 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000132872 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.297187+0000", 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000116916 2023-09-23T19:56:41.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.310335+0000", 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013148425 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.310355+0000", 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0049000000000001e-05 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527294+0000", 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21693880199999999 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527316+0000", 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1852e-05 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527334+0000", 2023-09-23T19:56:41.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7538e-05 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:222035 5.e 5:706c1a84:::benchmark_data_smithi135_284570_object106251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:24.305331+0000", 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.793765203, 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.222082695, 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:41.242 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 222035 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305331+0000", 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.243 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305331+0000", 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305330+0000", 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:41.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305336+0000", 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2090000000000002e-06 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305349+0000", 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3164e-05 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305356+0000", 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5289999999999997e-06 2023-09-23T19:56:41.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305939+0000", 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00058264799999999996 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.306055+0000", 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000116242 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.246 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.306190+0000", 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000134969 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.310385+0000", 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0041951389999999996 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.310977+0000", 2023-09-23T19:56:41.247 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00059236599999999999 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527370+0000", 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21639288600000001 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527395+0000", 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4952000000000001e-05 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:41.248 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527414+0000", 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8810000000000001e-05 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:41.249 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:56:41.249 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:41.282 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:41.326 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:41.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:41.429 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:41.455 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T19:56:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 156'14636 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: 5.9 starting backfill to osd.1 from (0'0,0'0] MAX to 156'14278 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[213871]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 156'14658 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 156'14636 2023-09-23T19:56:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: 5.9 starting backfill to osd.1 from (0'0,0'0] MAX to 156'14278 2023-09-23T19:56:41.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:41 smithi135 ceph-mon[221113]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 156'14658 2023-09-23T19:56:41.497 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:41.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:41.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:41.600 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:41.628 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T19:56:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 156'14636 2023-09-23T19:56:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: failed to encode map e159 with expected crc 2023-09-23T19:56:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: 5.9 starting backfill to osd.1 from (0'0,0'0] MAX to 156'14278 2023-09-23T19:56:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:41 smithi144 ceph-mon[194833]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 156'14658 2023-09-23T19:56:41.670 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:41.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:41.773 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:41.800 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:41.826 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:56:41.847 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:41.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:41.952 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:41.978 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:42.021 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:42.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:42.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:42.036 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:42.126 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:42.153 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:42.199 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:42.250 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:42.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:42.325 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_0' removed 2023-09-23T19:56:42.353 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:42.421 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: pgmap v404: 99 pgs: 2 remapped+peering, 6 peering, 6 active+undersized+degraded, 19 active+undersized, 66 active+clean; 16 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 88/627 objects degraded (14.035%) 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/724655998' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:42.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 88/627 objects degraded (14.035%), 6 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:42.442 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: pgmap v404: 99 pgs: 2 remapped+peering, 6 peering, 6 active+undersized+degraded, 19 active+undersized, 66 active+clean; 16 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 88/627 objects degraded (14.035%) 2023-09-23T19:56:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T19:56:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/724655998' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 88/627 objects degraded (14.035%), 6 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: pgmap v404: 99 pgs: 2 remapped+peering, 6 peering, 6 active+undersized+degraded, 19 active+undersized, 66 active+clean; 16 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 88/627 objects degraded (14.035%) 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T19:56:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: failed to encode map e160 with expected crc 2023-09-23T19:56:42.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/724655998' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:56:42.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 88/627 objects degraded (14.035%), 6 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:43.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/724655998' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:56:43.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T19:56:43.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.532 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.532 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.532 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:43 smithi144 ceph-mon[194833]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/724655998' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[213871]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/724655998' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:43 smithi135 ceph-mon[221113]: failed to encode map e161 with expected crc 2023-09-23T19:56:43.645 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:56:43.645 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:56:43.645 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T19:56:43.646 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T19:56:43.735 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:56:43.736 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T19:56:43.736 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T19:56:43.736 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:56:43.811 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:43.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:43.885 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:56:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:56:43] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T19:56:43.916 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:43.947 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:44.089 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:44.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:44.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:44.192 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:44.219 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:44.362 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:56:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:44 smithi135 ceph-mon[213871]: pgmap v407: 83 pgs: 1 active+undersized+remapped, 2 remapped+peering, 7 peering, 3 active+undersized+degraded, 12 active+undersized, 58 active+clean; 4.7 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9/84 objects degraded (10.714%) 2023-09-23T19:56:44.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:44 smithi135 ceph-mon[221113]: pgmap v407: 83 pgs: 1 active+undersized+remapped, 2 remapped+peering, 7 peering, 3 active+undersized+degraded, 12 active+undersized, 58 active+clean; 4.7 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9/84 objects degraded (10.714%) 2023-09-23T19:56:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:44 smithi144 ceph-mon[194833]: pgmap v407: 83 pgs: 1 active+undersized+remapped, 2 remapped+peering, 7 peering, 3 active+undersized+degraded, 12 active+undersized, 58 active+clean; 4.7 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9/84 objects degraded (10.714%) 2023-09-23T19:56:44.705 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:44.800 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:44.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:44.903 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:44.938 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:44.980 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:44.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:44.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:45.038 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:45.084 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:45.113 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:45.269 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:45.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:45.279 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 249 (0%) 2023-09-23T19:56:45.279 INFO:teuthology.orchestra.run.smithi135.stdout:avg 35.5714 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 5.45856 -> 5.45856 (expected baseline 5.52176) 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 25 -> 25 pgs (0.702811 -> 0.702811 * mean) 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.4 with 41 -> 41 pgs (1.15261 -> 1.15261 * mean) 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 3 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0029 2023-09-23T19:56:45.280 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0291 2023-09-23T19:56:45.282 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:56:45.309 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:56:45.372 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:45.400 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4048186928' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:45.401 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4048186928' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:45.405 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:45.506 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:56:45.525 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:56:45.543 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:56:45.563 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:56:45.582 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:56:45.601 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:56:45.620 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:56:45.620 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:56:45.620 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:56:45.620 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:56:45.621 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:56:45.621 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:56:45.621 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:56:45.621 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:56:45.621 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:56:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4048186928' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:56:46.193 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:46.193 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: pgmap v408: 83 pgs: 1 active+undersized+remapped, 2 remapped+peering, 1 peering, 2 active+undersized+degraded, 4 active+undersized, 73 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 915 B/s, 2 objects/s recovering 2023-09-23T19:56:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4048186928' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2429482004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: pgmap v408: 83 pgs: 1 active+undersized+remapped, 2 remapped+peering, 1 peering, 2 active+undersized+degraded, 4 active+undersized, 73 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 915 B/s, 2 objects/s recovering 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4048186928' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:46.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T19:56:46.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2429482004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:46.589 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:46.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:46.595 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T19:56:46.595 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T19:56:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: pgmap v408: 83 pgs: 1 active+undersized+remapped, 2 remapped+peering, 1 peering, 2 active+undersized+degraded, 4 active+undersized, 73 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 915 B/s, 2 objects/s recovering 2023-09-23T19:56:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4048186928' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:56:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T19:56:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: failed to encode map e162 with expected crc 2023-09-23T19:56:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2429482004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:46.693 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:46.725 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:46.835 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:56:46.847 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:46.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:46.953 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:46.979 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:47.183 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:47.336 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:47.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:47.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:47.405 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:56:47.406 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:56:47.440 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:47.468 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:47.563 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:47.565 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:56:47.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:47.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:47.666 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:47.694 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:47.787 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:47.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:47.890 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:47.918 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:48.154 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:48.163 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:48.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:48.269 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:48.298 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:48.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:48 smithi135 ceph-mon[221113]: pgmap v410: 83 pgs: 1 peering, 2 active+undersized+degraded, 4 active+undersized, 76 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 791 B/s, 4 objects/s recovering 2023-09-23T19:56:48.405 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:48 smithi135 ceph-mon[213871]: pgmap v410: 83 pgs: 1 peering, 2 active+undersized+degraded, 4 active+undersized, 76 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 791 B/s, 4 objects/s recovering 2023-09-23T19:56:48.464 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:48.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:48.481 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:48 smithi144 ceph-mon[194833]: pgmap v410: 83 pgs: 1 peering, 2 active+undersized+degraded, 4 active+undersized, 76 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 791 B/s, 4 objects/s recovering 2023-09-23T19:56:48.568 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:48.596 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:48.640 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:48.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:48.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:48.657 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:56:48.717 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:48.717 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:48.744 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:48.771 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:48.866 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:48.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:48.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:48.970 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:48.995 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:49.342 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:49.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:49.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:49.379 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:56:49.407 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:56:49.447 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:56:49.479 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:49.537 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:56:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:49.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:56:49.595 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:49.642 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:49.669 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:49.722 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:49.763 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:49.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:49.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:49.869 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:49.902 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:50.000 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:50.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:50.104 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:50.130 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:50.274 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:50.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:50.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:50.323 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_1' created 2023-09-23T19:56:50.378 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:50.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:50.458 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:50.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:50 smithi135 ceph-mon[221113]: pgmap v411: 83 pgs: 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 661 B/s, 3 objects/s recovering 2023-09-23T19:56:50.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/581561895' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T19:56:50.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:50 smithi135 ceph-mon[213871]: pgmap v411: 83 pgs: 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 661 B/s, 3 objects/s recovering 2023-09-23T19:56:50.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/581561895' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T19:56:50.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:50.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:50.562 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:50.588 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:50 smithi144 ceph-mon[194833]: pgmap v411: 83 pgs: 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 661 B/s, 3 objects/s recovering 2023-09-23T19:56:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/581561895' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T19:56:51.044 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:51.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:51.147 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:51.176 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:51.320 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[221113]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.320 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/581561895' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T19:56:51.320 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[221113]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T19:56:51.321 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[221113]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.321 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[221113]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.321 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[221113]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.321 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[221113]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:51.321 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'12661'}"}]: dispatch 2023-09-23T19:56:51.323 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T19:56:51.323 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:56:51.323 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 5.43608 -> 5.43608 (expected baseline 6.03053) 2023-09-23T19:56:51.323 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 35 -> 35 pgs (0.824916 -> 0.824916 * mean) 2023-09-23T19:56:51.323 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.3 with 50 -> 50 pgs (1.17845 -> 1.17845 * mean) 2023-09-23T19:56:51.323 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:56:51.324 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T19:56:51.324 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T19:56:51.324 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:56:51.324 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0029 2023-09-23T19:56:51.324 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0032 2023-09-23T19:56:51.324 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 0.2432 -> 0.1932 2023-09-23T19:56:51.325 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:51.325 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:51.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:51 smithi144 ceph-mon[194833]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/581561895' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T19:56:51.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:51 smithi144 ceph-mon[194833]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T19:56:51.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:51 smithi144 ceph-mon[194833]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:51 smithi144 ceph-mon[194833]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:51 smithi144 ceph-mon[194833]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:51 smithi144 ceph-mon[194833]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'12661'}"}]: dispatch 2023-09-23T19:56:51.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[213871]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/581561895' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T19:56:51.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[213871]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T19:56:51.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[213871]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[213871]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[213871]: failed to encode map e163 with expected crc 2023-09-23T19:56:51.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[213871]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:56:51.691 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'12661'}"}]: dispatch 2023-09-23T19:56:51.988 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:52.002 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T19:56:52.014 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T19:56:52.345 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:52.371 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:52.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:52.381 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T19:56:52.381 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T19:56:52.476 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:52.508 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:52.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:52 smithi144 ceph-mon[194833]: pgmap v413: 99 pgs: 16 unknown, 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 593 B/s, 3 objects/s recovering 2023-09-23T19:56:52.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'12661'}"}]': finished 2023-09-23T19:56:52.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:52 smithi144 ceph-mon[194833]: osdmap e164: 8 total, 7 up, 8 in 2023-09-23T19:56:52.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:52 smithi144 ceph-mon[194833]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:52 smithi144 ceph-mon[194833]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:52 smithi144 ceph-mon[194833]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:52.613 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:52.616 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[221113]: pgmap v413: 99 pgs: 16 unknown, 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 593 B/s, 3 objects/s recovering 2023-09-23T19:56:52.616 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'12661'}"}]': finished 2023-09-23T19:56:52.616 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[221113]: osdmap e164: 8 total, 7 up, 8 in 2023-09-23T19:56:52.616 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[221113]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.616 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[221113]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.617 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[221113]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.617 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:52.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:52.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:52.618 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[213871]: pgmap v413: 99 pgs: 16 unknown, 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%); 593 B/s, 3 objects/s recovering 2023-09-23T19:56:52.618 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'12661'}"}]': finished 2023-09-23T19:56:52.618 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[213871]: osdmap e164: 8 total, 7 up, 8 in 2023-09-23T19:56:52.618 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[213871]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.618 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[213871]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.618 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[213871]: failed to encode map e164 with expected crc 2023-09-23T19:56:52.618 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:56:52.646 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:52.718 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:52.747 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:52.946 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:52.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:53.049 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:53.077 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:53.118 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:53.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:53.224 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:53.268 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:53.461 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:56:53.511 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:53.513 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:53 smithi135 ceph-mon[213871]: failed to encode map e165 with expected crc 2023-09-23T19:56:53.513 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:53 smithi135 ceph-mon[213871]: osdmap e165: 8 total, 7 up, 8 in 2023-09-23T19:56:53.514 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3883700494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:53.514 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:53.514 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:53 smithi135 ceph-mon[221113]: failed to encode map e165 with expected crc 2023-09-23T19:56:53.514 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:53 smithi135 ceph-mon[221113]: osdmap e165: 8 total, 7 up, 8 in 2023-09-23T19:56:53.514 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3883700494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:53.514 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:53.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:53.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:53.616 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:53 smithi144 ceph-mon[194833]: failed to encode map e165 with expected crc 2023-09-23T19:56:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:53 smithi144 ceph-mon[194833]: osdmap e165: 8 total, 7 up, 8 in 2023-09-23T19:56:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3883700494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:56:53.646 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:53.695 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:53.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:53.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:53.800 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:53.828 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:53.872 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:53.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:53.881 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:56:53.881 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:56:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:56:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:56:53] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-09-23T19:56:53.977 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:54.006 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:54.098 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:54.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:54.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:54.201 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:54.228 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:54.270 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:54.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:54.374 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:54.400 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:54.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:54 smithi144 ceph-mon[194833]: pgmap v416: 99 pgs: 3 creating+peering, 11 unknown, 2 active+undersized+degraded, 4 active+undersized, 79 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:54.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:54 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:54.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:54 smithi144 ceph-mon[194833]: failed to encode map e166 with expected crc 2023-09-23T19:56:54.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:54 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:54.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:54 smithi144 ceph-mon[194833]: osdmap e166: 8 total, 7 up, 8 in 2023-09-23T19:56:54.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:54 smithi144 ceph-mon[194833]: 1.7 scrub starts 2023-09-23T19:56:54.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:54 smithi144 ceph-mon[194833]: 1.7 scrub ok 2023-09-23T19:56:54.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:54 smithi144 ceph-mon[194833]: failed to encode map e165 with expected crc 2023-09-23T19:56:54.617 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T19:56:54.631 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:54.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:54.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:54.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[221113]: pgmap v416: 99 pgs: 3 creating+peering, 11 unknown, 2 active+undersized+degraded, 4 active+undersized, 79 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:54.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:54.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[221113]: failed to encode map e166 with expected crc 2023-09-23T19:56:54.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:54.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[221113]: osdmap e166: 8 total, 7 up, 8 in 2023-09-23T19:56:54.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[221113]: 1.7 scrub starts 2023-09-23T19:56:54.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[221113]: 1.7 scrub ok 2023-09-23T19:56:54.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[221113]: failed to encode map e165 with expected crc 2023-09-23T19:56:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[213871]: pgmap v416: 99 pgs: 3 creating+peering, 11 unknown, 2 active+undersized+degraded, 4 active+undersized, 79 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:56:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[213871]: failed to encode map e166 with expected crc 2023-09-23T19:56:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:56:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[213871]: osdmap e166: 8 total, 7 up, 8 in 2023-09-23T19:56:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[213871]: 1.7 scrub starts 2023-09-23T19:56:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[213871]: 1.7 scrub ok 2023-09-23T19:56:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:54 smithi135 ceph-mon[213871]: failed to encode map e165 with expected crc 2023-09-23T19:56:54.736 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:54.762 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:54.808 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:54.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:54.913 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:54.939 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:54.981 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:56:55.015 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:55.033 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:55.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:55.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:55.136 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:55.163 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:55.209 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:55.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:55.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:55.313 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:55.346 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:55.425 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:56:55.448 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:56:55.470 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:56:55.491 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:56:55.510 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:56:55.528 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:56:55.544 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:55 smithi144 ceph-mon[194833]: failed to encode map e165 with expected crc 2023-09-23T19:56:55.544 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:55 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T19:56:55.545 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:55 smithi144 ceph-mon[194833]: 1.6 scrub ok 2023-09-23T19:56:55.547 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:56:55.547 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:56:55.547 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:56:55.547 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:56:55.547 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:56:55.547 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:56:55.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:56:55.548 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:56:55.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:56:55.709 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:55.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:55 smithi135 ceph-mon[213871]: failed to encode map e165 with expected crc 2023-09-23T19:56:55.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:55 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T19:56:55.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:55 smithi135 ceph-mon[213871]: 1.6 scrub ok 2023-09-23T19:56:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:55.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:55.713 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:55 smithi135 ceph-mon[221113]: failed to encode map e165 with expected crc 2023-09-23T19:56:55.713 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:55 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T19:56:55.713 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:55 smithi135 ceph-mon[221113]: 1.6 scrub ok 2023-09-23T19:56:55.814 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:55.843 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:55.951 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:55.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:56.055 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:56.084 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:56.178 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:56.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:56.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:56.282 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:56.310 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:56.346 INFO:teuthology.orchestra.run.smithi135.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T19:56:56.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:56 smithi135 ceph-mon[221113]: pgmap v418: 99 pgs: 3 creating+peering, 2 unknown, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:56.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4254600275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:56.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4242067934' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:56:56.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:56 smithi135 ceph-mon[213871]: pgmap v418: 99 pgs: 3 creating+peering, 2 unknown, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:56.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4254600275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:56.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4242067934' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:56:56.405 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:56.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:56.478 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:56:56.478 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:31957 5.4 5:20704a50:::benchmark_data_smithi135_284570_object31956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:52:40.121123+0000", 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 256.35506029700002, 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.22172633, 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.479 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 31957 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.121123+0000", 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.480 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.121123+0000", 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.121122+0000", 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.121152+0000", 2023-09-23T19:56:56.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9958999999999999e-05 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.121161+0000", 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1779999999999992e-06 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.121166+0000", 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3190000000000002e-06 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.482 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.122511+0000", 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001344193 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.122614+0000", 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103188 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.122750+0000", 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000135926 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.123750+0000", 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0010000899999999999 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.125156+0000", 2023-09-23T19:56:56.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014059840000000001 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.342698+0000", 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21754249000000001 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.342801+0000", 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102595 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.485 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:52:40.342849+0000", 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8205999999999999e-05 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:71282 5.4 5:2be2fa14:::benchmark_data_smithi135_284570_object71281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:53:08.912109+0000", 2023-09-23T19:56:56.486 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 227.56407438100001, 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.227427821, 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71282 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912109+0000", 2023-09-23T19:56:56.487 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.488 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.488 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.488 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.488 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912109+0000", 2023-09-23T19:56:56.489 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.489 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.489 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.489 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.489 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912107+0000", 2023-09-23T19:56:56.489 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:56:56.489 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.489 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912149+0000", 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2188999999999999e-05 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912170+0000", 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1308e-05 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.490 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912179+0000", 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4090000000000003e-06 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912227+0000", 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8327e-05 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912322+0000", 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4456999999999994e-05 2023-09-23T19:56:56.491 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.912482+0000", 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016005199999999999 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.914534+0000", 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020527890000000002 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:08.914791+0000", 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00025626300000000001 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:09.139435+0000", 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.22464457900000001 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:09.139488+0000", 2023-09-23T19:56:56.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3053000000000002e-05 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:53:09.139536+0000", 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8130999999999997e-05 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.494 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:125829 5.9 5:97318625:::benchmark_data_smithi135_284570_object10045:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:10.737529+0000", 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 165.73865386700001, 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.20346123399999999, 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 125829 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.495 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737529+0000", 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737529+0000", 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737529+0000", 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737531+0000", 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7630000000000001e-06 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737538+0000", 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9269999999999999e-06 2023-09-23T19:56:56.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737542+0000", 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9219999999999999e-06 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737594+0000", 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1947999999999998e-05 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737676+0000", 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1874999999999998e-05 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.737746+0000", 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9677000000000004e-05 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.741479+0000", 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037330129999999999 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.741585+0000", 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010571199999999999 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.940899+0000", 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.199314723 2023-09-23T19:56:56.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.940951+0000", 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2046999999999998e-05 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.940990+0000", 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9026999999999998e-05 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.501 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:125838 5.9 5:96e5a08b:::benchmark_data_smithi135_284570_object10054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:10.751140+0000", 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 165.72504334800001, 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19987365800000001, 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.502 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 125838 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751140+0000", 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751140+0000", 2023-09-23T19:56:56.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751139+0000", 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751144+0000", 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6189999999999999e-06 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751159+0000", 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4620000000000001e-05 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751166+0000", 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1430000000000001e-06 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751222+0000", 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6570999999999998e-05 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751354+0000", 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013145299999999999 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.751474+0000", 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000120479 2023-09-23T19:56:56.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.755424+0000", 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0039499419999999997 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.755441+0000", 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6855e-05 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.950926+0000", 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.195484615 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.950974+0000", 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7784000000000002e-05 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:10.951013+0000", 2023-09-23T19:56:56.508 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9669000000000003e-05 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:126108 5.4 5:21918726:::benchmark_data_smithi135_284570_object10324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:11.226095+0000", 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 165.250087826, 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28227993200000001, 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.509 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 126108 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226095+0000", 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.510 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226095+0000", 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226094+0000", 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.511 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226101+0000", 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5599999999999996e-06 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226128+0000", 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6851000000000001e-05 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226136+0000", 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5539999999999998e-06 2023-09-23T19:56:56.512 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226189+0000", 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2916e-05 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226304+0000", 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114191 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.513 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226421+0000", 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000116996 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.234910+0000", 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0084897110000000005 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.241278+0000", 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0063678600000000004 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.508288+0000", 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.26701033800000001 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.508332+0000", 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3680000000000002e-05 2023-09-23T19:56:56.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.508375+0000", 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2895000000000002e-05 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:138548 5.4 5:29e6e693:::benchmark_data_smithi135_284570_object22764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:28.691388+0000", 2023-09-23T19:56:56.516 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 147.78479522200001, 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21180558199999999, 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 138548 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.517 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.691388+0000", 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.691388+0000", 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.691387+0000", 2023-09-23T19:56:56.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.691391+0000", 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3780000000000003e-06 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.691408+0000", 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6772000000000001e-05 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.691413+0000", 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1220000000000003e-06 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.696906+0000", 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054926109999999997 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.697016+0000", 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000109852 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.697121+0000", 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105593 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.702091+0000", 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0049699599999999998 2023-09-23T19:56:56.521 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.708638+0000", 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0065467449999999996 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.903091+0000", 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19445322000000001 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.522 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.903146+0000", 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4707000000000003e-05 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.903193+0000", 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7274999999999997e-05 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.523 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:138554 5.4 5:2d6fb451:::benchmark_data_smithi135_284570_object22770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:28.704259+0000", 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 147.77192421800001, 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.199040671, 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 138554 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.524 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704259+0000", 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704259+0000", 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704257+0000", 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704265+0000", 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8269999999999998e-06 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704282+0000", 2023-09-23T19:56:56.526 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6529999999999999e-05 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704289+0000", 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9020000000000002e-06 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704334+0000", 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4984999999999997e-05 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.527 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704481+0000", 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014741 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.704614+0000", 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013350699999999999 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.528 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.708652+0000", 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040371560000000001 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.716206+0000", 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0075541430000000001 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.903229+0000", 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18702290499999999 2023-09-23T19:56:56.529 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.903272+0000", 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3170999999999997e-05 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:28.903299+0000", 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7610000000000002e-05 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.530 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149309 5.9 5:90e65886:::benchmark_data_smithi135_284570_object33525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.246164+0000", 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 132.23001897500001, 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.106800241, 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.531 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149309 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246164+0000", 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246164+0000", 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.532 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246162+0000", 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246172+0000", 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5470000000000002e-06 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246189+0000", 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6769000000000002e-05 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246197+0000", 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7260000000000008e-06 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246251+0000", 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.325e-05 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246392+0000", 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014115199999999999 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246525+0000", 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013331399999999999 2023-09-23T19:56:56.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.256018+0000", 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.009493352 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.265047+0000", 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090289589999999996 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.536 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.265056+0000", 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6139999999999992e-06 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352889+0000", 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0878331240000001 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352964+0000", 2023-09-23T19:56:56.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5154000000000006e-05 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149362 5.9 5:96e7d67a:::benchmark_data_smithi135_284570_object33578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.354858+0000", 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 132.121325412, 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.005834238, 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.538 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149362 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354858+0000", 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.539 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354858+0000", 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354857+0000", 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354866+0000", 2023-09-23T19:56:56.540 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8339999999999994e-06 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354879+0000", 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2714000000000001e-05 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354887+0000", 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0430000000000008e-06 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354939+0000", 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2045000000000001e-05 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354947+0000", 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3210000000000005e-06 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352991+0000", 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.998044025 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.353109+0000", 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011731999999999999 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.353230+0000", 2023-09-23T19:56:56.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000121514 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.358433+0000", 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052028939999999996 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360549+0000", 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021155750000000002 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360599+0000", 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9743000000000001e-05 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360692+0000", 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3247999999999999e-05 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149825 5.4 5:207530db:::benchmark_data_smithi135_284570_object34041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:45.852692+0000", 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 130.623491175, 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25976732600000002, 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149825 2023-09-23T19:56:56.546 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852692+0000", 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852692+0000", 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852690+0000", 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852698+0000", 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4900000000000003e-06 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852715+0000", 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6963e-05 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852721+0000", 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8590000000000001e-06 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852759+0000", 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8395000000000001e-05 2023-09-23T19:56:56.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852898+0000", 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000139073 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.853026+0000", 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000127899 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.550 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.856882+0000", 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038562650000000002 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.856968+0000", 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5603e-05 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112369+0000", 2023-09-23T19:56:56.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25540094400000002 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112417+0000", 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8297999999999997e-05 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112459+0000", 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2116999999999997e-05 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.552 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149832 5.4 5:26481272:::benchmark_data_smithi135_284570_object34048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:45.858049+0000", 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 130.61813347500001, 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.254506177, 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.553 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149832 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858049+0000", 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858049+0000", 2023-09-23T19:56:56.554 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858048+0000", 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858054+0000", 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5260000000000001e-06 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.555 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858066+0000", 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2330000000000001e-05 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858071+0000", 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7029999999999997e-06 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.556 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858108+0000", 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7404999999999998e-05 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858188+0000", 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9385e-05 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858256+0000", 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8514000000000002e-05 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.557 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.861360+0000", 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003103401 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.862145+0000", 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00078522800000000001 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.558 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112495+0000", 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25034988800000002 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112527+0000", 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2110999999999998e-05 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112556+0000", 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8765999999999999e-05 2023-09-23T19:56:56.559 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149838 5.4 5:2ac2db12:::benchmark_data_smithi135_284570_object34054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:45.872656+0000", 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 130.60352678199999, 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.23998376199999999, 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.560 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149838 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.872656+0000", 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.561 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.872656+0000", 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.872654+0000", 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.872665+0000", 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0722e-05 2023-09-23T19:56:56.562 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.872686+0000", 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1169e-05 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.872695+0000", 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4419999999999998e-06 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.563 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.872749+0000", 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4614999999999998e-05 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.872927+0000", 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017800199999999999 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.873078+0000", 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015108100000000001 2023-09-23T19:56:56.564 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.877521+0000", 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0044425460000000003 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.877533+0000", 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2075e-05 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.565 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112583+0000", 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.23504968500000001 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112612+0000", 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9892000000000001e-05 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112640+0000", 2023-09-23T19:56:56.566 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7444999999999999e-05 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:151030 5.4 5:2b25c154:::benchmark_data_smithi135_284570_object35246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:47.757786+0000", 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 128.718396794, 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28100605099999998, 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.567 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 151030 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757786+0000", 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.568 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757786+0000", 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757785+0000", 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757791+0000", 2023-09-23T19:56:56.569 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8100000000000003e-06 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757800+0000", 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9309999999999997e-06 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757804+0000", 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1710000000000004e-06 2023-09-23T19:56:56.570 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757843+0000", 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9303e-05 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757940+0000", 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6701999999999997e-05 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.571 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.758023+0000", 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3245999999999995e-05 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.760993+0000", 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002969377 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.767091+0000", 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0060981780000000001 2023-09-23T19:56:56.572 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:48.038692+0000", 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.271601384 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:48.038752+0000", 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9178999999999997e-05 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.573 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:48.038792+0000", 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0713000000000001e-05 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:165070 5.4 5:2a2864df:::benchmark_data_smithi135_284570_object49286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:07.061096+0000", 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 109.41508655200001, 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.203871369, 2023-09-23T19:56:56.574 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 165070 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.061096+0000", 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.575 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.061096+0000", 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.061096+0000", 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.576 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.061100+0000", 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3420000000000001e-06 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.061113+0000", 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2559e-05 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.577 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.061117+0000", 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7219999999999999e-06 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.063130+0000", 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002012397 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.063251+0000", 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012088599999999999 2023-09-23T19:56:56.578 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.063374+0000", 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000123709 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.067014+0000", 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0036397479999999999 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.579 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.067126+0000", 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111817 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.264872+0000", 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19774614200000001 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.264916+0000", 2023-09-23T19:56:56.580 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3733000000000003e-05 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:07.264968+0000", 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2058999999999997e-05 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:189881 5.4 5:25dcba3c:::benchmark_data_smithi135_284570_object74097:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:56.581 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:40.341356+0000", 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 76.134826715000003, 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19938323799999999, 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 189881 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.582 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.341356+0000", 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.341356+0000", 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.341355+0000", 2023-09-23T19:56:56.583 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.341361+0000", 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3349999999999997e-06 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.341372+0000", 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1649e-05 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.584 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.341379+0000", 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1190000000000002e-06 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.345360+0000", 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003981526 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.585 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.345495+0000", 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013486600000000001 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.345633+0000", 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013771899999999999 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.353475+0000", 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0078419869999999999 2023-09-23T19:56:56.586 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.355612+0000", 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021372790000000002 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.540675+0000", 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.185063477 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.587 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.540712+0000", 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6353000000000002e-05 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:40.540739+0000", 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7758000000000001e-05 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.588 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:195641 5.4 5:2f9c08d9:::benchmark_data_smithi135_284570_object79857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:48.122659+0000", 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 68.353523566000007, 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29659205, 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 195641 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.589 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122659+0000", 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122659+0000", 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.590 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122659+0000", 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122663+0000", 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.16e-06 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122673+0000", 2023-09-23T19:56:56.591 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9799999999999993e-06 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122677+0000", 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8469999999999998e-06 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122717+0000", 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9867999999999998e-05 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.592 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122810+0000", 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3264999999999996e-05 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122896+0000", 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6045999999999996e-05 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.125913+0000", 2023-09-23T19:56:56.593 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0030173159999999999 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.126349+0000", 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043560300000000001 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.419161+0000", 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29281219400000003 2023-09-23T19:56:56.594 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.419206+0000", 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5012000000000002e-05 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.419251+0000", 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5460000000000002e-05 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.595 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:195643 5.4 5:227f4ff3:::benchmark_data_smithi135_284570_object79859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:48.127393+0000", 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 68.348789940000003, 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29418176800000001, 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.596 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 195643 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127393+0000", 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127393+0000", 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.597 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127392+0000", 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127398+0000", 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5559999999999998e-06 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.598 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127410+0000", 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1887e-05 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127415+0000", 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0660000000000003e-06 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127447+0000", 2023-09-23T19:56:56.599 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2240000000000003e-05 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127533+0000", 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6580000000000001e-05 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127611+0000", 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7192000000000002e-05 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.600 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.131281+0000", 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003670151 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.131597+0000", 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000316621 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.601 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.421510+0000", 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28991222900000002 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.421540+0000", 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0307999999999998e-05 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.421575+0000", 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4882000000000002e-05 2023-09-23T19:56:56.602 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:195647 5.4 5:262c0f54:::benchmark_data_smithi135_284570_object79863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:48.132161+0000", 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 68.344021503999997, 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29250667499999999, 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.603 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 195647 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132161+0000", 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132161+0000", 2023-09-23T19:56:56.604 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132160+0000", 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132170+0000", 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0045e-05 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.605 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132183+0000", 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2731000000000001e-05 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132189+0000", 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5740000000000001e-06 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.606 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132234+0000", 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4857000000000001e-05 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132326+0000", 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2430999999999996e-05 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132406+0000", 2023-09-23T19:56:56.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9338000000000005e-05 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.135238+0000", 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028321750000000001 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.135418+0000", 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017962699999999999 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.424495+0000", 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28907732000000003 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.424576+0000", 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1153e-05 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.424668+0000", 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2083000000000006e-05 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:220413 5.4 5:23b6015e:::benchmark_data_smithi135_284570_object104629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:21.650921+0000", 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 34.825262361, 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.31297672500000001, 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.610 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 220413 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650921+0000", 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.611 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650921+0000", 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650919+0000", 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650926+0000", 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6760000000000001e-06 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650941+0000", 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5191000000000001e-05 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650948+0000", 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3219999999999998e-06 2023-09-23T19:56:56.613 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.653753+0000", 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002805714 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.653909+0000", 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000155725 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.614 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.654041+0000", 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013156800000000001 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.657951+0000", 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003910302 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.671727+0000", 2023-09-23T19:56:56.615 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013775597000000001 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.963804+0000", 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29207719100000001 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.963858+0000", 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3888000000000003e-05 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.963897+0000", 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9699000000000001e-05 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:220417 5.4 5:2b4f3313:::benchmark_data_smithi135_284570_object104633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:21.658119+0000", 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 34.818064090999997, 2023-09-23T19:56:56.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.317104999, 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 220417 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658119+0000", 2023-09-23T19:56:56.618 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658119+0000", 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658117+0000", 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658125+0000", 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3760000000000002e-06 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658141+0000", 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6640999999999999e-05 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658149+0000", 2023-09-23T19:56:56.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2060000000000001e-06 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658196+0000", 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7549000000000002e-05 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658334+0000", 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000137515 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.621 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658476+0000", 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014275299999999999 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.662437+0000", 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0039602600000000002 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.622 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.671760+0000", 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0093229220000000008 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.975140+0000", 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.30337997700000002 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.975186+0000", 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6607999999999998e-05 2023-09-23T19:56:56.623 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.975224+0000", 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7709000000000001e-05 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:56:56.624 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:56:56.624 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:56.625 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:56 smithi144 ceph-mon[194833]: pgmap v418: 99 pgs: 3 creating+peering, 2 unknown, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 4.7 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:56.626 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4254600275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:56:56.626 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4242067934' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:56:56.653 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:57.048 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:56:57.099 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:57.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:57.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:57.119 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:56:57.203 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:56:57.232 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:57.274 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:56:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:57.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:56:57.381 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4242067934' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:56:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:57 smithi144 ceph-mon[194833]: osdmap e167: 8 total, 7 up, 8 in 2023-09-23T19:56:57.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:57.490 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:56:57.491 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:56:57.552 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:56:57.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:57.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:56:57.656 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:57.682 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:57.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4242067934' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:56:57.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:57 smithi135 ceph-mon[213871]: osdmap e167: 8 total, 7 up, 8 in 2023-09-23T19:56:57.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4242067934' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:56:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:57 smithi135 ceph-mon[221113]: osdmap e167: 8 total, 7 up, 8 in 2023-09-23T19:56:57.724 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:57.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:57.831 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:57.858 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:57.899 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:57.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:57.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:58.007 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:56:58.034 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:58.094 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:58.130 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:56:58.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:58.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:56:58.149 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:56:58.236 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:56:58.264 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:58.350 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:56:58.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:58.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:56:58.402 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T19:56:58.402 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T19:56:58.402 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 5.43608 -> 5.43608 (expected baseline 6.03053) 2023-09-23T19:56:58.402 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 35 -> 35 pgs (0.824916 -> 0.824916 * mean) 2023-09-23T19:56:58.403 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.3 with 50 -> 50 pgs (1.17845 -> 1.17845 * mean) 2023-09-23T19:56:58.403 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:56:58.403 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T19:56:58.403 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T19:56:58.403 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:56:58.403 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0029 2023-09-23T19:56:58.403 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0291 2023-09-23T19:56:58.405 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:56:58.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:58 smithi135 ceph-mon[221113]: pgmap v420: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:58.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:58 smithi135 ceph-mon[221113]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:58 smithi135 ceph-mon[221113]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:56:58 smithi135 ceph-mon[221113]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.421 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:58 smithi135 ceph-mon[213871]: pgmap v420: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:58.421 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:58 smithi135 ceph-mon[213871]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.421 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:58 smithi135 ceph-mon[213871]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.421 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:56:58 smithi135 ceph-mon[213871]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.454 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:56:58.481 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:58.525 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:56:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:58.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:56:58.630 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:58 smithi144 ceph-mon[194833]: pgmap v420: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:56:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:58 smithi144 ceph-mon[194833]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:58 smithi144 ceph-mon[194833]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:56:58 smithi144 ceph-mon[194833]: failed to encode map e167 with expected crc 2023-09-23T19:56:58.656 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:58.697 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:58.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:58.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:58.804 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:56:58.807 DEBUG:teuthology.orchestra.run.smithi135:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T19:56:58.809 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:56:58.809 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:56:58.829 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:58.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:hints = 1 2023-09-23T19:56:58.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:56:58.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object prefix: benchmark_data_smithi135_344736 2023-09-23T19:56:58.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:56:58.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:56:58.880 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:56:58.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:56:58.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:56:58.986 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:56:59.018 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:56:59.491 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:56:59.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 1 16 2891 2875 179.678 179.688 0.00848187 0.00555048 2023-09-23T19:57:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:00 smithi144 ceph-mon[194833]: from='client.44933 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:57:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:00 smithi144 ceph-mon[194833]: pgmap v421: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:57:00.647 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:00 smithi135 ceph-mon[213871]: from='client.44933 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:57:00.648 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:00 smithi135 ceph-mon[213871]: pgmap v421: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:57:00.648 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:00 smithi135 ceph-mon[221113]: from='client.44933 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:57:00.648 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:00 smithi135 ceph-mon[221113]: pgmap v421: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T19:57:00.715 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:00.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:00.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:00.824 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:00.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 2 16 6068 6052 189.111 198.562 0.00248337 0.00528122 2023-09-23T19:57:00.864 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:00.932 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:00.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:00.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:01.039 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:01.080 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:01.148 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:01.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:01.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:01.253 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:01.306 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:01.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:01 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/825 objects degraded (0.606%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:01.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:01 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/825 objects degraded (0.606%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:01.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 3 16 9286 9270 193.109 201.125 0.00318838 0.00516896 2023-09-23T19:57:01.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:01 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/825 objects degraded (0.606%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:02.035 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:02.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:02.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:02.147 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:02.187 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:02.238 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:02.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:02.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:02.343 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:02.357 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:02.411 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:02.559 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:02.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:02.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:02 smithi144 ceph-mon[194833]: pgmap v422: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 20 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s wr, 25 op/s; 5/825 objects degraded (0.606%) 2023-09-23T19:57:02.668 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:02.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:02 smithi135 ceph-mon[213871]: pgmap v422: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 20 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s wr, 25 op/s; 5/825 objects degraded (0.606%) 2023-09-23T19:57:02.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:02 smithi135 ceph-mon[221113]: pgmap v422: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 20 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s wr, 25 op/s; 5/825 objects degraded (0.606%) 2023-09-23T19:57:02.760 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:02.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 4 16 11642 11626 181.64 147.25 0.00575835 0.00549785 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:11650 6.9 6:950a2c9d:::benchmark_data_smithi135_344736_object11649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:02.863660+0000", 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0080050959999999997, 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080389840000000008, 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:57:02.873 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 11650 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.863660+0000", 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.863660+0000", 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.863658+0000", 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.863686+0000", 2023-09-23T19:57:02.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7151000000000001e-05 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.863694+0000", 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3159999999999997e-06 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.863699+0000", 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3020000000000002e-06 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.863920+0000", 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00022089500000000001 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.864072+0000", 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015215199999999999 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:02.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.864280+0000", 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020786200000000001 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.865483+0000", 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001202945 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866225+0000", 2023-09-23T19:57:02.878 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00074153900000000004 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:11660 6.9 6:90443186:::benchmark_data_smithi135_344736_object11659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:02.866622+0000", 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0050428260000000003, 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005171124, 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:57:02.879 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 11660 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866622+0000", 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:02.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866622+0000", 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866621+0000", 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866654+0000", 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3108000000000002e-05 2023-09-23T19:57:02.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866673+0000", 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8845e-05 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866692+0000", 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8515000000000001e-05 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866817+0000", 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012520399999999999 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866976+0000", 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015941999999999999 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.867155+0000", 2023-09-23T19:57:02.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017847199999999999 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868306+0000", 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001151556 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:11661 6.9 6:901ecc9d:::benchmark_data_smithi135_344736_object11660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-23T19:57:02.884 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:02.866714+0000", 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0049510020000000004, 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005159205, 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 11661 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.885 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866714+0000", 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866714+0000", 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:02.886 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866714+0000", 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866745+0000", 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1424000000000003e-05 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866753+0000", 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0860000000000002e-06 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866759+0000", 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2720000000000003e-06 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.867708+0000", 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00094894800000000002 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:02.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.867819+0000", 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110402 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.867946+0000", 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012768499999999999 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.869633+0000", 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001687135 2023-09-23T19:57:02.889 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:11662 6.0 6:0a8e4838:::benchmark_data_smithi135_344736_object11661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:02.866764+0000", 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0049007149999999999, 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005213896, 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:02.890 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 11662 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866764+0000", 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:02.891 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866764+0000", 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866764+0000", 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.866783+0000", 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8981999999999999e-05 2023-09-23T19:57:02.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868493+0000", 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017097569999999999 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868503+0000", 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0502e-05 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.893 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868568+0000", 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4618999999999999e-05 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868722+0000", 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000153845 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,3", 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868849+0000", 2023-09-23T19:57:02.894 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012750899999999999 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:11663 6.0 6:0d30f93e:::benchmark_data_smithi135_344736_object11662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:02.868524+0000", 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0031407079999999999, 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0035347949999999999, 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:57:02.895 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 11663 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868524+0000", 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.896 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868524+0000", 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868524+0000", 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868549+0000", 2023-09-23T19:57:02.897 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5652000000000001e-05 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868558+0000", 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9430000000000006e-06 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.868564+0000", 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2009999999999996e-06 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870090+0000", 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015258660000000001 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870219+0000", 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012834399999999999 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,3", 2023-09-23T19:57:02.899 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870356+0000", 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013771899999999999 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:11668 6.a e167/163 6:58b3699a:::benchmark_data_smithi135_344736_object11667:head v 167'720, mlcod=167'718)", 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:02.870793+0000", 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.00087171899999999997, 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001341816, 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:02.900 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870793+0000", 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870793+0000", 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.901 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870792+0000", 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870886+0000", 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3646000000000004e-05 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870897+0000", 2023-09-23T19:57:02.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0569e-05 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870902+0000", 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2449999999999998e-06 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870972+0000", 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0505000000000003e-05 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:02.903 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:02.870990+0000", 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7391999999999999e-05 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 6 2023-09-23T19:57:02.904 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:57:03.019 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:03.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:03.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:03.126 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:03.203 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:03.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:03 smithi135 ceph-mon[213871]: pgmap v423: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 262 op/s; 5/7392 objects degraded (0.068%) 2023-09-23T19:57:03.686 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:57:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:57:03] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-09-23T19:57:03.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:03 smithi135 ceph-mon[221113]: pgmap v423: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 262 op/s; 5/7392 objects degraded (0.068%) 2023-09-23T19:57:03.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 5 16 14129 14113 176.397 155.438 0.00495811 0.00566036 2023-09-23T19:57:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:03 smithi144 ceph-mon[194833]: pgmap v423: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 262 op/s; 5/7392 objects degraded (0.068%) 2023-09-23T19:57:04.429 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/345284796' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:04.454 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:57:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/345284796' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:04.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 6 16 16511 16495 171.809 148.875 0.00177086 0.0058113 2023-09-23T19:57:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/345284796' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:57:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/345284796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[221113]: osdmap e168: 8 total, 7 up, 8 in 2023-09-23T19:57:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[221113]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[221113]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[221113]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[221113]: pgmap v425: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 595 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 5/28404 objects degraded (0.018%) 2023-09-23T19:57:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:57:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/345284796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[213871]: osdmap e168: 8 total, 7 up, 8 in 2023-09-23T19:57:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[213871]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[213871]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[213871]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:05 smithi135 ceph-mon[213871]: pgmap v425: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 595 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 5/28404 objects degraded (0.018%) 2023-09-23T19:57:05.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 7 16 18985 18969 169.352 154.625 0.0057651 0.0058993 2023-09-23T19:57:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:05 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:57:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/345284796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:05 smithi144 ceph-mon[194833]: osdmap e168: 8 total, 7 up, 8 in 2023-09-23T19:57:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:05 smithi144 ceph-mon[194833]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:05 smithi144 ceph-mon[194833]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:05 smithi144 ceph-mon[194833]: failed to encode map e168 with expected crc 2023-09-23T19:57:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:05 smithi144 ceph-mon[194833]: pgmap v425: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 595 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 5/28404 objects degraded (0.018%) 2023-09-23T19:57:06.263 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:06.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:06.368 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:06.433 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:06.563 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:06 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/41154 objects degraded (0.012%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:06.777 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:57:06.778 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:57:06.778 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-09-23T19:57:06.778 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:57:06.830 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:06 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/41154 objects degraded (0.012%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:06.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 8 16 21359 21343 166.728 148.375 0.00973196 0.00599041 2023-09-23T19:57:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:06 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/41154 objects degraded (0.012%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:07.006 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:07.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:07.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:07.110 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:07.148 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:07.563 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:07.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:07.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:07.669 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:07.694 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:07.857 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 9 16 22380 22364 155.292 63.8125 0.00344226 0.00643336 2023-09-23T19:57:08.228 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:57:08.271 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:08.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:08.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:08 smithi144 ceph-mon[194833]: pgmap v426: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 860 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 5/41154 objects degraded (0.012%) 2023-09-23T19:57:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:08.384 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:08.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:08 smithi135 ceph-mon[213871]: pgmap v426: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 860 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 5/41154 objects degraded (0.012%) 2023-09-23T19:57:08.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:08.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:08.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:08 smithi135 ceph-mon[221113]: pgmap v426: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 860 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 5/41154 objects degraded (0.012%) 2023-09-23T19:57:08.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:08.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:08.501 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:08.518 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:08.629 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:08.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:08.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:08.735 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:08.801 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:08.857 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 10 16 24049 24033 150.193 104.312 0.0102521 0.00665196 2023-09-23T19:57:09.108 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:09.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:09.224 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:09.268 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:09.277 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:57:09.323 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:57:09.328 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:09.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:09.363 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:57:09.397 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:57:09.433 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:09.434 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:57:09.468 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:57:09.503 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:09.505 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:57:09.505 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:57:09.506 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:57:09.506 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:57:09.506 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:57:09.506 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:57:09.506 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:57:09.507 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:57:09.507 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:57:09.589 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:09.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:09.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:09.692 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:09.733 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:09.800 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:09.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:09.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:09.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 11 16 25888 25872 146.973 114.938 0.00965542 0.00679823 2023-09-23T19:57:09.907 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:09.945 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:10.014 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:10.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:10.121 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:10.184 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:10.284 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:57:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:10 smithi144 ceph-mon[194833]: pgmap v427: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 5/63243 objects degraded (0.008%) 2023-09-23T19:57:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4198515955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:10.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:10 smithi135 ceph-mon[221113]: pgmap v427: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 5/63243 objects degraded (0.008%) 2023-09-23T19:57:10.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4198515955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:10.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:10 smithi135 ceph-mon[213871]: pgmap v427: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 5/63243 objects degraded (0.008%) 2023-09-23T19:57:10.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4198515955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:10.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 12 16 27477 27461 143 99.3125 0.00189681 0.00691867 2023-09-23T19:57:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:57:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:57:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:57:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:11.791 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:11.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:11.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:11.857 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:11.860 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 13 16 28809 28793 138.384 83.25 0.0146293 0.00721769 2023-09-23T19:57:11.884 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:57:11.884 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:57:11.900 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:11.961 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:12.099 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:12.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:12.206 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:12.278 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:12 smithi144 ceph-mon[194833]: pgmap v428: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 5/66786 objects degraded (0.007%) 2023-09-23T19:57:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:12 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:12 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:12.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:12 smithi135 ceph-mon[221113]: pgmap v428: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 5/66786 objects degraded (0.007%) 2023-09-23T19:57:12.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:12 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:12.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:12 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:12.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:12 smithi135 ceph-mon[213871]: pgmap v428: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 5/66786 objects degraded (0.007%) 2023-09-23T19:57:12.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:12 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:12.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:12 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:12.564 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:12.567 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:12.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:12.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:12.670 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:12.705 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:12.860 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 14 16 30338 30322 135.325 95.5625 0.00727735 0.00738344 2023-09-23T19:57:13.069 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:13.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:13.174 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:13.211 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:13.286 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:13.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:13.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:13 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/66786 objects degraded (0.007%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:13.391 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:13.444 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:13 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/66786 objects degraded (0.007%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:13.445 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:13 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/66786 objects degraded (0.007%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:13.448 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:13.503 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:57:13.505 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:57:13.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:57:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:57:13] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2023-09-23T19:57:13.860 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 15 16 31567 31551 131.423 76.8125 0.0117063 0.00760434 2023-09-23T19:57:13.885 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:57:14.008 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:14.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:14.124 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:14.226 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:14.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:14 smithi135 ceph-mon[221113]: pgmap v429: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 5/74286 objects degraded (0.007%) 2023-09-23T19:57:14.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:14 smithi135 ceph-mon[213871]: pgmap v429: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 5/74286 objects degraded (0.007%) 2023-09-23T19:57:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:14 smithi144 ceph-mon[194833]: pgmap v429: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 5/74286 objects degraded (0.007%) 2023-09-23T19:57:14.860 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 16 16 32791 32775 127.991 76.5 0.00667113 0.00777767 2023-09-23T19:57:15.121 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:15.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:15.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:15.219 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":429,"stamp":"2023-09-23T19:57:12.459046+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1625757619,"num_objects":24762,"num_object_clones":0,"num_object_copies":74286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24762,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":25641,"num_write_kb":1595701,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":4753,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27537,"ondisk_log_size":27537,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6888744,"kb_used_data":5967272,"kb_used_omap":32,"kb_used_meta":921375,"kb_avail":743072472,"statfs":{"total":767960285184,"available":760906211328,"internally_reserved":0,"allocated":6110486528,"data_stored":6105410046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":943488518},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,3,4,9,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1461321728,"num_objects":22298,"num_object_clones":0,"num_object_copies":66894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22298,"num_write_kb":1427072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006066"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":146,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484087+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:56:57.484087+0000","last_peered":"2023-09-23T19:56:57.484087+0000","last_clean":"2023-09-23T19:56:57.484087+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:56:57.484087+0000","last_undegraded":"2023-09-23T19:56:57.484087+0000","last_fullsized":"2023-09-23T19:56:57.484087+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":145,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341646+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:56:56.341646+0000","last_peered":"2023-09-23T19:56:56.341646+0000","last_clean":"2023-09-23T19:56:56.341646+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:56:56.341646+0000","last_undegraded":"2023-09-23T19:56:56.341646+0000","last_fullsized":"2023-09-23T19:56:56.341646+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":106,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:04.453005+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:57:04.453005+0000","last_peered":"2023-09-23T19:57:04.453005+0000","last_clean":"2023-09-23T19:57:04.453005+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:57:04.453005+0000","last_undegraded":"2023-09-23T19:57:04.453005+0000","last_fullsized":"2023-09-23T19:57:04.453005+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":113,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.388050+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:56:57.388050+0000","last_peered":"2023-09-23T19:56:57.388050+0000","last_clean":"2023-09-23T19:56:57.388050+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:56:57.388050+0000","last_undegraded":"2023-09-23T19:56:57.388050+0000","last_fullsized":"2023-09-23T19:56:57.388050+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":149,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341287+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:56:56.341287+0000","last_peered":"2023-09-23T19:56:56.341287+0000","last_clean":"2023-09-23T19:56:56.341287+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:56:56.341287+0000","last_undegraded":"2023-09-23T19:56:56.341287+0000","last_fullsized":"2023-09-23T19:56:56.341287+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":93,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486613+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T19:56:57.486613+0000","last_peered":"2023-09-23T19:56:57.486613+0000","last_clean":"2023-09-23T19:56:57.486613+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T19:56:57.486613+0000","last_undegraded":"2023-09-23T19:56:57.486613+0000","last_fullsized":"2023-09-23T19:56:57.486613+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:07.049200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":111,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.387972+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:56:57.387972+0000","last_peered":"2023-09-23T19:56:57.387972+0000","last_clean":"2023-09-23T19:56:57.387972+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:56:57.387972+0000","last_undegraded":"2023-09-23T19:56:57.387972+0000","last_fullsized":"2023-09-23T19:56:57.387972+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":146,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341374+0000","last_change":"2023-09-23T19:56:39.278349+0000","last_active":"2023-09-23T19:56:56.341374+0000","last_peered":"2023-09-23T19:56:56.341374+0000","last_clean":"2023-09-23T19:56:56.341374+0000","last_became_active":"2023-09-23T19:56:39.278068+0000","last_became_peered":"2023-09-23T19:56:39.278068+0000","last_unstale":"2023-09-23T19:56:56.341374+0000","last_undegraded":"2023-09-23T19:56:56.341374+0000","last_fullsized":"2023-09-23T19:56:56.341374+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:04.173721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486419+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:56:57.486419+0000","last_peered":"2023-09-23T19:56:57.486419+0000","last_clean":"2023-09-23T19:56:57.486419+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:56:57.486419+0000","last_undegraded":"2023-09-23T19:56:57.486419+0000","last_fullsized":"2023-09-23T19:56:57.486419+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":97,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341266+0000","last_change":"2023-09-23T19:56:39.273704+0000","last_active":"2023-09-23T19:56:56.341266+0000","last_peered":"2023-09-23T19:56:56.341266+0000","last_clean":"2023-09-23T19:56:56.341266+0000","last_became_active":"2023-09-23T19:56:39.273360+0000","last_became_peered":"2023-09-23T19:56:39.273360+0000","last_unstale":"2023-09-23T19:56:56.341266+0000","last_undegraded":"2023-09-23T19:56:56.341266+0000","last_fullsized":"2023-09-23T19:56:56.341266+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:17.542729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":131,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.485584+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:56:57.485584+0000","last_peered":"2023-09-23T19:56:57.485584+0000","last_clean":"2023-09-23T19:56:57.485584+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:56:57.485584+0000","last_undegraded":"2023-09-23T19:56:57.485584+0000","last_fullsized":"2023-09-23T19:56:57.485584+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":122,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483868+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T19:56:57.483868+0000","last_peered":"2023-09-23T19:56:57.483868+0000","last_clean":"2023-09-23T19:56:57.483868+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T19:56:57.483868+0000","last_undegraded":"2023-09-23T19:56:57.483868+0000","last_fullsized":"2023-09-23T19:56:57.483868+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:52.846135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":160,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483830+0000","last_change":"2023-09-23T19:56:39.292849+0000","last_active":"2023-09-23T19:56:57.483830+0000","last_peered":"2023-09-23T19:56:57.483830+0000","last_clean":"2023-09-23T19:56:57.483830+0000","last_became_active":"2023-09-23T19:56:39.284622+0000","last_became_peered":"2023-09-23T19:56:39.284622+0000","last_unstale":"2023-09-23T19:56:57.483830+0000","last_undegraded":"2023-09-23T19:56:57.483830+0000","last_fullsized":"2023-09-23T19:56:57.483830+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:24.088523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":134,"reported_epoch":167,"state":"active+undersized","last_fresh":"2023-09-23T19:56:57.483994+0000","last_change":"2023-09-23T19:54:45.327308+0000","last_active":"2023-09-23T19:56:57.483994+0000","last_peered":"2023-09-23T19:56:57.483994+0000","last_clean":"2023-09-23T19:54:37.307838+0000","last_became_active":"2023-09-23T19:54:45.327308+0000","last_became_peered":"2023-09-23T19:54:45.327308+0000","last_unstale":"2023-09-23T19:56:57.483994+0000","last_undegraded":"2023-09-23T19:56:57.483994+0000","last_fullsized":"2023-09-23T19:54:45.254218+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:34.491557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":130,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.485006+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:56:57.485006+0000","last_peered":"2023-09-23T19:56:57.485006+0000","last_clean":"2023-09-23T19:56:57.485006+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:56:57.485006+0000","last_undegraded":"2023-09-23T19:56:57.485006+0000","last_fullsized":"2023-09-23T19:56:57.485006+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":131,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.342185+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:56:56.342185+0000","last_peered":"2023-09-23T19:56:56.342185+0000","last_clean":"2023-09-23T19:56:56.342185+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:56:56.342185+0000","last_undegraded":"2023-09-23T19:56:56.342185+0000","last_fullsized":"2023-09-23T19:56:56.342185+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.387768+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:56:57.387768+0000","last_peered":"2023-09-23T19:56:57.387768+0000","last_clean":"2023-09-23T19:56:57.387768+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:56:57.387768+0000","last_undegraded":"2023-09-23T19:56:57.387768+0000","last_fullsized":"2023-09-23T19:56:57.387768+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":126,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486459+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:56:57.486459+0000","last_peered":"2023-09-23T19:56:57.486459+0000","last_clean":"2023-09-23T19:56:57.486459+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:56:57.486459+0000","last_undegraded":"2023-09-23T19:56:57.486459+0000","last_fullsized":"2023-09-23T19:56:57.486459+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":118,"reported_epoch":167,"state":"active+undersized","last_fresh":"2023-09-23T19:56:57.485100+0000","last_change":"2023-09-23T19:54:45.330750+0000","last_active":"2023-09-23T19:56:57.485100+0000","last_peered":"2023-09-23T19:56:57.485100+0000","last_clean":"2023-09-23T19:54:37.324687+0000","last_became_active":"2023-09-23T19:54:45.330750+0000","last_became_peered":"2023-09-23T19:54:45.330750+0000","last_unstale":"2023-09-23T19:56:57.485100+0000","last_undegraded":"2023-09-23T19:56:57.485100+0000","last_fullsized":"2023-09-23T19:54:45.253851+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:18.890717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":112,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.388010+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:56:57.388010+0000","last_peered":"2023-09-23T19:56:57.388010+0000","last_clean":"2023-09-23T19:56:57.388010+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:56:57.388010+0000","last_undegraded":"2023-09-23T19:56:57.388010+0000","last_fullsized":"2023-09-23T19:56:57.388010+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":44,"reported_epoch":168,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:57:04.453773+0000","last_change":"2023-09-23T19:54:45.331318+0000","last_active":"2023-09-23T19:57:04.453773+0000","last_peered":"2023-09-23T19:57:04.453773+0000","last_clean":"2023-09-23T19:41:41.886282+0000","last_became_active":"2023-09-23T19:54:45.331318+0000","last_became_peered":"2023-09-23T19:54:45.331318+0000","last_unstale":"2023-09-23T19:57:04.453773+0000","last_undegraded":"2023-09-23T19:54:45.255936+0000","last_fullsized":"2023-09-23T19:54:45.255813+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:16.760756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":131,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341975+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:56:56.341975+0000","last_peered":"2023-09-23T19:56:56.341975+0000","last_clean":"2023-09-23T19:56:56.341975+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:56:56.341975+0000","last_undegraded":"2023-09-23T19:56:56.341975+0000","last_fullsized":"2023-09-23T19:56:56.341975+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"168'1413","reported_seq":1425,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:08.258712+0000","last_change":"2023-09-23T19:56:51.336830+0000","last_active":"2023-09-23T19:57:08.258712+0000","last_peered":"2023-09-23T19:57:08.258712+0000","last_clean":"2023-09-23T19:57:08.258712+0000","last_became_active":"2023-09-23T19:56:51.336645+0000","last_became_peered":"2023-09-23T19:56:51.336645+0000","last_unstale":"2023-09-23T19:57:08.258712+0000","last_undegraded":"2023-09-23T19:57:08.258712+0000","last_fullsized":"2023-09-23T19:57:08.258712+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1413,"log_dups_size":0,"ondisk_log_size":1413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:27.681999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92602368,"num_objects":1413,"num_object_clones":0,"num_object_copies":4239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1413,"num_write_kb":90432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":129,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.485763+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:56:57.485763+0000","last_peered":"2023-09-23T19:56:57.485763+0000","last_clean":"2023-09-23T19:56:57.485763+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:56:57.485763+0000","last_undegraded":"2023-09-23T19:56:57.485763+0000","last_fullsized":"2023-09-23T19:56:57.485763+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":126,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486120+0000","last_change":"2023-09-23T19:56:39.275614+0000","last_active":"2023-09-23T19:56:57.486120+0000","last_peered":"2023-09-23T19:56:57.486120+0000","last_clean":"2023-09-23T19:56:57.486120+0000","last_became_active":"2023-09-23T19:56:39.275242+0000","last_became_peered":"2023-09-23T19:56:39.275242+0000","last_unstale":"2023-09-23T19:56:57.486120+0000","last_undegraded":"2023-09-23T19:56:57.486120+0000","last_fullsized":"2023-09-23T19:56:57.486120+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:55.343087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"168'1659","reported_seq":1673,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.715089+0000","last_change":"2023-09-23T19:56:51.328940+0000","last_active":"2023-09-23T19:57:10.715089+0000","last_peered":"2023-09-23T19:57:10.715089+0000","last_clean":"2023-09-23T19:57:10.715089+0000","last_became_active":"2023-09-23T19:56:51.328668+0000","last_became_peered":"2023-09-23T19:56:51.328668+0000","last_unstale":"2023-09-23T19:57:10.715089+0000","last_undegraded":"2023-09-23T19:57:10.715089+0000","last_fullsized":"2023-09-23T19:57:10.715089+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1659,"log_dups_size":0,"ondisk_log_size":1659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:01.871957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108724224,"num_objects":1659,"num_object_clones":0,"num_object_copies":4977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1659,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":130,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.485514+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:56:57.485514+0000","last_peered":"2023-09-23T19:56:57.485514+0000","last_clean":"2023-09-23T19:56:57.485514+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:56:57.485514+0000","last_undegraded":"2023-09-23T19:56:57.485514+0000","last_fullsized":"2023-09-23T19:56:57.485514+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483752+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:56:57.483752+0000","last_peered":"2023-09-23T19:56:57.483752+0000","last_clean":"2023-09-23T19:56:57.483752+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:56:57.483752+0000","last_undegraded":"2023-09-23T19:56:57.483752+0000","last_fullsized":"2023-09-23T19:56:57.483752+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"168'1438","reported_seq":1451,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:08.210319+0000","last_change":"2023-09-23T19:56:51.329607+0000","last_active":"2023-09-23T19:57:08.210319+0000","last_peered":"2023-09-23T19:57:08.210319+0000","last_clean":"2023-09-23T19:57:08.210319+0000","last_became_active":"2023-09-23T19:56:51.329330+0000","last_became_peered":"2023-09-23T19:56:51.329330+0000","last_unstale":"2023-09-23T19:57:08.210319+0000","last_undegraded":"2023-09-23T19:57:08.210319+0000","last_fullsized":"2023-09-23T19:57:08.210319+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1438,"log_dups_size":0,"ondisk_log_size":1438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:51.413448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94240768,"num_objects":1438,"num_object_clones":0,"num_object_copies":4314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1438,"num_write_kb":92032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":119,"reported_epoch":167,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:56:57.485435+0000","last_change":"2023-09-23T19:54:45.328405+0000","last_active":"2023-09-23T19:56:57.485435+0000","last_peered":"2023-09-23T19:56:57.485435+0000","last_clean":"2023-09-23T19:54:37.326665+0000","last_became_active":"2023-09-23T19:54:45.328405+0000","last_became_peered":"2023-09-23T19:54:45.328405+0000","last_unstale":"2023-09-23T19:56:57.485435+0000","last_undegraded":"2023-09-23T19:54:45.254169+0000","last_fullsized":"2023-09-23T19:54:45.254056+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:10.168687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483589+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T19:56:57.483589+0000","last_peered":"2023-09-23T19:56:57.483589+0000","last_clean":"2023-09-23T19:56:57.483589+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T19:56:57.483589+0000","last_undegraded":"2023-09-23T19:56:57.483589+0000","last_fullsized":"2023-09-23T19:56:57.483589+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:30.170186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"168'1766","reported_seq":1780,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:11.600766+0000","last_change":"2023-09-23T19:56:51.330015+0000","last_active":"2023-09-23T19:57:11.600766+0000","last_peered":"2023-09-23T19:57:11.600766+0000","last_clean":"2023-09-23T19:57:11.600766+0000","last_became_active":"2023-09-23T19:56:51.329727+0000","last_became_peered":"2023-09-23T19:56:51.329727+0000","last_unstale":"2023-09-23T19:57:11.600766+0000","last_undegraded":"2023-09-23T19:57:11.600766+0000","last_fullsized":"2023-09-23T19:57:11.600766+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1766,"log_dups_size":0,"ondisk_log_size":1766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:59.494181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115736576,"num_objects":1766,"num_object_clones":0,"num_object_copies":5298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1766,"num_write_kb":113024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486458+0000","last_change":"2023-09-23T19:56:39.274176+0000","last_active":"2023-09-23T19:56:57.486458+0000","last_peered":"2023-09-23T19:56:57.486458+0000","last_clean":"2023-09-23T19:56:57.486458+0000","last_became_active":"2023-09-23T19:56:39.273835+0000","last_became_peered":"2023-09-23T19:56:39.273835+0000","last_unstale":"2023-09-23T19:56:57.486458+0000","last_undegraded":"2023-09-23T19:56:57.486458+0000","last_fullsized":"2023-09-23T19:56:57.486458+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:51.379105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":112,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.388134+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:56:57.388134+0000","last_peered":"2023-09-23T19:56:57.388134+0000","last_clean":"2023-09-23T19:56:57.388134+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:56:57.388134+0000","last_undegraded":"2023-09-23T19:56:57.388134+0000","last_fullsized":"2023-09-23T19:56:57.388134+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"168'1413","reported_seq":1425,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:08.228856+0000","last_change":"2023-09-23T19:56:51.335737+0000","last_active":"2023-09-23T19:57:08.228856+0000","last_peered":"2023-09-23T19:57:08.228856+0000","last_clean":"2023-09-23T19:57:08.228856+0000","last_became_active":"2023-09-23T19:56:51.335399+0000","last_became_peered":"2023-09-23T19:56:51.335399+0000","last_unstale":"2023-09-23T19:57:08.228856+0000","last_undegraded":"2023-09-23T19:57:08.228856+0000","last_fullsized":"2023-09-23T19:57:08.228856+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1413,"log_dups_size":0,"ondisk_log_size":1413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:20.933908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92602368,"num_objects":1413,"num_object_clones":0,"num_object_copies":4239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1413,"num_write_kb":90432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":548,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484668+0000","last_change":"2023-09-23T19:52:57.349564+0000","last_active":"2023-09-23T19:56:57.484668+0000","last_peered":"2023-09-23T19:56:57.484668+0000","last_clean":"2023-09-23T19:56:57.484668+0000","last_became_active":"2023-09-23T19:52:57.349182+0000","last_became_peered":"2023-09-23T19:52:57.349182+0000","last_unstale":"2023-09-23T19:56:57.484668+0000","last_undegraded":"2023-09-23T19:56:57.484668+0000","last_fullsized":"2023-09-23T19:56:57.484668+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:25.262721+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":129,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341898+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:56:56.341898+0000","last_peered":"2023-09-23T19:56:56.341898+0000","last_clean":"2023-09-23T19:56:56.341898+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:56:56.341898+0000","last_undegraded":"2023-09-23T19:56:56.341898+0000","last_fullsized":"2023-09-23T19:56:56.341898+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483719+0000","last_change":"2023-09-23T19:56:39.284044+0000","last_active":"2023-09-23T19:56:57.483719+0000","last_peered":"2023-09-23T19:56:57.483719+0000","last_clean":"2023-09-23T19:56:57.483719+0000","last_became_active":"2023-09-23T19:56:39.283753+0000","last_became_peered":"2023-09-23T19:56:39.283753+0000","last_unstale":"2023-09-23T19:56:57.483719+0000","last_undegraded":"2023-09-23T19:56:57.483719+0000","last_fullsized":"2023-09-23T19:56:57.483719+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:51.940014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"168'1420","reported_seq":1433,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:07.578174+0000","last_change":"2023-09-23T19:56:51.332462+0000","last_active":"2023-09-23T19:57:07.578174+0000","last_peered":"2023-09-23T19:57:07.578174+0000","last_clean":"2023-09-23T19:57:07.578174+0000","last_became_active":"2023-09-23T19:56:51.332192+0000","last_became_peered":"2023-09-23T19:56:51.332192+0000","last_unstale":"2023-09-23T19:57:07.578174+0000","last_undegraded":"2023-09-23T19:57:07.578174+0000","last_fullsized":"2023-09-23T19:57:07.578174+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1420,"log_dups_size":0,"ondisk_log_size":1420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:42.318489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93061120,"num_objects":1420,"num_object_clones":0,"num_object_copies":4260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1420,"num_write_kb":90880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":479,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.485971+0000","last_change":"2023-09-23T19:52:58.803028+0000","last_active":"2023-09-23T19:56:57.485971+0000","last_peered":"2023-09-23T19:56:57.485971+0000","last_clean":"2023-09-23T19:56:57.485971+0000","last_became_active":"2023-09-23T19:52:57.353261+0000","last_became_peered":"2023-09-23T19:52:57.353261+0000","last_unstale":"2023-09-23T19:56:57.485971+0000","last_undegraded":"2023-09-23T19:56:57.485971+0000","last_fullsized":"2023-09-23T19:56:57.485971+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:53.333663+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":94,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484069+0000","last_change":"2023-09-23T19:56:39.280681+0000","last_active":"2023-09-23T19:56:57.484069+0000","last_peered":"2023-09-23T19:56:57.484069+0000","last_clean":"2023-09-23T19:56:57.484069+0000","last_became_active":"2023-09-23T19:56:39.280377+0000","last_became_peered":"2023-09-23T19:56:39.280377+0000","last_unstale":"2023-09-23T19:56:57.484069+0000","last_undegraded":"2023-09-23T19:56:57.484069+0000","last_fullsized":"2023-09-23T19:56:57.484069+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:42.514714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483778+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:56:57.483778+0000","last_peered":"2023-09-23T19:56:57.483778+0000","last_clean":"2023-09-23T19:56:57.483778+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:56:57.483778+0000","last_undegraded":"2023-09-23T19:56:57.483778+0000","last_fullsized":"2023-09-23T19:56:57.483778+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"168'1770","reported_seq":1784,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.735497+0000","last_change":"2023-09-23T19:56:51.334600+0000","last_active":"2023-09-23T19:57:10.735497+0000","last_peered":"2023-09-23T19:57:10.735497+0000","last_clean":"2023-09-23T19:57:10.735497+0000","last_became_active":"2023-09-23T19:56:51.334315+0000","last_became_peered":"2023-09-23T19:56:51.334315+0000","last_unstale":"2023-09-23T19:57:10.735497+0000","last_undegraded":"2023-09-23T19:57:10.735497+0000","last_fullsized":"2023-09-23T19:57:10.735497+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1770,"log_dups_size":0,"ondisk_log_size":1770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:47.078218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115998720,"num_objects":1770,"num_object_clones":0,"num_object_copies":5310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1770,"num_write_kb":113280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":128,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483520+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:56:57.483520+0000","last_peered":"2023-09-23T19:56:57.483520+0000","last_clean":"2023-09-23T19:56:57.483520+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:56:57.483520+0000","last_undegraded":"2023-09-23T19:56:57.483520+0000","last_fullsized":"2023-09-23T19:56:57.483520+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":154,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483853+0000","last_change":"2023-09-23T19:56:39.281487+0000","last_active":"2023-09-23T19:56:57.483853+0000","last_peered":"2023-09-23T19:56:57.483853+0000","last_clean":"2023-09-23T19:56:57.483853+0000","last_became_active":"2023-09-23T19:56:39.281248+0000","last_became_peered":"2023-09-23T19:56:39.281248+0000","last_unstale":"2023-09-23T19:56:57.483853+0000","last_undegraded":"2023-09-23T19:56:57.483853+0000","last_fullsized":"2023-09-23T19:56:57.483853+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:25.466372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"168'1473","reported_seq":1485,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:08.271019+0000","last_change":"2023-09-23T19:56:51.339206+0000","last_active":"2023-09-23T19:57:08.271019+0000","last_peered":"2023-09-23T19:57:08.271019+0000","last_clean":"2023-09-23T19:57:08.271019+0000","last_became_active":"2023-09-23T19:56:51.338963+0000","last_became_peered":"2023-09-23T19:56:51.338963+0000","last_unstale":"2023-09-23T19:57:08.271019+0000","last_undegraded":"2023-09-23T19:57:08.271019+0000","last_fullsized":"2023-09-23T19:57:08.271019+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1473,"log_dups_size":0,"ondisk_log_size":1473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:49.976592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1473,"num_write_kb":94272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":506,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486091+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T19:56:57.486091+0000","last_peered":"2023-09-23T19:56:57.486091+0000","last_clean":"2023-09-23T19:56:57.486091+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T19:56:57.486091+0000","last_undegraded":"2023-09-23T19:56:57.486091+0000","last_fullsized":"2023-09-23T19:56:57.486091+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.320089+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":132,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341103+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:56:56.341103+0000","last_peered":"2023-09-23T19:56:56.341103+0000","last_clean":"2023-09-23T19:56:56.341103+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:56:56.341103+0000","last_undegraded":"2023-09-23T19:56:56.341103+0000","last_fullsized":"2023-09-23T19:56:56.341103+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":107,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484193+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:56:57.484193+0000","last_peered":"2023-09-23T19:56:57.484193+0000","last_clean":"2023-09-23T19:56:57.484193+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:56:57.484193+0000","last_undegraded":"2023-09-23T19:56:57.484193+0000","last_fullsized":"2023-09-23T19:56:57.484193+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"168'1772","reported_seq":1786,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:11.593687+0000","last_change":"2023-09-23T19:56:51.330347+0000","last_active":"2023-09-23T19:57:11.593687+0000","last_peered":"2023-09-23T19:57:11.593687+0000","last_clean":"2023-09-23T19:57:11.593687+0000","last_became_active":"2023-09-23T19:56:51.330084+0000","last_became_peered":"2023-09-23T19:56:51.330084+0000","last_unstale":"2023-09-23T19:57:11.593687+0000","last_undegraded":"2023-09-23T19:57:11.593687+0000","last_fullsized":"2023-09-23T19:57:11.593687+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1772,"log_dups_size":0,"ondisk_log_size":1772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:08.671890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116129792,"num_objects":1772,"num_object_clones":0,"num_object_copies":5316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1772,"num_write_kb":113408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":480,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341741+0000","last_change":"2023-09-23T19:56:53.618146+0000","last_active":"2023-09-23T19:56:56.341741+0000","last_peered":"2023-09-23T19:56:56.341741+0000","last_clean":"2023-09-23T19:56:56.341741+0000","last_became_active":"2023-09-23T19:56:42.331700+0000","last_became_peered":"2023-09-23T19:56:42.331700+0000","last_unstale":"2023-09-23T19:56:56.341741+0000","last_undegraded":"2023-09-23T19:56:56.341741+0000","last_fullsized":"2023-09-23T19:56:56.341741+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:56:53.618048+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T19:56:53.618048+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.066107+0000","scrub_duration":0.001843293,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":503,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483974+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:56:57.483974+0000","last_peered":"2023-09-23T19:56:57.483974+0000","last_clean":"2023-09-23T19:56:57.483974+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:56:57.483974+0000","last_undegraded":"2023-09-23T19:56:57.483974+0000","last_fullsized":"2023-09-23T19:56:57.483974+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486513+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:56:57.486513+0000","last_peered":"2023-09-23T19:56:57.486513+0000","last_clean":"2023-09-23T19:56:57.486513+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:56:57.486513+0000","last_undegraded":"2023-09-23T19:56:57.486513+0000","last_fullsized":"2023-09-23T19:56:57.486513+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":92,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483824+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:56:57.483824+0000","last_peered":"2023-09-23T19:56:57.483824+0000","last_clean":"2023-09-23T19:56:57.483824+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:56:57.483824+0000","last_undegraded":"2023-09-23T19:56:57.483824+0000","last_fullsized":"2023-09-23T19:56:57.483824+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"168'1719","reported_seq":1733,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:10.738215+0000","last_change":"2023-09-23T19:56:51.333298+0000","last_active":"2023-09-23T19:57:10.738215+0000","last_peered":"2023-09-23T19:57:10.738215+0000","last_clean":"2023-09-23T19:57:10.738215+0000","last_became_active":"2023-09-23T19:56:51.333009+0000","last_became_peered":"2023-09-23T19:56:51.333009+0000","last_unstale":"2023-09-23T19:57:10.738215+0000","last_undegraded":"2023-09-23T19:57:10.738215+0000","last_fullsized":"2023-09-23T19:57:10.738215+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1719,"log_dups_size":0,"ondisk_log_size":1719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:12.985476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112656384,"num_objects":1719,"num_object_clones":0,"num_object_copies":5157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1719,"num_write_kb":110016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":479,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.485962+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T19:56:57.485962+0000","last_peered":"2023-09-23T19:56:57.485962+0000","last_clean":"2023-09-23T19:56:57.485962+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T19:56:57.485962+0000","last_undegraded":"2023-09-23T19:56:57.485962+0000","last_fullsized":"2023-09-23T19:56:57.485962+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":728,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486265+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T19:56:57.486265+0000","last_peered":"2023-09-23T19:56:57.486265+0000","last_clean":"2023-09-23T19:56:57.486265+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T19:56:57.486265+0000","last_undegraded":"2023-09-23T19:56:57.486265+0000","last_fullsized":"2023-09-23T19:56:57.486265+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:34.015520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":145,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484098+0000","last_change":"2023-09-23T19:56:39.283954+0000","last_active":"2023-09-23T19:56:57.484098+0000","last_peered":"2023-09-23T19:56:57.484098+0000","last_clean":"2023-09-23T19:56:57.484098+0000","last_became_active":"2023-09-23T19:56:39.283711+0000","last_became_peered":"2023-09-23T19:56:39.283711+0000","last_unstale":"2023-09-23T19:56:57.484098+0000","last_undegraded":"2023-09-23T19:56:57.484098+0000","last_fullsized":"2023-09-23T19:56:57.484098+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:55.890028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":122,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483928+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:56:57.483928+0000","last_peered":"2023-09-23T19:56:57.483928+0000","last_clean":"2023-09-23T19:56:57.483928+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:56:57.483928+0000","last_undegraded":"2023-09-23T19:56:57.483928+0000","last_fullsized":"2023-09-23T19:56:57.483928+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"168'1384","reported_seq":1397,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:08.208412+0000","last_change":"2023-09-23T19:56:51.329873+0000","last_active":"2023-09-23T19:57:08.208412+0000","last_peered":"2023-09-23T19:57:08.208412+0000","last_clean":"2023-09-23T19:57:08.208412+0000","last_became_active":"2023-09-23T19:56:51.329355+0000","last_became_peered":"2023-09-23T19:56:51.329355+0000","last_unstale":"2023-09-23T19:57:08.208412+0000","last_undegraded":"2023-09-23T19:57:08.208412+0000","last_fullsized":"2023-09-23T19:57:08.208412+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1384,"log_dups_size":0,"ondisk_log_size":1384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.152209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90701824,"num_objects":1384,"num_object_clones":0,"num_object_copies":4152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1384,"num_write_kb":88576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":548,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484801+0000","last_change":"2023-09-23T19:52:57.745972+0000","last_active":"2023-09-23T19:56:57.484801+0000","last_peered":"2023-09-23T19:56:57.484801+0000","last_clean":"2023-09-23T19:56:57.484801+0000","last_became_active":"2023-09-23T19:52:57.348142+0000","last_became_peered":"2023-09-23T19:52:57.348142+0000","last_unstale":"2023-09-23T19:56:57.484801+0000","last_undegraded":"2023-09-23T19:56:57.484801+0000","last_fullsized":"2023-09-23T19:56:57.484801+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.745894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.745894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.660195+0000","scrub_duration":0.0079262489999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":653,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.343038+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:56:56.343038+0000","last_peered":"2023-09-23T19:56:56.343038+0000","last_clean":"2023-09-23T19:56:56.343038+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:56:56.343038+0000","last_undegraded":"2023-09-23T19:56:56.343038+0000","last_fullsized":"2023-09-23T19:56:56.343038+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483485+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T19:56:57.483485+0000","last_peered":"2023-09-23T19:56:57.483485+0000","last_clean":"2023-09-23T19:56:57.483485+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T19:56:57.483485+0000","last_undegraded":"2023-09-23T19:56:57.483485+0000","last_fullsized":"2023-09-23T19:56:57.483485+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:46.217415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":136,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484151+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:56:57.484151+0000","last_peered":"2023-09-23T19:56:57.484151+0000","last_clean":"2023-09-23T19:56:57.484151+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:56:57.484151+0000","last_undegraded":"2023-09-23T19:56:57.484151+0000","last_fullsized":"2023-09-23T19:56:57.484151+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"168'1427","reported_seq":1439,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:08.276956+0000","last_change":"2023-09-23T19:56:51.335393+0000","last_active":"2023-09-23T19:57:08.276956+0000","last_peered":"2023-09-23T19:57:08.276956+0000","last_clean":"2023-09-23T19:57:08.276956+0000","last_became_active":"2023-09-23T19:56:51.335106+0000","last_became_peered":"2023-09-23T19:56:51.335106+0000","last_unstale":"2023-09-23T19:57:08.276956+0000","last_undegraded":"2023-09-23T19:57:08.276956+0000","last_fullsized":"2023-09-23T19:57:08.276956+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1427,"log_dups_size":0,"ondisk_log_size":1427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.493039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93519872,"num_objects":1427,"num_object_clones":0,"num_object_copies":4281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1427,"num_write_kb":91328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":976,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486273+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:56:57.486273+0000","last_peered":"2023-09-23T19:56:57.486273+0000","last_clean":"2023-09-23T19:56:57.486273+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:56:57.486273+0000","last_undegraded":"2023-09-23T19:56:57.486273+0000","last_fullsized":"2023-09-23T19:56:57.486273+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":556,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484878+0000","last_change":"2023-09-23T19:56:42.408346+0000","last_active":"2023-09-23T19:56:57.484878+0000","last_peered":"2023-09-23T19:56:57.484878+0000","last_clean":"2023-09-23T19:56:57.484878+0000","last_became_active":"2023-09-23T19:56:42.407370+0000","last_became_peered":"2023-09-23T19:56:42.407370+0000","last_unstale":"2023-09-23T19:56:57.484878+0000","last_undegraded":"2023-09-23T19:56:57.484878+0000","last_fullsized":"2023-09-23T19:56:57.484878+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:38.144096+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":97,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:04.448486+0000","last_change":"2023-09-23T19:56:39.277079+0000","last_active":"2023-09-23T19:57:04.448486+0000","last_peered":"2023-09-23T19:57:04.448486+0000","last_clean":"2023-09-23T19:57:04.448486+0000","last_became_active":"2023-09-23T19:56:39.276787+0000","last_became_peered":"2023-09-23T19:56:39.276787+0000","last_unstale":"2023-09-23T19:57:04.448486+0000","last_undegraded":"2023-09-23T19:57:04.448486+0000","last_fullsized":"2023-09-23T19:57:04.448486+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:00.570242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":43,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484018+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T19:56:57.484018+0000","last_peered":"2023-09-23T19:56:57.484018+0000","last_clean":"2023-09-23T19:56:57.484018+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T19:56:57.484018+0000","last_undegraded":"2023-09-23T19:56:57.484018+0000","last_fullsized":"2023-09-23T19:56:57.484018+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:31.068090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"168'1591","reported_seq":1604,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:09.531747+0000","last_change":"2023-09-23T19:56:51.335699+0000","last_active":"2023-09-23T19:57:09.531747+0000","last_peered":"2023-09-23T19:57:09.531747+0000","last_clean":"2023-09-23T19:57:09.531747+0000","last_became_active":"2023-09-23T19:56:51.335366+0000","last_became_peered":"2023-09-23T19:56:51.335366+0000","last_unstale":"2023-09-23T19:57:09.531747+0000","last_undegraded":"2023-09-23T19:57:09.531747+0000","last_fullsized":"2023-09-23T19:57:09.531747+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1591,"log_dups_size":0,"ondisk_log_size":1591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:18.373892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104267776,"num_objects":1591,"num_object_clones":0,"num_object_copies":4773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1591,"num_write_kb":101824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":505,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:04.455776+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T19:57:04.455776+0000","last_peered":"2023-09-23T19:57:04.455776+0000","last_clean":"2023-09-23T19:57:04.455776+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T19:57:04.455776+0000","last_undegraded":"2023-09-23T19:57:04.455776+0000","last_fullsized":"2023-09-23T19:57:04.455776+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:32.858007+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":886,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484873+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:56:57.484873+0000","last_peered":"2023-09-23T19:56:57.484873+0000","last_clean":"2023-09-23T19:56:57.484873+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:56:57.484873+0000","last_undegraded":"2023-09-23T19:56:57.484873+0000","last_fullsized":"2023-09-23T19:56:57.484873+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":112,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.387532+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:56:57.387532+0000","last_peered":"2023-09-23T19:56:57.387532+0000","last_clean":"2023-09-23T19:56:57.387532+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:56:57.387532+0000","last_undegraded":"2023-09-23T19:56:57.387532+0000","last_fullsized":"2023-09-23T19:56:57.387532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":125,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.388089+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:56:57.388089+0000","last_peered":"2023-09-23T19:56:57.388089+0000","last_clean":"2023-09-23T19:56:57.388089+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:56:57.388089+0000","last_undegraded":"2023-09-23T19:56:57.388089+0000","last_fullsized":"2023-09-23T19:56:57.388089+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"168'1595","reported_seq":1608,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:09.524212+0000","last_change":"2023-09-23T19:56:51.329836+0000","last_active":"2023-09-23T19:57:09.524212+0000","last_peered":"2023-09-23T19:57:09.524212+0000","last_clean":"2023-09-23T19:57:09.524212+0000","last_became_active":"2023-09-23T19:56:51.329518+0000","last_became_peered":"2023-09-23T19:56:51.329518+0000","last_unstale":"2023-09-23T19:57:09.524212+0000","last_undegraded":"2023-09-23T19:57:09.524212+0000","last_fullsized":"2023-09-23T19:57:09.524212+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1595,"log_dups_size":0,"ondisk_log_size":1595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:38.536519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104529920,"num_objects":1595,"num_object_clones":0,"num_object_copies":4785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1595,"num_write_kb":102080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":473,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.387962+0000","last_change":"2023-09-23T19:56:54.487207+0000","last_active":"2023-09-23T19:56:57.387962+0000","last_peered":"2023-09-23T19:56:57.387962+0000","last_clean":"2023-09-23T19:56:57.387962+0000","last_became_active":"2023-09-23T19:56:42.420361+0000","last_became_peered":"2023-09-23T19:56:42.420361+0000","last_unstale":"2023-09-23T19:56:57.387962+0000","last_undegraded":"2023-09-23T19:56:57.387962+0000","last_fullsized":"2023-09-23T19:56:57.387962+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:56:54.487110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:56:54.487110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:19.686156+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":246,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.485500+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:56:57.485500+0000","last_peered":"2023-09-23T19:56:57.485500+0000","last_clean":"2023-09-23T19:56:57.485500+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:56:57.485500+0000","last_undegraded":"2023-09-23T19:56:57.485500+0000","last_fullsized":"2023-09-23T19:56:57.485500+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":106,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484049+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:56:57.484049+0000","last_peered":"2023-09-23T19:56:57.484049+0000","last_clean":"2023-09-23T19:56:57.484049+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:56:57.484049+0000","last_undegraded":"2023-09-23T19:56:57.484049+0000","last_fullsized":"2023-09-23T19:56:57.484049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":98,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.388252+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:56:57.388252+0000","last_peered":"2023-09-23T19:56:57.388252+0000","last_clean":"2023-09-23T19:56:57.388252+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:56:57.388252+0000","last_undegraded":"2023-09-23T19:56:57.388252+0000","last_fullsized":"2023-09-23T19:56:57.388252+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"168'1417","reported_seq":1429,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:08.240222+0000","last_change":"2023-09-23T19:56:51.332314+0000","last_active":"2023-09-23T19:57:08.240222+0000","last_peered":"2023-09-23T19:57:08.240222+0000","last_clean":"2023-09-23T19:57:08.240222+0000","last_became_active":"2023-09-23T19:56:51.332106+0000","last_became_peered":"2023-09-23T19:56:51.332106+0000","last_unstale":"2023-09-23T19:57:08.240222+0000","last_undegraded":"2023-09-23T19:57:08.240222+0000","last_fullsized":"2023-09-23T19:57:08.240222+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1417,"log_dups_size":0,"ondisk_log_size":1417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:38.454829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92864512,"num_objects":1417,"num_object_clones":0,"num_object_copies":4251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1417,"num_write_kb":90688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":506,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483688+0000","last_change":"2023-09-23T19:56:42.336491+0000","last_active":"2023-09-23T19:56:57.483688+0000","last_peered":"2023-09-23T19:56:57.483688+0000","last_clean":"2023-09-23T19:56:57.483688+0000","last_became_active":"2023-09-23T19:56:42.336151+0000","last_became_peered":"2023-09-23T19:56:42.336151+0000","last_unstale":"2023-09-23T19:56:57.483688+0000","last_undegraded":"2023-09-23T19:56:57.483688+0000","last_fullsized":"2023-09-23T19:56:57.483688+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:52.411762+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":129,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341881+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:56:56.341881+0000","last_peered":"2023-09-23T19:56:56.341881+0000","last_clean":"2023-09-23T19:56:56.341881+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:56:56.341881+0000","last_undegraded":"2023-09-23T19:56:56.341881+0000","last_fullsized":"2023-09-23T19:56:56.341881+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":480,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484286+0000","last_change":"2023-09-23T19:56:42.312115+0000","last_active":"2023-09-23T19:56:57.484286+0000","last_peered":"2023-09-23T19:56:57.484286+0000","last_clean":"2023-09-23T19:56:57.484286+0000","last_became_active":"2023-09-23T19:56:42.290218+0000","last_became_peered":"2023-09-23T19:56:42.290218+0000","last_unstale":"2023-09-23T19:56:57.484286+0000","last_undegraded":"2023-09-23T19:56:57.484286+0000","last_fullsized":"2023-09-23T19:56:57.484286+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:24.023589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341198+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:56:56.341198+0000","last_peered":"2023-09-23T19:56:56.341198+0000","last_clean":"2023-09-23T19:56:56.341198+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:56:56.341198+0000","last_undegraded":"2023-09-23T19:56:56.341198+0000","last_fullsized":"2023-09-23T19:56:56.341198+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"168'1477","reported_seq":1490,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:08.169696+0000","last_change":"2023-09-23T19:56:51.331630+0000","last_active":"2023-09-23T19:57:08.169696+0000","last_peered":"2023-09-23T19:57:08.169696+0000","last_clean":"2023-09-23T19:57:08.169696+0000","last_became_active":"2023-09-23T19:56:51.331338+0000","last_became_peered":"2023-09-23T19:56:51.331338+0000","last_unstale":"2023-09-23T19:57:08.169696+0000","last_undegraded":"2023-09-23T19:57:08.169696+0000","last_fullsized":"2023-09-23T19:57:08.169696+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":1477,"log_dups_size":0,"ondisk_log_size":1477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:45.394928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96796672,"num_objects":1477,"num_object_clones":0,"num_object_copies":4431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1477,"num_write_kb":94528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":484,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T19:57:04.448811+0000","last_change":"2023-09-23T19:53:00.705074+0000","last_active":"2023-09-23T19:57:04.448811+0000","last_peered":"2023-09-23T19:57:04.448811+0000","last_clean":"2023-09-23T19:57:04.448811+0000","last_became_active":"2023-09-23T19:52:57.353333+0000","last_became_peered":"2023-09-23T19:52:57.353333+0000","last_unstale":"2023-09-23T19:57:04.448811+0000","last_undegraded":"2023-09-23T19:57:04.448811+0000","last_fullsized":"2023-09-23T19:57:04.448811+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T19:53:00.704900+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:24.642289+0000","scrub_duration":0.0090978159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":330,"reported_epoch":167,"state":"active+undersized","last_fresh":"2023-09-23T19:56:57.485679+0000","last_change":"2023-09-23T19:54:45.328104+0000","last_active":"2023-09-23T19:56:57.485679+0000","last_peered":"2023-09-23T19:56:57.485679+0000","last_clean":"2023-09-23T19:43:08.441095+0000","last_became_active":"2023-09-23T19:54:45.328104+0000","last_became_peered":"2023-09-23T19:54:45.328104+0000","last_unstale":"2023-09-23T19:56:57.485679+0000","last_undegraded":"2023-09-23T19:56:57.485679+0000","last_fullsized":"2023-09-23T19:54:45.325453+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:40.212909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":115,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.388102+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:56:57.388102+0000","last_peered":"2023-09-23T19:56:57.388102+0000","last_clean":"2023-09-23T19:56:57.388102+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:56:57.388102+0000","last_undegraded":"2023-09-23T19:56:57.388102+0000","last_fullsized":"2023-09-23T19:56:57.388102+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":117,"reported_epoch":167,"state":"active+undersized","last_fresh":"2023-09-23T19:56:57.483550+0000","last_change":"2023-09-23T19:54:45.330341+0000","last_active":"2023-09-23T19:56:57.483550+0000","last_peered":"2023-09-23T19:56:57.483550+0000","last_clean":"2023-09-23T19:54:37.305426+0000","last_became_active":"2023-09-23T19:54:45.330341+0000","last_became_peered":"2023-09-23T19:54:45.330341+0000","last_unstale":"2023-09-23T19:56:57.483550+0000","last_undegraded":"2023-09-23T19:56:57.483550+0000","last_fullsized":"2023-09-23T19:54:45.253699+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:22.534625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":100,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.388214+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:56:57.388214+0000","last_peered":"2023-09-23T19:56:57.388214+0000","last_clean":"2023-09-23T19:56:57.388214+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:56:57.388214+0000","last_undegraded":"2023-09-23T19:56:57.388214+0000","last_fullsized":"2023-09-23T19:56:57.388214+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":150,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.343320+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:56:56.343320+0000","last_peered":"2023-09-23T19:56:56.343320+0000","last_clean":"2023-09-23T19:56:56.343320+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:56:56.343320+0000","last_undegraded":"2023-09-23T19:56:56.343320+0000","last_fullsized":"2023-09-23T19:56:56.343320+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":129,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486555+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:56:57.486555+0000","last_peered":"2023-09-23T19:56:57.486555+0000","last_clean":"2023-09-23T19:56:57.486555+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:56:57.486555+0000","last_undegraded":"2023-09-23T19:56:57.486555+0000","last_fullsized":"2023-09-23T19:56:57.486555+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.484125+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:56:57.484125+0000","last_peered":"2023-09-23T19:56:57.484125+0000","last_clean":"2023-09-23T19:56:57.484125+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:56:57.484125+0000","last_undegraded":"2023-09-23T19:56:57.484125+0000","last_fullsized":"2023-09-23T19:56:57.484125+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":133,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.486536+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T19:56:57.486536+0000","last_peered":"2023-09-23T19:56:57.486536+0000","last_clean":"2023-09-23T19:56:57.486536+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T19:56:57.486536+0000","last_undegraded":"2023-09-23T19:56:57.486536+0000","last_fullsized":"2023-09-23T19:56:57.486536+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:03.354856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483962+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:56:57.483962+0000","last_peered":"2023-09-23T19:56:57.483962+0000","last_clean":"2023-09-23T19:56:57.483962+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:56:57.483962+0000","last_undegraded":"2023-09-23T19:56:57.483962+0000","last_fullsized":"2023-09-23T19:56:57.483962+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":153,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.342268+0000","last_change":"2023-09-23T19:56:39.276333+0000","last_active":"2023-09-23T19:56:56.342268+0000","last_peered":"2023-09-23T19:56:56.342268+0000","last_clean":"2023-09-23T19:56:56.342268+0000","last_became_active":"2023-09-23T19:56:39.275998+0000","last_became_peered":"2023-09-23T19:56:39.275998+0000","last_unstale":"2023-09-23T19:56:56.342268+0000","last_undegraded":"2023-09-23T19:56:56.342268+0000","last_fullsized":"2023-09-23T19:56:56.342268+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:48.085884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":152,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:56.341196+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:56:56.341196+0000","last_peered":"2023-09-23T19:56:56.341196+0000","last_clean":"2023-09-23T19:56:56.341196+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:56:56.341196+0000","last_undegraded":"2023-09-23T19:56:56.341196+0000","last_fullsized":"2023-09-23T19:56:56.341196+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":116,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.388003+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:56:57.388003+0000","last_peered":"2023-09-23T19:56:57.388003+0000","last_clean":"2023-09-23T19:56:57.388003+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:56:57.388003+0000","last_undegraded":"2023-09-23T19:56:57.388003+0000","last_fullsized":"2023-09-23T19:56:57.388003+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T19:56:57.483499+0000","last_change":"2023-09-23T19:56:39.281130+0000","last_active":"2023-09-23T19:56:57.483499+0000","last_peered":"2023-09-23T19:56:57.483499+0000","last_clean":"2023-09-23T19:56:57.483499+0000","last_became_active":"2023-09-23T19:56:39.280800+0000","last_became_peered":"2023-09-23T19:56:39.280800+0000","last_unstale":"2023-09-23T19:56:57.483499+0000","last_undegraded":"2023-09-23T19:56:57.483499+0000","last_fullsized":"2023-09-23T19:56:57.483499+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:03.511985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1620967424,"num_objects":24734,"num_object_clones":0,"num_object_copies":74202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":24734,"num_write_kb":1582976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4815585280,"data_stored":4815585280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":24734,"ondisk_log_size":24734,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":93,"acting":93,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234192,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":815180,"kb_used_data":728516,"kb_used_omap":0,"kb_used_meta":86656,"kb_avail":92929972,"statfs":{"total":95995035648,"available":95160291328,"internally_reserved":0,"allocated":746000384,"data_stored":745446974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":88735744},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":2.0009999999999999,"15min":1.0660000000000001},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.3470000000000004,"5min":46.476999999999997,"15min":46.476999999999997},"last":1.077},{"interface":"front","average":{"1min":2.528,"5min":2.669,"15min":1.3120000000000001},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":7.4580000000000002,"5min":46.823999999999998,"15min":46.823999999999998},"last":0.28299999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.7,"15min":0.96399999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.7519999999999998,"5min":11.093999999999999,"15min":11.093999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.262,"5min":1.8069999999999999,"15min":0.99299999999999999},"min":{"1min":0.247,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.2300000000000004,"5min":14.747,"15min":14.747},"last":0.82399999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:56:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":4.0149999999999997,"15min":1.907},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.4499999999999993,"5min":130.673,"15min":130.673},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.6429999999999998,"5min":3.976,"15min":1.716},"min":{"1min":0.36299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.3979999999999997,"5min":130.71100000000001,"15min":130.71100000000001},"last":0.61799999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:56:07 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.7390000000000001,"15min":1.137},"min":{"1min":0.33000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.7309999999999999,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":0.439},{"interface":"front","average":{"1min":2.6509999999999998,"5min":1.577,"15min":0.95499999999999996},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.5490000000000004,"5min":7.5490000000000004,"15min":7.5490000000000004},"last":0.497}]},{"osd":4,"last update":"Sat Sep 23 19:56:35 2023","interfaces":[{"interface":"back","average":{"1min":9.3689999999999998,"5min":4.0380000000000003,"15min":1.6699999999999999},"min":{"1min":0.29799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":143.255,"5min":143.255,"15min":143.255},"last":0.32700000000000001},{"interface":"front","average":{"1min":8.1479999999999997,"5min":3.8460000000000001,"15min":1.6439999999999999},"min":{"1min":0.30599999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":143.41200000000001,"5min":143.41200000000001,"15min":143.41200000000001},"last":0.47299999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:56:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":2.4340000000000002,"15min":1.167},"min":{"1min":0.32100000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.972,"5min":84.566999999999993,"15min":84.566999999999993},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.2229999999999999,"15min":1.113},"min":{"1min":0.32600000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8979999999999997,"5min":81.244,"15min":81.244},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.0189999999999999,"15min":0.71499999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.6449999999999996,"5min":5.6449999999999996,"15min":5.6449999999999996},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.121,"15min":0.76500000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.7720000000000002,"5min":5.7720000000000002,"15min":5.7720000000000002},"last":0.38900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495800,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379500,"kb_used_data":1334692,"kb_used_omap":0,"kb_used_meta":44800,"kb_avail":92365652,"statfs":{"total":95995035648,"available":94582427648,"internally_reserved":0,"allocated":1366724608,"data_stored":1366198307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":45875200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:52 2023","interfaces":[{"interface":"back","average":{"1min":4.415,"5min":1.3500000000000001,"15min":0.85199999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":60.738,"5min":60.738,"15min":60.738},"last":1.476},{"interface":"front","average":{"1min":4.202,"5min":1.333,"15min":0.88700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":60.722999999999999,"5min":60.722999999999999,"15min":60.722999999999999},"last":1.377}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.85799999999999998,"15min":0.69999999999999996},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6520000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.395,"5min":0.94299999999999995,"15min":0.69099999999999995},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":4.3029999999999999,"5min":2.7829999999999999,"15min":1.3149999999999999},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.245},"max":{"1min":53.667999999999999,"5min":61.155000000000001,"15min":61.155000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":4.024,"5min":2.7989999999999999,"15min":1.417},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":53.628,"5min":60.506,"15min":60.506},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":0.97899999999999998,"15min":0.76700000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":5.7619999999999996},"last":1.361},{"interface":"front","average":{"1min":2.1379999999999999,"5min":0.94299999999999995,"15min":0.69299999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":1.387}]},{"osd":4,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":1.0740000000000001,"15min":0.68600000000000005},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.4859999999999998,"5min":24.003,"15min":24.003},"last":0.377},{"interface":"front","average":{"1min":1.607,"5min":1.0920000000000001,"15min":0.70199999999999996},"min":{"1min":0.378,"5min":0.313,"15min":0.26500000000000001},"max":{"1min":6.6020000000000003,"5min":24.802,"15min":24.802},"last":1.3149999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.147,"15min":0.71099999999999997},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":14.378,"5min":24.065999999999999,"15min":24.065999999999999},"last":1.4079999999999999},{"interface":"front","average":{"1min":1.641,"5min":1.1359999999999999,"15min":0.74399999999999999},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":6.0529999999999999,"5min":24.841000000000001,"15min":24.841000000000001},"last":1.2769999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:54:39 2023","interfaces":[{"interface":"back","average":{"1min":6.1509999999999998,"5min":1.8149999999999999,"15min":0.98099999999999998},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":105.71299999999999,"5min":105.71299999999999,"15min":105.71299999999999},"last":2.1579999999999999},{"interface":"front","average":{"1min":6.5620000000000003,"5min":2.0259999999999998,"15min":1.075},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":103.206,"5min":103.206,"15min":103.206},"last":0.46100000000000002}]}]},{"osd":1,"up_from":99,"seq":425201762394,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":725000,"kb_used_data":598976,"kb_used_omap":7,"kb_used_meta":126008,"kb_avail":93020152,"statfs":{"total":95995035648,"available":95252635648,"internally_reserved":0,"allocated":613351424,"data_stored":612627425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":129033018},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":9.8149999999999995,"5min":7.5449999999999999,"15min":2.7530000000000001},"min":{"1min":0.151,"5min":0.151,"15min":0.151},"max":{"1min":186.84,"5min":202.917,"15min":202.917},"last":1.6559999999999999},{"interface":"front","average":{"1min":10.493,"5min":7.9180000000000001,"15min":2.8820000000000001},"min":{"1min":0.17999999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":191.322,"5min":203.34899999999999,"15min":203.34899999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":13.922000000000001,"5min":8.3569999999999993,"15min":3.0179999999999998},"min":{"1min":0.252,"5min":0.214,"15min":0.192},"max":{"1min":180.25200000000001,"5min":231.57499999999999,"15min":231.57499999999999},"last":5.1909999999999998},{"interface":"front","average":{"1min":12.314,"5min":7.548,"15min":2.75},"min":{"1min":0.22500000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":180.05000000000001,"5min":236.077,"15min":236.077},"last":1.708}]},{"osd":3,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1030000000000002,"5min":2.6389999999999998,"15min":1.127},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":26.388000000000002,"5min":26.388000000000002,"15min":26.388000000000002},"last":4.0469999999999997},{"interface":"front","average":{"1min":1.748,"5min":1.9239999999999999,"15min":0.90800000000000003},"min":{"1min":0.24299999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.507,"5min":20.166,"15min":20.166},"last":0.26300000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":3.71,"15min":1.6659999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":7.2149999999999999,"5min":161.68299999999999,"15min":161.68299999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":3.4990000000000001,"15min":1.5720000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.944,"5min":161.286,"15min":161.286},"last":0.38200000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":11.356999999999999,"5min":5.8170000000000002,"15min":2.3610000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":219.048,"5min":219.048,"15min":219.048},"last":0.41099999999999998},{"interface":"front","average":{"1min":12.51,"5min":6.4039999999999999,"15min":2.5659999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":218.69499999999999,"5min":218.69499999999999,"15min":218.69499999999999},"last":4.1159999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.097,"15min":0.80500000000000005},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.9649999999999999,"5min":7.9649999999999999,"15min":7.9649999999999999},"last":14.897},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.0800000000000001,"15min":0.82399999999999995},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":14.904999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.6400000000000001,"15min":1.3120000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":26.327999999999999,"5min":26.327999999999999,"15min":26.327999999999999},"last":4.085},{"interface":"front","average":{"1min":1.393,"5min":1.599,"15min":0.97599999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.1389999999999993,"5min":9.1389999999999993,"15min":9.1389999999999993},"last":1.5469999999999999}]}]},{"osd":0,"up_from":94,"seq":403726925929,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":604668,"kb_used_data":460596,"kb_used_omap":8,"kb_used_meta":144055,"kb_avail":93140484,"statfs":{"total":95995035648,"available":95375855616,"internally_reserved":0,"allocated":471650304,"data_stored":470937040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":147512929},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:56:08 2023","interfaces":[{"interface":"back","average":{"1min":4.835,"5min":1.6899999999999999,"15min":0.77900000000000003},"min":{"1min":0.21099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":74.751000000000005,"5min":74.751000000000005,"15min":74.751000000000005},"last":0.27000000000000002},{"interface":"front","average":{"1min":7.2869999999999999,"5min":2.298,"15min":0.99199999999999999},"min":{"1min":0.34799999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":74.998000000000005,"5min":74.998000000000005,"15min":74.998000000000005},"last":2.4529999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:56:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":3.0600000000000001,"15min":1.3029999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.7309999999999999,"5min":74.795000000000002,"15min":74.795000000000002},"last":1.649},{"interface":"front","average":{"1min":3.512,"5min":3.569,"15min":1.4410000000000001},"min":{"1min":0.245,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":20.189,"5min":78.366,"15min":78.366},"last":1.3220000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:56:08 2023","interfaces":[{"interface":"back","average":{"1min":7.149,"5min":3.4350000000000001,"15min":1.397},"min":{"1min":0.25800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":75.037999999999997,"5min":104.084,"15min":104.084},"last":1.6160000000000001},{"interface":"front","average":{"1min":6.9950000000000001,"5min":3.6160000000000001,"15min":1.454},"min":{"1min":0.40400000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":74.965999999999994,"5min":104.143,"15min":104.143},"last":2.246}]},{"osd":4,"last update":"Sat Sep 23 19:56:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":3.7610000000000001,"15min":1.679},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.952,"5min":110.312,"15min":110.312},"last":0.82699999999999996},{"interface":"front","average":{"1min":2.1749999999999998,"5min":3.4540000000000002,"15min":1.597},"min":{"1min":0.32800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.8140000000000001,"5min":110.256,"15min":110.256},"last":0.56799999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:56:52 2023","interfaces":[{"interface":"back","average":{"1min":8.9670000000000005,"5min":3.6469999999999998,"15min":1.6799999999999999},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":131.34399999999999,"5min":131.34399999999999,"15min":131.34399999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":9.2690000000000001,"5min":4.1159999999999997,"15min":1.835},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":131.386,"5min":131.386,"15min":131.386},"last":0.85599999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":0.89700000000000002,"15min":0.78300000000000003},"min":{"1min":0.40899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.9139999999999997,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":2.4100000000000001},{"interface":"front","average":{"1min":2.3410000000000002,"5min":0.996,"15min":0.81000000000000005},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.6920000000000002,"5min":8.6920000000000002,"15min":8.6920000000000002},"last":1.1679999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:56:52 2023","interfaces":[{"interface":"back","average":{"1min":1.415,"5min":2.431,"15min":1.2909999999999999},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.8490000000000002,"5min":77.132999999999996,"15min":77.132999999999996},"last":1.6850000000000001},{"interface":"front","average":{"1min":2.4119999999999999,"5min":2.855,"15min":1.425},"min":{"1min":0.34699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.9809999999999999,"5min":75.465999999999994,"15min":75.465999999999994},"last":1.3680000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598867,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1009988,"kb_used_data":861884,"kb_used_omap":12,"kb_used_meta":148083,"kb_avail":92735164,"statfs":{"total":95995035648,"available":94960807936,"internally_reserved":0,"allocated":882569216,"data_stored":881851201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":151638014},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":4.8700000000000001,"5min":2.7709999999999999,"15min":1.1579999999999999},"min":{"1min":0.154,"5min":0.154,"15min":0.154},"max":{"1min":50.506999999999998,"5min":50.506999999999998,"15min":50.506999999999998},"last":0.317},{"interface":"front","average":{"1min":5.5460000000000003,"5min":2.827,"15min":1.1910000000000001},"min":{"1min":0.21299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":50.802999999999997,"5min":50.802999999999997,"15min":50.802999999999997},"last":0.20899999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":4.3010000000000002,"5min":2.3220000000000001,"15min":1.0289999999999999},"min":{"1min":0.22900000000000001,"5min":0.215,"15min":0.214},"max":{"1min":50.061,"5min":50.061,"15min":50.061},"last":1.8779999999999999},{"interface":"front","average":{"1min":4.931,"5min":2.419,"15min":1.0649999999999999},"min":{"1min":0.26700000000000002,"5min":0.217,"15min":0.217},"max":{"1min":50.533999999999999,"5min":50.533999999999999,"15min":50.533999999999999},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:56:09 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":1.6639999999999999,"15min":0.81399999999999995},"min":{"1min":0.219,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":11.263,"5min":11.263,"15min":11.263},"last":1.921},{"interface":"front","average":{"1min":2.653,"5min":1.696,"15min":0.82999999999999996},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":18.765999999999998,"5min":18.765999999999998,"15min":18.765999999999998},"last":0.61599999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":4.3940000000000001,"5min":2.2490000000000001,"15min":1.173},"min":{"1min":0.33000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":50.640000000000001,"5min":50.640000000000001,"15min":50.640000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.6349999999999998,"5min":2.3900000000000001,"15min":1.2110000000000001},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":50.154000000000003,"5min":50.154000000000003,"15min":50.154000000000003},"last":1.903}]},{"osd":5,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":4.7949999999999999,"5min":2.681,"15min":1.3169999999999999},"min":{"1min":0.34599999999999997,"5min":0.308,"15min":0.308},"max":{"1min":50.616,"5min":50.616,"15min":50.616},"last":0.39300000000000002},{"interface":"front","average":{"1min":4.2960000000000003,"5min":2.3540000000000001,"15min":1.204},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":50.762,"5min":50.762,"15min":50.762},"last":2.2519999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.827,"5min":1.343,"15min":0.84999999999999998},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4740000000000002,"5min":7.4740000000000002,"15min":7.4740000000000002},"last":1.0660000000000001},{"interface":"front","average":{"1min":2.899,"5min":1.3120000000000001,"15min":0.85699999999999998},"min":{"1min":0.42399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.3029999999999999,"5min":7.3029999999999999,"15min":7.3029999999999999},"last":1.008}]},{"osd":7,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":3.7389999999999999,"5min":2.4529999999999998,"15min":1.2589999999999999},"min":{"1min":0.28799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":50.118000000000002,"5min":50.118000000000002,"15min":50.118000000000002},"last":2.1070000000000002},{"interface":"front","average":{"1min":4.3250000000000002,"5min":2.839,"15min":1.393},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":50.664999999999999,"5min":50.664999999999999,"15min":50.664999999999999},"last":0.51600000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435341,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1203488,"kb_used_data":1091480,"kb_used_omap":4,"kb_used_meta":111995,"kb_avail":92541664,"statfs":{"total":95995035648,"available":94762663936,"internally_reserved":0,"allocated":1117675520,"data_stored":1116928238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":114682989},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,6,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":7.2359999999999998,"15min":2.6469999999999998},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":3.2509999999999999,"5min":240.40600000000001,"15min":240.40600000000001},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.8779999999999999,"5min":7.8609999999999998,"15min":2.8620000000000001},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.1329999999999991,"5min":240.352,"15min":240.352},"last":1.696}]},{"osd":1,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.5860000000000001,"15min":0.78400000000000003},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.1619999999999999,"5min":8.5289999999999999,"15min":8.5289999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.4239999999999999,"15min":0.72799999999999998},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.6699999999999999,"5min":11.361000000000001,"15min":11.361000000000001},"last":1.2849999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":6.423,"5min":10.749000000000001,"15min":3.8420000000000001},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":83.590000000000003,"5min":264.97800000000001,"15min":264.97800000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":7.7910000000000004,"5min":10.734999999999999,"15min":3.8530000000000002},"min":{"1min":0.26100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":83.344999999999999,"5min":257.40899999999999,"15min":257.40899999999999},"last":1.2310000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":2.0649999999999999,"15min":1.077},"min":{"1min":0.30199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.7869999999999999,"5min":69.236000000000004,"15min":69.236000000000004},"last":1.1950000000000001},{"interface":"front","average":{"1min":1.714,"5min":2.7189999999999999,"15min":1.2969999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.2420000000000009,"5min":69.373000000000005,"15min":69.373000000000005},"last":0.47399999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":1.8480000000000001,"15min":1.0169999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.0830000000000002,"5min":13.185,"15min":13.185},"last":0.505},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.694,"15min":0.98399999999999999},"min":{"1min":0.35099999999999998,"5min":0.313,"15min":0.313},"max":{"1min":2.4089999999999998,"5min":10.977,"15min":10.977},"last":0.58699999999999997}]},{"osd":6,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":3.4809999999999999,"5min":1.2849999999999999,"15min":0.82099999999999995},"min":{"1min":0.38600000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.4399999999999995,"5min":9.4399999999999995,"15min":9.4399999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.1179999999999999,"5min":0.94599999999999995,"15min":0.72099999999999997},"min":{"1min":0.44400000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":10.932,"5min":10.932,"15min":10.932},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.784,"15min":1.0069999999999999},"min":{"1min":0.38,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.8839999999999999,"5min":12.720000000000001,"15min":12.720000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.4770000000000001,"15min":0.91100000000000003},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8199999999999998,"5min":9.8369999999999997,"15min":9.8369999999999997},"last":1.252}]}]},{"osd":4,"up_from":30,"seq":128849019102,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":570504,"kb_used_data":441920,"kb_used_omap":0,"kb_used_meta":128576,"kb_avail":93174648,"statfs":{"total":95995035648,"available":95410839552,"internally_reserved":0,"allocated":452526080,"data_stored":451976754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":131661824},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:54 2023","interfaces":[{"interface":"back","average":{"1min":6.6820000000000004,"5min":4.9279999999999999,"15min":2.0409999999999999},"min":{"1min":0.23300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":104.604,"5min":104.604,"15min":104.604},"last":1.6819999999999999},{"interface":"front","average":{"1min":6.8040000000000003,"5min":4.8109999999999999,"15min":2.0270000000000001},"min":{"1min":0.29699999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":104.687,"5min":104.687,"15min":104.687},"last":1.8200000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":2.1099999999999999,"15min":1.073},"min":{"1min":0.23400000000000001,"5min":0.216,"15min":0.216},"max":{"1min":12.718999999999999,"5min":12.718999999999999,"15min":12.718999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.5109999999999999,"15min":0.879},"min":{"1min":0.316,"5min":0.22,"15min":0.22},"max":{"1min":2.7679999999999998,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":1.7809999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:56:39 2023","interfaces":[{"interface":"back","average":{"1min":9.3209999999999997,"5min":3.9239999999999999,"15min":1.6819999999999999},"min":{"1min":0.27600000000000002,"5min":0.25,"15min":0.25},"max":{"1min":98.063999999999993,"5min":98.063999999999993,"15min":98.063999999999993},"last":0.42299999999999999},{"interface":"front","average":{"1min":9.1379999999999999,"5min":3.8210000000000002,"15min":1.637},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":97.921000000000006,"5min":97.921000000000006,"15min":97.921000000000006},"last":2.0059999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":5.9039999999999999,"5min":2.746,"15min":1.27},"min":{"1min":0.36099999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":94.141000000000005,"5min":94.141000000000005,"15min":94.141000000000005},"last":1.663},{"interface":"front","average":{"1min":6.7750000000000004,"5min":3.181,"15min":1.4550000000000001},"min":{"1min":0.435,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":94.037000000000006,"5min":94.037000000000006,"15min":94.037000000000006},"last":1.429}]},{"osd":5,"last update":"Sat Sep 23 19:56:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":1.3700000000000001,"15min":0.77200000000000002},"min":{"1min":0.28399999999999997,"5min":0.246,"15min":0.246},"max":{"1min":2.782,"5min":10.246,"15min":10.246},"last":0.31},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.6359999999999999,"15min":0.88800000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":8.7629999999999999,"5min":12.675000000000001,"15min":12.675000000000001},"last":0.433}]},{"osd":6,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":0.84699999999999998,"15min":0.60199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.867,"5min":5.867,"15min":5.867},"last":8.0150000000000006},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.80300000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":3.931}]},{"osd":7,"last update":"Sat Sep 23 19:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.494,"15min":0.874},"min":{"1min":0.28699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.2450000000000001,"5min":11.59,"15min":11.59},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.3899999999999999,"15min":0.85999999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.077,"5min":9.3089999999999993,"15min":9.3089999999999993},"last":0.48499999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757466,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":580416,"kb_used_data":449208,"kb_used_omap":0,"kb_used_meta":131200,"kb_avail":93164736,"statfs":{"total":95995035648,"available":95400689664,"internally_reserved":0,"allocated":459988992,"data_stored":459444107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":134348800},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0009999999999999,"5min":2.0030000000000001,"15min":1.042},"min":{"1min":0.219,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":18.651,"5min":24.852,"15min":24.852},"last":0.309},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.8839999999999999,"15min":0.99399999999999999},"min":{"1min":0.224,"5min":0.216,"15min":0.216},"max":{"1min":18.547999999999998,"5min":24.638999999999999,"15min":24.638999999999999},"last":0.23899999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.8879999999999999,"15min":0.97799999999999998},"min":{"1min":0.26300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":8.4879999999999995,"5min":9.9789999999999992,"15min":9.9789999999999992},"last":148.03800000000001},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.7829999999999999,"15min":0.94399999999999995},"min":{"1min":0.27900000000000003,"5min":0.249,"15min":0.249},"max":{"1min":6.0209999999999999,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":147.952}]},{"osd":2,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":11.048,"5min":6.649,"15min":2.5750000000000002},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":141.863,"5min":167.10400000000001,"15min":167.10400000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":11.08,"5min":6.6829999999999998,"15min":2.6200000000000001},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":141.91999999999999,"5min":167.167,"15min":167.167},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":2.052,"15min":1.048},"min":{"1min":0.34300000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.548,"5min":12.853999999999999,"15min":12.853999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.052,"5min":1.772,"15min":0.995},"min":{"1min":0.373,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2439999999999998,"5min":10.115,"15min":10.115},"last":0.32200000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:56:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.288,"15min":0.73799999999999999},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.23699999999999999},"max":{"1min":6.0519999999999996,"5min":10.077,"15min":10.077},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.395,"15min":0.80200000000000005},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.52,"5min":10.226000000000001,"15min":10.226000000000001},"last":0.34999999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:54:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":0.91600000000000004,"15min":0.63400000000000001},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":10.180999999999999,"5min":10.180999999999999,"15min":10.180999999999999},"last":4.2949999999999999},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.046,"15min":0.71299999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":10.406000000000001},"last":3.879}]},{"osd":7,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.365,"15min":0.82199999999999995},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7349999999999994,"5min":13.734999999999999,"15min":13.734999999999999},"last":0.373},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.5309999999999999,"15min":0.88100000000000001},"min":{"1min":0.34699999999999998,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":6.0540000000000003,"5min":10.041,"15min":10.041},"last":0.625}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":466485248,"data_stored":466485248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":623378432,"data_stored":623378432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":903217152,"data_stored":903217152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1121779712,"data_stored":1121779712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":470286336,"data_stored":470286336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":475529216,"data_stored":475529216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":754909184,"data_stored":754909184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:57:15.226 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-09-23T19:57:15.234 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:15.295 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:15.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:15 smithi135 ceph-mon[221113]: from='client.54772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:15.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:15 smithi135 ceph-mon[213871]: from='client.54772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:15.506 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:15.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:15.612 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:15 smithi144 ceph-mon[194833]: from='client.54772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:15.647 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:15.718 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:15.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:15.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:15.826 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:15.865 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 17 16 34266 34250 125.851 92.1875 0.00522558 0.00794142 2023-09-23T19:57:15.895 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:15.998 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:16.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:16.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:16.103 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:16.182 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:16 smithi135 ceph-mon[221113]: pgmap v430: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 5/87918 objects degraded (0.006%) 2023-09-23T19:57:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:16 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:16 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:16 smithi135 ceph-mon[213871]: pgmap v430: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 5/87918 objects degraded (0.006%) 2023-09-23T19:57:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:16 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:16 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:16.517 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:57:16.518 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:57:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:16 smithi144 ceph-mon[194833]: pgmap v430: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 5/87918 objects degraded (0.006%) 2023-09-23T19:57:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:16 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:16 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:16.865 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 18 16 35410 35394 122.832 71.5 0.0101633 0.00813719 2023-09-23T19:57:16.943 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:16.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:16.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:17.053 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:17.085 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:17.142 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:17.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:17.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:17.249 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:17.286 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:17.335 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:17.339 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/87918 objects degraded (0.006%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:17.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:17.441 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:17.493 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:17.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/87918 objects degraded (0.006%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:17.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/87918 objects degraded (0.006%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:17.865 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 19 16 36755 36739 120.792 84.0625 0.012203 0.00827228 2023-09-23T19:57:17.885 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:17.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:17.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:17.916 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:57:17.996 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:18.027 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:18.028 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:18.079 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:18.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:18.185 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:18.273 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:18.325 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:18.330 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:18 smithi144 ceph-mon[194833]: pgmap v431: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s; 5/94302 objects degraded (0.005%) 2023-09-23T19:57:18.650 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:18 smithi135 ceph-mon[213871]: pgmap v431: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s; 5/94302 objects degraded (0.005%) 2023-09-23T19:57:18.651 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:18 smithi135 ceph-mon[221113]: pgmap v431: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s; 5/94302 objects degraded (0.005%) 2023-09-23T19:57:18.832 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:18.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:18.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:18.865 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T19:57:18.866592+0000 min lat: 0.001133 max lat: 0.232634 avg lat: 0.00842236 2023-09-23T19:57:18.865 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:57:18.866 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 20 16 38005 37989 118.659 78.125 0.0130162 0.00842236 2023-09-23T19:57:18.939 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:19.016 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:19.374 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:57:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2913718318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:19.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2913718318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:19.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2913718318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:19.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:57:19.865 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 21 16 39508 39492 117.482 93.9375 0.00228701 0.00850204 2023-09-23T19:57:20.303 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:20.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:20.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:20.410 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:20.459 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:20 smithi144 ceph-mon[194833]: pgmap v432: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 5/107511 objects degraded (0.005%) 2023-09-23T19:57:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:20 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:20 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:57:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:20 smithi144 ceph-mon[194833]: osdmap e169: 8 total, 7 up, 8 in 2023-09-23T19:57:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:20 smithi144 ceph-mon[194833]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:20 smithi144 ceph-mon[194833]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:20 smithi144 ceph-mon[194833]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:20 smithi144 ceph-mon[194833]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[213871]: pgmap v432: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 5/107511 objects degraded (0.005%) 2023-09-23T19:57:20.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:20.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:57:20.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[213871]: osdmap e169: 8 total, 7 up, 8 in 2023-09-23T19:57:20.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[213871]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[213871]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[213871]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[213871]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[221113]: pgmap v432: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 5/107511 objects degraded (0.005%) 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[221113]: osdmap e169: 8 total, 7 up, 8 in 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[221113]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[221113]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[221113]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:20 smithi135 ceph-mon[221113]: failed to encode map e169 with expected crc 2023-09-23T19:57:20.866 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 22 16 40585 40569 115.202 67.3125 0.0157806 0.00867248 2023-09-23T19:57:21.196 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:21.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:21.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:21.302 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:21.393 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:21.578 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:21.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:21.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:21.627 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:21.691 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:21.763 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:57:21.765 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:21.817 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:57:21.866 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 23 16 41890 41874 113.739 81.5625 0.0259746 0.00878677 2023-09-23T19:57:21.868 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:57:21.920 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:57:21.923 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:21.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:21.951 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:57:21.983 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:57:22.022 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:57:22.022 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:57:22.022 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:57:22.023 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:57:22.023 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:57:22.023 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:57:22.023 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:57:22.023 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:57:22.023 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:57:22.029 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:22.067 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:22.132 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:22.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:22.237 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:22.303 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:22 smithi144 ceph-mon[194833]: pgmap v434: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 5/110061 objects degraded (0.005%) 2023-09-23T19:57:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:22 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3653953345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/110061 objects degraded (0.005%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[221113]: pgmap v434: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 5/110061 objects degraded (0.005%) 2023-09-23T19:57:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3653953345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:22.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:22.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/110061 objects degraded (0.005%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:22.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[213871]: pgmap v434: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 5/110061 objects degraded (0.005%) 2023-09-23T19:57:22.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3653953345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:22.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:22.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:22.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/110061 objects degraded (0.005%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:22.866 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 24 16 43152 43136 112.287 78.875 0.00852459 0.00890017 2023-09-23T19:57:23.213 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:57:23.642 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:23.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:23.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:23.753 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:23.786 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:23.836 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:23.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:23.866 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 25 16 44278 44262 110.608 70.375 0.0186139 0.00903609 2023-09-23T19:57:23.941 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:23.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:57:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:57:23] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.43.0" 2023-09-23T19:57:23.999 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:24.358 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:24 smithi135 ceph-mon[213871]: pgmap v435: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 5/115779 objects degraded (0.004%) 2023-09-23T19:57:24.587 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:24.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:24 smithi144 ceph-mon[194833]: pgmap v435: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 5/115779 objects degraded (0.004%) 2023-09-23T19:57:24.693 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:24.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:24 smithi135 ceph-mon[221113]: pgmap v435: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 5/115779 objects degraded (0.004%) 2023-09-23T19:57:24.730 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:24.867 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 26 16 45487 45471 109.257 75.5625 0.0159548 0.00914674 2023-09-23T19:57:24.940 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:57:24.941 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:57:25.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:25.142 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:25.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:25.243 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:25.346 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:25.492 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:25.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:25.601 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:25.634 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:25.697 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:25.862 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:25.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:25.871 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 27 16 46694 46678 107.992 75.4375 0.0101477 0.00925363 2023-09-23T19:57:25.968 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:26.007 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:57:26.007 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:126107 5.5 5:a10e4646:::benchmark_data_smithi135_284570_object10323:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.218209+0000", 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 194.78609648, 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27198878900000001, 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.008 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 126107 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218209+0000", 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.009 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218209+0000", 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218208+0000", 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.010 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218214+0000", 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5260000000000001e-06 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218222+0000", 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4260000000000003e-06 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218227+0000", 2023-09-23T19:57:26.011 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4559999999999999e-06 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218277+0000", 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9412000000000001e-05 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218389+0000", 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000112487 2023-09-23T19:57:26.012 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.218456+0000", 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6709000000000001e-05 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490024+0000", 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27156842399999997 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.013 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490066+0000", 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1647999999999998e-05 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490111+0000", 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4467e-05 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.014 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490160+0000", 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9153000000000002e-05 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490198+0000", 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8263999999999998e-05 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.015 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:126107 5.5 e138/112 ondisk, result = 0)", 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.221247+0000", 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 194.78305893699999, 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.268783465, 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.016 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.017 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.221247+0000", 2023-09-23T19:57:26.017 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.017 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.017 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.017 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.017 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.221247+0000", 2023-09-23T19:57:26.017 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.017 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.221246+0000", 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.221251+0000", 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3669999999999996e-06 2023-09-23T19:57:26.018 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.484397+0000", 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26314538700000001 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.484413+0000", 2023-09-23T19:57:26.019 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6208000000000001e-05 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490008+0000", 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0055951769999999998 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.020 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490017+0000", 2023-09-23T19:57:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3579999999999992e-06 2023-09-23T19:57:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490030+0000", 2023-09-23T19:57:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2850999999999999e-05 2023-09-23T19:57:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.021 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:126101 5.2 e138/112 5:43b10799:::benchmark_data_smithi135_284570_object10317:head v 138'7963, mlcod=138'7959)", 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.222103+0000", 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 194.78220267, 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.267958474, 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.022 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.023 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.023 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222103+0000", 2023-09-23T19:57:26.023 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.023 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.023 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.023 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.023 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222103+0000", 2023-09-23T19:57:26.024 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.024 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.024 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.024 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.024 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222102+0000", 2023-09-23T19:57:26.024 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.024 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.024 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.025 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.025 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.222108+0000", 2023-09-23T19:57:26.025 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.541e-06 2023-09-23T19:57:26.025 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.025 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.026 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.026 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.484399+0000", 2023-09-23T19:57:26.026 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26229089 2023-09-23T19:57:26.026 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.026 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.484413+0000", 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3932e-05 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.484470+0000", 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7510000000000003e-05 2023-09-23T19:57:26.028 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.484495+0000", 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4856000000000001e-05 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490007+0000", 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0055121340000000001 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.029 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.490061+0000", 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4199000000000001e-05 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149320 5.3 5:cd92c7f8:::benchmark_data_smithi135_284570_object33536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.280804+0000", 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.72350177300001, 2023-09-23T19:57:26.030 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0708736379999999, 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149320 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280804+0000", 2023-09-23T19:57:26.031 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280804+0000", 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280803+0000", 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.032 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280809+0000", 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.161e-06 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280822+0000", 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3189e-05 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.033 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280830+0000", 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5580000000000001e-06 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.282819+0000", 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019887889999999999 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.282828+0000", 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3950000000000001e-06 2023-09-23T19:57:26.034 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.347419+0000", 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.06459086 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.347570+0000", 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015181 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.347717+0000", 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000146434 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351560+0000", 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038436 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351566+0000", 2023-09-23T19:57:26.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7300000000000002e-06 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351603+0000", 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7171e-05 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351678+0000", 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4312000000000003e-05 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.037 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149323 5.2 5:47d02f96:::benchmark_data_smithi135_284570_object33539:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.280843+0000", 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.72346273700001, 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0859534829999999, 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.038 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149323 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280843+0000", 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280843+0000", 2023-09-23T19:57:26.039 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280842+0000", 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280846+0000", 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.179e-06 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280852+0000", 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4479999999999999e-06 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280857+0000", 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.383e-06 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280898+0000", 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0759000000000001e-05 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280923+0000", 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4875000000000002e-05 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.352747+0000", 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.071823787 2023-09-23T19:57:26.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.355785+0000", 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030379999999999999 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.356129+0000", 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034367799999999998 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.365029+0000", 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0089008030000000005 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366669+0000", 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016391800000000001 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366715+0000", 2023-09-23T19:57:26.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6289999999999999e-05 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366796+0000", 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1630000000000003e-05 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149324 5.3 5:ce00d33f:::benchmark_data_smithi135_284570_object33540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:57:26.045 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.280870+0000", 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.72343564400001, 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.085749139, 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149324 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.046 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280870+0000", 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280870+0000", 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.047 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280870+0000", 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280872+0000", 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.384e-06 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280878+0000", 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9699999999999996e-06 2023-09-23T19:57:26.048 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280883+0000", 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9520000000000002e-06 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.282849+0000", 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001965751 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.049 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:57:26.050 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.282854+0000", 2023-09-23T19:57:26.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3460000000000001e-06 2023-09-23T19:57:26.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.050 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.050 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351709+0000", 2023-09-23T19:57:26.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0688548769999999 2023-09-23T19:57:26.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351843+0000", 2023-09-23T19:57:26.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013402800000000001 2023-09-23T19:57:26.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351994+0000", 2023-09-23T19:57:26.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015095700000000001 2023-09-23T19:57:26.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366391+0000", 2023-09-23T19:57:26.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014397092 2023-09-23T19:57:26.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366490+0000", 2023-09-23T19:57:26.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9444000000000005e-05 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366537+0000", 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6789999999999998e-05 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366619+0000", 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1923999999999997e-05 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.057 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149332 5.3 5:c5fd6d95:::benchmark_data_smithi135_284570_object33548:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.280890+0000", 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.723415526, 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0858478300000001, 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.058 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.059 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.059 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.059 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.059 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149332 2023-09-23T19:57:26.059 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.059 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.059 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.060 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.060 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280890+0000", 2023-09-23T19:57:26.060 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.060 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.060 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.060 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.060 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280890+0000", 2023-09-23T19:57:26.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280890+0000", 2023-09-23T19:57:26.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280893+0000", 2023-09-23T19:57:26.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0180000000000002e-06 2023-09-23T19:57:26.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280899+0000", 2023-09-23T19:57:26.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0610000000000001e-06 2023-09-23T19:57:26.063 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.063 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280921+0000", 2023-09-23T19:57:26.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2518e-05 2023-09-23T19:57:26.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.284960+0000", 2023-09-23T19:57:26.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0040385450000000002 2023-09-23T19:57:26.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:57:26.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.284971+0000", 2023-09-23T19:57:26.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0883000000000001e-05 2023-09-23T19:57:26.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.355670+0000", 2023-09-23T19:57:26.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.070698873 2023-09-23T19:57:26.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.066 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.066 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.066 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.355783+0000", 2023-09-23T19:57:26.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011296199999999999 2023-09-23T19:57:26.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.355910+0000", 2023-09-23T19:57:26.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000127749 2023-09-23T19:57:26.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366479+0000", 2023-09-23T19:57:26.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010569017999999999 2023-09-23T19:57:26.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366629+0000", 2023-09-23T19:57:26.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000149727 2023-09-23T19:57:26.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.069 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366667+0000", 2023-09-23T19:57:26.069 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7789e-05 2023-09-23T19:57:26.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.070 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.070 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366738+0000", 2023-09-23T19:57:26.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1017000000000001e-05 2023-09-23T19:57:26.070 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.070 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.070 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.071 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.071 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149341 5.3 5:ce411052:::benchmark_data_smithi135_284570_object33557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:57:26.071 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.310999+0000", 2023-09-23T19:57:26.071 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.69330675699999, 2023-09-23T19:57:26.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0559260479999999, 2023-09-23T19:57:26.071 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149341 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.072 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.310999+0000", 2023-09-23T19:57:26.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.310999+0000", 2023-09-23T19:57:26.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.074 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.074 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.310998+0000", 2023-09-23T19:57:26.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:26.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.075 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.075 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311003+0000", 2023-09-23T19:57:26.075 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9860000000000002e-06 2023-09-23T19:57:26.075 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.075 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.075 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.075 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311012+0000", 2023-09-23T19:57:26.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9889999999999998e-06 2023-09-23T19:57:26.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.076 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.076 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.076 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311017+0000", 2023-09-23T19:57:26.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6710000000000004e-06 2023-09-23T19:57:26.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.076 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311078+0000", 2023-09-23T19:57:26.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0819000000000002e-05 2023-09-23T19:57:26.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:57:26.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311090+0000", 2023-09-23T19:57:26.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.167e-05 2023-09-23T19:57:26.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.359565+0000", 2023-09-23T19:57:26.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0484753490000001 2023-09-23T19:57:26.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.079 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.359659+0000", 2023-09-23T19:57:26.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3467000000000005e-05 2023-09-23T19:57:26.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.079 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.359738+0000", 2023-09-23T19:57:26.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9252000000000006e-05 2023-09-23T19:57:26.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366748+0000", 2023-09-23T19:57:26.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0070098000000000001 2023-09-23T19:57:26.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.081 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366795+0000", 2023-09-23T19:57:26.081 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7707999999999998e-05 2023-09-23T19:57:26.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.081 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366838+0000", 2023-09-23T19:57:26.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2372e-05 2023-09-23T19:57:26.082 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.082 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.082 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366925+0000", 2023-09-23T19:57:26.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7219999999999995e-05 2023-09-23T19:57:26.082 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.083 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.083 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.083 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149342 5.3 5:c85e3fd6:::benchmark_data_smithi135_284570_object33558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:57:26.083 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.311033+0000", 2023-09-23T19:57:26.083 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.69327235200001, 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0636845029999999, 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149342 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.084 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311033+0000", 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311033+0000", 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311033+0000", 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311036+0000", 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5859999999999999e-06 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311041+0000", 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7690000000000002e-06 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311047+0000", 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4960000000000004e-06 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311117+0000", 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0098999999999994e-05 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:57:26.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311124+0000", 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.17e-06 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.362787+0000", 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0516628180000001 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.362915+0000", 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000128133 2023-09-23T19:57:26.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.363035+0000", 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011958199999999999 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366969+0000", 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003934033 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374590+0000", 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0076213009999999996 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374640+0000", 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0034999999999998e-05 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374718+0000", 2023-09-23T19:57:26.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7803e-05 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149346 5.2 5:4e06899e:::benchmark_data_smithi135_284570_object33562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.315466+0000", 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.68883956400001, 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0514979799999999, 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.091 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149346 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315466+0000", 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.092 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315466+0000", 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315465+0000", 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315473+0000", 2023-09-23T19:57:26.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3769999999999996e-06 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315485+0000", 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1821000000000001e-05 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315493+0000", 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7610000000000007e-06 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.318492+0000", 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002999191 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.318504+0000", 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1802e-05 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.361234+0000", 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0427307020000001 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.361347+0000", 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000112432 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.361477+0000", 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000129951 2023-09-23T19:57:26.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.365236+0000", 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0037598369999999998 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366836+0000", 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015994659999999999 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366877+0000", 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0794999999999998e-05 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366964+0000", 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7371000000000001e-05 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149353 5.3 5:c1a9dfba:::benchmark_data_smithi135_284570_object33569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.332016+0000", 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.67228996099999, 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.042929658, 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149353 2023-09-23T19:57:26.099 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332016+0000", 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332016+0000", 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332014+0000", 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332022+0000", 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0299999999999994e-06 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332041+0000", 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9198000000000002e-05 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332051+0000", 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0376e-05 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332106+0000", 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4449e-05 2023-09-23T19:57:26.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332119+0000", 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3286e-05 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366991+0000", 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0348714619999999 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.367054+0000", 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3359000000000001e-05 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.367121+0000", 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7280999999999995e-05 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374731+0000", 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0076100459999999997 2023-09-23T19:57:26.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374817+0000", 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6079e-05 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374862+0000", 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4888000000000001e-05 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374945+0000", 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3095000000000003e-05 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203710 5.3 5:c64083ef:::benchmark_data_smithi135_284570_object87926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.291899+0000", 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.712407139999996, 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28919233900000002, 2023-09-23T19:57:26.106 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203710 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.291899+0000", 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.291899+0000", 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.291897+0000", 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.291915+0000", 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7411999999999999e-05 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.291927+0000", 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2072000000000001e-05 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.291935+0000", 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1750000000000005e-06 2023-09-23T19:57:26.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.291982+0000", 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7562000000000003e-05 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.292094+0000", 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011186100000000001 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.292204+0000", 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011017400000000001 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.580169+0000", 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28796411500000002 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.580955+0000", 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00078692400000000004 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.581003+0000", 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7224000000000001e-05 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.581091+0000", 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8195999999999994e-05 2023-09-23T19:57:26.112 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203714 5.3 5:c51e5fb3:::benchmark_data_smithi135_284570_object87930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.292095+0000", 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.712210620999997, 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28910571099999999, 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.113 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203714 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.292095+0000", 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.114 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.292095+0000", 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.292094+0000", 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.292099+0000", 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0599999999999998e-06 2023-09-23T19:57:26.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.292108+0000", 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7090000000000008e-06 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.292114+0000", 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1879999999999999e-06 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.298955+0000", 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0068405239999999997 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.299072+0000", 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117624 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.299189+0000", 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000116878 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.580210+0000", 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28102023700000001 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.581099+0000", 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00088963200000000001 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.581133+0000", 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3359999999999999e-05 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.581201+0000", 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8232999999999995e-05 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:203710 5.3 e152/144 ondisk, result = 0)", 2023-09-23T19:57:26.119 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.299941+0000", 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.704364583, 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.280234857, 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.299941+0000", 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.299941+0000", 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.299940+0000", 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.299946+0000", 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3199999999999996e-06 2023-09-23T19:57:26.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.299957+0000", 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0614e-05 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.299964+0000", 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.99e-06 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.579815+0000", 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.279851144 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.580161+0000", 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034556099999999998 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.580176+0000", 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5221999999999999e-05 2023-09-23T19:57:26.123 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203719 5.3 5:c2ca6f5a:::benchmark_data_smithi135_284570_object87935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.301152+0000", 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.703153841000002, 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30138194899999998, 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.124 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203719 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301152+0000", 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301152+0000", 2023-09-23T19:57:26.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301151+0000", 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301155+0000", 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2010000000000001e-06 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301164+0000", 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4579999999999993e-06 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301169+0000", 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8189999999999997e-06 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.581278+0000", 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28010825900000003 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.581403+0000", 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000125487 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.581509+0000", 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010550100000000001 2023-09-23T19:57:26.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602394+0000", 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.020885813999999999 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602420+0000", 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6055999999999999e-05 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602454+0000", 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3730999999999999e-05 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602534+0000", 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9585000000000005e-05 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.130 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203721 5.3 5:cfbf513c:::benchmark_data_smithi135_284570_object87937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.301178+0000", 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.703128082000006, 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30152889100000002, 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203721 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301178+0000", 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301178+0000", 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301177+0000", 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301180+0000", 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3309999999999998e-06 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301186+0000", 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4790000000000001e-06 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301191+0000", 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9529999999999998e-06 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.585593+0000", 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28440167300000002 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.585719+0000", 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012650399999999999 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.585850+0000", 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013128200000000001 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602545+0000", 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016694229000000001 2023-09-23T19:57:26.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602592+0000", 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7463000000000003e-05 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602630+0000", 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8025999999999998e-05 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602707+0000", 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6346000000000004e-05 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203722 5.2 5:4e520d86:::benchmark_data_smithi135_284570_object87938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.301195+0000", 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.703110241999994, 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28732858, 2023-09-23T19:57:26.137 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203722 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301195+0000", 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301195+0000", 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301195+0000", 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301197+0000", 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1569999999999998e-06 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301203+0000", 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2680000000000002e-06 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301207+0000", 2023-09-23T19:57:26.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5669999999999996e-06 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301253+0000", 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6007000000000002e-05 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301361+0000", 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107768 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.579829+0000", 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27846784600000002 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.588386+0000", 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085570539999999997 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.588396+0000", 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9750000000000002e-06 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.588439+0000", 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3086e-05 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.588524+0000", 2023-09-23T19:57:26.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5198000000000007e-05 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203724 5.3 5:c7db1c20:::benchmark_data_smithi135_284570_object87940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.301224+0000", 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.703082073999994, 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30165318699999999, 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.144 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203724 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301224+0000", 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301224+0000", 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301223+0000", 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301226+0000", 2023-09-23T19:57:26.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7319999999999998e-06 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301232+0000", 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4150000000000004e-06 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301239+0000", 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4609999999999996e-06 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.590412+0000", 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28917298800000002 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.590557+0000", 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000145463 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.590676+0000", 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000119067 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602550+0000", 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011873711 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602757+0000", 2023-09-23T19:57:26.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020686599999999999 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602801+0000", 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4097999999999997e-05 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602877+0000", 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5827000000000004e-05 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.150 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203725 5.3 5:cd7d81d5:::benchmark_data_smithi135_284570_object87941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.301446+0000", 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.70285982, 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31410600700000002, 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:26.151 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203725 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301446+0000", 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301446+0000", 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301445+0000", 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301450+0000", 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9470000000000002e-06 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.579691+0000", 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27824077600000002 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.579711+0000", 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9721e-05 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.594999+0000", 2023-09-23T19:57:26.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01528876 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.595109+0000", 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010924100000000001 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.595218+0000", 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000109893 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602555+0000", 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073367709999999997 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.615415+0000", 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012859628 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:26.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.615463+0000", 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8649999999999997e-05 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.615552+0000", 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8417000000000004e-05 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.157 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:203714 5.3 e152/144 ondisk, result = 0)", 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.303531+0000", 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 86.700774850000002, 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.276680967, 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.303531+0000", 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.303531+0000", 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.303530+0000", 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:26.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.303538+0000", 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7209999999999994e-06 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.579739+0000", 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.276200795 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.579747+0000", 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8869999999999992e-06 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.580197+0000", 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045060800000000002 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.580204+0000", 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8360000000000003e-06 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:26.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.580212+0000", 2023-09-23T19:57:26.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7100000000000007e-06 2023-09-23T19:57:26.162 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.162 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.162 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.162 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:26.162 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:26.162 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:57:26.163 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:26.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:26 smithi135 ceph-mon[221113]: pgmap v436: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 5/126522 objects degraded (0.004%) 2023-09-23T19:57:26.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2960800610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:57:26.388 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:57:26.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:26 smithi135 ceph-mon[213871]: pgmap v436: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 5/126522 objects degraded (0.004%) 2023-09-23T19:57:26.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2960800610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:57:26.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:57:26.392 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 6 pgp_num to 16 2023-09-23T19:57:26.871 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 28 16 47974 47958 106.992 80 0.00978369 0.00933994 2023-09-23T19:57:26.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:26 smithi144 ceph-mon[194833]: pgmap v436: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 5/126522 objects degraded (0.004%) 2023-09-23T19:57:26.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2960800610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:57:26.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:57:26.942 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:57:27.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:57:27.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[213871]: osdmap e170: 8 total, 7 up, 8 in 2023-09-23T19:57:27.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[213871]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[213871]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[213871]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[213871]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[213871]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/126522 objects degraded (0.004%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:27.643 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:27.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:27.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:27.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:57:27.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[221113]: osdmap e170: 8 total, 7 up, 8 in 2023-09-23T19:57:27.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[221113]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[221113]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[221113]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[221113]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[221113]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/126522 objects degraded (0.004%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:27.754 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:27.836 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:27.879 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 29 16 49317 49301 106.166 83.9375 0.0156952 0.00941341 2023-09-23T19:57:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:27 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:57:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:27 smithi144 ceph-mon[194833]: osdmap e170: 8 total, 7 up, 8 in 2023-09-23T19:57:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:27 smithi144 ceph-mon[194833]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:27 smithi144 ceph-mon[194833]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:27 smithi144 ceph-mon[194833]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:27 smithi144 ceph-mon[194833]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:27 smithi144 ceph-mon[194833]: failed to encode map e170 with expected crc 2023-09-23T19:57:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/126522 objects degraded (0.004%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:28.369 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:57:28.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:28 smithi135 ceph-mon[213871]: pgmap v438: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5/132192 objects degraded (0.004%) 2023-09-23T19:57:28.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:28 smithi135 ceph-mon[221113]: pgmap v438: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5/132192 objects degraded (0.004%) 2023-09-23T19:57:28.879 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 30 16 50502 50486 105.096 74.0625 0.00716356 0.00951037 2023-09-23T19:57:28.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:28 smithi144 ceph-mon[194833]: pgmap v438: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5/132192 objects degraded (0.004%) 2023-09-23T19:57:28.891 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:28.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:28.998 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:29.030 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:29.086 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:29.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:29.192 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:29.288 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:29.514 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:29.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:29.528 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:29.534 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:29.622 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:29.737 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:29.880 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 31 16 51665 51649 104.051 72.6875 0.0137471 0.00960378 2023-09-23T19:57:30.001 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:30.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:30.115 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:30.150 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:30.202 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:30.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:30.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:30.307 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:30.380 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:30.411 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:30 smithi144 ceph-mon[194833]: pgmap v439: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 5/145257 objects degraded (0.003%) 2023-09-23T19:57:30.688 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:30.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:30.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:30 smithi135 ceph-mon[213871]: pgmap v439: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 5/145257 objects degraded (0.003%) 2023-09-23T19:57:30.701 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:30 smithi135 ceph-mon[221113]: pgmap v439: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 5/145257 objects degraded (0.003%) 2023-09-23T19:57:30.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:30.803 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:30.880 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 32 16 52859 52843 103.132 74.625 0.0119945 0.00969083 2023-09-23T19:57:30.893 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:31.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:31 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1657010411' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:31.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:31 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:31.460 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:57:31.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:31 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1657010411' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:31.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:31 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:31.881 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 33 16 54025 54009 102.213 72.875 0.00618769 0.00977883 2023-09-23T19:57:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:31 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1657010411' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:31 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:32.306 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:32.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:32.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:32.412 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:32.449 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:32.499 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:32.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:32.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:32.609 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:32.646 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:32.707 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:32.712 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:32 smithi144 ceph-mon[194833]: pgmap v440: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 5/147489 objects degraded (0.003%) 2023-09-23T19:57:32.712 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:32 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:32.712 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:32 smithi144 ceph-mon[194833]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.712 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:32 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:32.713 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:32 smithi144 ceph-mon[194833]: osdmap e171: 8 total, 7 up, 8 in 2023-09-23T19:57:32.713 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:32 smithi144 ceph-mon[194833]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.713 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:32 smithi144 ceph-mon[194833]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.713 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/147489 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:32.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[213871]: pgmap v440: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 5/147489 objects degraded (0.003%) 2023-09-23T19:57:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[213871]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[213871]: osdmap e171: 8 total, 7 up, 8 in 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[213871]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[213871]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/147489 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[221113]: pgmap v440: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 5/147489 objects degraded (0.003%) 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[221113]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[221113]: osdmap e171: 8 total, 7 up, 8 in 2023-09-23T19:57:32.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[221113]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[221113]: failed to encode map e171 with expected crc 2023-09-23T19:57:32.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/147489 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:32.814 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:32.881 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:32.884 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 34 16 55297 55281 101.541 79.5 0.00713635 0.00984363 2023-09-23T19:57:33.718 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:57:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:57:33] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.43.0" 2023-09-23T19:57:33.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:33 smithi135 ceph-mon[213871]: pgmap v442: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 5/153264 objects degraded (0.003%) 2023-09-23T19:57:33.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:33 smithi135 ceph-mon[221113]: pgmap v442: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 5/153264 objects degraded (0.003%) 2023-09-23T19:57:33.741 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:33.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:33.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:33.847 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:33.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:33 smithi144 ceph-mon[194833]: pgmap v442: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 5/153264 objects degraded (0.003%) 2023-09-23T19:57:33.882 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 35 16 56242 56226 100.326 59.0625 0.0140894 0.0099605 2023-09-23T19:57:33.945 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:34.097 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:57:34.097 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:57:34.097 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T19:57:34.097 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:57:34.170 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:34.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:34.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:34.283 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:34.426 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:34.596 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:57:34.596 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:57:34.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.596 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:56936 6.1 6:8c64c073:::benchmark_data_smithi135_344736_object56935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:57:34.596 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:34.583771+0000", 2023-09-23T19:57:34.596 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.011578283, 2023-09-23T19:57:34.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011624450999999999, 2023-09-23T19:57:34.596 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 56936 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583771+0000", 2023-09-23T19:57:34.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583771+0000", 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583769+0000", 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:57:34.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583795+0000", 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6109e-05 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583823+0000", 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7199000000000002e-05 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583843+0000", 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0769999999999999e-05 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583899+0000", 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5714000000000002e-05 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.584033+0000", 2023-09-23T19:57:34.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013409399999999999 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.584168+0000", 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013524899999999999 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.591033+0000", 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0068642019999999998 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.592357+0000", 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001323959 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:56938 6.4 6:2d4331a0:::benchmark_data_smithi135_344736_object56937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:34.583863+0000", 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.011486344000000001, 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011640091999999999, 2023-09-23T19:57:34.602 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 56938 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583863+0000", 2023-09-23T19:57:34.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583863+0000", 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583863+0000", 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583890+0000", 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7273000000000001e-05 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583918+0000", 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8130999999999999e-05 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:34.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583925+0000", 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4170000000000003e-06 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.589497+0000", 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0055715210000000003 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.589607+0000", 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011026 2023-09-23T19:57:34.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.589725+0000", 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000118045 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.591743+0000", 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020182329999999999 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:34.607 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:56939 6.1 6:8375c62b:::benchmark_data_smithi135_344736_object56938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:34.583945+0000", 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.011404955, 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01164201, 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:34.608 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 56939 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583945+0000", 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583945+0000", 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:34.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583944+0000", 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583967+0000", 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2724000000000001e-05 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583975+0000", 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3410000000000003e-06 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.583981+0000", 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1079999999999998e-06 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:34.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.588726+0000", 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047447189999999997 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.588877+0000", 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000151426 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.589035+0000", 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000157405 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.592342+0000", 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033075859999999999 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.592362+0000", 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9652000000000001e-05 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:56947 6.d e171/163)", 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:34.595344+0000", 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 5.417e-06, 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000322997, 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.595344+0000", 2023-09-23T19:57:34.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.595344+0000", 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.595343+0000", 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.595379+0000", 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6730999999999998e-05 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.595400+0000", 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0460999999999999e-05 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:34.595409+0000", 2023-09-23T19:57:34.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9800000000000004e-06 2023-09-23T19:57:34.617 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.617 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:34.617 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.617 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:34.617 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T19:57:34.617 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 4 2023-09-23T19:57:34.617 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:57:34.883 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 36 16 57276 57260 99.3352 64.625 0.0064496 0.0100619 2023-09-23T19:57:34.891 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:34.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:34.997 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:35.104 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:35.764 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:57:35.764 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:57:35.765 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:57:35.773 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:57:35.809 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:57:35.843 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:57:35.874 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:57:35.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:35 smithi144 ceph-mon[194833]: pgmap v443: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 5/163155 objects degraded (0.003%) 2023-09-23T19:57:35.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:35 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2984964310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:35.883 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 37 16 58142 58126 98.1137 54.125 0.0169094 0.0101861 2023-09-23T19:57:35.898 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:57:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:57:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:57:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:57:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:57:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:57:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:57:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:57:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:57:35.917 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:35 smithi135 ceph-mon[221113]: pgmap v443: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 5/163155 objects degraded (0.003%) 2023-09-23T19:57:35.918 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:35 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2984964310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:35.920 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:35 smithi135 ceph-mon[213871]: pgmap v443: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 5/163155 objects degraded (0.003%) 2023-09-23T19:57:35.920 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:35 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2984964310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:36.088 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:36.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:36.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:36.194 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:36.268 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:36.314 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:57:36.427 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:36.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:36.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:36.532 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:36.558 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:36.598 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:36.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:36.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:36.703 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:36.764 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:36.886 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 38 16 59008 58992 96.9492 54.125 0.0175812 0.0103083 2023-09-23T19:57:37.083 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:37.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:37.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:37.128 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:57:37.129 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:57:37.192 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:37.235 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:37.270 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:37.331 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:37.390 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:37.401 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:37.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:37.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:37.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:37.408 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:37.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:37.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:37.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:37.510 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:37.542 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:37.608 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:37.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:37.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:37.718 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:37.753 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:37.818 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:37.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:37.889 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 39 16 60040 60024 96.1096 64.5 0.016041 0.0103999 2023-09-23T19:57:37.926 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:38.028 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[213871]: pgmap v444: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 5/168075 objects degraded (0.003%) 2023-09-23T19:57:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:57:38.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:57:38.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:38.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/168075 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:38.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[221113]: pgmap v444: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 5/168075 objects degraded (0.003%) 2023-09-23T19:57:38.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:57:38.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:57:38.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:38.227 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/168075 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:38.403 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:38.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:38.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:38.507 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:38.538 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:38 smithi144 ceph-mon[194833]: pgmap v444: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 5/168075 objects degraded (0.003%) 2023-09-23T19:57:38.538 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:57:38.538 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:57:38.538 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:57:38.538 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/168075 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:38.540 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:38.603 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:38.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:38.709 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:38.748 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:38.809 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:38.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:38.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:38.889 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T19:57:38.890803+0000 min lat: 0.001133 max lat: 0.365714 avg lat: 0.0104579 2023-09-23T19:57:38.889 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:57:38.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 40 16 61236 61220 95.5757 74.75 0.0166612 0.0104579 2023-09-23T19:57:38.917 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:38.956 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:39.026 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:39.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:39.130 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:57:39.134 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:39.165 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:39.215 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:39.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:39.321 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:39.347 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:39.399 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:39.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:39.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:39.507 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:39.544 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:39.603 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:39.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:39.711 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:39.791 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:39.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 41 16 62303 62287 94.8714 66.6875 0.00656498 0.0105353 2023-09-23T19:57:40.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:40 smithi135 ceph-mon[213871]: pgmap v445: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 5/178008 objects degraded (0.003%) 2023-09-23T19:57:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:40 smithi135 ceph-mon[221113]: pgmap v445: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 5/178008 objects degraded (0.003%) 2023-09-23T19:57:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:40 smithi144 ceph-mon[194833]: pgmap v445: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 5/178008 objects degraded (0.003%) 2023-09-23T19:57:40.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 42 16 63258 63242 94.0341 59.6875 0.0129685 0.0106285 2023-09-23T19:57:40.894 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:40.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:41.009 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:41.040 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:41.101 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:41.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:41.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:41.206 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:41.280 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:41.382 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:41.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:41.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:41.489 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:41.589 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:41.803 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:41.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 43 16 64286 64270 93.3418 64.25 0.0225424 0.0107055 2023-09-23T19:57:41.969 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:41.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:42.084 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:42.150 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:42.237 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:57:42.237 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:57:42.350 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:42.352 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:57:42.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:42.381 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:57:42.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:42 smithi135 ceph-mon[213871]: pgmap v446: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5/179967 objects degraded (0.003%) 2023-09-23T19:57:42.423 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:42 smithi135 ceph-mon[221113]: pgmap v446: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5/179967 objects degraded (0.003%) 2023-09-23T19:57:42.454 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:42.516 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:42 smithi144 ceph-mon[194833]: pgmap v446: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5/179967 objects degraded (0.003%) 2023-09-23T19:57:42.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 44 16 65200 65184 92.5191 57.125 0.0133175 0.010803 2023-09-23T19:57:43.291 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:43.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:43.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:43.319 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:57:43.407 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:43.449 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:43 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/179967 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:43.449 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:43 smithi144 ceph-mon[194833]: from='client.54796 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:43.449 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3774105305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:57:43.449 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:43 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:57:43.451 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:43 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/179967 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:43 smithi135 ceph-mon[213871]: from='client.54796 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3774105305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:57:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:43 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:57:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:43 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/179967 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:43.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:43 smithi135 ceph-mon[221113]: from='client.54796 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:57:43.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3774105305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:57:43.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:43 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:57:43.510 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:43.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:43.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:43.617 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:43.654 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:43.714 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:43.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:43.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:43.821 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:43.849 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:43.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 45 16 66040 66024 91.6303 52.5 0.00306775 0.0108896 2023-09-23T19:57:43.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:57:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:57:43] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T19:57:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[213871]: pgmap v447: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5/184776 objects degraded (0.003%) 2023-09-23T19:57:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:57:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[213871]: osdmap e172: 8 total, 7 up, 8 in 2023-09-23T19:57:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[213871]: failed to encode map e172 with expected crc 2023-09-23T19:57:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[213871]: failed to encode map e172 with expected crc 2023-09-23T19:57:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[221113]: pgmap v447: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5/184776 objects degraded (0.003%) 2023-09-23T19:57:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:57:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[221113]: osdmap e172: 8 total, 7 up, 8 in 2023-09-23T19:57:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[221113]: failed to encode map e172 with expected crc 2023-09-23T19:57:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:44 smithi135 ceph-mon[221113]: failed to encode map e172 with expected crc 2023-09-23T19:57:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:44 smithi144 ceph-mon[194833]: pgmap v447: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5/184776 objects degraded (0.003%) 2023-09-23T19:57:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:44 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:57:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:44 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:57:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:44 smithi144 ceph-mon[194833]: osdmap e172: 8 total, 7 up, 8 in 2023-09-23T19:57:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:44 smithi144 ceph-mon[194833]: failed to encode map e172 with expected crc 2023-09-23T19:57:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:44 smithi144 ceph-mon[194833]: failed to encode map e172 with expected crc 2023-09-23T19:57:44.891 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 46 15 66872 66857 90.7693 52.0625 0.0153299 0.0110128 2023-09-23T19:57:45.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:45 smithi135 ceph-mon[213871]: failed to encode map e172 with expected crc 2023-09-23T19:57:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:45 smithi135 ceph-mon[221113]: failed to encode map e172 with expected crc 2023-09-23T19:57:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:45 smithi144 ceph-mon[194833]: failed to encode map e172 with expected crc 2023-09-23T19:57:45.793 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:57:45.809 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:45.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:45.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:45.843 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":446,"stamp":"2023-09-23T19:57:40.473982+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3934394291,"num_objects":59989,"num_object_clones":0,"num_object_copies":179967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59989,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":60868,"num_write_kb":3850229,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":4753,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62764,"ondisk_log_size":62764,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13808944,"kb_used_data":12751472,"kb_used_omap":32,"kb_used_meta":1057375,"kb_avail":736152272,"statfs":{"total":767960285184,"available":753819926528,"internally_reserved":0,"allocated":13057507328,"data_stored":13052399184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":1082752518},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,3,0,8],"upper_bound":512},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":709492736,"num_objects":10826,"num_object_clones":0,"num_object_copies":32478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10826,"num_write_kb":692864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006222"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":149,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.397351+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:57:26.397351+0000","last_peered":"2023-09-23T19:57:26.397351+0000","last_clean":"2023-09-23T19:57:26.397351+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:57:26.397351+0000","last_undegraded":"2023-09-23T19:57:26.397351+0000","last_fullsized":"2023-09-23T19:57:26.397351+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":148,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.376577+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:57:26.376577+0000","last_peered":"2023-09-23T19:57:26.376577+0000","last_clean":"2023-09-23T19:57:26.376577+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:57:26.376577+0000","last_undegraded":"2023-09-23T19:57:26.376577+0000","last_fullsized":"2023-09-23T19:57:26.376577+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":109,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401814+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:57:26.401814+0000","last_peered":"2023-09-23T19:57:26.401814+0000","last_clean":"2023-09-23T19:57:26.401814+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:57:26.401814+0000","last_undegraded":"2023-09-23T19:57:26.401814+0000","last_fullsized":"2023-09-23T19:57:26.401814+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":116,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387185+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:57:26.387185+0000","last_peered":"2023-09-23T19:57:26.387185+0000","last_clean":"2023-09-23T19:57:26.387185+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:57:26.387185+0000","last_undegraded":"2023-09-23T19:57:26.387185+0000","last_fullsized":"2023-09-23T19:57:26.387185+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":152,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.378206+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:57:26.378206+0000","last_peered":"2023-09-23T19:57:26.378206+0000","last_clean":"2023-09-23T19:57:26.378206+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:57:26.378206+0000","last_undegraded":"2023-09-23T19:57:26.378206+0000","last_fullsized":"2023-09-23T19:57:26.378206+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.386169+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T19:57:26.386169+0000","last_peered":"2023-09-23T19:57:26.386169+0000","last_clean":"2023-09-23T19:57:26.386169+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T19:57:26.386169+0000","last_undegraded":"2023-09-23T19:57:26.386169+0000","last_fullsized":"2023-09-23T19:57:26.386169+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:07.049200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":114,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387590+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:57:26.387590+0000","last_peered":"2023-09-23T19:57:26.387590+0000","last_clean":"2023-09-23T19:57:26.387590+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:57:26.387590+0000","last_undegraded":"2023-09-23T19:57:26.387590+0000","last_fullsized":"2023-09-23T19:57:26.387590+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":149,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.378279+0000","last_change":"2023-09-23T19:56:39.278349+0000","last_active":"2023-09-23T19:57:26.378279+0000","last_peered":"2023-09-23T19:57:26.378279+0000","last_clean":"2023-09-23T19:57:26.378279+0000","last_became_active":"2023-09-23T19:56:39.278068+0000","last_became_peered":"2023-09-23T19:56:39.278068+0000","last_unstale":"2023-09-23T19:57:26.378279+0000","last_undegraded":"2023-09-23T19:57:26.378279+0000","last_fullsized":"2023-09-23T19:57:26.378279+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:04.173721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":130,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.385876+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:57:26.385876+0000","last_peered":"2023-09-23T19:57:26.385876+0000","last_clean":"2023-09-23T19:57:26.385876+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:57:26.385876+0000","last_undegraded":"2023-09-23T19:57:26.385876+0000","last_fullsized":"2023-09-23T19:57:26.385876+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":100,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.378195+0000","last_change":"2023-09-23T19:56:39.273704+0000","last_active":"2023-09-23T19:57:26.378195+0000","last_peered":"2023-09-23T19:57:26.378195+0000","last_clean":"2023-09-23T19:57:26.378195+0000","last_became_active":"2023-09-23T19:56:39.273360+0000","last_became_peered":"2023-09-23T19:56:39.273360+0000","last_unstale":"2023-09-23T19:57:26.378195+0000","last_undegraded":"2023-09-23T19:57:26.378195+0000","last_fullsized":"2023-09-23T19:57:26.378195+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:17.542729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":134,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.399273+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:57:26.399273+0000","last_peered":"2023-09-23T19:57:26.399273+0000","last_clean":"2023-09-23T19:57:26.399273+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:57:26.399273+0000","last_undegraded":"2023-09-23T19:57:26.399273+0000","last_fullsized":"2023-09-23T19:57:26.399273+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401739+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T19:57:26.401739+0000","last_peered":"2023-09-23T19:57:26.401739+0000","last_clean":"2023-09-23T19:57:26.401739+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T19:57:26.401739+0000","last_undegraded":"2023-09-23T19:57:26.401739+0000","last_fullsized":"2023-09-23T19:57:26.401739+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:52.846135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":163,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396715+0000","last_change":"2023-09-23T19:56:39.292849+0000","last_active":"2023-09-23T19:57:26.396715+0000","last_peered":"2023-09-23T19:57:26.396715+0000","last_clean":"2023-09-23T19:57:26.396715+0000","last_became_active":"2023-09-23T19:56:39.284622+0000","last_became_peered":"2023-09-23T19:56:39.284622+0000","last_unstale":"2023-09-23T19:57:26.396715+0000","last_undegraded":"2023-09-23T19:57:26.396715+0000","last_fullsized":"2023-09-23T19:57:26.396715+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:24.088523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":137,"reported_epoch":170,"state":"active+undersized","last_fresh":"2023-09-23T19:57:26.396646+0000","last_change":"2023-09-23T19:54:45.327308+0000","last_active":"2023-09-23T19:57:26.396646+0000","last_peered":"2023-09-23T19:57:26.396646+0000","last_clean":"2023-09-23T19:54:37.307838+0000","last_became_active":"2023-09-23T19:54:45.327308+0000","last_became_peered":"2023-09-23T19:54:45.327308+0000","last_unstale":"2023-09-23T19:57:26.396646+0000","last_undegraded":"2023-09-23T19:57:26.396646+0000","last_fullsized":"2023-09-23T19:54:45.254218+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:34.491557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":133,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.399004+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:57:26.399004+0000","last_peered":"2023-09-23T19:57:26.399004+0000","last_clean":"2023-09-23T19:57:26.399004+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:57:26.399004+0000","last_undegraded":"2023-09-23T19:57:26.399004+0000","last_fullsized":"2023-09-23T19:57:26.399004+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":134,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.376713+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:57:26.376713+0000","last_peered":"2023-09-23T19:57:26.376713+0000","last_clean":"2023-09-23T19:57:26.376713+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:57:26.376713+0000","last_undegraded":"2023-09-23T19:57:26.376713+0000","last_fullsized":"2023-09-23T19:57:26.376713+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":114,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387395+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:57:26.387395+0000","last_peered":"2023-09-23T19:57:26.387395+0000","last_clean":"2023-09-23T19:57:26.387395+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:57:26.387395+0000","last_undegraded":"2023-09-23T19:57:26.387395+0000","last_fullsized":"2023-09-23T19:57:26.387395+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.385998+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:57:26.385998+0000","last_peered":"2023-09-23T19:57:26.385998+0000","last_clean":"2023-09-23T19:57:26.385998+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:57:26.385998+0000","last_undegraded":"2023-09-23T19:57:26.385998+0000","last_fullsized":"2023-09-23T19:57:26.385998+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":170,"state":"active+undersized","last_fresh":"2023-09-23T19:57:26.398138+0000","last_change":"2023-09-23T19:54:45.330750+0000","last_active":"2023-09-23T19:57:26.398138+0000","last_peered":"2023-09-23T19:57:26.398138+0000","last_clean":"2023-09-23T19:54:37.324687+0000","last_became_active":"2023-09-23T19:54:45.330750+0000","last_became_peered":"2023-09-23T19:54:45.330750+0000","last_unstale":"2023-09-23T19:57:26.398138+0000","last_undegraded":"2023-09-23T19:57:26.398138+0000","last_fullsized":"2023-09-23T19:54:45.253851+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:18.890717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":115,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387114+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:57:26.387114+0000","last_peered":"2023-09-23T19:57:26.387114+0000","last_clean":"2023-09-23T19:57:26.387114+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:57:26.387114+0000","last_undegraded":"2023-09-23T19:57:26.387114+0000","last_fullsized":"2023-09-23T19:57:26.387114+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":47,"reported_epoch":170,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:57:26.386709+0000","last_change":"2023-09-23T19:54:45.331318+0000","last_active":"2023-09-23T19:57:26.386709+0000","last_peered":"2023-09-23T19:57:26.386709+0000","last_clean":"2023-09-23T19:41:41.886282+0000","last_became_active":"2023-09-23T19:54:45.331318+0000","last_became_peered":"2023-09-23T19:54:45.331318+0000","last_unstale":"2023-09-23T19:57:26.386709+0000","last_undegraded":"2023-09-23T19:54:45.255936+0000","last_fullsized":"2023-09-23T19:54:45.255813+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:16.760756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":134,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.380650+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:57:26.380650+0000","last_peered":"2023-09-23T19:57:26.380650+0000","last_clean":"2023-09-23T19:57:26.380650+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:57:26.380650+0000","last_undegraded":"2023-09-23T19:57:26.380650+0000","last_fullsized":"2023-09-23T19:57:26.380650+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"171'3741","reported_seq":3756,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:38.246110+0000","last_change":"2023-09-23T19:56:51.336830+0000","last_active":"2023-09-23T19:57:38.246110+0000","last_peered":"2023-09-23T19:57:38.246110+0000","last_clean":"2023-09-23T19:57:38.246110+0000","last_became_active":"2023-09-23T19:56:51.336645+0000","last_became_peered":"2023-09-23T19:56:51.336645+0000","last_unstale":"2023-09-23T19:57:38.246110+0000","last_undegraded":"2023-09-23T19:57:38.246110+0000","last_fullsized":"2023-09-23T19:57:38.246110+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3741,"log_dups_size":0,"ondisk_log_size":3741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:27.681999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245170176,"num_objects":3741,"num_object_clones":0,"num_object_copies":11223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3741,"num_write_kb":239424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.399458+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:57:26.399458+0000","last_peered":"2023-09-23T19:57:26.399458+0000","last_clean":"2023-09-23T19:57:26.399458+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:57:26.399458+0000","last_undegraded":"2023-09-23T19:57:26.399458+0000","last_fullsized":"2023-09-23T19:57:26.399458+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.385615+0000","last_change":"2023-09-23T19:56:39.275614+0000","last_active":"2023-09-23T19:57:26.385615+0000","last_peered":"2023-09-23T19:57:26.385615+0000","last_clean":"2023-09-23T19:57:26.385615+0000","last_became_active":"2023-09-23T19:56:39.275242+0000","last_became_peered":"2023-09-23T19:56:39.275242+0000","last_unstale":"2023-09-23T19:57:26.385615+0000","last_undegraded":"2023-09-23T19:57:26.385615+0000","last_fullsized":"2023-09-23T19:57:26.385615+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:55.343087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"171'3505","reported_seq":3522,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:35.881751+0000","last_change":"2023-09-23T19:56:51.328940+0000","last_active":"2023-09-23T19:57:35.881751+0000","last_peered":"2023-09-23T19:57:35.881751+0000","last_clean":"2023-09-23T19:57:35.881751+0000","last_became_active":"2023-09-23T19:56:51.328668+0000","last_became_peered":"2023-09-23T19:56:51.328668+0000","last_unstale":"2023-09-23T19:57:35.881751+0000","last_undegraded":"2023-09-23T19:57:35.881751+0000","last_fullsized":"2023-09-23T19:57:35.881751+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3505,"log_dups_size":0,"ondisk_log_size":3505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:01.871957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229703680,"num_objects":3505,"num_object_clones":0,"num_object_copies":10515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3505,"num_write_kb":224320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":133,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.399182+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:57:26.399182+0000","last_peered":"2023-09-23T19:57:26.399182+0000","last_clean":"2023-09-23T19:57:26.399182+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:57:26.399182+0000","last_undegraded":"2023-09-23T19:57:26.399182+0000","last_fullsized":"2023-09-23T19:57:26.399182+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396741+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:57:26.396741+0000","last_peered":"2023-09-23T19:57:26.396741+0000","last_clean":"2023-09-23T19:57:26.396741+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:57:26.396741+0000","last_undegraded":"2023-09-23T19:57:26.396741+0000","last_fullsized":"2023-09-23T19:57:26.396741+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"171'3795","reported_seq":3810,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:38.223879+0000","last_change":"2023-09-23T19:56:51.329607+0000","last_active":"2023-09-23T19:57:38.223879+0000","last_peered":"2023-09-23T19:57:38.223879+0000","last_clean":"2023-09-23T19:57:38.223879+0000","last_became_active":"2023-09-23T19:56:51.329330+0000","last_became_peered":"2023-09-23T19:56:51.329330+0000","last_unstale":"2023-09-23T19:57:38.223879+0000","last_undegraded":"2023-09-23T19:57:38.223879+0000","last_fullsized":"2023-09-23T19:57:38.223879+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3795,"log_dups_size":0,"ondisk_log_size":3795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:51.413448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248709120,"num_objects":3795,"num_object_clones":0,"num_object_copies":11385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3795,"num_write_kb":242880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":122,"reported_epoch":170,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:57:26.399204+0000","last_change":"2023-09-23T19:54:45.328405+0000","last_active":"2023-09-23T19:57:26.399204+0000","last_peered":"2023-09-23T19:57:26.399204+0000","last_clean":"2023-09-23T19:54:37.326665+0000","last_became_active":"2023-09-23T19:54:45.328405+0000","last_became_peered":"2023-09-23T19:54:45.328405+0000","last_unstale":"2023-09-23T19:57:26.399204+0000","last_undegraded":"2023-09-23T19:54:45.254169+0000","last_fullsized":"2023-09-23T19:54:45.254056+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:10.168687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401515+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T19:57:26.401515+0000","last_peered":"2023-09-23T19:57:26.401515+0000","last_clean":"2023-09-23T19:57:26.401515+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T19:57:26.401515+0000","last_undegraded":"2023-09-23T19:57:26.401515+0000","last_fullsized":"2023-09-23T19:57:26.401515+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:30.170186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"171'3654","reported_seq":3671,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:36.488361+0000","last_change":"2023-09-23T19:56:51.330015+0000","last_active":"2023-09-23T19:57:36.488361+0000","last_peered":"2023-09-23T19:57:36.488361+0000","last_clean":"2023-09-23T19:57:36.488361+0000","last_became_active":"2023-09-23T19:56:51.329727+0000","last_became_peered":"2023-09-23T19:56:51.329727+0000","last_unstale":"2023-09-23T19:57:36.488361+0000","last_undegraded":"2023-09-23T19:57:36.488361+0000","last_fullsized":"2023-09-23T19:57:36.488361+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3654,"log_dups_size":0,"ondisk_log_size":3654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:59.494181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239468544,"num_objects":3654,"num_object_clones":0,"num_object_copies":10962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3654,"num_write_kb":233856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.385979+0000","last_change":"2023-09-23T19:56:39.274176+0000","last_active":"2023-09-23T19:57:26.385979+0000","last_peered":"2023-09-23T19:57:26.385979+0000","last_clean":"2023-09-23T19:57:26.385979+0000","last_became_active":"2023-09-23T19:56:39.273835+0000","last_became_peered":"2023-09-23T19:56:39.273835+0000","last_unstale":"2023-09-23T19:57:26.385979+0000","last_undegraded":"2023-09-23T19:57:26.385979+0000","last_fullsized":"2023-09-23T19:57:26.385979+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:51.379105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":116,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:31.454980+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:57:31.454980+0000","last_peered":"2023-09-23T19:57:31.454980+0000","last_clean":"2023-09-23T19:57:31.454980+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:57:31.454980+0000","last_undegraded":"2023-09-23T19:57:31.454980+0000","last_fullsized":"2023-09-23T19:57:31.454980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"171'3817","reported_seq":3832,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:38.231273+0000","last_change":"2023-09-23T19:56:51.335737+0000","last_active":"2023-09-23T19:57:38.231273+0000","last_peered":"2023-09-23T19:57:38.231273+0000","last_clean":"2023-09-23T19:57:38.231273+0000","last_became_active":"2023-09-23T19:56:51.335399+0000","last_became_peered":"2023-09-23T19:56:51.335399+0000","last_unstale":"2023-09-23T19:57:38.231273+0000","last_undegraded":"2023-09-23T19:57:38.231273+0000","last_fullsized":"2023-09-23T19:57:38.231273+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3817,"log_dups_size":0,"ondisk_log_size":3817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:20.933908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250150912,"num_objects":3817,"num_object_clones":0,"num_object_copies":11451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3817,"num_write_kb":244288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":551,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.398018+0000","last_change":"2023-09-23T19:52:57.349564+0000","last_active":"2023-09-23T19:57:26.398018+0000","last_peered":"2023-09-23T19:57:26.398018+0000","last_clean":"2023-09-23T19:57:26.398018+0000","last_became_active":"2023-09-23T19:52:57.349182+0000","last_became_peered":"2023-09-23T19:52:57.349182+0000","last_unstale":"2023-09-23T19:57:26.398018+0000","last_undegraded":"2023-09-23T19:57:26.398018+0000","last_fullsized":"2023-09-23T19:57:26.398018+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:25.262721+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.376470+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:57:26.376470+0000","last_peered":"2023-09-23T19:57:26.376470+0000","last_clean":"2023-09-23T19:57:26.376470+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:57:26.376470+0000","last_undegraded":"2023-09-23T19:57:26.376470+0000","last_fullsized":"2023-09-23T19:57:26.376470+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396624+0000","last_change":"2023-09-23T19:56:39.284044+0000","last_active":"2023-09-23T19:57:26.396624+0000","last_peered":"2023-09-23T19:57:26.396624+0000","last_clean":"2023-09-23T19:57:26.396624+0000","last_became_active":"2023-09-23T19:56:39.283753+0000","last_became_peered":"2023-09-23T19:56:39.283753+0000","last_unstale":"2023-09-23T19:57:26.396624+0000","last_undegraded":"2023-09-23T19:57:26.396624+0000","last_fullsized":"2023-09-23T19:57:26.396624+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:51.940014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"171'3826","reported_seq":3842,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:37.745584+0000","last_change":"2023-09-23T19:56:51.332462+0000","last_active":"2023-09-23T19:57:37.745584+0000","last_peered":"2023-09-23T19:57:37.745584+0000","last_clean":"2023-09-23T19:57:37.745584+0000","last_became_active":"2023-09-23T19:56:51.332192+0000","last_became_peered":"2023-09-23T19:56:51.332192+0000","last_unstale":"2023-09-23T19:57:37.745584+0000","last_undegraded":"2023-09-23T19:57:37.745584+0000","last_fullsized":"2023-09-23T19:57:37.745584+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3826,"log_dups_size":0,"ondisk_log_size":3826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:42.318489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250740736,"num_objects":3826,"num_object_clones":0,"num_object_copies":11478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3826,"num_write_kb":244864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":482,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.385616+0000","last_change":"2023-09-23T19:52:58.803028+0000","last_active":"2023-09-23T19:57:26.385616+0000","last_peered":"2023-09-23T19:57:26.385616+0000","last_clean":"2023-09-23T19:57:26.385616+0000","last_became_active":"2023-09-23T19:52:57.353261+0000","last_became_peered":"2023-09-23T19:52:57.353261+0000","last_unstale":"2023-09-23T19:57:26.385616+0000","last_undegraded":"2023-09-23T19:57:26.385616+0000","last_fullsized":"2023-09-23T19:57:26.385616+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:53.333663+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":97,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.397300+0000","last_change":"2023-09-23T19:56:39.280681+0000","last_active":"2023-09-23T19:57:26.397300+0000","last_peered":"2023-09-23T19:57:26.397300+0000","last_clean":"2023-09-23T19:57:26.397300+0000","last_became_active":"2023-09-23T19:56:39.280377+0000","last_became_peered":"2023-09-23T19:56:39.280377+0000","last_unstale":"2023-09-23T19:57:26.397300+0000","last_undegraded":"2023-09-23T19:57:26.397300+0000","last_fullsized":"2023-09-23T19:57:26.397300+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:42.514714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396709+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:57:26.396709+0000","last_peered":"2023-09-23T19:57:26.396709+0000","last_clean":"2023-09-23T19:57:26.396709+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:57:26.396709+0000","last_undegraded":"2023-09-23T19:57:26.396709+0000","last_fullsized":"2023-09-23T19:57:26.396709+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"171'3636","reported_seq":3653,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:35.885537+0000","last_change":"2023-09-23T19:56:51.334600+0000","last_active":"2023-09-23T19:57:35.885537+0000","last_peered":"2023-09-23T19:57:35.885537+0000","last_clean":"2023-09-23T19:57:35.885537+0000","last_became_active":"2023-09-23T19:56:51.334315+0000","last_became_peered":"2023-09-23T19:56:51.334315+0000","last_unstale":"2023-09-23T19:57:35.885537+0000","last_undegraded":"2023-09-23T19:57:35.885537+0000","last_fullsized":"2023-09-23T19:57:35.885537+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3636,"log_dups_size":0,"ondisk_log_size":3636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:47.078218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238288896,"num_objects":3636,"num_object_clones":0,"num_object_copies":10908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3636,"num_write_kb":232704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":131,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396442+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:57:26.396442+0000","last_peered":"2023-09-23T19:57:26.396442+0000","last_clean":"2023-09-23T19:57:26.396442+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:57:26.396442+0000","last_undegraded":"2023-09-23T19:57:26.396442+0000","last_fullsized":"2023-09-23T19:57:26.396442+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":157,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.397046+0000","last_change":"2023-09-23T19:56:39.281487+0000","last_active":"2023-09-23T19:57:26.397046+0000","last_peered":"2023-09-23T19:57:26.397046+0000","last_clean":"2023-09-23T19:57:26.397046+0000","last_became_active":"2023-09-23T19:56:39.281248+0000","last_became_peered":"2023-09-23T19:56:39.281248+0000","last_unstale":"2023-09-23T19:57:26.397046+0000","last_undegraded":"2023-09-23T19:57:26.397046+0000","last_fullsized":"2023-09-23T19:57:26.397046+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:25.466372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"171'3864","reported_seq":3879,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:38.287750+0000","last_change":"2023-09-23T19:56:51.339206+0000","last_active":"2023-09-23T19:57:38.287750+0000","last_peered":"2023-09-23T19:57:38.287750+0000","last_clean":"2023-09-23T19:57:38.287750+0000","last_became_active":"2023-09-23T19:56:51.338963+0000","last_became_peered":"2023-09-23T19:56:51.338963+0000","last_unstale":"2023-09-23T19:57:38.287750+0000","last_undegraded":"2023-09-23T19:57:38.287750+0000","last_fullsized":"2023-09-23T19:57:38.287750+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3864,"log_dups_size":0,"ondisk_log_size":3864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:49.976592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253231104,"num_objects":3864,"num_object_clones":0,"num_object_copies":11592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3864,"num_write_kb":247296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":509,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.385823+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T19:57:26.385823+0000","last_peered":"2023-09-23T19:57:26.385823+0000","last_clean":"2023-09-23T19:57:26.385823+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T19:57:26.385823+0000","last_undegraded":"2023-09-23T19:57:26.385823+0000","last_fullsized":"2023-09-23T19:57:26.385823+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.320089+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":136,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.378051+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:57:26.378051+0000","last_peered":"2023-09-23T19:57:26.378051+0000","last_clean":"2023-09-23T19:57:26.378051+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:57:26.378051+0000","last_undegraded":"2023-09-23T19:57:26.378051+0000","last_fullsized":"2023-09-23T19:57:26.378051+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":110,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.403291+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:57:26.403291+0000","last_peered":"2023-09-23T19:57:26.403291+0000","last_clean":"2023-09-23T19:57:26.403291+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:57:26.403291+0000","last_undegraded":"2023-09-23T19:57:26.403291+0000","last_fullsized":"2023-09-23T19:57:26.403291+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"171'3646","reported_seq":3663,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:36.641924+0000","last_change":"2023-09-23T19:56:51.330347+0000","last_active":"2023-09-23T19:57:36.641924+0000","last_peered":"2023-09-23T19:57:36.641924+0000","last_clean":"2023-09-23T19:57:36.641924+0000","last_became_active":"2023-09-23T19:56:51.330084+0000","last_became_peered":"2023-09-23T19:56:51.330084+0000","last_unstale":"2023-09-23T19:57:36.641924+0000","last_undegraded":"2023-09-23T19:57:36.641924+0000","last_fullsized":"2023-09-23T19:57:36.641924+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3646,"log_dups_size":0,"ondisk_log_size":3646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:08.671890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238944256,"num_objects":3646,"num_object_clones":0,"num_object_copies":10938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3646,"num_write_kb":233344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":483,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.380526+0000","last_change":"2023-09-23T19:56:53.618146+0000","last_active":"2023-09-23T19:57:26.380526+0000","last_peered":"2023-09-23T19:57:26.380526+0000","last_clean":"2023-09-23T19:57:26.380526+0000","last_became_active":"2023-09-23T19:56:42.331700+0000","last_became_peered":"2023-09-23T19:56:42.331700+0000","last_unstale":"2023-09-23T19:57:26.380526+0000","last_undegraded":"2023-09-23T19:57:26.380526+0000","last_fullsized":"2023-09-23T19:57:26.380526+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:56:53.618048+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T19:56:53.618048+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.066107+0000","scrub_duration":0.001843293,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":506,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401911+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:57:26.401911+0000","last_peered":"2023-09-23T19:57:26.401911+0000","last_clean":"2023-09-23T19:57:26.401911+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:57:26.401911+0000","last_undegraded":"2023-09-23T19:57:26.401911+0000","last_fullsized":"2023-09-23T19:57:26.401911+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.386068+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:57:26.386068+0000","last_peered":"2023-09-23T19:57:26.386068+0000","last_clean":"2023-09-23T19:57:26.386068+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:57:26.386068+0000","last_undegraded":"2023-09-23T19:57:26.386068+0000","last_fullsized":"2023-09-23T19:57:26.386068+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":95,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401665+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:57:26.401665+0000","last_peered":"2023-09-23T19:57:26.401665+0000","last_clean":"2023-09-23T19:57:26.401665+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:57:26.401665+0000","last_undegraded":"2023-09-23T19:57:26.401665+0000","last_fullsized":"2023-09-23T19:57:26.401665+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"171'3580","reported_seq":3597,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:35.837630+0000","last_change":"2023-09-23T19:56:51.333298+0000","last_active":"2023-09-23T19:57:35.837630+0000","last_peered":"2023-09-23T19:57:35.837630+0000","last_clean":"2023-09-23T19:57:35.837630+0000","last_became_active":"2023-09-23T19:56:51.333009+0000","last_became_peered":"2023-09-23T19:56:51.333009+0000","last_unstale":"2023-09-23T19:57:35.837630+0000","last_undegraded":"2023-09-23T19:57:35.837630+0000","last_fullsized":"2023-09-23T19:57:35.837630+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3580,"log_dups_size":0,"ondisk_log_size":3580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:12.985476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234618880,"num_objects":3580,"num_object_clones":0,"num_object_copies":10740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3580,"num_write_kb":229120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":482,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.385724+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T19:57:26.385724+0000","last_peered":"2023-09-23T19:57:26.385724+0000","last_clean":"2023-09-23T19:57:26.385724+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T19:57:26.385724+0000","last_undegraded":"2023-09-23T19:57:26.385724+0000","last_fullsized":"2023-09-23T19:57:26.385724+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":731,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.386067+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T19:57:26.386067+0000","last_peered":"2023-09-23T19:57:26.386067+0000","last_clean":"2023-09-23T19:57:26.386067+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T19:57:26.386067+0000","last_undegraded":"2023-09-23T19:57:26.386067+0000","last_fullsized":"2023-09-23T19:57:26.386067+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:34.015520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":148,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396977+0000","last_change":"2023-09-23T19:56:39.283954+0000","last_active":"2023-09-23T19:57:26.396977+0000","last_peered":"2023-09-23T19:57:26.396977+0000","last_clean":"2023-09-23T19:57:26.396977+0000","last_became_active":"2023-09-23T19:56:39.283711+0000","last_became_peered":"2023-09-23T19:56:39.283711+0000","last_unstale":"2023-09-23T19:57:26.396977+0000","last_undegraded":"2023-09-23T19:57:26.396977+0000","last_fullsized":"2023-09-23T19:57:26.396977+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:55.890028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401833+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:57:26.401833+0000","last_peered":"2023-09-23T19:57:26.401833+0000","last_clean":"2023-09-23T19:57:26.401833+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:57:26.401833+0000","last_undegraded":"2023-09-23T19:57:26.401833+0000","last_fullsized":"2023-09-23T19:57:26.401833+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"171'3719","reported_seq":3735,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:38.198161+0000","last_change":"2023-09-23T19:56:51.329873+0000","last_active":"2023-09-23T19:57:38.198161+0000","last_peered":"2023-09-23T19:57:38.198161+0000","last_clean":"2023-09-23T19:57:38.198161+0000","last_became_active":"2023-09-23T19:56:51.329355+0000","last_became_peered":"2023-09-23T19:56:51.329355+0000","last_unstale":"2023-09-23T19:57:38.198161+0000","last_undegraded":"2023-09-23T19:57:38.198161+0000","last_fullsized":"2023-09-23T19:57:38.198161+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3719,"log_dups_size":0,"ondisk_log_size":3719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.152209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243728384,"num_objects":3719,"num_object_clones":0,"num_object_copies":11157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3719,"num_write_kb":238016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":551,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.398115+0000","last_change":"2023-09-23T19:52:57.745972+0000","last_active":"2023-09-23T19:57:26.398115+0000","last_peered":"2023-09-23T19:57:26.398115+0000","last_clean":"2023-09-23T19:57:26.398115+0000","last_became_active":"2023-09-23T19:52:57.348142+0000","last_became_peered":"2023-09-23T19:52:57.348142+0000","last_unstale":"2023-09-23T19:57:26.398115+0000","last_undegraded":"2023-09-23T19:57:26.398115+0000","last_fullsized":"2023-09-23T19:57:26.398115+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.745894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.745894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.660195+0000","scrub_duration":0.0079262489999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":656,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.391028+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:57:26.391028+0000","last_peered":"2023-09-23T19:57:26.391028+0000","last_clean":"2023-09-23T19:57:26.391028+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:57:26.391028+0000","last_undegraded":"2023-09-23T19:57:26.391028+0000","last_fullsized":"2023-09-23T19:57:26.391028+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":124,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401394+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T19:57:26.401394+0000","last_peered":"2023-09-23T19:57:26.401394+0000","last_clean":"2023-09-23T19:57:26.401394+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T19:57:26.401394+0000","last_undegraded":"2023-09-23T19:57:26.401394+0000","last_fullsized":"2023-09-23T19:57:26.401394+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:46.217415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":139,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.403217+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:57:26.403217+0000","last_peered":"2023-09-23T19:57:26.403217+0000","last_clean":"2023-09-23T19:57:26.403217+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:57:26.403217+0000","last_undegraded":"2023-09-23T19:57:26.403217+0000","last_fullsized":"2023-09-23T19:57:26.403217+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"171'3789","reported_seq":3804,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:38.198692+0000","last_change":"2023-09-23T19:56:51.335393+0000","last_active":"2023-09-23T19:57:38.198692+0000","last_peered":"2023-09-23T19:57:38.198692+0000","last_clean":"2023-09-23T19:57:38.198692+0000","last_became_active":"2023-09-23T19:56:51.335106+0000","last_became_peered":"2023-09-23T19:56:51.335106+0000","last_unstale":"2023-09-23T19:57:38.198692+0000","last_undegraded":"2023-09-23T19:57:38.198692+0000","last_fullsized":"2023-09-23T19:57:38.198692+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3789,"log_dups_size":0,"ondisk_log_size":3789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.493039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248315904,"num_objects":3789,"num_object_clones":0,"num_object_copies":11367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3789,"num_write_kb":242496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":979,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.385793+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:57:26.385793+0000","last_peered":"2023-09-23T19:57:26.385793+0000","last_clean":"2023-09-23T19:57:26.385793+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:57:26.385793+0000","last_undegraded":"2023-09-23T19:57:26.385793+0000","last_fullsized":"2023-09-23T19:57:26.385793+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":559,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.398004+0000","last_change":"2023-09-23T19:56:42.408346+0000","last_active":"2023-09-23T19:57:26.398004+0000","last_peered":"2023-09-23T19:57:26.398004+0000","last_clean":"2023-09-23T19:57:26.398004+0000","last_became_active":"2023-09-23T19:56:42.407370+0000","last_became_peered":"2023-09-23T19:56:42.407370+0000","last_unstale":"2023-09-23T19:57:26.398004+0000","last_undegraded":"2023-09-23T19:57:26.398004+0000","last_fullsized":"2023-09-23T19:57:26.398004+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:38.144096+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":100,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.380333+0000","last_change":"2023-09-23T19:56:39.277079+0000","last_active":"2023-09-23T19:57:26.380333+0000","last_peered":"2023-09-23T19:57:26.380333+0000","last_clean":"2023-09-23T19:57:26.380333+0000","last_became_active":"2023-09-23T19:56:39.276787+0000","last_became_peered":"2023-09-23T19:56:39.276787+0000","last_unstale":"2023-09-23T19:57:26.380333+0000","last_undegraded":"2023-09-23T19:57:26.380333+0000","last_fullsized":"2023-09-23T19:57:26.380333+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:00.570242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396919+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T19:57:26.396919+0000","last_peered":"2023-09-23T19:57:26.396919+0000","last_clean":"2023-09-23T19:57:26.396919+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T19:57:26.396919+0000","last_undegraded":"2023-09-23T19:57:26.396919+0000","last_fullsized":"2023-09-23T19:57:26.396919+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:31.068090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"171'3883","reported_seq":3899,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:39.540372+0000","last_change":"2023-09-23T19:56:51.335699+0000","last_active":"2023-09-23T19:57:39.540372+0000","last_peered":"2023-09-23T19:57:39.540372+0000","last_clean":"2023-09-23T19:57:39.540372+0000","last_became_active":"2023-09-23T19:56:51.335366+0000","last_became_peered":"2023-09-23T19:56:51.335366+0000","last_unstale":"2023-09-23T19:57:39.540372+0000","last_undegraded":"2023-09-23T19:57:39.540372+0000","last_fullsized":"2023-09-23T19:57:39.540372+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3883,"log_dups_size":0,"ondisk_log_size":3883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:18.373892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254476288,"num_objects":3883,"num_object_clones":0,"num_object_copies":11649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3883,"num_write_kb":248512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":508,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401824+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T19:57:26.401824+0000","last_peered":"2023-09-23T19:57:26.401824+0000","last_clean":"2023-09-23T19:57:26.401824+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T19:57:26.401824+0000","last_undegraded":"2023-09-23T19:57:26.401824+0000","last_fullsized":"2023-09-23T19:57:26.401824+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:32.858007+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":889,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.398020+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:57:26.398020+0000","last_peered":"2023-09-23T19:57:26.398020+0000","last_clean":"2023-09-23T19:57:26.398020+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:57:26.398020+0000","last_undegraded":"2023-09-23T19:57:26.398020+0000","last_fullsized":"2023-09-23T19:57:26.398020+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":115,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.386566+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:57:26.386566+0000","last_peered":"2023-09-23T19:57:26.386566+0000","last_clean":"2023-09-23T19:57:26.386566+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:57:26.386566+0000","last_undegraded":"2023-09-23T19:57:26.386566+0000","last_fullsized":"2023-09-23T19:57:26.386566+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387341+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:57:26.387341+0000","last_peered":"2023-09-23T19:57:26.387341+0000","last_clean":"2023-09-23T19:57:26.387341+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:57:26.387341+0000","last_undegraded":"2023-09-23T19:57:26.387341+0000","last_fullsized":"2023-09-23T19:57:26.387341+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"171'3890","reported_seq":3906,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:39.566050+0000","last_change":"2023-09-23T19:56:51.329836+0000","last_active":"2023-09-23T19:57:39.566050+0000","last_peered":"2023-09-23T19:57:39.566050+0000","last_clean":"2023-09-23T19:57:39.566050+0000","last_became_active":"2023-09-23T19:56:51.329518+0000","last_became_peered":"2023-09-23T19:56:51.329518+0000","last_unstale":"2023-09-23T19:57:39.566050+0000","last_undegraded":"2023-09-23T19:57:39.566050+0000","last_fullsized":"2023-09-23T19:57:39.566050+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3890,"log_dups_size":0,"ondisk_log_size":3890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:38.536519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254935040,"num_objects":3890,"num_object_clones":0,"num_object_copies":11670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3890,"num_write_kb":248960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":476,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387035+0000","last_change":"2023-09-23T19:56:54.487207+0000","last_active":"2023-09-23T19:57:26.387035+0000","last_peered":"2023-09-23T19:57:26.387035+0000","last_clean":"2023-09-23T19:57:26.387035+0000","last_became_active":"2023-09-23T19:56:42.420361+0000","last_became_peered":"2023-09-23T19:56:42.420361+0000","last_unstale":"2023-09-23T19:57:26.387035+0000","last_undegraded":"2023-09-23T19:57:26.387035+0000","last_fullsized":"2023-09-23T19:57:26.387035+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:56:54.487110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:56:54.487110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:19.686156+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":249,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.399179+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:57:26.399179+0000","last_peered":"2023-09-23T19:57:26.399179+0000","last_clean":"2023-09-23T19:57:26.399179+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:57:26.399179+0000","last_undegraded":"2023-09-23T19:57:26.399179+0000","last_fullsized":"2023-09-23T19:57:26.399179+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":109,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.401914+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:57:26.401914+0000","last_peered":"2023-09-23T19:57:26.401914+0000","last_clean":"2023-09-23T19:57:26.401914+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:57:26.401914+0000","last_undegraded":"2023-09-23T19:57:26.401914+0000","last_fullsized":"2023-09-23T19:57:26.401914+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":101,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387781+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:57:26.387781+0000","last_peered":"2023-09-23T19:57:26.387781+0000","last_clean":"2023-09-23T19:57:26.387781+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:57:26.387781+0000","last_undegraded":"2023-09-23T19:57:26.387781+0000","last_fullsized":"2023-09-23T19:57:26.387781+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"171'3730","reported_seq":3745,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:38.253467+0000","last_change":"2023-09-23T19:56:51.332314+0000","last_active":"2023-09-23T19:57:38.253467+0000","last_peered":"2023-09-23T19:57:38.253467+0000","last_clean":"2023-09-23T19:57:38.253467+0000","last_became_active":"2023-09-23T19:56:51.332106+0000","last_became_peered":"2023-09-23T19:56:51.332106+0000","last_unstale":"2023-09-23T19:57:38.253467+0000","last_undegraded":"2023-09-23T19:57:38.253467+0000","last_fullsized":"2023-09-23T19:57:38.253467+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3730,"log_dups_size":0,"ondisk_log_size":3730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:38.454829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244449280,"num_objects":3730,"num_object_clones":0,"num_object_copies":11190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3730,"num_write_kb":238720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":509,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396905+0000","last_change":"2023-09-23T19:56:42.336491+0000","last_active":"2023-09-23T19:57:26.396905+0000","last_peered":"2023-09-23T19:57:26.396905+0000","last_clean":"2023-09-23T19:57:26.396905+0000","last_became_active":"2023-09-23T19:56:42.336151+0000","last_became_peered":"2023-09-23T19:56:42.336151+0000","last_unstale":"2023-09-23T19:57:26.396905+0000","last_undegraded":"2023-09-23T19:57:26.396905+0000","last_fullsized":"2023-09-23T19:57:26.396905+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:52.411762+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":133,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.380678+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:57:26.380678+0000","last_peered":"2023-09-23T19:57:26.380678+0000","last_clean":"2023-09-23T19:57:26.380678+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:57:26.380678+0000","last_undegraded":"2023-09-23T19:57:26.380678+0000","last_fullsized":"2023-09-23T19:57:26.380678+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":483,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.404255+0000","last_change":"2023-09-23T19:56:42.312115+0000","last_active":"2023-09-23T19:57:26.404255+0000","last_peered":"2023-09-23T19:57:26.404255+0000","last_clean":"2023-09-23T19:57:26.404255+0000","last_became_active":"2023-09-23T19:56:42.290218+0000","last_became_peered":"2023-09-23T19:56:42.290218+0000","last_unstale":"2023-09-23T19:57:26.404255+0000","last_undegraded":"2023-09-23T19:57:26.404255+0000","last_fullsized":"2023-09-23T19:57:26.404255+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:24.023589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":130,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.378132+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:57:26.378132+0000","last_peered":"2023-09-23T19:57:26.378132+0000","last_clean":"2023-09-23T19:57:26.378132+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:57:26.378132+0000","last_undegraded":"2023-09-23T19:57:26.378132+0000","last_fullsized":"2023-09-23T19:57:26.378132+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"171'3886","reported_seq":3902,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T19:57:38.153798+0000","last_change":"2023-09-23T19:56:51.331630+0000","last_active":"2023-09-23T19:57:38.153798+0000","last_peered":"2023-09-23T19:57:38.153798+0000","last_clean":"2023-09-23T19:57:38.153798+0000","last_became_active":"2023-09-23T19:56:51.331338+0000","last_became_peered":"2023-09-23T19:56:51.331338+0000","last_unstale":"2023-09-23T19:57:38.153798+0000","last_undegraded":"2023-09-23T19:57:38.153798+0000","last_fullsized":"2023-09-23T19:57:38.153798+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":3886,"log_dups_size":0,"ondisk_log_size":3886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:45.394928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254672896,"num_objects":3886,"num_object_clones":0,"num_object_copies":11658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3886,"num_write_kb":248704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":487,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.377024+0000","last_change":"2023-09-23T19:53:00.705074+0000","last_active":"2023-09-23T19:57:26.377024+0000","last_peered":"2023-09-23T19:57:26.377024+0000","last_clean":"2023-09-23T19:57:26.377024+0000","last_became_active":"2023-09-23T19:52:57.353333+0000","last_became_peered":"2023-09-23T19:52:57.353333+0000","last_unstale":"2023-09-23T19:57:26.377024+0000","last_undegraded":"2023-09-23T19:57:26.377024+0000","last_fullsized":"2023-09-23T19:57:26.377024+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T19:53:00.704900+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:24.642289+0000","scrub_duration":0.0090978159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":333,"reported_epoch":170,"state":"active+undersized","last_fresh":"2023-09-23T19:57:26.399350+0000","last_change":"2023-09-23T19:54:45.328104+0000","last_active":"2023-09-23T19:57:26.399350+0000","last_peered":"2023-09-23T19:57:26.399350+0000","last_clean":"2023-09-23T19:43:08.441095+0000","last_became_active":"2023-09-23T19:54:45.328104+0000","last_became_peered":"2023-09-23T19:54:45.328104+0000","last_unstale":"2023-09-23T19:57:26.399350+0000","last_undegraded":"2023-09-23T19:57:26.399350+0000","last_fullsized":"2023-09-23T19:54:45.325453+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:40.212909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":119,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387267+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:57:26.387267+0000","last_peered":"2023-09-23T19:57:26.387267+0000","last_clean":"2023-09-23T19:57:26.387267+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:57:26.387267+0000","last_undegraded":"2023-09-23T19:57:26.387267+0000","last_fullsized":"2023-09-23T19:57:26.387267+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":120,"reported_epoch":170,"state":"active+undersized","last_fresh":"2023-09-23T19:57:26.396547+0000","last_change":"2023-09-23T19:54:45.330341+0000","last_active":"2023-09-23T19:57:26.396547+0000","last_peered":"2023-09-23T19:57:26.396547+0000","last_clean":"2023-09-23T19:54:37.305426+0000","last_became_active":"2023-09-23T19:54:45.330341+0000","last_became_peered":"2023-09-23T19:54:45.330341+0000","last_unstale":"2023-09-23T19:57:26.396547+0000","last_undegraded":"2023-09-23T19:57:26.396547+0000","last_fullsized":"2023-09-23T19:54:45.253699+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:22.534625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":103,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.386745+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:57:26.386745+0000","last_peered":"2023-09-23T19:57:26.386745+0000","last_clean":"2023-09-23T19:57:26.386745+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:57:26.386745+0000","last_undegraded":"2023-09-23T19:57:26.386745+0000","last_fullsized":"2023-09-23T19:57:26.386745+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":153,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.391009+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:57:26.391009+0000","last_peered":"2023-09-23T19:57:26.391009+0000","last_clean":"2023-09-23T19:57:26.391009+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:57:26.391009+0000","last_undegraded":"2023-09-23T19:57:26.391009+0000","last_fullsized":"2023-09-23T19:57:26.391009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.386177+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:57:26.386177+0000","last_peered":"2023-09-23T19:57:26.386177+0000","last_clean":"2023-09-23T19:57:26.386177+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:57:26.386177+0000","last_undegraded":"2023-09-23T19:57:26.386177+0000","last_fullsized":"2023-09-23T19:57:26.386177+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396903+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:57:26.396903+0000","last_peered":"2023-09-23T19:57:26.396903+0000","last_clean":"2023-09-23T19:57:26.396903+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:57:26.396903+0000","last_undegraded":"2023-09-23T19:57:26.396903+0000","last_fullsized":"2023-09-23T19:57:26.396903+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":136,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.386116+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T19:57:26.386116+0000","last_peered":"2023-09-23T19:57:26.386116+0000","last_clean":"2023-09-23T19:57:26.386116+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T19:57:26.386116+0000","last_undegraded":"2023-09-23T19:57:26.386116+0000","last_fullsized":"2023-09-23T19:57:26.386116+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:03.354856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.397266+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:57:26.397266+0000","last_peered":"2023-09-23T19:57:26.397266+0000","last_clean":"2023-09-23T19:57:26.397266+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:57:26.397266+0000","last_undegraded":"2023-09-23T19:57:26.397266+0000","last_fullsized":"2023-09-23T19:57:26.397266+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":156,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.376213+0000","last_change":"2023-09-23T19:56:39.276333+0000","last_active":"2023-09-23T19:57:26.376213+0000","last_peered":"2023-09-23T19:57:26.376213+0000","last_clean":"2023-09-23T19:57:26.376213+0000","last_became_active":"2023-09-23T19:56:39.275998+0000","last_became_peered":"2023-09-23T19:56:39.275998+0000","last_unstale":"2023-09-23T19:57:26.376213+0000","last_undegraded":"2023-09-23T19:57:26.376213+0000","last_fullsized":"2023-09-23T19:57:26.376213+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:48.085884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":155,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.378126+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:57:26.378126+0000","last_peered":"2023-09-23T19:57:26.378126+0000","last_clean":"2023-09-23T19:57:26.378126+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:57:26.378126+0000","last_undegraded":"2023-09-23T19:57:26.378126+0000","last_fullsized":"2023-09-23T19:57:26.378126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":119,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.387668+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:57:26.387668+0000","last_peered":"2023-09-23T19:57:26.387668+0000","last_clean":"2023-09-23T19:57:26.387668+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:57:26.387668+0000","last_undegraded":"2023-09-23T19:57:26.387668+0000","last_fullsized":"2023-09-23T19:57:26.387668+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T19:57:26.396404+0000","last_change":"2023-09-23T19:56:39.281130+0000","last_active":"2023-09-23T19:57:26.396404+0000","last_peered":"2023-09-23T19:57:26.396404+0000","last_clean":"2023-09-23T19:57:26.396404+0000","last_became_active":"2023-09-23T19:56:39.280800+0000","last_became_peered":"2023-09-23T19:56:39.280800+0000","last_unstale":"2023-09-23T19:57:26.396404+0000","last_undegraded":"2023-09-23T19:57:26.396404+0000","last_fullsized":"2023-09-23T19:57:26.396404+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:03.511985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3929604096,"num_objects":59961,"num_object_clones":0,"num_object_copies":179883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":59961,"num_write_kb":3837504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11776491520,"data_stored":11776491520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59961,"ondisk_log_size":59961,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":93,"acting":93,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234198,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2022180,"kb_used_data":1913116,"kb_used_omap":0,"kb_used_meta":109056,"kb_avail":91722972,"statfs":{"total":95995035648,"available":93924323328,"internally_reserved":0,"allocated":1959030784,"data_stored":1958477008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":111673344},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":2.0009999999999999,"15min":1.0660000000000001},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.3470000000000004,"5min":46.476999999999997,"15min":46.476999999999997},"last":62.061999999999998},{"interface":"front","average":{"1min":2.528,"5min":2.669,"15min":1.3120000000000001},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":7.4580000000000002,"5min":46.823999999999998,"15min":46.823999999999998},"last":62.131}]},{"osd":1,"last update":"Sat Sep 23 19:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.7,"15min":0.96399999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.7519999999999998,"5min":11.093999999999999,"15min":11.093999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.262,"5min":1.8069999999999999,"15min":0.99299999999999999},"min":{"1min":0.247,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.2300000000000004,"5min":14.747,"15min":14.747},"last":0.41399999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":4.0270000000000001,"15min":1.8919999999999999},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.0609999999999999,"5min":130.673,"15min":130.673},"last":94.629000000000005},{"interface":"front","average":{"1min":1.2230000000000001,"5min":4.1020000000000003,"15min":1.758},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.9269999999999996,"5min":130.71100000000001,"15min":130.71100000000001},"last":94.620000000000005}]},{"osd":3,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.7689999999999999,"15min":1.147},"min":{"1min":0.33200000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.4500000000000002,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":76.393000000000001},{"interface":"front","average":{"1min":1.849,"5min":1.8180000000000001,"15min":1.0349999999999999},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":13.515000000000001,"5min":13.515000000000001,"15min":13.515000000000001},"last":77.974000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:56:35 2023","interfaces":[{"interface":"back","average":{"1min":9.3689999999999998,"5min":4.0380000000000003,"15min":1.6699999999999999},"min":{"1min":0.29799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":143.255,"5min":143.255,"15min":143.255},"last":0.27700000000000002},{"interface":"front","average":{"1min":8.1479999999999997,"5min":3.8460000000000001,"15min":1.6439999999999999},"min":{"1min":0.30599999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":143.41200000000001,"5min":143.41200000000001,"15min":143.41200000000001},"last":0.32100000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:56:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":2.4340000000000002,"15min":1.167},"min":{"1min":0.32100000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.972,"5min":84.566999999999993,"15min":84.566999999999993},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.2229999999999999,"15min":1.113},"min":{"1min":0.32600000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8979999999999997,"5min":81.244,"15min":81.244},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.0189999999999999,"15min":0.71499999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.6449999999999996,"5min":5.6449999999999996,"15min":5.6449999999999996},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.121,"15min":0.76500000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.7720000000000002,"5min":5.7720000000000002,"15min":5.7720000000000002},"last":0.38900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495800,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379500,"kb_used_data":1334692,"kb_used_omap":0,"kb_used_meta":44800,"kb_avail":92365652,"statfs":{"total":95995035648,"available":94582427648,"internally_reserved":0,"allocated":1366724608,"data_stored":1366198307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":45875200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:52 2023","interfaces":[{"interface":"back","average":{"1min":4.415,"5min":1.3500000000000001,"15min":0.85199999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":60.738,"5min":60.738,"15min":60.738},"last":1.476},{"interface":"front","average":{"1min":4.202,"5min":1.333,"15min":0.88700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":60.722999999999999,"5min":60.722999999999999,"15min":60.722999999999999},"last":1.377}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.85799999999999998,"15min":0.69999999999999996},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6520000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.395,"5min":0.94299999999999995,"15min":0.69099999999999995},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":4.3029999999999999,"5min":2.7829999999999999,"15min":1.3149999999999999},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.245},"max":{"1min":53.667999999999999,"5min":61.155000000000001,"15min":61.155000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":4.024,"5min":2.7989999999999999,"15min":1.417},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":53.628,"5min":60.506,"15min":60.506},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":0.97899999999999998,"15min":0.76700000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":5.7619999999999996},"last":1.361},{"interface":"front","average":{"1min":2.1379999999999999,"5min":0.94299999999999995,"15min":0.69299999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":1.387}]},{"osd":4,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":1.0740000000000001,"15min":0.68600000000000005},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.4859999999999998,"5min":24.003,"15min":24.003},"last":0.377},{"interface":"front","average":{"1min":1.607,"5min":1.0920000000000001,"15min":0.70199999999999996},"min":{"1min":0.378,"5min":0.313,"15min":0.26500000000000001},"max":{"1min":6.6020000000000003,"5min":24.802,"15min":24.802},"last":1.3149999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.147,"15min":0.71099999999999997},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":14.378,"5min":24.065999999999999,"15min":24.065999999999999},"last":1.4079999999999999},{"interface":"front","average":{"1min":1.641,"5min":1.1359999999999999,"15min":0.74399999999999999},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":6.0529999999999999,"5min":24.841000000000001,"15min":24.841000000000001},"last":1.2769999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:54:39 2023","interfaces":[{"interface":"back","average":{"1min":6.1509999999999998,"5min":1.8149999999999999,"15min":0.98099999999999998},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":105.71299999999999,"5min":105.71299999999999,"15min":105.71299999999999},"last":2.1579999999999999},{"interface":"front","average":{"1min":6.5620000000000003,"5min":2.0259999999999998,"15min":1.075},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":103.206,"5min":103.206,"15min":103.206},"last":0.46100000000000002}]}]},{"osd":1,"up_from":99,"seq":425201762400,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1609512,"kb_used_data":1491680,"kb_used_omap":7,"kb_used_meta":117816,"kb_avail":92135640,"statfs":{"total":95995035648,"available":94346895360,"internally_reserved":0,"allocated":1527480320,"data_stored":1526748425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":120644410},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":9.8149999999999995,"5min":7.5449999999999999,"15min":2.7530000000000001},"min":{"1min":0.151,"5min":0.151,"15min":0.151},"max":{"1min":186.84,"5min":202.917,"15min":202.917},"last":1.365},{"interface":"front","average":{"1min":10.493,"5min":7.9180000000000001,"15min":2.8820000000000001},"min":{"1min":0.17999999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":191.322,"5min":203.34899999999999,"15min":203.34899999999999},"last":6.1550000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:57:38 2023","interfaces":[{"interface":"back","average":{"1min":8.5419999999999998,"5min":9.9860000000000007,"15min":3.5640000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.192},"max":{"1min":62.697000000000003,"5min":231.57499999999999,"15min":231.57499999999999},"last":3.4380000000000002},{"interface":"front","average":{"1min":9.1219999999999999,"5min":9.2959999999999994,"15min":3.335},"min":{"1min":0.254,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":61.008000000000003,"5min":236.077,"15min":236.077},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1030000000000002,"5min":2.6389999999999998,"15min":1.127},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":26.388000000000002,"5min":26.388000000000002,"15min":26.388000000000002},"last":2.0960000000000001},{"interface":"front","average":{"1min":1.748,"5min":1.9239999999999999,"15min":0.90800000000000003},"min":{"1min":0.24299999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.507,"5min":20.166,"15min":20.166},"last":6.9859999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":3.71,"15min":1.6659999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":7.2149999999999999,"5min":161.68299999999999,"15min":161.68299999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.0309999999999999,"5min":3.4990000000000001,"15min":1.5720000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.944,"5min":161.286,"15min":161.286},"last":2.3220000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":11.356999999999999,"5min":5.8170000000000002,"15min":2.3610000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":219.048,"5min":219.048,"15min":219.048},"last":2.3570000000000002},{"interface":"front","average":{"1min":12.51,"5min":6.4039999999999999,"15min":2.5659999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":218.69499999999999,"5min":218.69499999999999,"15min":218.69499999999999},"last":2.133}]},{"osd":6,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.097,"15min":0.80500000000000005},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.9649999999999999,"5min":7.9649999999999999,"15min":7.9649999999999999},"last":14.897},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.0800000000000001,"15min":0.82399999999999995},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":14.904999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.6400000000000001,"15min":1.3120000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":26.327999999999999,"5min":26.327999999999999,"15min":26.327999999999999},"last":2.1619999999999999},{"interface":"front","average":{"1min":1.393,"5min":1.599,"15min":0.97599999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.1389999999999993,"5min":9.1389999999999993,"15min":9.1389999999999993},"last":0.36599999999999999}]}]},{"osd":0,"up_from":94,"seq":403726925934,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1118620,"kb_used_data":934356,"kb_used_omap":8,"kb_used_meta":184247,"kb_avail":92626532,"statfs":{"total":95995035648,"available":94849568768,"internally_reserved":0,"allocated":956780544,"data_stored":956059724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":188669537},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:57:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.921,"15min":0.85499999999999998},"min":{"1min":0.20399999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":7.2569999999999997,"5min":74.751000000000005,"15min":74.751000000000005},"last":15.4},{"interface":"front","average":{"1min":1.532,"5min":2.5390000000000001,"15min":1.071},"min":{"1min":0.20599999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":7.6379999999999999,"5min":74.998000000000005,"15min":74.998000000000005},"last":15.531000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:56:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":3.0600000000000001,"15min":1.3029999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.7309999999999999,"5min":74.795000000000002,"15min":74.795000000000002},"last":0.23799999999999999},{"interface":"front","average":{"1min":3.512,"5min":3.569,"15min":1.4410000000000001},"min":{"1min":0.245,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":20.189,"5min":78.366,"15min":78.366},"last":0.25700000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:57:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4099999999999999,"5min":3.641,"15min":1.4650000000000001},"min":{"1min":0.29799999999999999,"5min":0.224,"15min":0.224},"max":{"1min":4.984,"5min":104.084,"15min":104.084},"last":120.642},{"interface":"front","average":{"1min":2.3889999999999998,"5min":4.0190000000000001,"15min":1.589},"min":{"1min":0.35399999999999998,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":8.2319999999999993,"5min":104.143,"15min":104.143},"last":120.61199999999999}]},{"osd":4,"last update":"Sat Sep 23 19:56:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":3.7610000000000001,"15min":1.679},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.952,"5min":110.312,"15min":110.312},"last":0.627},{"interface":"front","average":{"1min":2.1749999999999998,"5min":3.4540000000000002,"15min":1.597},"min":{"1min":0.32800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.8140000000000001,"5min":110.256,"15min":110.256},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:56:52 2023","interfaces":[{"interface":"back","average":{"1min":8.9670000000000005,"5min":3.6469999999999998,"15min":1.6799999999999999},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":131.34399999999999,"5min":131.34399999999999,"15min":131.34399999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":9.2690000000000001,"5min":4.1159999999999997,"15min":1.835},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":131.386,"5min":131.386,"15min":131.386},"last":0.497}]},{"osd":6,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":0.89700000000000002,"15min":0.78300000000000003},"min":{"1min":0.40899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.9139999999999997,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":2.4100000000000001},{"interface":"front","average":{"1min":2.3410000000000002,"5min":0.996,"15min":0.81000000000000005},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.6920000000000002,"5min":8.6920000000000002,"15min":8.6920000000000002},"last":1.1679999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:56:52 2023","interfaces":[{"interface":"back","average":{"1min":1.415,"5min":2.431,"15min":1.2909999999999999},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.8490000000000002,"5min":77.132999999999996,"15min":77.132999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.4119999999999999,"5min":2.855,"15min":1.425},"min":{"1min":0.34699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.9809999999999999,"5min":75.465999999999994,"15min":75.465999999999994},"last":0.59099999999999997}]}]},{"osd":2,"up_from":104,"seq":446676598873,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2559840,"kb_used_data":2379736,"kb_used_omap":12,"kb_used_meta":180083,"kb_avail":91185312,"statfs":{"total":95995035648,"available":93373759488,"internally_reserved":0,"allocated":2436849664,"data_stored":2436127858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":184406014},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":4.8700000000000001,"5min":2.7709999999999999,"15min":1.1579999999999999},"min":{"1min":0.154,"5min":0.154,"15min":0.154},"max":{"1min":50.506999999999998,"5min":50.506999999999998,"15min":50.506999999999998},"last":2.2440000000000002},{"interface":"front","average":{"1min":5.5460000000000003,"5min":2.827,"15min":1.1910000000000001},"min":{"1min":0.21299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":50.802999999999997,"5min":50.802999999999997,"15min":50.802999999999997},"last":0.38100000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":4.3010000000000002,"5min":2.3220000000000001,"15min":1.0289999999999999},"min":{"1min":0.22900000000000001,"5min":0.215,"15min":0.214},"max":{"1min":50.061,"5min":50.061,"15min":50.061},"last":0.65900000000000003},{"interface":"front","average":{"1min":4.931,"5min":2.419,"15min":1.0649999999999999},"min":{"1min":0.26700000000000002,"5min":0.217,"15min":0.217},"max":{"1min":50.533999999999999,"5min":50.533999999999999,"15min":50.533999999999999},"last":6.4240000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:57:09 2023","interfaces":[{"interface":"back","average":{"1min":5.9539999999999997,"5min":2.7770000000000001,"15min":1.1850000000000001},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":50.186999999999998,"5min":50.186999999999998,"15min":50.186999999999998},"last":0.68999999999999995},{"interface":"front","average":{"1min":6.3159999999999998,"5min":2.8799999999999999,"15min":1.224},"min":{"1min":0.26000000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":50.585000000000001,"5min":50.585000000000001,"15min":50.585000000000001},"last":3.8500000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":4.3940000000000001,"5min":2.2490000000000001,"15min":1.173},"min":{"1min":0.33000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":50.640000000000001,"5min":50.640000000000001,"15min":50.640000000000001},"last":2.2229999999999999},{"interface":"front","average":{"1min":3.6349999999999998,"5min":2.3900000000000001,"15min":1.2110000000000001},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":50.154000000000003,"5min":50.154000000000003,"15min":50.154000000000003},"last":0.71399999999999997}]},{"osd":5,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":4.7949999999999999,"5min":2.681,"15min":1.3169999999999999},"min":{"1min":0.34599999999999997,"5min":0.308,"15min":0.308},"max":{"1min":50.616,"5min":50.616,"15min":50.616},"last":2.1629999999999998},{"interface":"front","average":{"1min":4.2960000000000003,"5min":2.3540000000000001,"15min":1.204},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":50.762,"5min":50.762,"15min":50.762},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.827,"5min":1.343,"15min":0.84999999999999998},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4740000000000002,"5min":7.4740000000000002,"15min":7.4740000000000002},"last":1.0660000000000001},{"interface":"front","average":{"1min":2.899,"5min":1.3120000000000001,"15min":0.85699999999999998},"min":{"1min":0.42399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.3029999999999999,"5min":7.3029999999999999,"15min":7.3029999999999999},"last":1.008}]},{"osd":7,"last update":"Sat Sep 23 19:56:41 2023","interfaces":[{"interface":"back","average":{"1min":3.7389999999999999,"5min":2.4529999999999998,"15min":1.2589999999999999},"min":{"1min":0.28799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":50.118000000000002,"5min":50.118000000000002,"15min":50.118000000000002},"last":0.74299999999999999},{"interface":"front","average":{"1min":4.3250000000000002,"5min":2.839,"15min":1.393},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":50.664999999999999,"5min":50.664999999999999,"15min":50.664999999999999},"last":2.2000000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435346,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2420676,"kb_used_data":2290620,"kb_used_omap":4,"kb_used_meta":130043,"kb_avail":91324476,"statfs":{"total":95995035648,"available":93516263424,"internally_reserved":0,"allocated":2345594880,"data_stored":2344836277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":133164141},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":7.2359999999999998,"15min":2.6469999999999998},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":3.2509999999999999,"5min":240.40600000000001,"15min":240.40600000000001},"last":0.505},{"interface":"front","average":{"1min":1.8779999999999999,"5min":7.8609999999999998,"15min":2.8620000000000001},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.1329999999999991,"5min":240.352,"15min":240.352},"last":2.2959999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.5860000000000001,"15min":0.78400000000000003},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.1619999999999999,"5min":8.5289999999999999,"15min":8.5289999999999999},"last":1.159},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.4239999999999999,"15min":0.72799999999999998},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.6699999999999999,"5min":11.361000000000001,"15min":11.361000000000001},"last":2.077}]},{"osd":2,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":6.423,"5min":10.749000000000001,"15min":3.8420000000000001},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":83.590000000000003,"5min":264.97800000000001,"15min":264.97800000000001},"last":1.6579999999999999},{"interface":"front","average":{"1min":7.7910000000000004,"5min":10.734999999999999,"15min":3.8530000000000002},"min":{"1min":0.26100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":83.344999999999999,"5min":257.40899999999999,"15min":257.40899999999999},"last":3.9399999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":2.0649999999999999,"15min":1.077},"min":{"1min":0.30199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.7869999999999999,"5min":69.236000000000004,"15min":69.236000000000004},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.714,"5min":2.7189999999999999,"15min":1.2969999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.2420000000000009,"5min":69.373000000000005,"15min":69.373000000000005},"last":2.2690000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":1.8480000000000001,"15min":1.0169999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.0830000000000002,"5min":13.185,"15min":13.185},"last":2.2349999999999999},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.694,"15min":0.98399999999999999},"min":{"1min":0.35099999999999998,"5min":0.313,"15min":0.313},"max":{"1min":2.4089999999999998,"5min":10.977,"15min":10.977},"last":1.1850000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":3.4809999999999999,"5min":1.2849999999999999,"15min":0.82099999999999995},"min":{"1min":0.38600000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.4399999999999995,"5min":9.4399999999999995,"15min":9.4399999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.1179999999999999,"5min":0.94599999999999995,"15min":0.72099999999999997},"min":{"1min":0.44400000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":10.932,"5min":10.932,"15min":10.932},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.784,"15min":1.0069999999999999},"min":{"1min":0.38,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.8839999999999999,"5min":12.720000000000001,"15min":12.720000000000001},"last":1.208},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.4770000000000001,"15min":0.91100000000000003},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8199999999999998,"5min":9.8369999999999997,"15min":9.8369999999999997},"last":0.48299999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019108,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1333472,"kb_used_data":1197336,"kb_used_omap":0,"kb_used_meta":136128,"kb_avail":92411680,"statfs":{"total":95995035648,"available":94629560320,"internally_reserved":0,"allocated":1226072064,"data_stored":1225522372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":139395072},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:54 2023","interfaces":[{"interface":"back","average":{"1min":6.6820000000000004,"5min":4.9279999999999999,"15min":2.0409999999999999},"min":{"1min":0.23300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":104.604,"5min":104.604,"15min":104.604},"last":1.6870000000000001},{"interface":"front","average":{"1min":6.8040000000000003,"5min":4.8109999999999999,"15min":2.0270000000000001},"min":{"1min":0.29699999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":104.687,"5min":104.687,"15min":104.687},"last":1.7030000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":2.1099999999999999,"15min":1.073},"min":{"1min":0.23400000000000001,"5min":0.216,"15min":0.216},"max":{"1min":12.718999999999999,"5min":12.718999999999999,"15min":12.718999999999999},"last":1.3100000000000001},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.5109999999999999,"15min":0.879},"min":{"1min":0.316,"5min":0.22,"15min":0.22},"max":{"1min":2.7679999999999998,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":1.115}]},{"osd":2,"last update":"Sat Sep 23 19:56:39 2023","interfaces":[{"interface":"back","average":{"1min":9.3209999999999997,"5min":3.9239999999999999,"15min":1.6819999999999999},"min":{"1min":0.27600000000000002,"5min":0.25,"15min":0.25},"max":{"1min":98.063999999999993,"5min":98.063999999999993,"15min":98.063999999999993},"last":4.9279999999999999},{"interface":"front","average":{"1min":9.1379999999999999,"5min":3.8210000000000002,"15min":1.637},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":97.921000000000006,"5min":97.921000000000006,"15min":97.921000000000006},"last":1.0289999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":5.9039999999999999,"5min":2.746,"15min":1.27},"min":{"1min":0.36099999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":94.141000000000005,"5min":94.141000000000005,"15min":94.141000000000005},"last":4.8579999999999997},{"interface":"front","average":{"1min":6.7750000000000004,"5min":3.181,"15min":1.4550000000000001},"min":{"1min":0.435,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":94.037000000000006,"5min":94.037000000000006,"15min":94.037000000000006},"last":10.930999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:56:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":1.3700000000000001,"15min":0.77200000000000002},"min":{"1min":0.28399999999999997,"5min":0.246,"15min":0.246},"max":{"1min":2.782,"5min":10.246,"15min":10.246},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.6359999999999999,"15min":0.88800000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":8.7629999999999999,"5min":12.675000000000001,"15min":12.675000000000001},"last":0.33600000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":0.84699999999999998,"15min":0.60199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.867,"5min":5.867,"15min":5.867},"last":8.0150000000000006},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.80300000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":3.931}]},{"osd":7,"last update":"Sat Sep 23 19:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.4710000000000001,"15min":0.872},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.248,"5min":11.59,"15min":11.59},"last":0.373},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.3759999999999999,"15min":0.86099999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.2970000000000002,"5min":9.3089999999999993,"15min":9.3089999999999993},"last":0.32300000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757472,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1365144,"kb_used_data":1209936,"kb_used_omap":0,"kb_used_meta":155200,"kb_avail":92380008,"statfs":{"total":95995035648,"available":94597128192,"internally_reserved":0,"allocated":1238974464,"data_stored":1238429213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":158924800},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:56:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0009999999999999,"5min":2.0030000000000001,"15min":1.042},"min":{"1min":0.219,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":18.651,"5min":24.852,"15min":24.852},"last":2.8239999999999998},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.8839999999999999,"15min":0.99399999999999999},"min":{"1min":0.224,"5min":0.216,"15min":0.216},"max":{"1min":18.547999999999998,"5min":24.638999999999999,"15min":24.638999999999999},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.8879999999999999,"15min":0.97799999999999998},"min":{"1min":0.26300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":8.4879999999999995,"5min":9.9789999999999992,"15min":9.9789999999999992},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.7829999999999999,"15min":0.94399999999999995},"min":{"1min":0.27900000000000003,"5min":0.249,"15min":0.249},"max":{"1min":6.0209999999999999,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":11.048,"5min":6.649,"15min":2.5750000000000002},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":141.863,"5min":167.10400000000001,"15min":167.10400000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":11.08,"5min":6.6829999999999998,"15min":2.6200000000000001},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":141.91999999999999,"5min":167.167,"15min":167.167},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":2.052,"15min":1.048},"min":{"1min":0.34300000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.548,"5min":12.853999999999999,"15min":12.853999999999999},"last":2.4239999999999999},{"interface":"front","average":{"1min":1.052,"5min":1.772,"15min":0.995},"min":{"1min":0.373,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2439999999999998,"5min":10.115,"15min":10.115},"last":0.80800000000000005}]},{"osd":4,"last update":"Sat Sep 23 19:56:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.288,"15min":0.73799999999999999},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.23699999999999999},"max":{"1min":6.0519999999999996,"5min":10.077,"15min":10.077},"last":0.37},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.395,"15min":0.80200000000000005},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.52,"5min":10.226000000000001,"15min":10.226000000000001},"last":0.34000000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:54:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":0.91600000000000004,"15min":0.63400000000000001},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":10.180999999999999,"5min":10.180999999999999,"15min":10.180999999999999},"last":4.2949999999999999},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.046,"15min":0.71299999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":10.406000000000001},"last":3.879}]},{"osd":7,"last update":"Sat Sep 23 19:56:38 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.365,"15min":0.82199999999999995},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7349999999999994,"5min":13.734999999999999,"15min":13.734999999999999},"last":0.307},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.5309999999999999,"15min":0.88100000000000001},"min":{"1min":0.34699999999999998,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":6.0540000000000003,"5min":10.041,"15min":10.041},"last":2.0270000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":962002944,"data_stored":962002944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1530527744,"data_stored":1530527744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2456748032,"data_stored":2456748032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2372009984,"data_stored":2372009984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1240989696,"data_stored":1240989696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1242300416,"data_stored":1242300416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1971912704,"data_stored":1971912704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:57:45.851 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-09-23T19:57:45.891 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 47 16 67896 67880 90.1987 63.9375 0.0121997 0.0110813 2023-09-23T19:57:45.918 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:46.024 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:46.212 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:46.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:46.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:46.326 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:46.393 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:46 smithi135 ceph-mon[213871]: pgmap v449: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5/193536 objects degraded (0.003%) 2023-09-23T19:57:46.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:46 smithi135 ceph-mon[221113]: pgmap v449: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5/193536 objects degraded (0.003%) 2023-09-23T19:57:46.400 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:46 smithi144 ceph-mon[194833]: pgmap v449: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5/193536 objects degraded (0.003%) 2023-09-23T19:57:46.652 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:46.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:46.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:46.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:46.852 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:46.891 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 48 16 68821 68805 89.5243 57.8125 0.0239707 0.0111646 2023-09-23T19:57:46.946 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:57:46.952 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:46.983 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:57:47.027 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:47.029 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:57:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:47.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:47.046 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:47.069 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:57:47.106 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:57:47.136 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:47.142 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:57:47.184 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:57:47.219 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:47.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2242816884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:47.393 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2242816884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2242816884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:57:47.794 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:57:47.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 49 16 69875 69859 89.0391 65.875 0.0237105 0.0112259 2023-09-23T19:57:48.571 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:48.575 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:48 smithi135 ceph-mon[221113]: pgmap v450: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 5/197760 objects degraded (0.003%) 2023-09-23T19:57:48.575 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:48 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/197760 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:48.576 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:48 smithi135 ceph-mon[213871]: pgmap v450: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 5/197760 objects degraded (0.003%) 2023-09-23T19:57:48.576 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:48 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/197760 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:48.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:48.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:48 smithi144 ceph-mon[194833]: pgmap v450: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 5/197760 objects degraded (0.003%) 2023-09-23T19:57:48.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:48 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/197760 objects degraded (0.003%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:48.678 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:48.711 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:48.765 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:48.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:48.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:48.872 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:57:48.873 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:57:48.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 50 16 70749 70733 88.3511 54.625 0.0169496 0.0113136 2023-09-23T19:57:48.902 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:48.960 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:48.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:48.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:49.066 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:49.097 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:49.152 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:49.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:49.258 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:49.337 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:49.695 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:49.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:49.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:49.801 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:49.827 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:49.868 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:49.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:49.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 51 16 71702 71686 87.7869 59.5625 0.00308691 0.0113599 2023-09-23T19:57:49.971 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:50.041 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:50.292 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:50.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:50.300 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:50 smithi135 ceph-mon[221113]: pgmap v451: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 5/207030 objects degraded (0.002%) 2023-09-23T19:57:50.303 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:50 smithi135 ceph-mon[213871]: pgmap v451: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 5/207030 objects degraded (0.002%) 2023-09-23T19:57:50.401 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:50.482 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:50 smithi144 ceph-mon[194833]: pgmap v451: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 5/207030 objects degraded (0.002%) 2023-09-23T19:57:50.651 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:50.709 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:50.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:50.823 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:50.874 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:57:50.880 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:50.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 52 16 72616 72600 87.1976 57.125 0.0151253 0.0114625 2023-09-23T19:57:51.042 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:51.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:51.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:51.153 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:51.266 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:51.354 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:57:51.354 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:114614 5.c 5:3bdfca03:::benchmark_data_smithi135_284570_object114613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:53:56.071208+0000", 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 235.27965460600001, 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22332307600000001, 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.355 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 114614 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071208+0000", 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071208+0000", 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071206+0000", 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071227+0000", 2023-09-23T19:57:51.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0344999999999999e-05 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071236+0000", 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2450000000000004e-06 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071241+0000", 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5199999999999997e-06 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071281+0000", 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9801000000000001e-05 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071370+0000", 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8949999999999994e-05 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.071455+0000", 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4414999999999997e-05 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.074979+0000", 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0035246019999999999 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.078084+0000", 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031047700000000002 2023-09-23T19:57:51.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.294440+0000", 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21635629100000001 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.294490+0000", 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9446000000000001e-05 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:53:56.294531+0000", 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1065000000000001e-05 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:126118 5.c 5:380143a9:::benchmark_data_smithi135_284570_object10334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:11.241628+0000", 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 220.10923441200001, 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24666907800000001, 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.362 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 126118 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241628+0000", 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241628+0000", 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241627+0000", 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241632+0000", 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.31e-06 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241641+0000", 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2660000000000007e-06 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241647+0000", 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8579999999999996e-06 2023-09-23T19:57:51.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241685+0000", 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7818999999999998e-05 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241766+0000", 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0822000000000006e-05 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.241835+0000", 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8881999999999995e-05 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.244659+0000", 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028244749999999999 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.246015+0000", 2023-09-23T19:57:51.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0013558470000000001 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.488191+0000", 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24217628999999999 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.488251+0000", 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9348999999999997e-05 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:11.488297+0000", 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6428999999999999e-05 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:148373 5.c 5:3fc24fd7:::benchmark_data_smithi135_284570_object32589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.657866+0000", 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 188.69299637200001, 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.230255707, 2023-09-23T19:57:51.369 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148373 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657866+0000", 2023-09-23T19:57:51.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657866+0000", 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657865+0000", 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657870+0000", 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4219999999999999e-06 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657883+0000", 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2418999999999999e-05 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657888+0000", 2023-09-23T19:57:51.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7320000000000003e-06 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.657948+0000", 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9524999999999997e-05 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658053+0000", 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000105282 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658154+0000", 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101173 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887885+0000", 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.229730404 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887926+0000", 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1737999999999999e-05 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888044+0000", 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117816 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888083+0000", 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8380999999999997e-05 2023-09-23T19:57:51.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888122+0000", 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9079000000000003e-05 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:148374 5.c 5:31a4f63e:::benchmark_data_smithi135_284570_object32590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.658148+0000", 2023-09-23T19:57:51.376 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 188.692714776, 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.23127130900000001, 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148374 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658148+0000", 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658148+0000", 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658147+0000", 2023-09-23T19:57:51.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658151+0000", 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1200000000000004e-06 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658163+0000", 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.224e-05 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.658169+0000", 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.015e-06 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661984+0000", 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038152279999999999 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662082+0000", 2023-09-23T19:57:51.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8055999999999998e-05 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662151+0000", 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8774000000000003e-05 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888037+0000", 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22588576199999999 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888126+0000", 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9195000000000002e-05 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889336+0000", 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012101569999999999 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889381+0000", 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4808000000000002e-05 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889419+0000", 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7645000000000003e-05 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.383 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:148373 5.c e143/112 ondisk, result = 0)", 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.661352+0000", 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 188.689509845, 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22653495600000001, 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661352+0000", 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661352+0000", 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661352+0000", 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661357+0000", 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8050000000000003e-06 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661368+0000", 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0984e-05 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.661374+0000", 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8810000000000001e-06 2023-09-23T19:57:51.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887872+0000", 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.226497741 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887879+0000", 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0419999999999997e-06 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887887+0000", 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4079999999999998e-06 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:148377 5.5 e143/112 5:a6d497bb:::benchmark_data_smithi135_284570_object32593:head v 143'9284, mlcod=143'9282)", 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.662002+0000", 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 188.68885991299999, 2023-09-23T19:57:51.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.225817616, 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662002+0000", 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662002+0000", 2023-09-23T19:57:51.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662002+0000", 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662008+0000", 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9850000000000003e-06 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662015+0000", 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5950000000000001e-06 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662020+0000", 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7219999999999999e-06 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662064+0000", 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3977000000000003e-05 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.662082+0000", 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8164000000000001e-05 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664929+0000", 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028465999999999999 2023-09-23T19:57:51.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887820+0000", 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.222891216 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:148380 5.0 e143/112 5:05c78c8a:::benchmark_data_smithi135_284570_object32596:head v 143'9387, mlcod=143'9385)", 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.664519+0000", 2023-09-23T19:57:51.393 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 188.686342846, 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.223721211, 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664519+0000", 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664519+0000", 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664519+0000", 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664526+0000", 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4739999999999999e-06 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664538+0000", 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2330000000000001e-05 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664546+0000", 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3429999999999998e-06 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664589+0000", 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3686000000000002e-05 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664610+0000", 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0885000000000001e-05 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888206+0000", 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22359541199999999 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.888241+0000", 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4885000000000002e-05 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:148373 5.c e143/112 ondisk, result = 0)", 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.666374+0000", 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 188.68448821199999, 2023-09-23T19:57:51.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.221554634, 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.666374+0000", 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.666374+0000", 2023-09-23T19:57:51.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.666373+0000", 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.666381+0000", 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6020000000000002e-06 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887631+0000", 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22124992099999999 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887647+0000", 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5733999999999999e-05 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887915+0000", 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00026879200000000001 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887921+0000", 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8359999999999997e-06 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.887929+0000", 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4189999999999996e-06 2023-09-23T19:57:51.402 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:189893 5.c 5:3cf87985:::benchmark_data_smithi135_284570_object74109:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:40.360813+0000", 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 130.99004923300001, 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20771811300000001, 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.403 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 189893 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360813+0000", 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360813+0000", 2023-09-23T19:57:51.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360812+0000", 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360817+0000", 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5049999999999998e-06 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360830+0000", 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2425999999999999e-05 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360835+0000", 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5949999999999998e-06 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360873+0000", 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7963000000000002e-05 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.360983+0000", 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010943899999999999 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.361050+0000", 2023-09-23T19:57:51.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6923999999999998e-05 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.363988+0000", 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029381730000000001 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.365200+0000", 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012125459999999999 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.568446+0000", 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20324604499999999 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.568493+0000", 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6761000000000002e-05 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:40.568531+0000", 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8022999999999998e-05 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220921 5.c 5:34d3c8ae:::benchmark_data_smithi135_284570_object105137:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:22.506521+0000", 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 88.844341545000006, 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20951647000000001, 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.410 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220921 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506521+0000", 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506521+0000", 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506519+0000", 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506526+0000", 2023-09-23T19:57:51.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7240000000000003e-06 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506538+0000", 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1460999999999999e-05 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506545+0000", 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8450000000000003e-06 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506592+0000", 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7048000000000001e-05 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506718+0000", 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012579299999999999 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.506836+0000", 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117941 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.510313+0000", 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034770640000000002 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.510375+0000", 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1398999999999999e-05 2023-09-23T19:57:51.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.715943+0000", 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.205568326 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.715994+0000", 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0586e-05 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:22.716037+0000", 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3714000000000002e-05 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:222026 5.c 5:30c5ba68:::benchmark_data_smithi135_284570_object106242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:24.295255+0000", 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 87.055607770999998, 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.230908946, 2023-09-23T19:57:51.417 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 222026 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295255+0000", 2023-09-23T19:57:51.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295255+0000", 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295253+0000", 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.419 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295260+0000", 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6069999999999996e-06 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295271+0000", 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1369000000000001e-05 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295278+0000", 2023-09-23T19:57:51.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8379999999999996e-06 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295324+0000", 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5710000000000001e-05 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295428+0000", 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000104897 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.295505+0000", 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6792000000000006e-05 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.304664+0000", 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0091589099999999993 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.304679+0000", 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5150999999999999e-05 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526068+0000", 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22138866300000001 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526122+0000", 2023-09-23T19:57:51.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4168e-05 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526163+0000", 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1326999999999999e-05 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:222031 5.c 5:36db52b5:::benchmark_data_smithi135_284570_object106247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:24.299986+0000", 2023-09-23T19:57:51.424 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 87.050876158999998, 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22627581699999999, 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 222031 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.299986+0000", 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.299986+0000", 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.299984+0000", 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:57:51.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.299993+0000", 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1730000000000001e-06 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.300011+0000", 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7504999999999999e-05 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.300021+0000", 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0020999999999999e-05 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.300084+0000", 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2970000000000002e-05 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.300218+0000", 2023-09-23T19:57:51.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013407800000000001 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.300344+0000", 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012555800000000001 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.304775+0000", 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0044315910000000003 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.310738+0000", 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0059629289999999996 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526197+0000", 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21545847700000001 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526231+0000", 2023-09-23T19:57:51.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4669999999999998e-05 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526262+0000", 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.078e-05 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:222036 5.c 5:32c6dd63:::benchmark_data_smithi135_284570_object106252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:57:51.431 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:24.305360+0000", 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 87.045501950000002, 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22098923200000001, 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 222036 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305360+0000", 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305360+0000", 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305359+0000", 2023-09-23T19:57:51.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305367+0000", 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1070000000000005e-06 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305381+0000", 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4147e-05 2023-09-23T19:57:51.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305389+0000", 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1969999999999996e-06 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305427+0000", 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8170999999999998e-05 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305520+0000", 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2276999999999998e-05 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.305597+0000", 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6843999999999998e-05 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.308848+0000", 2023-09-23T19:57:51.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003251 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.310843+0000", 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019949820000000002 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526290+0000", 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.215447946 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526321+0000", 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1016000000000003e-05 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:24.526350+0000", 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8113e-05 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:43436 6.4 6:2dbcd109:::benchmark_data_smithi135_344736_object43435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:23.056996+0000", 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 28.293865809, 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.208080072, 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 43436 2023-09-23T19:57:51.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.056996+0000", 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.056996+0000", 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.056995+0000", 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.057024+0000", 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8756e-05 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.057054+0000", 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9918000000000001e-05 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.057069+0000", 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5071e-05 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.057112+0000", 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3497e-05 2023-09-23T19:57:51.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.057246+0000", 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013409699999999999 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.057379+0000", 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0001328 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.059113+0000", 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017339429999999999 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.061176+0000", 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020629860000000002 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.264971+0000", 2023-09-23T19:57:51.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20379550399999999 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.265030+0000", 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8498999999999997e-05 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:23.265077+0000", 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6607000000000003e-05 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.445 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:57966 6.1 6:8fb0d209:::benchmark_data_smithi135_344736_object57965:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.402082+0000", 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 15.948780073, 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22303630399999999, 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:51.446 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 57966 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402082+0000", 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402082+0000", 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402081+0000", 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402105+0000", 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4189000000000002e-05 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402116+0000", 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0932e-05 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402122+0000", 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9699999999999996e-06 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402165+0000", 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3198999999999998e-05 2023-09-23T19:57:51.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402274+0000", 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010880299999999999 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.402374+0000", 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100399 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.624850+0000", 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22247588700000001 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.624896+0000", 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5954000000000001e-05 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.625022+0000", 2023-09-23T19:57:51.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012610200000000001 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.625074+0000", 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2070000000000001e-05 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.625119+0000", 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4434999999999997e-05 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.452 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:57968 6.e e171/163 6:7f9f7c6b:::benchmark_data_smithi135_344736_object57967:head v 171'3648, mlcod=171'3646)", 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.403012+0000", 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 15.947850637, 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22243292000000001, 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.403012+0000", 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.403012+0000", 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.403011+0000", 2023-09-23T19:57:51.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.403065+0000", 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3946000000000002e-05 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.623192+0000", 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22012759600000001 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.623206+0000", 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3380999999999999e-05 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.623261+0000", 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.524e-05 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.623287+0000", 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5819000000000001e-05 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.625410+0000", 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0021231919999999999 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.625445+0000", 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4687999999999997e-05 2023-09-23T19:57:51.457 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44942.0:57966 6.1 e171/163 ondisk, result = 0)", 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.404228+0000", 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 15.946634076, 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22062600199999999, 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:51.458 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.404228+0000", 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.404228+0000", 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.404228+0000", 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.404234+0000", 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2419999999999997e-06 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.623197+0000", 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21896268799999999 2023-09-23T19:57:51.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.623208+0000", 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.114e-05 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.624839+0000", 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001631609 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.624846+0000", 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5899999999999996e-06 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.624854+0000", 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3720000000000005e-06 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.462 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:69994 6.1 6:860f4b7d:::benchmark_data_smithi135_344736_object69993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:48.009763+0000", 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.341099367, 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21653482600000001, 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 69994 2023-09-23T19:57:51.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.009763+0000", 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.009763+0000", 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.009761+0000", 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.009787+0000", 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6027e-05 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.009801+0000", 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3705e-05 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.009809+0000", 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3599999999999996e-06 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.009867+0000", 2023-09-23T19:57:51.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7200000000000001e-05 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.010025+0000", 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015859399999999999 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.010151+0000", 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012589599999999999 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.015310+0000", 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005158368 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021184+0000", 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058749170000000003 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.468 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.226203+0000", 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20501829299999999 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.226258+0000", 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.524e-05 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.226298+0000", 2023-09-23T19:57:51.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.981e-05 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:70003 6.1 6:8a08750a:::benchmark_data_smithi135_344736_object70002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:48.018289+0000", 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.3325734900000001, 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20811128700000001, 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.470 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 70003 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018289+0000", 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.471 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018289+0000", 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018287+0000", 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018313+0000", 2023-09-23T19:57:51.472 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6222000000000001e-05 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018326+0000", 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2867999999999999e-05 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018334+0000", 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4359999999999996e-06 2023-09-23T19:57:51.473 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018390+0000", 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5940999999999999e-05 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018499+0000", 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000108855 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.018646+0000", 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014703200000000001 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021048+0000", 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024021950000000002 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021247+0000", 2023-09-23T19:57:51.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019960999999999999 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.226336+0000", 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20508901600000001 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.226371+0000", 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4513e-05 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.226400+0000", 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9206000000000001e-05 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:70009 6.1 6:8933e7fa:::benchmark_data_smithi135_344736_object70008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:48.021727+0000", 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 3.329134941, 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.207282461, 2023-09-23T19:57:51.477 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 70009 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021727+0000", 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021727+0000", 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021726+0000", 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021752+0000", 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6137999999999999e-05 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021761+0000", 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3009999999999995e-06 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021769+0000", 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5730000000000002e-06 2023-09-23T19:57:51.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021817+0000", 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7929000000000001e-05 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.021987+0000", 2023-09-23T19:57:51.481 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017016700000000001 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.022148+0000", 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000161003 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:57:51.482 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.024814+0000", 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002665859 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.025144+0000", 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00033047099999999999 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.228913+0000", 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20376850199999999 2023-09-23T19:57:51.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.228967+0000", 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3946000000000002e-05 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:48.229010+0000", 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3006999999999997e-05 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.484 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.485 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:51.485 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:51.485 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:57:51.734 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:51.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:51.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:51.838 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:51.878 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:51.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 53 16 73511 73495 86.608 55.9375 0.0244495 0.0115406 2023-09-23T19:57:51.937 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:51.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:51.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:52.044 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:52.127 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:52.310 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 1 pgp_num to 11 2023-09-23T19:57:52.601 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:52.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:52.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:52 smithi135 ceph-mon[213871]: pgmap v452: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 5/208986 objects degraded (0.002%) 2023-09-23T19:57:52.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:52.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1758291430' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:57:52.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:57:52.609 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:52 smithi135 ceph-mon[221113]: pgmap v452: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 5/208986 objects degraded (0.002%) 2023-09-23T19:57:52.609 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:52.609 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1758291430' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:57:52.609 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:57:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:52 smithi144 ceph-mon[194833]: pgmap v452: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 5/208986 objects degraded (0.002%) 2023-09-23T19:57:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:57:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1758291430' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:57:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T19:57:52.709 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:52.748 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:52.810 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:52.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:52.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:52.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 54 16 74349 74333 85.9738 52.375 0.0113836 0.0116265 2023-09-23T19:57:52.915 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:52.991 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:53.288 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/208986 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:53.288 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[213871]: failed to encode map e173 with expected crc 2023-09-23T19:57:53.295 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/208986 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:53.295 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[221113]: failed to encode map e173 with expected crc 2023-09-23T19:57:53.295 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T19:57:53.295 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[221113]: osdmap e173: 8 total, 7 up, 8 in 2023-09-23T19:57:53.295 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[221113]: failed to encode map e173 with expected crc 2023-09-23T19:57:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/208986 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:53 smithi144 ceph-mon[194833]: failed to encode map e173 with expected crc 2023-09-23T19:57:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T19:57:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:53 smithi144 ceph-mon[194833]: osdmap e173: 8 total, 7 up, 8 in 2023-09-23T19:57:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:53 smithi144 ceph-mon[194833]: failed to encode map e173 with expected crc 2023-09-23T19:57:53.661 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T19:57:53.661 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[213871]: osdmap e173: 8 total, 7 up, 8 in 2023-09-23T19:57:53.661 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:53 smithi135 ceph-mon[213871]: failed to encode map e173 with expected crc 2023-09-23T19:57:53.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 55 16 75230 75214 85.412 55.0625 0.00381315 0.01169 2023-09-23T19:57:53.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:57:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:57:53] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.43.0" 2023-09-23T19:57:54.007 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:54.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:54.122 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:57:54.122 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:54.206 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:54.295 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:54 smithi135 ceph-mon[213871]: pgmap v454: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 5/213249 objects degraded (0.002%) 2023-09-23T19:57:54.506 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:54.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:54.620 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:54 smithi144 ceph-mon[194833]: pgmap v454: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 5/213249 objects degraded (0.002%) 2023-09-23T19:57:54.653 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:54.653 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:54.712 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:54.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:54.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:54.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:54 smithi135 ceph-mon[221113]: pgmap v454: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 5/213249 objects degraded (0.002%) 2023-09-23T19:57:54.817 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:57:54.884 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:54.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 56 16 76108 76092 84.8669 54.875 0.0323553 0.0117772 2023-09-23T19:57:55.034 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:57:55.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:55.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:57:55.140 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:55.236 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:55.361 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:57:55.421 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:55.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:55.525 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:57:55.594 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:55.894 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 57 16 77008 76992 84.3651 56.25 0.00911571 0.0118483 2023-09-23T19:57:56.007 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:57:56.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:56.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:57:56.114 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:56.154 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:56.221 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:56.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:56.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:56.330 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:56.333 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:57:56.358 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:56.419 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:56.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:56 smithi144 ceph-mon[194833]: pgmap v455: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s; 5/221010 objects degraded (0.002%) 2023-09-23T19:57:56.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3398610438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.422 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:56 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:56.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:56.524 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:56.562 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:56.614 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:56.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:56.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:56.721 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:56 smithi135 ceph-mon[221113]: pgmap v455: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s; 5/221010 objects degraded (0.002%) 2023-09-23T19:57:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3398610438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:56 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:56 smithi135 ceph-mon[213871]: pgmap v455: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s; 5/221010 objects degraded (0.002%) 2023-09-23T19:57:56.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3398610438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:56 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:57:56.751 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:56.815 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:56.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:56.894 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 58 16 78045 78029 84.028 64.8125 0.00285586 0.0118914 2023-09-23T19:57:56.921 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:56.949 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:57:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: osdmap e174: 8 total, 7 up, 8 in 2023-09-23T19:57:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/221010 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:57.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:57:57.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:57.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: osdmap e174: 8 total, 7 up, 8 in 2023-09-23T19:57:57.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/221010 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:57:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:57:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: osdmap e174: 8 total, 7 up, 8 in 2023-09-23T19:57:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: failed to encode map e174 with expected crc 2023-09-23T19:57:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/221010 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:57:57.894 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 59 16 78890 78874 83.4992 52.8125 0.0123439 0.0119712 2023-09-23T19:57:58.207 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:58.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:58.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:58.317 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:57:58.331 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:57:58 smithi135 ceph-mon[213871]: pgmap v457: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 5/225147 objects degraded (0.002%) 2023-09-23T19:57:58.410 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:57:58 smithi144 ceph-mon[194833]: pgmap v457: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 5/225147 objects degraded (0.002%) 2023-09-23T19:57:58.681 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:57:58 smithi135 ceph-mon[221113]: pgmap v457: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 5/225147 objects degraded (0.002%) 2023-09-23T19:57:58.769 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:57:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:58.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:57:58.890 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:57:58.895 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T19:57:58.896936+0000 min lat: 0.001133 max lat: 0.365714 avg lat: 0.0120441 2023-09-23T19:57:58.896 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:57:58.896 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 60 16 79743 79727 82.9938 53.3125 0.00714391 0.0120441 2023-09-23T19:57:58.967 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:59.071 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:57:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:59.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:57:59.180 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:57:59.211 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:59.256 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:57:59.256 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:57:59.256 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-09-23T19:57:59.256 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:57:59.272 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:57:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:59.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:57:59.378 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:57:59.407 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:59.459 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:57:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:59.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:57:59.568 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:59.570 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:57:59.599 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:59.615 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:57:59.655 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:80364 6.a e174/163 6:561b6b54:::benchmark_data_smithi135_344736_object80363:head v 174'5001, mlcod=174'4998)", 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:59.639837+0000", 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.011859791, 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011901559000000001, 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:59.656 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.639837+0000", 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.639837+0000", 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.639836+0000", 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.639878+0000", 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1980000000000001e-05 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:59.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.639890+0000", 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.218e-05 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.639898+0000", 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2740000000000007e-06 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.640113+0000", 2023-09-23T19:57:59.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00021511499999999999 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.640134+0000", 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0712000000000001e-05 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:80368 6.2 e174/163 6:42a3c977:::benchmark_data_smithi135_344736_object80367:head v 174'4928, mlcod=174'4926)", 2023-09-23T19:57:59.660 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:59.642620+0000", 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0090768159999999997, 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0092006829999999994, 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.642620+0000", 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.642620+0000", 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.642619+0000", 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:59.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.642723+0000", 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000104202 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.642733+0000", 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0548e-05 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.642742+0000", 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2800000000000003e-06 2023-09-23T19:57:59.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.642795+0000", 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3380000000000001e-05 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.642815+0000", 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0271999999999998e-05 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:59.664 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:80378 6.2 e174/163 6:476b68c0:::benchmark_data_smithi135_344736_object80377:head v 174'4929, mlcod=174'4926)", 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:59.650462+0000", 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.001234497, 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014308229999999999, 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:59.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.650462+0000", 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.650462+0000", 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.650461+0000", 2023-09-23T19:57:59.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.650494+0000", 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3300999999999998e-05 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.650513+0000", 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8794e-05 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.650522+0000", 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2150000000000007e-06 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.650960+0000", 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043773 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:59.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.650978+0000", 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7966e-05 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:80379 6.3 e174/163 6:cb8d0971:::benchmark_data_smithi135_344736_object80378:head v 174'5080, mlcod=174'5077)", 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:59.651235+0000", 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.00046116499999999998, 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00072258999999999995, 2023-09-23T19:57:59.669 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:59.670 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:57:59.670 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:59.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:59.670 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651235+0000", 2023-09-23T19:57:59.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651235+0000", 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651228+0000", 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999928 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:59.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651289+0000", 2023-09-23T19:57:59.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0593000000000001e-05 2023-09-23T19:57:59.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:59.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651305+0000", 2023-09-23T19:57:59.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6483000000000001e-05 2023-09-23T19:57:59.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651312+0000", 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0099999999999998e-06 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651411+0000", 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9117999999999995e-05 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:57:59.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651431+0000", 2023-09-23T19:57:59.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9649000000000001e-05 2023-09-23T19:57:59.674 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.674 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:59.674 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.674 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:80374 6.5 e174/163)", 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:59.651326+0000", 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.00037034600000000002, 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00068678000000000001, 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651326+0000", 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651326+0000", 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651326+0000", 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:57:59.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651355+0000", 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9754e-05 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651362+0000", 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3060000000000003e-06 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:59.651366+0000", 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.544e-06 2023-09-23T19:57:59.677 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.678 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:57:59.678 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.678 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:57:59.678 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T19:57:59.678 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 5 2023-09-23T19:57:59.678 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:57:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:59.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:59.680 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:57:59.723 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:57:59.764 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:57:59.780 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:57:59.810 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:57:59.816 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:57:59.854 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:57:59.855 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:57:59.855 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:57:59.855 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:57:59.855 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:57:59.855 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:57:59.855 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:57:59.855 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:57:59.855 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:57:59.870 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:57:59.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:57:59.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:57:59.896 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 61 16 80590 80574 82.5008 52.9375 0.00999542 0.0121157 2023-09-23T19:57:59.978 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:00.038 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:00 smithi144 ceph-mon[194833]: pgmap v458: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 5/234522 objects degraded (0.002%) 2023-09-23T19:58:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:00 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3069870927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:00 smithi135 ceph-mon[221113]: pgmap v458: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 5/234522 objects degraded (0.002%) 2023-09-23T19:58:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:00 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3069870927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:00.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:00 smithi135 ceph-mon[213871]: pgmap v458: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 5/234522 objects degraded (0.002%) 2023-09-23T19:58:00.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:00 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3069870927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:00.780 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:58:00.896 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 62 16 81576 81560 82.1642 61.625 0.0222183 0.0121656 2023-09-23T19:58:00.912 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:00.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:00.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:01.018 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:01.052 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:01.406 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:58:01.700 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:01.705 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:58:01.706 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:58:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:01.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:01.845 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:01.857 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:01.899 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 63 16 82306 82290 81.5814 45.625 0.0233197 0.0122528 2023-09-23T19:58:01.930 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:01.980 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:02.230 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:02.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:02.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:02.337 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:02.420 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:02.449 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:02 smithi135 ceph-mon[213871]: pgmap v459: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 5/236211 objects degraded (0.002%) 2023-09-23T19:58:02.449 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:02 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/236211 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:02.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:02 smithi135 ceph-mon[221113]: pgmap v459: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 5/236211 objects degraded (0.002%) 2023-09-23T19:58:02.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:02 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/236211 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:02 smithi144 ceph-mon[194833]: pgmap v459: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 5/236211 objects degraded (0.002%) 2023-09-23T19:58:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:02 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/236211 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:02.710 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:02.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:02.818 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:02.853 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:02.899 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 64 16 83198 83182 81.1779 55.75 0.0299055 0.0123126 2023-09-23T19:58:02.907 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:02.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:02.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:03.017 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:03.045 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:03.707 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:58:03.902 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 65 16 83940 83924 80.6416 46.375 0.0247796 0.0123951 2023-09-23T19:58:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:58:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:58:03] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.43.0" 2023-09-23T19:58:04.376 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:04 smithi135 ceph-mon[213871]: pgmap v460: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s; 5/240429 objects degraded (0.002%) 2023-09-23T19:58:04.588 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:04.590 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:04.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:04.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:04 smithi144 ceph-mon[194833]: pgmap v460: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s; 5/240429 objects degraded (0.002%) 2023-09-23T19:58:04.715 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:04.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:04 smithi135 ceph-mon[221113]: pgmap v460: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s; 5/240429 objects degraded (0.002%) 2023-09-23T19:58:04.751 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:04.806 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:04.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:04.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:04.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 66 16 84757 84741 80.1923 51.0625 0.0095838 0.0124654 2023-09-23T19:58:04.915 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:05.012 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:05.394 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:05.407 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:05.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:05.533 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:05.646 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:05.776 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:05.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:05.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:05.799 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:58:05.802 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:58:05.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:05.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 67 16 85520 85504 79.7073 47.6875 0.0101477 0.0125409 2023-09-23T19:58:05.965 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:06.424 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:58:06.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:06 smithi135 ceph-mon[221113]: pgmap v461: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s; 5/247770 objects degraded (0.002%) 2023-09-23T19:58:06.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2011996983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:06.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:06 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:06.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:06 smithi135 ceph-mon[213871]: pgmap v461: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s; 5/247770 objects degraded (0.002%) 2023-09-23T19:58:06.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2011996983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:06.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:06 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:06 smithi144 ceph-mon[194833]: pgmap v461: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s; 5/247770 objects degraded (0.002%) 2023-09-23T19:58:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2011996983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:06 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:06.902 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 68 16 86321 86305 79.27 50.0625 0.0153604 0.0126095 2023-09-23T19:58:07.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: from='client.54817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:07.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:07.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: osdmap e175: 8 total, 7 up, 8 in 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/247770 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: from='client.54817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:07.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: osdmap e175: 8 total, 7 up, 8 in 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:07.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:07 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/247770 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: from='client.54817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: osdmap e175: 8 total, 7 up, 8 in 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: failed to encode map e175 with expected crc 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:07 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/247770 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:07.902 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 69 16 87112 87096 78.8378 49.4375 0.012798 0.0126784 2023-09-23T19:58:08.142 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":461,"stamp":"2023-09-23T19:58:04.491870+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5415573427,"num_objects":82590,"num_object_clones":0,"num_object_copies":247770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82590,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":83469,"num_write_kb":5296693,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":4753,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85365,"ondisk_log_size":85365,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18272384,"kb_used_data":17109440,"kb_used_omap":32,"kb_used_meta":1162847,"kb_avail":731688832,"statfs":{"total":767960285184,"available":749249363968,"internally_reserved":0,"allocated":17520066560,"data_stored":17514919684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":1190755846},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,5,2,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":584581120,"num_objects":8920,"num_object_clones":0,"num_object_copies":26760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8920,"num_write_kb":570880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005453"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":152,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276522+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:57:52.276522+0000","last_peered":"2023-09-23T19:57:52.276522+0000","last_clean":"2023-09-23T19:57:52.276522+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:57:52.276522+0000","last_undegraded":"2023-09-23T19:57:52.276522+0000","last_fullsized":"2023-09-23T19:57:52.276522+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":151,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274483+0000","last_change":"2023-09-23T19:50:57.792804+0000","last_active":"2023-09-23T19:57:52.274483+0000","last_peered":"2023-09-23T19:57:52.274483+0000","last_clean":"2023-09-23T19:57:52.274483+0000","last_became_active":"2023-09-23T19:50:57.792333+0000","last_became_peered":"2023-09-23T19:50:57.792333+0000","last_unstale":"2023-09-23T19:57:52.274483+0000","last_undegraded":"2023-09-23T19:57:52.274483+0000","last_fullsized":"2023-09-23T19:57:52.274483+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:51.510792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":112,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.324116+0000","last_change":"2023-09-23T19:50:57.794931+0000","last_active":"2023-09-23T19:57:52.324116+0000","last_peered":"2023-09-23T19:57:52.324116+0000","last_clean":"2023-09-23T19:57:52.324116+0000","last_became_active":"2023-09-23T19:50:57.794642+0000","last_became_peered":"2023-09-23T19:50:57.794642+0000","last_unstale":"2023-09-23T19:57:52.324116+0000","last_undegraded":"2023-09-23T19:57:52.324116+0000","last_fullsized":"2023-09-23T19:57:52.324116+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:00.118588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":119,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.458776+0000","last_change":"2023-09-23T19:50:57.791240+0000","last_active":"2023-09-23T19:57:52.458776+0000","last_peered":"2023-09-23T19:57:52.458776+0000","last_clean":"2023-09-23T19:57:52.458776+0000","last_became_active":"2023-09-23T19:50:57.790587+0000","last_became_peered":"2023-09-23T19:50:57.790587+0000","last_unstale":"2023-09-23T19:57:52.458776+0000","last_undegraded":"2023-09-23T19:57:52.458776+0000","last_fullsized":"2023-09-23T19:57:52.458776+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.741360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":155,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274236+0000","last_change":"2023-09-23T19:50:57.789164+0000","last_active":"2023-09-23T19:57:52.274236+0000","last_peered":"2023-09-23T19:57:52.274236+0000","last_clean":"2023-09-23T19:57:52.274236+0000","last_became_active":"2023-09-23T19:50:57.788595+0000","last_became_peered":"2023-09-23T19:50:57.788595+0000","last_unstale":"2023-09-23T19:57:52.274236+0000","last_undegraded":"2023-09-23T19:57:52.274236+0000","last_fullsized":"2023-09-23T19:57:52.274236+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.927661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":99,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.318981+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T19:57:52.318981+0000","last_peered":"2023-09-23T19:57:52.318981+0000","last_clean":"2023-09-23T19:57:52.318981+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T19:57:52.318981+0000","last_undegraded":"2023-09-23T19:57:52.318981+0000","last_fullsized":"2023-09-23T19:57:52.318981+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:07.049200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":117,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.326014+0000","last_change":"2023-09-23T19:50:57.788864+0000","last_active":"2023-09-23T19:57:52.326014+0000","last_peered":"2023-09-23T19:57:52.326014+0000","last_clean":"2023-09-23T19:57:52.326014+0000","last_became_active":"2023-09-23T19:50:57.788556+0000","last_became_peered":"2023-09-23T19:50:57.788556+0000","last_unstale":"2023-09-23T19:57:52.326014+0000","last_undegraded":"2023-09-23T19:57:52.326014+0000","last_fullsized":"2023-09-23T19:57:52.326014+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:32.868586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.912473+0000","last_change":"2023-09-23T19:56:39.278349+0000","last_active":"2023-09-23T19:57:55.912473+0000","last_peered":"2023-09-23T19:57:55.912473+0000","last_clean":"2023-09-23T19:57:55.912473+0000","last_became_active":"2023-09-23T19:56:39.278068+0000","last_became_peered":"2023-09-23T19:56:39.278068+0000","last_unstale":"2023-09-23T19:57:55.912473+0000","last_undegraded":"2023-09-23T19:57:55.912473+0000","last_fullsized":"2023-09-23T19:57:55.912473+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:04.173721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":133,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.317916+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:57:52.317916+0000","last_peered":"2023-09-23T19:57:52.317916+0000","last_clean":"2023-09-23T19:57:52.317916+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:57:52.317916+0000","last_undegraded":"2023-09-23T19:57:52.317916+0000","last_fullsized":"2023-09-23T19:57:52.317916+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":104,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.912500+0000","last_change":"2023-09-23T19:56:39.273704+0000","last_active":"2023-09-23T19:57:55.912500+0000","last_peered":"2023-09-23T19:57:55.912500+0000","last_clean":"2023-09-23T19:57:55.912500+0000","last_became_active":"2023-09-23T19:56:39.273360+0000","last_became_peered":"2023-09-23T19:56:39.273360+0000","last_unstale":"2023-09-23T19:57:55.912500+0000","last_undegraded":"2023-09-23T19:57:55.912500+0000","last_fullsized":"2023-09-23T19:57:55.912500+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:17.542729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":137,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.320878+0000","last_change":"2023-09-23T19:50:57.793984+0000","last_active":"2023-09-23T19:57:52.320878+0000","last_peered":"2023-09-23T19:57:52.320878+0000","last_clean":"2023-09-23T19:57:52.320878+0000","last_became_active":"2023-09-23T19:50:57.793735+0000","last_became_peered":"2023-09-23T19:50:57.793735+0000","last_unstale":"2023-09-23T19:57:52.320878+0000","last_undegraded":"2023-09-23T19:57:52.320878+0000","last_fullsized":"2023-09-23T19:57:52.320878+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:21.712030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":128,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321597+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T19:57:52.321597+0000","last_peered":"2023-09-23T19:57:52.321597+0000","last_clean":"2023-09-23T19:57:52.321597+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T19:57:52.321597+0000","last_undegraded":"2023-09-23T19:57:52.321597+0000","last_fullsized":"2023-09-23T19:57:52.321597+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:52.846135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":166,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276509+0000","last_change":"2023-09-23T19:56:39.292849+0000","last_active":"2023-09-23T19:57:52.276509+0000","last_peered":"2023-09-23T19:57:52.276509+0000","last_clean":"2023-09-23T19:57:52.276509+0000","last_became_active":"2023-09-23T19:56:39.284622+0000","last_became_peered":"2023-09-23T19:56:39.284622+0000","last_unstale":"2023-09-23T19:57:52.276509+0000","last_undegraded":"2023-09-23T19:57:52.276509+0000","last_fullsized":"2023-09-23T19:57:52.276509+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:24.088523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":140,"reported_epoch":173,"state":"active+undersized","last_fresh":"2023-09-23T19:57:52.276465+0000","last_change":"2023-09-23T19:54:45.327308+0000","last_active":"2023-09-23T19:57:52.276465+0000","last_peered":"2023-09-23T19:57:52.276465+0000","last_clean":"2023-09-23T19:54:37.307838+0000","last_became_active":"2023-09-23T19:54:45.327308+0000","last_became_peered":"2023-09-23T19:54:45.327308+0000","last_unstale":"2023-09-23T19:57:52.276465+0000","last_undegraded":"2023-09-23T19:57:52.276465+0000","last_fullsized":"2023-09-23T19:54:45.254218+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:34.491557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":136,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.320595+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:57:52.320595+0000","last_peered":"2023-09-23T19:57:52.320595+0000","last_clean":"2023-09-23T19:57:52.320595+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:57:52.320595+0000","last_undegraded":"2023-09-23T19:57:52.320595+0000","last_fullsized":"2023-09-23T19:57:52.320595+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":138,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:55.914902+0000","last_change":"2023-09-23T19:50:57.790499+0000","last_active":"2023-09-23T19:57:55.914902+0000","last_peered":"2023-09-23T19:57:55.914902+0000","last_clean":"2023-09-23T19:57:55.914902+0000","last_became_active":"2023-09-23T19:50:57.790199+0000","last_became_peered":"2023-09-23T19:50:57.790199+0000","last_unstale":"2023-09-23T19:57:55.914902+0000","last_undegraded":"2023-09-23T19:57:55.914902+0000","last_fullsized":"2023-09-23T19:57:55.914902+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:29.998102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":118,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:57:56.317118+0000","last_change":"2023-09-23T19:50:57.788133+0000","last_active":"2023-09-23T19:57:56.317118+0000","last_peered":"2023-09-23T19:57:56.317118+0000","last_clean":"2023-09-23T19:57:56.317118+0000","last_became_active":"2023-09-23T19:50:57.787841+0000","last_became_peered":"2023-09-23T19:50:57.787841+0000","last_unstale":"2023-09-23T19:57:56.317118+0000","last_undegraded":"2023-09-23T19:57:56.317118+0000","last_fullsized":"2023-09-23T19:57:56.317118+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:13.293073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":132,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.318959+0000","last_change":"2023-09-23T19:50:57.790293+0000","last_active":"2023-09-23T19:57:52.318959+0000","last_peered":"2023-09-23T19:57:52.318959+0000","last_clean":"2023-09-23T19:57:52.318959+0000","last_became_active":"2023-09-23T19:50:57.790022+0000","last_became_peered":"2023-09-23T19:50:57.790022+0000","last_unstale":"2023-09-23T19:57:52.318959+0000","last_undegraded":"2023-09-23T19:57:52.318959+0000","last_fullsized":"2023-09-23T19:57:52.318959+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:52.355088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":124,"reported_epoch":173,"state":"active+undersized","last_fresh":"2023-09-23T19:57:52.320430+0000","last_change":"2023-09-23T19:54:45.330750+0000","last_active":"2023-09-23T19:57:52.320430+0000","last_peered":"2023-09-23T19:57:52.320430+0000","last_clean":"2023-09-23T19:54:37.324687+0000","last_became_active":"2023-09-23T19:54:45.330750+0000","last_became_peered":"2023-09-23T19:54:45.330750+0000","last_unstale":"2023-09-23T19:57:52.320430+0000","last_undegraded":"2023-09-23T19:57:52.320430+0000","last_fullsized":"2023-09-23T19:54:45.253851+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:18.890717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":118,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.326009+0000","last_change":"2023-09-23T19:50:57.791429+0000","last_active":"2023-09-23T19:57:52.326009+0000","last_peered":"2023-09-23T19:57:52.326009+0000","last_clean":"2023-09-23T19:57:52.326009+0000","last_became_active":"2023-09-23T19:50:57.790955+0000","last_became_peered":"2023-09-23T19:50:57.790955+0000","last_unstale":"2023-09-23T19:57:52.326009+0000","last_undegraded":"2023-09-23T19:57:52.326009+0000","last_fullsized":"2023-09-23T19:57:52.326009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:20.666540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":50,"reported_epoch":173,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:57:52.325661+0000","last_change":"2023-09-23T19:54:45.331318+0000","last_active":"2023-09-23T19:57:52.325661+0000","last_peered":"2023-09-23T19:57:52.325661+0000","last_clean":"2023-09-23T19:41:41.886282+0000","last_became_active":"2023-09-23T19:54:45.331318+0000","last_became_peered":"2023-09-23T19:54:45.331318+0000","last_unstale":"2023-09-23T19:57:52.325661+0000","last_undegraded":"2023-09-23T19:54:45.255936+0000","last_fullsized":"2023-09-23T19:54:45.255813+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:16.760756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":137,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274947+0000","last_change":"2023-09-23T19:50:57.789346+0000","last_active":"2023-09-23T19:57:52.274947+0000","last_peered":"2023-09-23T19:57:52.274947+0000","last_clean":"2023-09-23T19:57:52.274947+0000","last_became_active":"2023-09-23T19:50:57.788698+0000","last_became_peered":"2023-09-23T19:50:57.788698+0000","last_unstale":"2023-09-23T19:57:52.274947+0000","last_undegraded":"2023-09-23T19:57:52.274947+0000","last_fullsized":"2023-09-23T19:57:52.274947+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:53.668684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"174'5194","reported_seq":5212,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:03.279839+0000","last_change":"2023-09-23T19:56:51.336830+0000","last_active":"2023-09-23T19:58:03.279839+0000","last_peered":"2023-09-23T19:58:03.279839+0000","last_clean":"2023-09-23T19:58:03.279839+0000","last_became_active":"2023-09-23T19:56:51.336645+0000","last_became_peered":"2023-09-23T19:56:51.336645+0000","last_unstale":"2023-09-23T19:58:03.279839+0000","last_undegraded":"2023-09-23T19:58:03.279839+0000","last_fullsized":"2023-09-23T19:58:03.279839+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5194,"log_dups_size":0,"ondisk_log_size":5194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:27.681999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340393984,"num_objects":5194,"num_object_clones":0,"num_object_copies":15582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5194,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":135,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321092+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:57:52.321092+0000","last_peered":"2023-09-23T19:57:52.321092+0000","last_clean":"2023-09-23T19:57:52.321092+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:57:52.321092+0000","last_undegraded":"2023-09-23T19:57:52.321092+0000","last_fullsized":"2023-09-23T19:57:52.321092+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":132,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.317750+0000","last_change":"2023-09-23T19:56:39.275614+0000","last_active":"2023-09-23T19:57:52.317750+0000","last_peered":"2023-09-23T19:57:52.317750+0000","last_clean":"2023-09-23T19:57:52.317750+0000","last_became_active":"2023-09-23T19:56:39.275242+0000","last_became_peered":"2023-09-23T19:56:39.275242+0000","last_unstale":"2023-09-23T19:57:52.317750+0000","last_undegraded":"2023-09-23T19:57:52.317750+0000","last_fullsized":"2023-09-23T19:57:52.317750+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:55.343087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"174'4907","reported_seq":4927,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:00.891894+0000","last_change":"2023-09-23T19:56:51.328940+0000","last_active":"2023-09-23T19:58:00.891894+0000","last_peered":"2023-09-23T19:58:00.891894+0000","last_clean":"2023-09-23T19:58:00.891894+0000","last_became_active":"2023-09-23T19:56:51.328668+0000","last_became_peered":"2023-09-23T19:56:51.328668+0000","last_unstale":"2023-09-23T19:58:00.891894+0000","last_undegraded":"2023-09-23T19:58:00.891894+0000","last_fullsized":"2023-09-23T19:58:00.891894+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":4907,"log_dups_size":0,"ondisk_log_size":4907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:01.871957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321585152,"num_objects":4907,"num_object_clones":0,"num_object_copies":14721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4907,"num_write_kb":314048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":136,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.320852+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:57:52.320852+0000","last_peered":"2023-09-23T19:57:52.320852+0000","last_clean":"2023-09-23T19:57:52.320852+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:57:52.320852+0000","last_undegraded":"2023-09-23T19:57:52.320852+0000","last_fullsized":"2023-09-23T19:57:52.320852+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276305+0000","last_change":"2023-09-23T19:50:57.795290+0000","last_active":"2023-09-23T19:57:52.276305+0000","last_peered":"2023-09-23T19:57:52.276305+0000","last_clean":"2023-09-23T19:57:52.276305+0000","last_became_active":"2023-09-23T19:50:57.795045+0000","last_became_peered":"2023-09-23T19:50:57.795045+0000","last_unstale":"2023-09-23T19:57:52.276305+0000","last_undegraded":"2023-09-23T19:57:52.276305+0000","last_fullsized":"2023-09-23T19:57:52.276305+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:38.535751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"174'5280","reported_seq":5299,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:03.137661+0000","last_change":"2023-09-23T19:56:51.329607+0000","last_active":"2023-09-23T19:58:03.137661+0000","last_peered":"2023-09-23T19:58:03.137661+0000","last_clean":"2023-09-23T19:58:03.137661+0000","last_became_active":"2023-09-23T19:56:51.329330+0000","last_became_peered":"2023-09-23T19:56:51.329330+0000","last_unstale":"2023-09-23T19:58:03.137661+0000","last_undegraded":"2023-09-23T19:58:03.137661+0000","last_fullsized":"2023-09-23T19:58:03.137661+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5280,"log_dups_size":0,"ondisk_log_size":5280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:51.413448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346030080,"num_objects":5280,"num_object_clones":0,"num_object_copies":15840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5280,"num_write_kb":337920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":125,"reported_epoch":173,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:57:52.320755+0000","last_change":"2023-09-23T19:54:45.328405+0000","last_active":"2023-09-23T19:57:52.320755+0000","last_peered":"2023-09-23T19:57:52.320755+0000","last_clean":"2023-09-23T19:54:37.326665+0000","last_became_active":"2023-09-23T19:54:45.328405+0000","last_became_peered":"2023-09-23T19:54:45.328405+0000","last_unstale":"2023-09-23T19:57:52.320755+0000","last_undegraded":"2023-09-23T19:54:45.254169+0000","last_fullsized":"2023-09-23T19:54:45.254056+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:10.168687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321355+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T19:57:52.321355+0000","last_peered":"2023-09-23T19:57:52.321355+0000","last_clean":"2023-09-23T19:57:52.321355+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T19:57:52.321355+0000","last_undegraded":"2023-09-23T19:57:52.321355+0000","last_fullsized":"2023-09-23T19:57:52.321355+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:30.170186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"174'5141","reported_seq":5161,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:01.614454+0000","last_change":"2023-09-23T19:56:51.330015+0000","last_active":"2023-09-23T19:58:01.614454+0000","last_peered":"2023-09-23T19:58:01.614454+0000","last_clean":"2023-09-23T19:58:01.614454+0000","last_became_active":"2023-09-23T19:56:51.329727+0000","last_became_peered":"2023-09-23T19:56:51.329727+0000","last_unstale":"2023-09-23T19:58:01.614454+0000","last_undegraded":"2023-09-23T19:58:01.614454+0000","last_fullsized":"2023-09-23T19:58:01.614454+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5141,"log_dups_size":0,"ondisk_log_size":5141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:59.494181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336920576,"num_objects":5141,"num_object_clones":0,"num_object_copies":15423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5141,"num_write_kb":329024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.318472+0000","last_change":"2023-09-23T19:56:39.274176+0000","last_active":"2023-09-23T19:57:52.318472+0000","last_peered":"2023-09-23T19:57:52.318472+0000","last_clean":"2023-09-23T19:57:52.318472+0000","last_became_active":"2023-09-23T19:56:39.273835+0000","last_became_peered":"2023-09-23T19:56:39.273835+0000","last_unstale":"2023-09-23T19:57:52.318472+0000","last_undegraded":"2023-09-23T19:57:52.318472+0000","last_fullsized":"2023-09-23T19:57:52.318472+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:51.379105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":119,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.325753+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:57:52.325753+0000","last_peered":"2023-09-23T19:57:52.325753+0000","last_clean":"2023-09-23T19:57:52.325753+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:57:52.325753+0000","last_undegraded":"2023-09-23T19:57:52.325753+0000","last_fullsized":"2023-09-23T19:57:52.325753+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"174'5274","reported_seq":5292,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:03.257888+0000","last_change":"2023-09-23T19:56:51.335737+0000","last_active":"2023-09-23T19:58:03.257888+0000","last_peered":"2023-09-23T19:58:03.257888+0000","last_clean":"2023-09-23T19:58:03.257888+0000","last_became_active":"2023-09-23T19:56:51.335399+0000","last_became_peered":"2023-09-23T19:56:51.335399+0000","last_unstale":"2023-09-23T19:58:03.257888+0000","last_undegraded":"2023-09-23T19:58:03.257888+0000","last_fullsized":"2023-09-23T19:58:03.257888+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5274,"log_dups_size":0,"ondisk_log_size":5274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:20.933908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345636864,"num_objects":5274,"num_object_clones":0,"num_object_copies":15822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5274,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":554,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.320329+0000","last_change":"2023-09-23T19:52:57.349564+0000","last_active":"2023-09-23T19:57:52.320329+0000","last_peered":"2023-09-23T19:57:52.320329+0000","last_clean":"2023-09-23T19:57:52.320329+0000","last_became_active":"2023-09-23T19:52:57.349182+0000","last_became_peered":"2023-09-23T19:52:57.349182+0000","last_unstale":"2023-09-23T19:57:52.320329+0000","last_undegraded":"2023-09-23T19:57:52.320329+0000","last_fullsized":"2023-09-23T19:57:52.320329+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:25.262721+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":136,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274455+0000","last_change":"2023-09-23T19:50:57.793011+0000","last_active":"2023-09-23T19:57:52.274455+0000","last_peered":"2023-09-23T19:57:52.274455+0000","last_clean":"2023-09-23T19:57:52.274455+0000","last_became_active":"2023-09-23T19:50:57.792495+0000","last_became_peered":"2023-09-23T19:50:57.792495+0000","last_unstale":"2023-09-23T19:57:52.274455+0000","last_undegraded":"2023-09-23T19:57:52.274455+0000","last_fullsized":"2023-09-23T19:57:52.274455+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:06.392388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276489+0000","last_change":"2023-09-23T19:56:39.284044+0000","last_active":"2023-09-23T19:57:52.276489+0000","last_peered":"2023-09-23T19:57:52.276489+0000","last_clean":"2023-09-23T19:57:52.276489+0000","last_became_active":"2023-09-23T19:56:39.283753+0000","last_became_peered":"2023-09-23T19:56:39.283753+0000","last_unstale":"2023-09-23T19:57:52.276489+0000","last_undegraded":"2023-09-23T19:57:52.276489+0000","last_fullsized":"2023-09-23T19:57:52.276489+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:51.940014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"174'5309","reported_seq":5328,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:02.751212+0000","last_change":"2023-09-23T19:56:51.332462+0000","last_active":"2023-09-23T19:58:02.751212+0000","last_peered":"2023-09-23T19:58:02.751212+0000","last_clean":"2023-09-23T19:58:02.751212+0000","last_became_active":"2023-09-23T19:56:51.332192+0000","last_became_peered":"2023-09-23T19:56:51.332192+0000","last_unstale":"2023-09-23T19:58:02.751212+0000","last_undegraded":"2023-09-23T19:58:02.751212+0000","last_fullsized":"2023-09-23T19:58:02.751212+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5309,"log_dups_size":0,"ondisk_log_size":5309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:42.318489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5309,"num_write_kb":339776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":485,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.317771+0000","last_change":"2023-09-23T19:52:58.803028+0000","last_active":"2023-09-23T19:57:52.317771+0000","last_peered":"2023-09-23T19:57:52.317771+0000","last_clean":"2023-09-23T19:57:52.317771+0000","last_became_active":"2023-09-23T19:52:57.353261+0000","last_became_peered":"2023-09-23T19:52:57.353261+0000","last_unstale":"2023-09-23T19:57:52.317771+0000","last_undegraded":"2023-09-23T19:57:52.317771+0000","last_fullsized":"2023-09-23T19:57:52.317771+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:53.333663+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":100,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276588+0000","last_change":"2023-09-23T19:56:39.280681+0000","last_active":"2023-09-23T19:57:52.276588+0000","last_peered":"2023-09-23T19:57:52.276588+0000","last_clean":"2023-09-23T19:57:52.276588+0000","last_became_active":"2023-09-23T19:56:39.280377+0000","last_became_peered":"2023-09-23T19:56:39.280377+0000","last_unstale":"2023-09-23T19:57:52.276588+0000","last_undegraded":"2023-09-23T19:57:52.276588+0000","last_fullsized":"2023-09-23T19:57:52.276588+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:42.514714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276658+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:57:52.276658+0000","last_peered":"2023-09-23T19:57:52.276658+0000","last_clean":"2023-09-23T19:57:52.276658+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:57:52.276658+0000","last_undegraded":"2023-09-23T19:57:52.276658+0000","last_fullsized":"2023-09-23T19:57:52.276658+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"174'5056","reported_seq":5075,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:00.904790+0000","last_change":"2023-09-23T19:56:51.334600+0000","last_active":"2023-09-23T19:58:00.904790+0000","last_peered":"2023-09-23T19:58:00.904790+0000","last_clean":"2023-09-23T19:58:00.904790+0000","last_became_active":"2023-09-23T19:56:51.334315+0000","last_became_peered":"2023-09-23T19:56:51.334315+0000","last_unstale":"2023-09-23T19:58:00.904790+0000","last_undegraded":"2023-09-23T19:58:00.904790+0000","last_fullsized":"2023-09-23T19:58:00.904790+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5056,"log_dups_size":0,"ondisk_log_size":5056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:47.078218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331350016,"num_objects":5056,"num_object_clones":0,"num_object_copies":15168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5056,"num_write_kb":323584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":134,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276171+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:57:52.276171+0000","last_peered":"2023-09-23T19:57:52.276171+0000","last_clean":"2023-09-23T19:57:52.276171+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:57:52.276171+0000","last_undegraded":"2023-09-23T19:57:52.276171+0000","last_fullsized":"2023-09-23T19:57:52.276171+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":160,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276651+0000","last_change":"2023-09-23T19:56:39.281487+0000","last_active":"2023-09-23T19:57:52.276651+0000","last_peered":"2023-09-23T19:57:52.276651+0000","last_clean":"2023-09-23T19:57:52.276651+0000","last_became_active":"2023-09-23T19:56:39.281248+0000","last_became_peered":"2023-09-23T19:56:39.281248+0000","last_unstale":"2023-09-23T19:57:52.276651+0000","last_undegraded":"2023-09-23T19:57:52.276651+0000","last_fullsized":"2023-09-23T19:57:52.276651+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:25.466372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"174'5263","reported_seq":5281,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:03.283056+0000","last_change":"2023-09-23T19:56:51.339206+0000","last_active":"2023-09-23T19:58:03.283056+0000","last_peered":"2023-09-23T19:58:03.283056+0000","last_clean":"2023-09-23T19:58:03.283056+0000","last_became_active":"2023-09-23T19:56:51.338963+0000","last_became_peered":"2023-09-23T19:56:51.338963+0000","last_unstale":"2023-09-23T19:58:03.283056+0000","last_undegraded":"2023-09-23T19:58:03.283056+0000","last_fullsized":"2023-09-23T19:58:03.283056+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5263,"log_dups_size":0,"ondisk_log_size":5263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:49.976592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344915968,"num_objects":5263,"num_object_clones":0,"num_object_copies":15789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5263,"num_write_kb":336832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":512,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.318151+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T19:57:52.318151+0000","last_peered":"2023-09-23T19:57:52.318151+0000","last_clean":"2023-09-23T19:57:52.318151+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T19:57:52.318151+0000","last_undegraded":"2023-09-23T19:57:52.318151+0000","last_fullsized":"2023-09-23T19:57:52.318151+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.320089+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274145+0000","last_change":"2023-09-23T19:50:57.793102+0000","last_active":"2023-09-23T19:57:52.274145+0000","last_peered":"2023-09-23T19:57:52.274145+0000","last_clean":"2023-09-23T19:57:52.274145+0000","last_became_active":"2023-09-23T19:50:57.792630+0000","last_became_peered":"2023-09-23T19:50:57.792630+0000","last_unstale":"2023-09-23T19:57:52.274145+0000","last_undegraded":"2023-09-23T19:57:52.274145+0000","last_fullsized":"2023-09-23T19:57:52.274145+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:16.145228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":113,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321090+0000","last_change":"2023-09-23T19:50:57.790403+0000","last_active":"2023-09-23T19:57:52.321090+0000","last_peered":"2023-09-23T19:57:52.321090+0000","last_clean":"2023-09-23T19:57:52.321090+0000","last_became_active":"2023-09-23T19:50:57.789958+0000","last_became_peered":"2023-09-23T19:50:57.789958+0000","last_unstale":"2023-09-23T19:57:52.321090+0000","last_undegraded":"2023-09-23T19:57:52.321090+0000","last_fullsized":"2023-09-23T19:57:52.321090+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:05.976487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"174'5094","reported_seq":5113,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:01.629077+0000","last_change":"2023-09-23T19:56:51.330347+0000","last_active":"2023-09-23T19:58:01.629077+0000","last_peered":"2023-09-23T19:58:01.629077+0000","last_clean":"2023-09-23T19:58:01.629077+0000","last_became_active":"2023-09-23T19:56:51.330084+0000","last_became_peered":"2023-09-23T19:56:51.330084+0000","last_unstale":"2023-09-23T19:58:01.629077+0000","last_undegraded":"2023-09-23T19:58:01.629077+0000","last_fullsized":"2023-09-23T19:58:01.629077+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5094,"log_dups_size":0,"ondisk_log_size":5094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:08.671890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333840384,"num_objects":5094,"num_object_clones":0,"num_object_copies":15282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5094,"num_write_kb":326016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":486,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274847+0000","last_change":"2023-09-23T19:56:53.618146+0000","last_active":"2023-09-23T19:57:52.274847+0000","last_peered":"2023-09-23T19:57:52.274847+0000","last_clean":"2023-09-23T19:57:52.274847+0000","last_became_active":"2023-09-23T19:56:42.331700+0000","last_became_peered":"2023-09-23T19:56:42.331700+0000","last_unstale":"2023-09-23T19:57:52.274847+0000","last_undegraded":"2023-09-23T19:57:52.274847+0000","last_fullsized":"2023-09-23T19:57:52.274847+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:56:53.618048+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T19:56:53.618048+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.066107+0000","scrub_duration":0.001843293,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":509,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321407+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:57:52.321407+0000","last_peered":"2023-09-23T19:57:52.321407+0000","last_clean":"2023-09-23T19:57:52.321407+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:57:52.321407+0000","last_undegraded":"2023-09-23T19:57:52.321407+0000","last_fullsized":"2023-09-23T19:57:52.321407+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.319022+0000","last_change":"2023-09-23T19:50:57.787126+0000","last_active":"2023-09-23T19:57:52.319022+0000","last_peered":"2023-09-23T19:57:52.319022+0000","last_clean":"2023-09-23T19:57:52.319022+0000","last_became_active":"2023-09-23T19:50:57.786738+0000","last_became_peered":"2023-09-23T19:50:57.786738+0000","last_unstale":"2023-09-23T19:57:52.319022+0000","last_undegraded":"2023-09-23T19:57:52.319022+0000","last_fullsized":"2023-09-23T19:57:52.319022+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:03.063499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":98,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321530+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:57:52.321530+0000","last_peered":"2023-09-23T19:57:52.321530+0000","last_clean":"2023-09-23T19:57:52.321530+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:57:52.321530+0000","last_undegraded":"2023-09-23T19:57:52.321530+0000","last_fullsized":"2023-09-23T19:57:52.321530+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"174'5092","reported_seq":5112,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:00.862960+0000","last_change":"2023-09-23T19:56:51.333298+0000","last_active":"2023-09-23T19:58:00.862960+0000","last_peered":"2023-09-23T19:58:00.862960+0000","last_clean":"2023-09-23T19:58:00.862960+0000","last_became_active":"2023-09-23T19:56:51.333009+0000","last_became_peered":"2023-09-23T19:56:51.333009+0000","last_unstale":"2023-09-23T19:58:00.862960+0000","last_undegraded":"2023-09-23T19:58:00.862960+0000","last_fullsized":"2023-09-23T19:58:00.862960+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5092,"log_dups_size":0,"ondisk_log_size":5092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:12.985476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333709312,"num_objects":5092,"num_object_clones":0,"num_object_copies":15276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5092,"num_write_kb":325888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":485,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.318032+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T19:57:52.318032+0000","last_peered":"2023-09-23T19:57:52.318032+0000","last_clean":"2023-09-23T19:57:52.318032+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T19:57:52.318032+0000","last_undegraded":"2023-09-23T19:57:52.318032+0000","last_fullsized":"2023-09-23T19:57:52.318032+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":734,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.318296+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T19:57:52.318296+0000","last_peered":"2023-09-23T19:57:52.318296+0000","last_clean":"2023-09-23T19:57:52.318296+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T19:57:52.318296+0000","last_undegraded":"2023-09-23T19:57:52.318296+0000","last_fullsized":"2023-09-23T19:57:52.318296+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:34.015520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276536+0000","last_change":"2023-09-23T19:56:39.283954+0000","last_active":"2023-09-23T19:57:52.276536+0000","last_peered":"2023-09-23T19:57:52.276536+0000","last_clean":"2023-09-23T19:57:52.276536+0000","last_became_active":"2023-09-23T19:56:39.283711+0000","last_became_peered":"2023-09-23T19:56:39.283711+0000","last_unstale":"2023-09-23T19:57:52.276536+0000","last_undegraded":"2023-09-23T19:57:52.276536+0000","last_fullsized":"2023-09-23T19:57:52.276536+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:55.890028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":128,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321333+0000","last_change":"2023-09-23T19:50:57.794107+0000","last_active":"2023-09-23T19:57:52.321333+0000","last_peered":"2023-09-23T19:57:52.321333+0000","last_clean":"2023-09-23T19:57:52.321333+0000","last_became_active":"2023-09-23T19:50:57.793788+0000","last_became_peered":"2023-09-23T19:50:57.793788+0000","last_unstale":"2023-09-23T19:57:52.321333+0000","last_undegraded":"2023-09-23T19:57:52.321333+0000","last_fullsized":"2023-09-23T19:57:52.321333+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:41.139598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"174'5185","reported_seq":5203,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:03.242082+0000","last_change":"2023-09-23T19:56:51.329873+0000","last_active":"2023-09-23T19:58:03.242082+0000","last_peered":"2023-09-23T19:58:03.242082+0000","last_clean":"2023-09-23T19:58:03.242082+0000","last_became_active":"2023-09-23T19:56:51.329355+0000","last_became_peered":"2023-09-23T19:56:51.329355+0000","last_unstale":"2023-09-23T19:58:03.242082+0000","last_undegraded":"2023-09-23T19:58:03.242082+0000","last_fullsized":"2023-09-23T19:58:03.242082+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5185,"log_dups_size":0,"ondisk_log_size":5185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.152209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339804160,"num_objects":5185,"num_object_clones":0,"num_object_copies":15555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5185,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":554,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.320335+0000","last_change":"2023-09-23T19:52:57.745972+0000","last_active":"2023-09-23T19:57:52.320335+0000","last_peered":"2023-09-23T19:57:52.320335+0000","last_clean":"2023-09-23T19:57:52.320335+0000","last_became_active":"2023-09-23T19:52:57.348142+0000","last_became_peered":"2023-09-23T19:52:57.348142+0000","last_unstale":"2023-09-23T19:57:52.320335+0000","last_undegraded":"2023-09-23T19:57:52.320335+0000","last_fullsized":"2023-09-23T19:57:52.320335+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.745894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.745894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.660195+0000","scrub_duration":0.0079262489999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":659,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.317173+0000","last_change":"2023-09-23T19:50:57.786571+0000","last_active":"2023-09-23T19:57:52.317173+0000","last_peered":"2023-09-23T19:57:52.317173+0000","last_clean":"2023-09-23T19:57:52.317173+0000","last_became_active":"2023-09-23T19:50:57.784926+0000","last_became_peered":"2023-09-23T19:50:57.784926+0000","last_unstale":"2023-09-23T19:57:52.317173+0000","last_undegraded":"2023-09-23T19:57:52.317173+0000","last_fullsized":"2023-09-23T19:57:52.317173+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:30.760569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":128,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321111+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T19:57:52.321111+0000","last_peered":"2023-09-23T19:57:52.321111+0000","last_clean":"2023-09-23T19:57:52.321111+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T19:57:52.321111+0000","last_undegraded":"2023-09-23T19:57:52.321111+0000","last_fullsized":"2023-09-23T19:57:52.321111+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:46.217415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":142,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321022+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:57:52.321022+0000","last_peered":"2023-09-23T19:57:52.321022+0000","last_clean":"2023-09-23T19:57:52.321022+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:57:52.321022+0000","last_undegraded":"2023-09-23T19:57:52.321022+0000","last_fullsized":"2023-09-23T19:57:52.321022+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"174'5248","reported_seq":5266,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:03.264490+0000","last_change":"2023-09-23T19:56:51.335393+0000","last_active":"2023-09-23T19:58:03.264490+0000","last_peered":"2023-09-23T19:58:03.264490+0000","last_clean":"2023-09-23T19:58:03.264490+0000","last_became_active":"2023-09-23T19:56:51.335106+0000","last_became_peered":"2023-09-23T19:56:51.335106+0000","last_unstale":"2023-09-23T19:58:03.264490+0000","last_undegraded":"2023-09-23T19:58:03.264490+0000","last_fullsized":"2023-09-23T19:58:03.264490+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5248,"log_dups_size":0,"ondisk_log_size":5248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.493039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343932928,"num_objects":5248,"num_object_clones":0,"num_object_copies":15744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5248,"num_write_kb":335872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":982,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.317995+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:57:52.317995+0000","last_peered":"2023-09-23T19:57:52.317995+0000","last_clean":"2023-09-23T19:57:52.317995+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:57:52.317995+0000","last_undegraded":"2023-09-23T19:57:52.317995+0000","last_fullsized":"2023-09-23T19:57:52.317995+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":562,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.320343+0000","last_change":"2023-09-23T19:56:42.408346+0000","last_active":"2023-09-23T19:57:52.320343+0000","last_peered":"2023-09-23T19:57:52.320343+0000","last_clean":"2023-09-23T19:57:52.320343+0000","last_became_active":"2023-09-23T19:56:42.407370+0000","last_became_peered":"2023-09-23T19:56:42.407370+0000","last_unstale":"2023-09-23T19:57:52.320343+0000","last_undegraded":"2023-09-23T19:57:52.320343+0000","last_fullsized":"2023-09-23T19:57:52.320343+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:38.144096+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":103,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274341+0000","last_change":"2023-09-23T19:56:39.277079+0000","last_active":"2023-09-23T19:57:52.274341+0000","last_peered":"2023-09-23T19:57:52.274341+0000","last_clean":"2023-09-23T19:57:52.274341+0000","last_became_active":"2023-09-23T19:56:39.276787+0000","last_became_peered":"2023-09-23T19:56:39.276787+0000","last_unstale":"2023-09-23T19:57:52.274341+0000","last_undegraded":"2023-09-23T19:57:52.274341+0000","last_fullsized":"2023-09-23T19:57:52.274341+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:00.570242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":49,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276830+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T19:57:52.276830+0000","last_peered":"2023-09-23T19:57:52.276830+0000","last_clean":"2023-09-23T19:57:52.276830+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T19:57:52.276830+0000","last_undegraded":"2023-09-23T19:57:52.276830+0000","last_fullsized":"2023-09-23T19:57:52.276830+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:31.068090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"174'5068","reported_seq":5086,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:57:59.552638+0000","last_change":"2023-09-23T19:56:51.335699+0000","last_active":"2023-09-23T19:57:59.552638+0000","last_peered":"2023-09-23T19:57:59.552638+0000","last_clean":"2023-09-23T19:57:59.552638+0000","last_became_active":"2023-09-23T19:56:51.335366+0000","last_became_peered":"2023-09-23T19:56:51.335366+0000","last_unstale":"2023-09-23T19:57:59.552638+0000","last_undegraded":"2023-09-23T19:57:59.552638+0000","last_fullsized":"2023-09-23T19:57:59.552638+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5068,"log_dups_size":0,"ondisk_log_size":5068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:18.373892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332136448,"num_objects":5068,"num_object_clones":0,"num_object_copies":15204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5068,"num_write_kb":324352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":511,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.325779+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T19:57:52.325779+0000","last_peered":"2023-09-23T19:57:52.325779+0000","last_clean":"2023-09-23T19:57:52.325779+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T19:57:52.325779+0000","last_undegraded":"2023-09-23T19:57:52.325779+0000","last_fullsized":"2023-09-23T19:57:52.325779+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:32.858007+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":892,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.320317+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:57:52.320317+0000","last_peered":"2023-09-23T19:57:52.320317+0000","last_clean":"2023-09-23T19:57:52.320317+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:57:52.320317+0000","last_undegraded":"2023-09-23T19:57:52.320317+0000","last_fullsized":"2023-09-23T19:57:52.320317+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":118,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.325545+0000","last_change":"2023-09-23T19:50:57.794256+0000","last_active":"2023-09-23T19:57:52.325545+0000","last_peered":"2023-09-23T19:57:52.325545+0000","last_clean":"2023-09-23T19:57:52.325545+0000","last_became_active":"2023-09-23T19:50:57.794030+0000","last_became_peered":"2023-09-23T19:50:57.794030+0000","last_unstale":"2023-09-23T19:57:52.325545+0000","last_undegraded":"2023-09-23T19:57:52.325545+0000","last_fullsized":"2023-09-23T19:57:52.325545+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:50.908307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":132,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.458963+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:57:52.458963+0000","last_peered":"2023-09-23T19:57:52.458963+0000","last_clean":"2023-09-23T19:57:52.458963+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:57:52.458963+0000","last_undegraded":"2023-09-23T19:57:52.458963+0000","last_fullsized":"2023-09-23T19:57:52.458963+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"174'5019","reported_seq":5038,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:57:59.566149+0000","last_change":"2023-09-23T19:56:51.329836+0000","last_active":"2023-09-23T19:57:59.566149+0000","last_peered":"2023-09-23T19:57:59.566149+0000","last_clean":"2023-09-23T19:57:59.566149+0000","last_became_active":"2023-09-23T19:56:51.329518+0000","last_became_peered":"2023-09-23T19:56:51.329518+0000","last_unstale":"2023-09-23T19:57:59.566149+0000","last_undegraded":"2023-09-23T19:57:59.566149+0000","last_fullsized":"2023-09-23T19:57:59.566149+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5019,"log_dups_size":0,"ondisk_log_size":5019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:38.536519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328925184,"num_objects":5019,"num_object_clones":0,"num_object_copies":15057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5019,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":479,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.325699+0000","last_change":"2023-09-23T19:56:54.487207+0000","last_active":"2023-09-23T19:57:52.325699+0000","last_peered":"2023-09-23T19:57:52.325699+0000","last_clean":"2023-09-23T19:57:52.325699+0000","last_became_active":"2023-09-23T19:56:42.420361+0000","last_became_peered":"2023-09-23T19:56:42.420361+0000","last_unstale":"2023-09-23T19:57:52.325699+0000","last_undegraded":"2023-09-23T19:57:52.325699+0000","last_fullsized":"2023-09-23T19:57:52.325699+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:56:54.487110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:56:54.487110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:19.686156+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":252,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.320781+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:57:52.320781+0000","last_peered":"2023-09-23T19:57:52.320781+0000","last_clean":"2023-09-23T19:57:52.320781+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:57:52.320781+0000","last_undegraded":"2023-09-23T19:57:52.320781+0000","last_fullsized":"2023-09-23T19:57:52.320781+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":112,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321482+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:57:52.321482+0000","last_peered":"2023-09-23T19:57:52.321482+0000","last_clean":"2023-09-23T19:57:52.321482+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:57:52.321482+0000","last_undegraded":"2023-09-23T19:57:52.321482+0000","last_fullsized":"2023-09-23T19:57:52.321482+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":104,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.326082+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:57:52.326082+0000","last_peered":"2023-09-23T19:57:52.326082+0000","last_clean":"2023-09-23T19:57:52.326082+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:57:52.326082+0000","last_undegraded":"2023-09-23T19:57:52.326082+0000","last_fullsized":"2023-09-23T19:57:52.326082+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"174'5133","reported_seq":5151,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:03.235126+0000","last_change":"2023-09-23T19:56:51.332314+0000","last_active":"2023-09-23T19:58:03.235126+0000","last_peered":"2023-09-23T19:58:03.235126+0000","last_clean":"2023-09-23T19:58:03.235126+0000","last_became_active":"2023-09-23T19:56:51.332106+0000","last_became_peered":"2023-09-23T19:56:51.332106+0000","last_unstale":"2023-09-23T19:58:03.235126+0000","last_undegraded":"2023-09-23T19:58:03.235126+0000","last_fullsized":"2023-09-23T19:58:03.235126+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5133,"log_dups_size":0,"ondisk_log_size":5133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:38.454829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336396288,"num_objects":5133,"num_object_clones":0,"num_object_copies":15399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5133,"num_write_kb":328512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":512,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276509+0000","last_change":"2023-09-23T19:56:42.336491+0000","last_active":"2023-09-23T19:57:52.276509+0000","last_peered":"2023-09-23T19:57:52.276509+0000","last_clean":"2023-09-23T19:57:52.276509+0000","last_became_active":"2023-09-23T19:56:42.336151+0000","last_became_peered":"2023-09-23T19:56:42.336151+0000","last_unstale":"2023-09-23T19:57:52.276509+0000","last_undegraded":"2023-09-23T19:57:52.276509+0000","last_fullsized":"2023-09-23T19:57:52.276509+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:52.411762+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":136,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274948+0000","last_change":"2023-09-23T19:50:57.789467+0000","last_active":"2023-09-23T19:57:52.274948+0000","last_peered":"2023-09-23T19:57:52.274948+0000","last_clean":"2023-09-23T19:57:52.274948+0000","last_became_active":"2023-09-23T19:50:57.789136+0000","last_became_peered":"2023-09-23T19:50:57.789136+0000","last_unstale":"2023-09-23T19:57:52.274948+0000","last_undegraded":"2023-09-23T19:57:52.274948+0000","last_fullsized":"2023-09-23T19:57:52.274948+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:07.697652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":486,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.321899+0000","last_change":"2023-09-23T19:56:42.312115+0000","last_active":"2023-09-23T19:57:52.321899+0000","last_peered":"2023-09-23T19:57:52.321899+0000","last_clean":"2023-09-23T19:57:52.321899+0000","last_became_active":"2023-09-23T19:56:42.290218+0000","last_became_peered":"2023-09-23T19:56:42.290218+0000","last_unstale":"2023-09-23T19:57:52.321899+0000","last_undegraded":"2023-09-23T19:57:52.321899+0000","last_fullsized":"2023-09-23T19:57:52.321899+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:24.023589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":133,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274180+0000","last_change":"2023-09-23T19:50:57.793288+0000","last_active":"2023-09-23T19:57:52.274180+0000","last_peered":"2023-09-23T19:57:52.274180+0000","last_clean":"2023-09-23T19:57:52.274180+0000","last_became_active":"2023-09-23T19:50:57.792059+0000","last_became_peered":"2023-09-23T19:50:57.792059+0000","last_unstale":"2023-09-23T19:57:52.274180+0000","last_undegraded":"2023-09-23T19:57:52.274180+0000","last_fullsized":"2023-09-23T19:57:52.274180+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:20.551562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"174'5299","reported_seq":5318,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T19:58:03.220838+0000","last_change":"2023-09-23T19:56:51.331630+0000","last_active":"2023-09-23T19:58:03.220838+0000","last_peered":"2023-09-23T19:58:03.220838+0000","last_clean":"2023-09-23T19:58:03.220838+0000","last_became_active":"2023-09-23T19:56:51.331338+0000","last_became_peered":"2023-09-23T19:56:51.331338+0000","last_unstale":"2023-09-23T19:58:03.220838+0000","last_undegraded":"2023-09-23T19:58:03.220838+0000","last_fullsized":"2023-09-23T19:58:03.220838+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5299,"log_dups_size":0,"ondisk_log_size":5299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:45.394928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347275264,"num_objects":5299,"num_object_clones":0,"num_object_copies":15897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5299,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":490,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274806+0000","last_change":"2023-09-23T19:53:00.705074+0000","last_active":"2023-09-23T19:57:52.274806+0000","last_peered":"2023-09-23T19:57:52.274806+0000","last_clean":"2023-09-23T19:57:52.274806+0000","last_became_active":"2023-09-23T19:52:57.353333+0000","last_became_peered":"2023-09-23T19:52:57.353333+0000","last_unstale":"2023-09-23T19:57:52.274806+0000","last_undegraded":"2023-09-23T19:57:52.274806+0000","last_fullsized":"2023-09-23T19:57:52.274806+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T19:53:00.704900+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:24.642289+0000","scrub_duration":0.0090978159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":336,"reported_epoch":173,"state":"active+undersized","last_fresh":"2023-09-23T19:57:52.321022+0000","last_change":"2023-09-23T19:54:45.328104+0000","last_active":"2023-09-23T19:57:52.321022+0000","last_peered":"2023-09-23T19:57:52.321022+0000","last_clean":"2023-09-23T19:43:08.441095+0000","last_became_active":"2023-09-23T19:54:45.328104+0000","last_became_peered":"2023-09-23T19:54:45.328104+0000","last_unstale":"2023-09-23T19:57:52.321022+0000","last_undegraded":"2023-09-23T19:57:52.321022+0000","last_fullsized":"2023-09-23T19:54:45.325453+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:40.212909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":122,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.458863+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:57:52.458863+0000","last_peered":"2023-09-23T19:57:52.458863+0000","last_clean":"2023-09-23T19:57:52.458863+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:57:52.458863+0000","last_undegraded":"2023-09-23T19:57:52.458863+0000","last_fullsized":"2023-09-23T19:57:52.458863+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":123,"reported_epoch":173,"state":"active+undersized","last_fresh":"2023-09-23T19:57:52.276223+0000","last_change":"2023-09-23T19:54:45.330341+0000","last_active":"2023-09-23T19:57:52.276223+0000","last_peered":"2023-09-23T19:57:52.276223+0000","last_clean":"2023-09-23T19:54:37.305426+0000","last_became_active":"2023-09-23T19:54:45.330341+0000","last_became_peered":"2023-09-23T19:54:45.330341+0000","last_unstale":"2023-09-23T19:57:52.276223+0000","last_undegraded":"2023-09-23T19:57:52.276223+0000","last_fullsized":"2023-09-23T19:54:45.253699+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:22.534625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":106,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.326008+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:57:52.326008+0000","last_peered":"2023-09-23T19:57:52.326008+0000","last_clean":"2023-09-23T19:57:52.326008+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:57:52.326008+0000","last_undegraded":"2023-09-23T19:57:52.326008+0000","last_fullsized":"2023-09-23T19:57:52.326008+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":157,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.779211+0000","last_change":"2023-09-23T19:50:57.790667+0000","last_active":"2023-09-23T19:57:52.779211+0000","last_peered":"2023-09-23T19:57:52.779211+0000","last_clean":"2023-09-23T19:57:52.779211+0000","last_became_active":"2023-09-23T19:50:57.790375+0000","last_became_peered":"2023-09-23T19:50:57.790375+0000","last_unstale":"2023-09-23T19:57:52.779211+0000","last_undegraded":"2023-09-23T19:57:52.779211+0000","last_fullsized":"2023-09-23T19:57:52.779211+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:06.699384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":135,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.318599+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:57:52.318599+0000","last_peered":"2023-09-23T19:57:52.318599+0000","last_clean":"2023-09-23T19:57:52.318599+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:57:52.318599+0000","last_undegraded":"2023-09-23T19:57:52.318599+0000","last_fullsized":"2023-09-23T19:57:52.318599+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276613+0000","last_change":"2023-09-23T19:50:57.789973+0000","last_active":"2023-09-23T19:57:52.276613+0000","last_peered":"2023-09-23T19:57:52.276613+0000","last_clean":"2023-09-23T19:57:52.276613+0000","last_became_active":"2023-09-23T19:50:57.789421+0000","last_became_peered":"2023-09-23T19:50:57.789421+0000","last_unstale":"2023-09-23T19:57:52.276613+0000","last_undegraded":"2023-09-23T19:57:52.276613+0000","last_fullsized":"2023-09-23T19:57:52.276613+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:16.268531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.319081+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T19:57:52.319081+0000","last_peered":"2023-09-23T19:57:52.319081+0000","last_clean":"2023-09-23T19:57:52.319081+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T19:57:52.319081+0000","last_undegraded":"2023-09-23T19:57:52.319081+0000","last_fullsized":"2023-09-23T19:57:52.319081+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:03.354856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276373+0000","last_change":"2023-09-23T19:50:57.790948+0000","last_active":"2023-09-23T19:57:52.276373+0000","last_peered":"2023-09-23T19:57:52.276373+0000","last_clean":"2023-09-23T19:57:52.276373+0000","last_became_active":"2023-09-23T19:50:57.790560+0000","last_became_peered":"2023-09-23T19:50:57.790560+0000","last_unstale":"2023-09-23T19:57:52.276373+0000","last_undegraded":"2023-09-23T19:57:52.276373+0000","last_fullsized":"2023-09-23T19:57:52.276373+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:28.073026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":159,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274725+0000","last_change":"2023-09-23T19:56:39.276333+0000","last_active":"2023-09-23T19:57:52.274725+0000","last_peered":"2023-09-23T19:57:52.274725+0000","last_clean":"2023-09-23T19:57:52.274725+0000","last_became_active":"2023-09-23T19:56:39.275998+0000","last_became_peered":"2023-09-23T19:56:39.275998+0000","last_unstale":"2023-09-23T19:57:52.274725+0000","last_undegraded":"2023-09-23T19:57:52.274725+0000","last_fullsized":"2023-09-23T19:57:52.274725+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:48.085884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":158,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.274187+0000","last_change":"2023-09-23T19:50:57.785311+0000","last_active":"2023-09-23T19:57:52.274187+0000","last_peered":"2023-09-23T19:57:52.274187+0000","last_clean":"2023-09-23T19:57:52.274187+0000","last_became_active":"2023-09-23T19:50:57.784967+0000","last_became_peered":"2023-09-23T19:50:57.784967+0000","last_unstale":"2023-09-23T19:57:52.274187+0000","last_undegraded":"2023-09-23T19:57:52.274187+0000","last_fullsized":"2023-09-23T19:57:52.274187+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:51.972076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":122,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.326103+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:57:52.326103+0000","last_peered":"2023-09-23T19:57:52.326103+0000","last_clean":"2023-09-23T19:57:52.326103+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:57:52.326103+0000","last_undegraded":"2023-09-23T19:57:52.326103+0000","last_fullsized":"2023-09-23T19:57:52.326103+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T19:57:52.276159+0000","last_change":"2023-09-23T19:56:39.281130+0000","last_active":"2023-09-23T19:57:52.276159+0000","last_peered":"2023-09-23T19:57:52.276159+0000","last_clean":"2023-09-23T19:57:52.276159+0000","last_became_active":"2023-09-23T19:56:39.280800+0000","last_became_peered":"2023-09-23T19:56:39.280800+0000","last_unstale":"2023-09-23T19:57:52.276159+0000","last_undegraded":"2023-09-23T19:57:52.276159+0000","last_fullsized":"2023-09-23T19:57:52.276159+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:03.511985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5410783232,"num_objects":82562,"num_object_clones":0,"num_object_copies":247686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82562,"num_write_kb":5283968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16218652672,"data_stored":16218652672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82562,"ondisk_log_size":82562,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":93,"acting":93,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234203,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2772220,"kb_used_data":2655796,"kb_used_omap":0,"kb_used_meta":116416,"kb_avail":90972932,"statfs":{"total":95995035648,"available":93156282368,"internally_reserved":0,"allocated":2719535104,"data_stored":2718980962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":119209984},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:58:00 2023","interfaces":[{"interface":"back","average":{"1min":3.8370000000000002,"5min":2.6179999999999999,"15min":1.282},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":62.061999999999998,"5min":62.061999999999998,"15min":62.061999999999998},"last":0.29599999999999999},{"interface":"front","average":{"1min":3.875,"5min":3.3079999999999998,"15min":1.528},"min":{"1min":0.216,"5min":0.215,"15min":0.215},"max":{"1min":62.131,"5min":62.131,"15min":62.131},"last":0.32400000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.7,"15min":0.96399999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.7519999999999998,"5min":11.093999999999999,"15min":11.093999999999999},"last":15.491},{"interface":"front","average":{"1min":1.262,"5min":1.8069999999999999,"15min":0.99299999999999999},"min":{"1min":0.247,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.2300000000000004,"5min":14.747,"15min":14.747},"last":15.558}]},{"osd":2,"last update":"Sat Sep 23 19:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":4.0270000000000001,"15min":1.8919999999999999},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.0609999999999999,"5min":130.673,"15min":130.673},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.2230000000000001,"5min":4.1020000000000003,"15min":1.758},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.9269999999999996,"5min":130.71100000000001,"15min":130.71100000000001},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.7689999999999999,"15min":1.147},"min":{"1min":0.33200000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.4500000000000002,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.849,"5min":1.8180000000000001,"15min":1.0349999999999999},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":13.515000000000001,"5min":13.515000000000001,"15min":13.515000000000001},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:57:39 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":4.008,"15min":1.661},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.48499999999999999,"5min":143.255,"15min":143.255},"last":0.378},{"interface":"front","average":{"1min":0.39600000000000002,"5min":3.806,"15min":1.6359999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.53000000000000003,"5min":143.41200000000001,"15min":143.41200000000001},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:39 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":2.411,"15min":1.1639999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.821,"5min":84.566999999999993,"15min":84.566999999999993},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.38500000000000001,"5min":2.194,"15min":1.1020000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.61399999999999999,"5min":81.244,"15min":81.244},"last":0.34000000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.0189999999999999,"15min":0.71499999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.6449999999999996,"5min":5.6449999999999996,"15min":5.6449999999999996},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.121,"15min":0.76500000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.7720000000000002,"5min":5.7720000000000002,"15min":5.7720000000000002},"last":0.38900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495800,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379500,"kb_used_data":1334692,"kb_used_omap":0,"kb_used_meta":44800,"kb_avail":92365652,"statfs":{"total":95995035648,"available":94582427648,"internally_reserved":0,"allocated":1366724608,"data_stored":1366198307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":45875200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:52 2023","interfaces":[{"interface":"back","average":{"1min":4.415,"5min":1.3500000000000001,"15min":0.85199999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":60.738,"5min":60.738,"15min":60.738},"last":1.476},{"interface":"front","average":{"1min":4.202,"5min":1.333,"15min":0.88700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":60.722999999999999,"5min":60.722999999999999,"15min":60.722999999999999},"last":1.377}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.85799999999999998,"15min":0.69999999999999996},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6520000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.395,"5min":0.94299999999999995,"15min":0.69099999999999995},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":4.3029999999999999,"5min":2.7829999999999999,"15min":1.3149999999999999},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.245},"max":{"1min":53.667999999999999,"5min":61.155000000000001,"15min":61.155000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":4.024,"5min":2.7989999999999999,"15min":1.417},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":53.628,"5min":60.506,"15min":60.506},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":0.97899999999999998,"15min":0.76700000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":5.7619999999999996},"last":1.361},{"interface":"front","average":{"1min":2.1379999999999999,"5min":0.94299999999999995,"15min":0.69299999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":1.387}]},{"osd":4,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":1.0740000000000001,"15min":0.68600000000000005},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.4859999999999998,"5min":24.003,"15min":24.003},"last":0.377},{"interface":"front","average":{"1min":1.607,"5min":1.0920000000000001,"15min":0.70199999999999996},"min":{"1min":0.378,"5min":0.313,"15min":0.26500000000000001},"max":{"1min":6.6020000000000003,"5min":24.802,"15min":24.802},"last":1.3149999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.147,"15min":0.71099999999999997},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":14.378,"5min":24.065999999999999,"15min":24.065999999999999},"last":1.4079999999999999},{"interface":"front","average":{"1min":1.641,"5min":1.1359999999999999,"15min":0.74399999999999999},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":6.0529999999999999,"5min":24.841000000000001,"15min":24.841000000000001},"last":1.2769999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:54:39 2023","interfaces":[{"interface":"back","average":{"1min":6.1509999999999998,"5min":1.8149999999999999,"15min":0.98099999999999998},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":105.71299999999999,"5min":105.71299999999999,"15min":105.71299999999999},"last":2.1579999999999999},{"interface":"front","average":{"1min":6.5620000000000003,"5min":2.0259999999999998,"15min":1.075},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":103.206,"5min":103.206,"15min":103.206},"last":0.46100000000000002}]}]},{"osd":1,"up_from":99,"seq":425201762404,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2073412,"kb_used_data":1929340,"kb_used_omap":7,"kb_used_meta":144056,"kb_avail":91671740,"statfs":{"total":95995035648,"available":93871861760,"internally_reserved":0,"allocated":1975644160,"data_stored":1974908504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":147514170},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":9.8149999999999995,"5min":7.5449999999999999,"15min":2.7530000000000001},"min":{"1min":0.151,"5min":0.151,"15min":0.151},"max":{"1min":186.84,"5min":202.917,"15min":202.917},"last":3.4500000000000002},{"interface":"front","average":{"1min":10.493,"5min":7.9180000000000001,"15min":2.8820000000000001},"min":{"1min":0.17999999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":191.322,"5min":203.34899999999999,"15min":203.34899999999999},"last":4.9059999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:57:38 2023","interfaces":[{"interface":"back","average":{"1min":8.5419999999999998,"5min":9.9860000000000007,"15min":3.5640000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.192},"max":{"1min":62.697000000000003,"5min":231.57499999999999,"15min":231.57499999999999},"last":0.215},{"interface":"front","average":{"1min":9.1219999999999999,"5min":9.2959999999999994,"15min":3.335},"min":{"1min":0.254,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":61.008000000000003,"5min":236.077,"15min":236.077},"last":3.3149999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1030000000000002,"5min":2.6389999999999998,"15min":1.127},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":26.388000000000002,"5min":26.388000000000002,"15min":26.388000000000002},"last":3.4900000000000002},{"interface":"front","average":{"1min":1.748,"5min":1.9239999999999999,"15min":0.90800000000000003},"min":{"1min":0.24299999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.507,"5min":20.166,"15min":20.166},"last":4.9429999999999996}]},{"osd":4,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":3.71,"15min":1.6659999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":7.2149999999999999,"5min":161.68299999999999,"15min":161.68299999999999},"last":3.375},{"interface":"front","average":{"1min":1.0309999999999999,"5min":3.4990000000000001,"15min":1.5720000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.944,"5min":161.286,"15min":161.286},"last":4.8419999999999996}]},{"osd":5,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":11.356999999999999,"5min":5.8170000000000002,"15min":2.3610000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":219.048,"5min":219.048,"15min":219.048},"last":4.8680000000000003},{"interface":"front","average":{"1min":12.51,"5min":6.4039999999999999,"15min":2.5659999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":218.69499999999999,"5min":218.69499999999999,"15min":218.69499999999999},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.097,"15min":0.80500000000000005},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.9649999999999999,"5min":7.9649999999999999,"15min":7.9649999999999999},"last":14.897},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.0800000000000001,"15min":0.82399999999999995},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":14.904999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.6400000000000001,"15min":1.3120000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":26.327999999999999,"5min":26.327999999999999,"15min":26.327999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.393,"5min":1.599,"15min":0.97599999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.1389999999999993,"5min":9.1389999999999993,"15min":9.1389999999999993},"last":3.415}]}]},{"osd":0,"up_from":94,"seq":403726925939,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1481472,"kb_used_data":1309496,"kb_used_omap":8,"kb_used_meta":171959,"kb_avail":92263680,"statfs":{"total":95995035648,"available":94478008320,"internally_reserved":0,"allocated":1340923904,"data_stored":1340191793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":176086625},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:57:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.921,"15min":0.85499999999999998},"min":{"1min":0.20399999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":7.2569999999999997,"5min":74.751000000000005,"15min":74.751000000000005},"last":0.19500000000000001},{"interface":"front","average":{"1min":1.532,"5min":2.5390000000000001,"15min":1.071},"min":{"1min":0.20599999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":7.6379999999999999,"5min":74.998000000000005,"15min":74.998000000000005},"last":3.5649999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:57:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":3.6070000000000002,"15min":1.48},"min":{"1min":0.214,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":24.611999999999998,"5min":74.795000000000002,"15min":74.795000000000002},"last":11.180999999999999},{"interface":"front","average":{"1min":3.5059999999999998,"5min":4.2030000000000003,"15min":1.649},"min":{"1min":0.24199999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":24.677,"5min":78.366,"15min":78.366},"last":0.91200000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:57:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4099999999999999,"5min":3.641,"15min":1.4650000000000001},"min":{"1min":0.29799999999999999,"5min":0.224,"15min":0.224},"max":{"1min":4.984,"5min":104.084,"15min":104.084},"last":0.81399999999999995},{"interface":"front","average":{"1min":2.3889999999999998,"5min":4.0190000000000001,"15min":1.589},"min":{"1min":0.35399999999999998,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":8.2319999999999993,"5min":104.143,"15min":104.143},"last":1.298}]},{"osd":4,"last update":"Sat Sep 23 19:57:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":3.931,"15min":1.748},"min":{"1min":0.32100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7919999999999998,"5min":110.312,"15min":110.312},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.1000000000000001,"5min":3.528,"15min":1.6240000000000001},"min":{"1min":0.29399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7669999999999999,"5min":110.256,"15min":110.256},"last":0.89800000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:57:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":3.7639999999999998,"15min":1.7210000000000001},"min":{"1min":0.35499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.7320000000000002,"5min":131.34399999999999,"15min":131.34399999999999},"last":11.246},{"interface":"front","average":{"1min":1.8180000000000001,"5min":4.3200000000000003,"15min":1.9079999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":7.8280000000000003,"5min":131.386,"15min":131.386},"last":0.94099999999999995}]},{"osd":6,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":0.89700000000000002,"15min":0.78300000000000003},"min":{"1min":0.40899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.9139999999999997,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":2.4100000000000001},{"interface":"front","average":{"1min":2.3410000000000002,"5min":0.996,"15min":0.81000000000000005},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.6920000000000002,"5min":8.6920000000000002,"15min":8.6920000000000002},"last":1.1679999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:57:53 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":2.5840000000000001,"15min":1.3360000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.7709999999999999,"5min":77.132999999999996,"15min":77.132999999999996},"last":11.222},{"interface":"front","average":{"1min":1.978,"5min":3.0659999999999998,"15min":1.508},"min":{"1min":0.39000000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.8550000000000004,"5min":75.465999999999994,"15min":75.465999999999994},"last":2.79}]}]},{"osd":2,"up_from":104,"seq":446676598878,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3572868,"kb_used_data":3316476,"kb_used_omap":12,"kb_used_meta":256371,"kb_avail":90172284,"statfs":{"total":95995035648,"available":92336418816,"internally_reserved":0,"allocated":3396071424,"data_stored":3395338327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":262524926},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":2.8559999999999999,"15min":1.196},"min":{"1min":0.16900000000000001,"5min":0.154,"15min":0.154},"max":{"1min":4.4809999999999999,"5min":50.506999999999998,"15min":50.506999999999998},"last":2.3210000000000002},{"interface":"front","average":{"1min":1.0429999999999999,"5min":2.9329999999999998,"15min":1.2350000000000001},"min":{"1min":0.192,"5min":0.187,"15min":0.187},"max":{"1min":6.0629999999999997,"5min":50.802999999999997,"15min":50.802999999999997},"last":4.2569999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":2.5249999999999999,"15min":1.0960000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":6.0759999999999996,"5min":50.061,"15min":50.061},"last":6.7069999999999999},{"interface":"front","average":{"1min":1.7589999999999999,"5min":2.6960000000000002,"15min":1.1559999999999999},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":6.4240000000000004,"5min":50.533999999999999,"15min":50.533999999999999},"last":6.7240000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:57:09 2023","interfaces":[{"interface":"back","average":{"1min":5.9539999999999997,"5min":2.7770000000000001,"15min":1.1850000000000001},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":50.186999999999998,"5min":50.186999999999998,"15min":50.186999999999998},"last":0.218},{"interface":"front","average":{"1min":6.3159999999999998,"5min":2.8799999999999999,"15min":1.224},"min":{"1min":0.26000000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":50.585000000000001,"5min":50.585000000000001,"15min":50.585000000000001},"last":2.2909999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":2.3969999999999998,"15min":1.216},"min":{"1min":0.38200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.3170000000000002,"5min":50.640000000000001,"15min":50.640000000000001},"last":2.222},{"interface":"front","average":{"1min":0.93200000000000005,"5min":2.4510000000000001,"15min":1.232},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.1240000000000001,"5min":50.154000000000003,"15min":50.154000000000003},"last":1.2789999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.7850000000000001,"15min":1.3500000000000001},"min":{"1min":0.38,"5min":0.308,"15min":0.308},"max":{"1min":5.2670000000000003,"5min":50.616,"15min":50.616},"last":2.2690000000000001},{"interface":"front","average":{"1min":1.369,"5min":2.5059999999999998,"15min":1.2529999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.2169999999999996,"5min":50.762,"15min":50.762},"last":4.2149999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.827,"5min":1.343,"15min":0.84999999999999998},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4740000000000002,"5min":7.4740000000000002,"15min":7.4740000000000002},"last":1.0660000000000001},{"interface":"front","average":{"1min":2.899,"5min":1.3120000000000001,"15min":0.85699999999999998},"min":{"1min":0.42399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.3029999999999999,"5min":7.3029999999999999,"15min":7.3029999999999999},"last":1.008}]},{"osd":7,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":2.4980000000000002,"15min":1.274},"min":{"1min":0.30199999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.1070000000000002,"5min":50.118000000000002,"15min":50.118000000000002},"last":5.0419999999999998},{"interface":"front","average":{"1min":1.2070000000000001,"5min":2.956,"15min":1.429},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.3799999999999999,"5min":50.664999999999999,"15min":50.664999999999999},"last":5.266}]}]},{"osd":3,"up_from":109,"seq":468151435351,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3364520,"kb_used_data":3228704,"kb_used_omap":4,"kb_used_meta":135803,"kb_avail":90380632,"statfs":{"total":95995035648,"available":92549767168,"internally_reserved":0,"allocated":3306192896,"data_stored":3305423002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":139062381},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":7.2359999999999998,"15min":2.6469999999999998},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":3.2509999999999999,"5min":240.40600000000001,"15min":240.40600000000001},"last":4.0650000000000004},{"interface":"front","average":{"1min":1.8779999999999999,"5min":7.8609999999999998,"15min":2.8620000000000001},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.1329999999999991,"5min":240.352,"15min":240.352},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.5860000000000001,"15min":0.78400000000000003},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.1619999999999999,"5min":8.5289999999999999,"15min":8.5289999999999999},"last":2.915},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.4239999999999999,"15min":0.72799999999999998},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.6699999999999999,"5min":11.361000000000001,"15min":11.361000000000001},"last":5.3159999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":6.423,"5min":10.749000000000001,"15min":3.8420000000000001},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":83.590000000000003,"5min":264.97800000000001,"15min":264.97800000000001},"last":2.8860000000000001},{"interface":"front","average":{"1min":7.7910000000000004,"5min":10.734999999999999,"15min":3.8530000000000002},"min":{"1min":0.26100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":83.344999999999999,"5min":257.40899999999999,"15min":257.40899999999999},"last":4.1589999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":2.0649999999999999,"15min":1.077},"min":{"1min":0.30199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.7869999999999999,"5min":69.236000000000004,"15min":69.236000000000004},"last":3.996},{"interface":"front","average":{"1min":1.714,"5min":2.7189999999999999,"15min":1.2969999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.2420000000000009,"5min":69.373000000000005,"15min":69.373000000000005},"last":0.46800000000000003}]},{"osd":5,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":1.8480000000000001,"15min":1.0169999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.0830000000000002,"5min":13.185,"15min":13.185},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.694,"15min":0.98399999999999999},"min":{"1min":0.35099999999999998,"5min":0.313,"15min":0.313},"max":{"1min":2.4089999999999998,"5min":10.977,"15min":10.977},"last":2.9369999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:54:02 2023","interfaces":[{"interface":"back","average":{"1min":3.4809999999999999,"5min":1.2849999999999999,"15min":0.82099999999999995},"min":{"1min":0.38600000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.4399999999999995,"5min":9.4399999999999995,"15min":9.4399999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.1179999999999999,"5min":0.94599999999999995,"15min":0.72099999999999997},"min":{"1min":0.44400000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":10.932,"5min":10.932,"15min":10.932},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 19:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.784,"15min":1.0069999999999999},"min":{"1min":0.38,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.8839999999999999,"5min":12.720000000000001,"15min":12.720000000000001},"last":2.9660000000000002},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.4770000000000001,"15min":0.91100000000000003},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8199999999999998,"5min":9.8369999999999997,"15min":9.8369999999999997},"last":4.0369999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019113,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1804152,"kb_used_data":1664880,"kb_used_omap":0,"kb_used_meta":139264,"kb_avail":91941000,"statfs":{"total":95995035648,"available":94147584000,"internally_reserved":0,"allocated":1704837120,"data_stored":1704287062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":142606336},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:57 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":5.1139999999999999,"15min":2.097},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.9800000000000004,"5min":104.604,"15min":104.604},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.4199999999999999,"5min":4.9889999999999999,"15min":2.0790000000000002},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.9640000000000004,"5min":104.687,"15min":104.687},"last":0.70799999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":2.1099999999999999,"15min":1.073},"min":{"1min":0.23400000000000001,"5min":0.216,"15min":0.216},"max":{"1min":12.718999999999999,"5min":12.718999999999999,"15min":12.718999999999999},"last":3.6829999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.5109999999999999,"15min":0.879},"min":{"1min":0.316,"5min":0.22,"15min":0.22},"max":{"1min":2.7679999999999998,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":7.569}]},{"osd":2,"last update":"Sat Sep 23 19:57:41 2023","interfaces":[{"interface":"back","average":{"1min":4.4029999999999996,"5min":4.6920000000000002,"15min":1.9379999999999999},"min":{"1min":0.26900000000000002,"5min":0.25,"15min":0.25},"max":{"1min":69.989000000000004,"5min":98.063999999999993,"15min":98.063999999999993},"last":3.8039999999999998},{"interface":"front","average":{"1min":4.077,"5min":4.5199999999999996,"15min":1.873},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":69.799000000000007,"5min":97.921000000000006,"15min":97.921000000000006},"last":0.28999999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:57:05 2023","interfaces":[{"interface":"back","average":{"1min":5.9039999999999999,"5min":2.746,"15min":1.27},"min":{"1min":0.36099999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":94.141000000000005,"5min":94.141000000000005,"15min":94.141000000000005},"last":6.319},{"interface":"front","average":{"1min":6.7750000000000004,"5min":3.181,"15min":1.4550000000000001},"min":{"1min":0.435,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":94.037000000000006,"5min":94.037000000000006,"15min":94.037000000000006},"last":4.7199999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:57:57 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.296,"15min":0.76600000000000001},"min":{"1min":0.253,"5min":0.246,"15min":0.246},"max":{"1min":0.64300000000000002,"5min":10.246,"15min":10.246},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.434,"5min":1.6220000000000001,"15min":0.88400000000000001},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":1.2150000000000001,"5min":12.675000000000001,"15min":12.675000000000001},"last":0.31900000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":0.84699999999999998,"15min":0.60199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.867,"5min":5.867,"15min":5.867},"last":8.0150000000000006},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.80300000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":3.931}]},{"osd":7,"last update":"Sat Sep 23 19:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.4710000000000001,"15min":0.872},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.248,"5min":11.59,"15min":11.59},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.3759999999999999,"15min":0.86099999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.2970000000000002,"5min":9.3089999999999993,"15min":9.3089999999999993},"last":0.377}]}]},{"osd":5,"up_from":38,"seq":163208757477,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1824240,"kb_used_data":1670056,"kb_used_omap":0,"kb_used_meta":154176,"kb_avail":91920912,"statfs":{"total":95995035648,"available":94127013888,"internally_reserved":0,"allocated":1710137344,"data_stored":1709591727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":157876224},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":2.069,"15min":1.0740000000000001},"min":{"1min":0.219,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.5830000000000002,"5min":24.852,"15min":24.852},"last":2.024},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.917,"15min":1.0129999999999999},"min":{"1min":0.23899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":4.0289999999999999,"5min":24.638999999999999,"15min":24.638999999999999},"last":1.863}]},{"osd":1,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.8879999999999999,"15min":0.97799999999999998},"min":{"1min":0.26300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":8.4879999999999995,"5min":9.9789999999999992,"15min":9.9789999999999992},"last":1.222},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.7829999999999999,"15min":0.94399999999999995},"min":{"1min":0.27900000000000003,"5min":0.249,"15min":0.249},"max":{"1min":6.0209999999999999,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":1.954}]},{"osd":2,"last update":"Sat Sep 23 19:57:42 2023","interfaces":[{"interface":"back","average":{"1min":5.4960000000000004,"5min":7.641,"15min":2.9060000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":58.246000000000002,"5min":167.10400000000001,"15min":167.10400000000001},"last":0.94599999999999995},{"interface":"front","average":{"1min":5.5250000000000004,"5min":7.6699999999999999,"15min":2.9489999999999998},"min":{"1min":0.34499999999999997,"5min":0.31900000000000001,"15min":0.317},"max":{"1min":58.268999999999998,"5min":167.167,"15min":167.167},"last":0.97299999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":2.052,"15min":1.048},"min":{"1min":0.34300000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.548,"5min":12.853999999999999,"15min":12.853999999999999},"last":4.718},{"interface":"front","average":{"1min":1.052,"5min":1.772,"15min":0.995},"min":{"1min":0.373,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2439999999999998,"5min":10.115,"15min":10.115},"last":1.9430000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.246,"15min":0.73499999999999999},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.23699999999999999},"max":{"1min":1.8169999999999999,"5min":10.077,"15min":10.077},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":1.339,"15min":0.79200000000000004},"min":{"1min":0.26900000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.91700000000000004,"5min":10.226000000000001,"15min":10.226000000000001},"last":0.29399999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:54:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":0.91600000000000004,"15min":0.63400000000000001},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":10.180999999999999,"5min":10.180999999999999,"15min":10.180999999999999},"last":4.2949999999999999},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.046,"15min":0.71299999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":10.406000000000001},"last":3.879}]},{"osd":7,"last update":"Sat Sep 23 19:57:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.351,"15min":0.81899999999999995},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.105,"5min":13.734999999999999,"15min":13.734999999999999},"last":0.314},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.524,"15min":0.88},"min":{"1min":0.32200000000000001,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":2.0270000000000001,"5min":10.041,"15min":10.041},"last":2.3809999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1344602112,"data_stored":1344602112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1975189504,"data_stored":1975189504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3413311488,"data_stored":3413311488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3332898816,"data_stored":3332898816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1716977664,"data_stored":1716977664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1712455680,"data_stored":1712455680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2723217408,"data_stored":2723217408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:58:08.149 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-09-23T19:58:08.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:08 smithi135 ceph-mon[213871]: pgmap v463: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 5/251364 objects degraded (0.002%) 2023-09-23T19:58:08.561 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:08.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:08.676 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:08.695 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:08 smithi135 ceph-mon[221113]: pgmap v463: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 5/251364 objects degraded (0.002%) 2023-09-23T19:58:08.706 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:08 smithi144 ceph-mon[194833]: pgmap v463: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 5/251364 objects degraded (0.002%) 2023-09-23T19:58:08.708 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:08.760 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:08.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:08.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:08.865 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:08.893 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:08.903 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 70 16 87985 87969 78.4911 54.5625 0.00645087 0.0127303 2023-09-23T19:58:09.362 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:09.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:09.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:09.466 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:09.540 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:09.660 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:09.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:09.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:09.775 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:09.857 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:09.903 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 71 16 88785 88769 78.09 50 0.0200754 0.0128004 2023-09-23T19:58:10.000 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:10.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:10.047 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:58:10.047 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:58:10.106 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:10.198 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:10.308 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:10.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:10.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:10.418 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:10.437 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:10.522 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:10.601 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:58:10.618 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:58:10.635 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:10.637 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:10 smithi135 ceph-mon[213871]: pgmap v464: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 5/259572 objects degraded (0.002%) 2023-09-23T19:58:10.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:10.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:10.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:10 smithi135 ceph-mon[221113]: pgmap v464: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 5/259572 objects degraded (0.002%) 2023-09-23T19:58:10.667 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:58:10.710 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:58:10.740 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:10.756 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:58:10.767 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:10 smithi144 ceph-mon[194833]: pgmap v464: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 5/259572 objects degraded (0.002%) 2023-09-23T19:58:10.770 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:10.812 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:58:10.824 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:10.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:10.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:10.835 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:58:10.903 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 72 16 89562 89546 77.6801 48.5625 0.021704 0.0128653 2023-09-23T19:58:10.931 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:10.958 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:11.077 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:58:11.077 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:58:11.077 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:58:11.077 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:58:11.078 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:58:11.078 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:58:11.078 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:58:11.078 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:58:11.078 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:58:11.457 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:11.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:11.571 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:11.617 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:11.676 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:11.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:11.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:11 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2034700216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:11.735 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:11 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2034700216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:11.761 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:58:11.788 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:11.826 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:11 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2034700216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:11.830 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:11.881 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:11.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:11.906 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 73 16 90308 90292 77.2542 46.625 0.0371341 0.0129398 2023-09-23T19:58:11.987 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:12.018 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:12.066 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:12.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:12.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:12.171 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:12.196 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:12 smithi135 ceph-mon[213871]: pgmap v465: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 5/261147 objects degraded (0.002%) 2023-09-23T19:58:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/261147 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:12.619 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:12.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:12.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:12 smithi135 ceph-mon[221113]: pgmap v465: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 5/261147 objects degraded (0.002%) 2023-09-23T19:58:12.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/261147 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:12.729 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:58:12.764 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:58:12.835 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:12 smithi144 ceph-mon[194833]: pgmap v465: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 5/261147 objects degraded (0.002%) 2023-09-23T19:58:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/261147 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:12.904 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 74 16 91007 90991 76.8008 43.6875 0.00842434 0.0130145 2023-09-23T19:58:13.054 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:13.824 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:13.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:13.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:13.841 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:13 smithi135 ceph-mon[213871]: pgmap v466: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 5/265032 objects degraded (0.002%) 2023-09-23T19:58:13.841 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:58:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:58:13] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T19:58:13.841 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:13 smithi135 ceph-mon[221113]: pgmap v466: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 5/265032 objects degraded (0.002%) 2023-09-23T19:58:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:13 smithi144 ceph-mon[194833]: pgmap v466: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 5/265032 objects degraded (0.002%) 2023-09-23T19:58:13.908 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 75 16 91824 91808 76.4538 51.0625 0.0208714 0.013075 2023-09-23T19:58:13.934 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:13.967 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:14.024 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:14.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:14.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:14.068 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:14.132 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:14.210 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:14.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:14.542 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:14.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:14.643 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:14.674 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:14.733 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:14.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:14.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:14.765 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:58:14.845 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:14.880 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:14.908 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 76 16 92631 92615 76.1116 50.4375 0.0123443 0.0131332 2023-09-23T19:58:14.930 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:14.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:14.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:15.039 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:15.120 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:15.239 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:15.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:15.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:15.346 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:15.433 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:15.581 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 6 pgp_num to 16 2023-09-23T19:58:15.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:15 smithi135 ceph-mon[213871]: pgmap v467: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 5/271392 objects degraded (0.002%) 2023-09-23T19:58:15.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3851873146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:58:15.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:15 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:58:15.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:15 smithi135 ceph-mon[221113]: pgmap v467: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 5/271392 objects degraded (0.002%) 2023-09-23T19:58:15.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3851873146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:58:15.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:15 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:58:15.730 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:15.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:15.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:15.839 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:15.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:15 smithi144 ceph-mon[194833]: pgmap v467: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 5/271392 objects degraded (0.002%) 2023-09-23T19:58:15.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3851873146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:58:15.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:15 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T19:58:15.901 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:15.908 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 77 16 93438 93422 75.7783 50.4375 0.0129356 0.0131923 2023-09-23T19:58:16.107 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:16.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:16.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:16.212 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:16.302 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:16.769 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:16.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:16.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:16.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:58:16.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: osdmap e176: 8 total, 7 up, 8 in 2023-09-23T19:58:16.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.885 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:16 smithi144 ceph-mon[194833]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:16.910 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:58:16.910 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: osdmap e176: 8 total, 7 up, 8 in 2023-09-23T19:58:16.910 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.910 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.910 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.910 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.910 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.911 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.911 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[221113]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.913 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T19:58:16.913 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: osdmap e176: 8 total, 7 up, 8 in 2023-09-23T19:58:16.913 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.913 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.913 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:16 smithi135 ceph-mon[213871]: failed to encode map e176 with expected crc 2023-09-23T19:58:16.914 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 78 16 94261 94245 75.4614 51.4375 0.0137422 0.013246 2023-09-23T19:58:16.961 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:17.465 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:17.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:17.575 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:17.710 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:17.856 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:17.875 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:17.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:17 smithi144 ceph-mon[194833]: pgmap v469: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 5/275136 objects degraded (0.002%) 2023-09-23T19:58:17.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/275136 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:17.888 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:17 smithi135 ceph-mon[213871]: pgmap v469: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 5/275136 objects degraded (0.002%) 2023-09-23T19:58:17.888 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/275136 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:17.889 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:17 smithi135 ceph-mon[221113]: pgmap v469: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 5/275136 objects degraded (0.002%) 2023-09-23T19:58:17.889 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/275136 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:17.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:17.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:17.913 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 79 16 95015 94999 75.1029 47.125 0.040767 0.0133097 2023-09-23T19:58:17.996 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:18.086 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:18.370 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:18.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:18.474 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:18.546 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195610 5.8 5:13ab778b:::benchmark_data_smithi135_284570_object79826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086316+0000", 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.45701786800001, 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32473328200000001, 2023-09-23T19:58:18.547 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195610 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086316+0000", 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.548 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086316+0000", 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086315+0000", 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.549 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086318+0000", 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.616e-06 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086324+0000", 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0000000000000002e-06 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.550 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086329+0000", 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0629999999999996e-06 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099057+0000", 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012727894999999999 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099206+0000", 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014897700000000001 2023-09-23T19:58:18.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099330+0000", 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012414300000000001 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.105020+0000", 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056901069999999998 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410872+0000", 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30585215500000001 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410959+0000", 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7231999999999994e-05 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411010+0000", 2023-09-23T19:58:18.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.024e-05 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411049+0000", 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9240999999999999e-05 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.554 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195612 5.a 5:51964534:::benchmark_data_smithi135_284570_object79828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086334+0000", 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.45699915899999, 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31885195999999999, 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195612 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.555 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086336+0000", 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2479999999999999e-06 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086341+0000", 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0610000000000003e-06 2023-09-23T19:58:18.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086346+0000", 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5310000000000003e-06 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086419+0000", 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3072999999999996e-05 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086523+0000", 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000103767 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086633+0000", 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110221 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.095308+0000", 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0086747639999999997 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099671+0000", 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043637499999999996 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405096+0000", 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30542450199999999 2023-09-23T19:58:18.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405140+0000", 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3736000000000003e-05 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405186+0000", 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6621e-05 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.561 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195613 5.a 5:51e9d81c:::benchmark_data_smithi135_284570_object79829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086357+0000", 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.456976008, 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.321475978, 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.562 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195613 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T19:58:18.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086360+0000", 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6060000000000001e-06 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086365+0000", 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8440000000000003e-06 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086371+0000", 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.096e-06 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.090840+0000", 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004469794 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.091046+0000", 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000205508 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.091154+0000", 2023-09-23T19:58:18.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107816 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099701+0000", 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085474149999999992 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099715+0000", 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4365999999999999e-05 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405243+0000", 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30552804300000003 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405285+0000", 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1223999999999997e-05 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.407833+0000", 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025487410000000002 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195623 5.8 5:1605c7cf:::benchmark_data_smithi135_284570_object79839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106079+0000", 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.43725430399999, 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30509230399999998, 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.569 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195623 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106079+0000", 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106079+0000", 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106078+0000", 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106087+0000", 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8689999999999993e-06 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106101+0000", 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3971000000000001e-05 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106110+0000", 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9800000000000004e-06 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106177+0000", 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7464000000000003e-05 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106310+0000", 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000132712 2023-09-23T19:58:18.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106435+0000", 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000125499 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111791+0000", 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053554220000000003 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410920+0000", 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29912940799999999 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411095+0000", 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017488599999999999 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411136+0000", 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1517999999999998e-05 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411171+0000", 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5009999999999999e-05 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.576 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195626 5.6 5:6edb6a83:::benchmark_data_smithi135_284570_object79842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106123+0000", 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.43721094399999, 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30425704100000001, 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195626 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106123+0000", 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106123+0000", 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106122+0000", 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106126+0000", 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1300000000000003e-06 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106133+0000", 2023-09-23T19:58:18.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8109999999999997e-06 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106138+0000", 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2569999999999998e-06 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106419+0000", 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028042200000000001 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106516+0000", 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7001000000000002e-05 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106616+0000", 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100272 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405132+0000", 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.298516432 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405142+0000", 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2189999999999993e-06 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410291+0000", 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051497000000000001 2023-09-23T19:58:18.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410341+0000", 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0207000000000002e-05 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410380+0000", 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8109999999999999e-05 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.583 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:195612 5.a e151/112 ondisk, result = 0)", 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106148+0000", 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.43718518599999, 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29904286699999999, 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T19:58:18.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106152+0000", 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1230000000000002e-06 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106163+0000", 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0951000000000001e-05 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106169+0000", 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8220000000000001e-06 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405078+0000", 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.298908224 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405088+0000", 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0686999999999999e-05 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405191+0000", 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010278 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.588 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195629 5.8 5:14b98469:::benchmark_data_smithi135_284570_object79845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.110999+0000", 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.43233465899999, 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31090222000000001, 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195629 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110999+0000", 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110999+0000", 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110998+0000", 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111004+0000", 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8809999999999998e-06 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111022+0000", 2023-09-23T19:58:18.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7354999999999999e-05 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111032+0000", 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0251000000000001e-05 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111900+0000", 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00086782700000000005 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112014+0000", 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000113835 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112123+0000", 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010916000000000001 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411233+0000", 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.299110243 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411284+0000", 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0732999999999997e-05 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421815+0000", 2023-09-23T19:58:18.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010530813999999999 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421859+0000", 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4240000000000003e-05 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421901+0000", 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2107000000000002e-05 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.595 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195630 5.6 5:65c744e5:::benchmark_data_smithi135_284570_object79846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.111039+0000", 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.432294157, 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.304778728, 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195630 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T19:58:18.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111042+0000", 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.507e-06 2023-09-23T19:58:18.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111049+0000", 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9399999999999996e-06 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111055+0000", 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4439999999999999e-06 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111663+0000", 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060795100000000002 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111801+0000", 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013784600000000001 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111940+0000", 2023-09-23T19:58:18.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000139574 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405147+0000", 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29320712999999998 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410463+0000", 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053154830000000002 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415730+0000", 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052668560000000003 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415777+0000", 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7123000000000003e-05 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415818+0000", 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1347999999999998e-05 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:195610 5.8 e151/112 ondisk, result = 0)", 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.112245+0000", 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.431088002, 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29862952999999998, 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.603 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112250+0000", 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0350000000000001e-06 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405043+0000", 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29279347100000003 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405055+0000", 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1627000000000001e-05 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410857+0000", 2023-09-23T19:58:18.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058025580000000002 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410866+0000", 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3450000000000006e-06 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410875+0000", 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.217e-06 2023-09-23T19:58:18.607 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195633 5.b 5:d886e22d:::benchmark_data_smithi135_284570_object79849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.113988+0000", 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 150.42934593800001, 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32055229600000001, 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.608 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195633 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113988+0000", 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.609 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113988+0000", 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113986+0000", 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113993+0000", 2023-09-23T19:58:18.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5889999999999999e-06 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405037+0000", 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29104433699999999 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405056+0000", 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8836999999999999e-05 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405147+0000", 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1032000000000005e-05 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405287+0000", 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014026500000000001 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405410+0000", 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000122636 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415431+0000", 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010021077 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.423733+0000", 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083021039999999994 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434462+0000", 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010729140999999999 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434502+0000", 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9712000000000002e-05 2023-09-23T19:58:18.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434540+0000", 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7759999999999998e-05 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203734 5.b 5:dd140fcc:::benchmark_data_smithi135_284570_object87950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.309502+0000", 2023-09-23T19:58:18.615 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 139.23383187600001, 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29768655599999999, 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203734 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309502+0000", 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309502+0000", 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309501+0000", 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309504+0000", 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.225e-06 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309513+0000", 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4309999999999994e-06 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309517+0000", 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0199999999999996e-06 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309557+0000", 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9762999999999999e-05 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309632+0000", 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5908999999999994e-05 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309704+0000", 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1668000000000006e-05 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.313325+0000", 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036211889999999999 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.313868+0000", 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00054303499999999998 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607103+0000", 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.293234781 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607154+0000", 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0914000000000001e-05 2023-09-23T19:58:18.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607188+0000", 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4112999999999997e-05 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220400 5.b 5:d413819b:::benchmark_data_smithi135_284570_object104616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:58:18.622 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.640447+0000", 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.902886185, 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32328775900000001, 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220400 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640447+0000", 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640447+0000", 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640446+0000", 2023-09-23T19:58:18.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640451+0000", 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4940000000000003e-06 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640463+0000", 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1299e-05 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640471+0000", 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8220000000000007e-06 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640519+0000", 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8375999999999999e-05 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640629+0000", 2023-09-23T19:58:18.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011054 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640741+0000", 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111857 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650179+0000", 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094374469999999998 2023-09-23T19:58:18.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963581+0000", 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31340195599999998 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963649+0000", 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8036999999999999e-05 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963693+0000", 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4162000000000002e-05 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963735+0000", 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2098999999999998e-05 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220403 5.b 5:d5744800:::benchmark_data_smithi135_284570_object104619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.640988+0000", 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.90234501099999, 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32284452200000002, 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220403 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.630 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640993+0000", 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.604e-06 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.641005+0000", 2023-09-23T19:58:18.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1365e-05 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.641012+0000", 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1470000000000004e-06 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645957+0000", 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049455829999999999 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.646045+0000", 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7979999999999995e-05 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.646112+0000", 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6251e-05 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650257+0000", 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041449950000000003 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963620+0000", 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31336345500000001 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963767+0000", 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000147169 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963799+0000", 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2060000000000001e-05 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963833+0000", 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3733999999999999e-05 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.636 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220406 5.d 5:b474da8d:::benchmark_data_smithi135_284570_object104622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.645154+0000", 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.898179756, 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32365115, 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.637 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220406 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645154+0000", 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645154+0000", 2023-09-23T19:58:18.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645152+0000", 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645158+0000", 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9819999999999996e-06 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645168+0000", 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7580000000000004e-06 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645175+0000", 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9009999999999997e-06 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645213+0000", 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8535999999999998e-05 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645311+0000", 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7647999999999997e-05 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645397+0000", 2023-09-23T19:58:18.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5791999999999994e-05 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650541+0000", 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051441569999999999 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968678+0000", 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.318137369 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968716+0000", 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7765999999999997e-05 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968765+0000", 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8520000000000003e-05 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968805+0000", 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0170000000000003e-05 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220411 5.6 5:65202dbd:::benchmark_data_smithi135_284570_object104627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.650884+0000", 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.89244972, 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31285145399999997, 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.644 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220411 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650884+0000", 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650884+0000", 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650883+0000", 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650889+0000", 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7460000000000003e-06 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650902+0000", 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2266000000000001e-05 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650936+0000", 2023-09-23T19:58:18.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4878e-05 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650982+0000", 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5259000000000002e-05 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651089+0000", 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107784 2023-09-23T19:58:18.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651201+0000", 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111297 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963506+0000", 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.312305098 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963534+0000", 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8495000000000001e-05 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963649+0000", 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011424 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963692+0000", 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2907000000000001e-05 2023-09-23T19:58:18.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963735+0000", 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.367e-05 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220414 5.d 5:b1d7f748:::benchmark_data_smithi135_284570_object104630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.650962+0000", 2023-09-23T19:58:18.651 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.892371865, 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.317977958, 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220414 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650962+0000", 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650962+0000", 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650961+0000", 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650965+0000", 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.997e-06 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650972+0000", 2023-09-23T19:58:18.654 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0190000000000001e-06 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650977+0000", 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1340000000000003e-06 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651027+0000", 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9509000000000003e-05 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651141+0000", 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000114848 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651250+0000", 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000108455 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968686+0000", 2023-09-23T19:58:18.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31743582999999997 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968839+0000", 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015357700000000001 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968869+0000", 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9839e-05 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968902+0000", 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2449000000000001e-05 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968940+0000", 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7911999999999999e-05 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.658 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220415 5.d 5:bcbf5941:::benchmark_data_smithi135_284570_object104631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.654307+0000", 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.889026562, 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33038375800000003, 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220415 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654307+0000", 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654307+0000", 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654305+0000", 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654313+0000", 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0530000000000007e-06 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654330+0000", 2023-09-23T19:58:18.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6290000000000002e-05 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654339+0000", 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.499e-06 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655392+0000", 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010529370000000001 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655521+0000", 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012935900000000001 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655599+0000", 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7442999999999997e-05 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.979556+0000", 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32395747899999999 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.979597+0000", 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1085999999999999e-05 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984634+0000", 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005036769 2023-09-23T19:58:18.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984665+0000", 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0960000000000002e-05 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984691+0000", 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5607999999999999e-05 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.665 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:220413 5.4 e154/112 5:23b6015e:::benchmark_data_smithi135_284570_object104629:head v 154'13837, mlcod=154'13835)", 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.654925+0000", 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.88840886200001, 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.308583475, 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654925+0000", 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654925+0000", 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654924+0000", 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:18.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654932+0000", 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9510000000000006e-06 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654942+0000", 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0047999999999999e-05 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654947+0000", 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2089999999999996e-06 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654985+0000", 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8016999999999999e-05 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654999+0000", 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.454e-05 2023-09-23T19:58:18.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963414+0000", 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30841476099999998 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963508+0000", 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3911000000000003e-05 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.670 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:220406 5.d e154/112 ondisk, result = 0)", 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.657056+0000", 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.886277647, 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31163112799999998, 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657056+0000", 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657056+0000", 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657055+0000", 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657061+0000", 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4999999999999996e-06 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963465+0000", 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30640367400000001 2023-09-23T19:58:18.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963476+0000", 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0861e-05 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968664+0000", 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051881879999999998 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968672+0000", 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0690000000000002e-06 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968687+0000", 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4715000000000001e-05 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:220411 5.6 e154/112 ondisk, result = 0)", 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.658010+0000", 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.885322952, 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30549902800000001, 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:18.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T19:58:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T19:58:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658015+0000", 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6190000000000002e-06 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963397+0000", 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30538196699999998 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963412+0000", 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4925e-05 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963488+0000", 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6477000000000007e-05 2023-09-23T19:58:18.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963498+0000", 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0478e-05 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963510+0000", 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1058e-05 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.680 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.681 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:18.681 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:18.681 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:58:18.681 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:18.681 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:58:18.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1097693554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:18.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:18.913 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T19:58:18.914559+0000 min lat: 0.001133 max lat: 0.365714 avg lat: 0.0133735 2023-09-23T19:58:18.913 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:58:18.913 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 80 16 95758 95742 74.7447 46.4375 0.0290265 0.0133735 2023-09-23T19:58:18.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1097693554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:18.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:18.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1097693554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:18.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:19.596 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:19.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:19.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:19.715 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:19.744 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:19.812 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:19.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:19.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:19 smithi144 ceph-mon[194833]: pgmap v470: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 5/283212 objects degraded (0.002%) 2023-09-23T19:58:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:19 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:19 smithi144 ceph-mon[194833]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:19 smithi144 ceph-mon[194833]: osdmap e177: 8 total, 7 up, 8 in 2023-09-23T19:58:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:19 smithi144 ceph-mon[194833]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:19 smithi144 ceph-mon[194833]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.913 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 81 16 96490 96474 74.3869 45.75 0.00311095 0.0134283 2023-09-23T19:58:19.920 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:19.949 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[221113]: pgmap v470: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 5/283212 objects degraded (0.002%) 2023-09-23T19:58:19.949 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:19.950 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[221113]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.950 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:19.950 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[221113]: osdmap e177: 8 total, 7 up, 8 in 2023-09-23T19:58:19.950 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[221113]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.950 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[221113]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.950 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[213871]: pgmap v470: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 5/283212 objects degraded (0.002%) 2023-09-23T19:58:19.950 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:19.950 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[213871]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:19.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[213871]: osdmap e177: 8 total, 7 up, 8 in 2023-09-23T19:58:19.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[213871]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:19 smithi135 ceph-mon[213871]: failed to encode map e177 with expected crc 2023-09-23T19:58:19.969 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:20.643 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:20.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:20.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:20.749 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:20.803 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:58:20.866 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:20.913 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 82 16 97176 97160 74.0028 42.875 0.00891703 0.0135085 2023-09-23T19:58:21.080 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:21.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:21.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:21.185 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:21.268 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:21.393 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:21.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:21.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:21.467 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:58:21.506 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:21.522 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:58:21.575 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:58:21.606 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:21.622 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:58:21.629 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:21.671 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:58:21.712 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:58:21.737 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:21.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:21.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:21.757 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:58:21.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:58:21.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:58:21.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:58:21.758 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:58:21.758 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:58:21.758 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:58:21.758 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:58:21.758 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:58:21.848 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:21.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:21 smithi144 ceph-mon[194833]: pgmap v472: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 5/284703 objects degraded (0.002%) 2023-09-23T19:58:21.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2705227509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:21.885 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:21.914 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 83 16 97925 97909 73.6753 46.8125 0.0203178 0.0135687 2023-09-23T19:58:21.950 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:21.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:21.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:21.962 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:58:21.962 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:58:21.962 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T19:58:21.962 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T19:58:21.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:21 smithi135 ceph-mon[221113]: pgmap v472: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 5/284703 objects degraded (0.002%) 2023-09-23T19:58:21.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2705227509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:21.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:21 smithi135 ceph-mon[213871]: pgmap v472: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 5/284703 objects degraded (0.002%) 2023-09-23T19:58:21.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2705227509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:22.056 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:22.086 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:22.142 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:22.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:22.250 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:22.282 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:22.342 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:22.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:22.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:22.451 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:22.514 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:22.913 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:58:22.914 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 84 16 98598 98582 73.2991 42.0625 0.00770337 0.0136386 2023-09-23T19:58:22.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:22.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/284703 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:22.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:22.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/284703 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:23.099 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:23.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:23.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:23.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:23.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/284703 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:23.205 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:58:23.208 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:58:23.232 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:23.288 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:23.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:23.395 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:23.510 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:23.697 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:58:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:58:23] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T19:58:23.747 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:23.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:23.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:23.860 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:23.886 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:23.914 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 85 16 99317 99301 72.9656 44.9375 0.00382223 0.013688 2023-09-23T19:58:23.926 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:23.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:23.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:24.034 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:24.109 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:24.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:24 smithi144 ceph-mon[194833]: pgmap v473: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 5/288243 objects degraded (0.002%) 2023-09-23T19:58:24.341 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:24.345 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:24.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:24.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:24.453 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:24.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:24 smithi135 ceph-mon[213871]: pgmap v473: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 5/288243 objects degraded (0.002%) 2023-09-23T19:58:24.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:24 smithi135 ceph-mon[221113]: pgmap v473: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 5/288243 objects degraded (0.002%) 2023-09-23T19:58:24.485 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:24.553 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:24.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:24.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:24.658 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:24.734 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:24.914 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 86 16 100090 100074 72.6791 48.3125 0.0172519 0.0137545 2023-09-23T19:58:25.042 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:25.209 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:58:25.290 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:58:25.290 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:58:25.305 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:25.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:25.426 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:25.504 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:25.914 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 87 16 100827 100811 72.3733 46.0625 0.00400759 0.013805 2023-09-23T19:58:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:26 smithi144 ceph-mon[194833]: pgmap v474: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s; 5/294327 objects degraded (0.002%) 2023-09-23T19:58:26.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:26 smithi135 ceph-mon[221113]: pgmap v474: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s; 5/294327 objects degraded (0.002%) 2023-09-23T19:58:26.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:26 smithi135 ceph-mon[213871]: pgmap v474: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s; 5/294327 objects degraded (0.002%) 2023-09-23T19:58:26.756 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:26.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:26.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:26.867 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:26.908 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:26.916 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 88 16 101635 101619 72.1229 50.5 0.0249141 0.0138593 2023-09-23T19:58:26.968 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:26.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:26.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:27.076 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:27.167 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:27.347 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:58:27.371 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/294327 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:27.434 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/294327 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:27.434 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/294327 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:27.515 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:27.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:27.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:27.626 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:27.767 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:27.917 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 89 15 102370 102355 71.8295 46 0.0189233 0.0139163 2023-09-23T19:58:28.398 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:28 smithi135 conmon[213845]: 2023-09-23T19:58:28.138+0000 7f9708921700 -1 mon.a@0(leader).osd e177 definitely_dead 0 2023-09-23T19:58:28.398 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:28 smithi135 ceph-mon[213871]: pgmap v475: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s; 5/297732 objects degraded (0.002%) 2023-09-23T19:58:28.398 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1865346907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T19:58:28.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:28 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T19:58:28.399 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:28 smithi135 ceph-mon[213871]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T19:58:28.399 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:28 smithi135 ceph-mon[221113]: pgmap v475: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s; 5/297732 objects degraded (0.002%) 2023-09-23T19:58:28.399 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1865346907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T19:58:28.399 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:28 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T19:58:28.399 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:28 smithi135 ceph-mon[221113]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T19:58:28.408 INFO:teuthology.orchestra.run.smithi135.stderr:marked out osd.3. 2023-09-23T19:58:28.537 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:28.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:28.641 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:28.665 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:28 smithi144 ceph-mon[194833]: pgmap v475: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s; 5/297732 objects degraded (0.002%) 2023-09-23T19:58:28.665 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1865346907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T19:58:28.666 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:28 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T19:58:28.666 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:28 smithi144 ceph-mon[194833]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T19:58:28.668 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:28.709 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:28.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:28.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:28.817 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:28.843 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:28.883 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:28.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:28.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:28.917 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 90 16 102653 102637 71.2275 17.625 0.0149576 0.0139521 2023-09-23T19:58:28.990 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:29.017 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:29.058 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:29.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:29.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:29.128 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:29.165 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:29.194 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:29.322 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:29.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:29.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:29.427 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:29.452 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:29.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[213871]: failed to encode map e178 with expected crc 2023-09-23T19:58:29.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T19:58:29.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[213871]: osdmap e178: 8 total, 7 up, 7 in 2023-09-23T19:58:29.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[213871]: failed to encode map e178 with expected crc 2023-09-23T19:58:29.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[213871]: osdmap e179: 8 total, 7 up, 7 in 2023-09-23T19:58:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[221113]: failed to encode map e178 with expected crc 2023-09-23T19:58:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T19:58:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[221113]: osdmap e178: 8 total, 7 up, 7 in 2023-09-23T19:58:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[221113]: failed to encode map e178 with expected crc 2023-09-23T19:58:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:29 smithi135 ceph-mon[221113]: osdmap e179: 8 total, 7 up, 7 in 2023-09-23T19:58:29.493 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:29.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:29.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:29.599 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:29.624 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:29.664 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:29.670 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:29 smithi144 ceph-mon[194833]: failed to encode map e178 with expected crc 2023-09-23T19:58:29.670 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:29 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T19:58:29.670 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:29 smithi144 ceph-mon[194833]: osdmap e178: 8 total, 7 up, 7 in 2023-09-23T19:58:29.671 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:29 smithi144 ceph-mon[194833]: failed to encode map e178 with expected crc 2023-09-23T19:58:29.671 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:29 smithi144 ceph-mon[194833]: osdmap e179: 8 total, 7 up, 7 in 2023-09-23T19:58:29.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:29.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:29.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:29.803 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:29.826 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:29.850 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:29.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:29.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:29.917 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 91 16 102653 102637 70.4453 0 - 0.0139521 2023-09-23T19:58:29.953 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:29.980 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:30.022 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:30.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:30.127 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:30.152 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:30.193 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:30.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:30.299 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:30.327 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:30.459 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:58:30.619 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:30.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: pgmap v477: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 5/305397 objects degraded (0.002%) 2023-09-23T19:58:30.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3182922422' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:30.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:30.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3182922422' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:58:30.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[221113]: osdmap e180: 8 total, 7 up, 7 in 2023-09-23T19:58:30.623 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: pgmap v477: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 5/305397 objects degraded (0.002%) 2023-09-23T19:58:30.623 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.623 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3182922422' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:30.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:30.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3182922422' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:58:30.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:30 smithi135 ceph-mon[213871]: osdmap e180: 8 total, 7 up, 7 in 2023-09-23T19:58:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:30.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:30.726 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:30.755 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: pgmap v477: 99 pgs: 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 5/305397 objects degraded (0.002%) 2023-09-23T19:58:30.755 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: failed to encode map e179 with expected crc 2023-09-23T19:58:30.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3182922422' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:58:30.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:30.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3182922422' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:58:30.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:30 smithi144 ceph-mon[194833]: osdmap e180: 8 total, 7 up, 7 in 2023-09-23T19:58:30.758 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:30.806 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:30.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:30.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:30.912 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:30.917 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total time run: 91.8819 2023-09-23T19:58:30.917 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total writes made: 102653 2023-09-23T19:58:30.917 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Write size: 65536 2023-09-23T19:58:30.917 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object size: 65536 2023-09-23T19:58:30.917 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Bandwidth (MB/sec): 69.8268 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Bandwidth: 35.1012 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max bandwidth (MB/sec): 201.125 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min bandwidth (MB/sec): 0 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average IOPS: 1117 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev IOPS: 561.618 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max IOPS: 3218 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min IOPS: 0 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average Latency(s): 0.0143015 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Latency(s): 0.0321493 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max latency(s): 2.35054 2023-09-23T19:58:30.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min latency(s): 0.001133 2023-09-23T19:58:30.925 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:58:30.949 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:31.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 177'6415 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: 6.b starting backfill to osd.4 from (0'0,0'0] MAX to 177'6213 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 177'6365 2023-09-23T19:58:31.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[213871]: osdmap e181: 8 total, 7 up, 7 in 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 177'6415 2023-09-23T19:58:31.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: 6.b starting backfill to osd.4 from (0'0,0'0] MAX to 177'6213 2023-09-23T19:58:31.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 177'6365 2023-09-23T19:58:31.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:31 smithi135 ceph-mon[221113]: osdmap e181: 8 total, 7 up, 7 in 2023-09-23T19:58:31.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 177'6415 2023-09-23T19:58:31.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: failed to encode map e180 with expected crc 2023-09-23T19:58:31.885 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: 6.b starting backfill to osd.4 from (0'0,0'0] MAX to 177'6213 2023-09-23T19:58:31.885 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 177'6365 2023-09-23T19:58:31.885 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:31 smithi144 ceph-mon[194833]: osdmap e181: 8 total, 7 up, 7 in 2023-09-23T19:58:32.493 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:32.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:32.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:32.611 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:32.655 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:32.698 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:32.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:32 smithi144 ceph-mon[194833]: pgmap v480: 99 pgs: 3 remapped+peering, 2 active+undersized+degraded, 4 active+undersized, 90 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 5/306501 objects degraded (0.002%) 2023-09-23T19:58:32.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:32 smithi144 ceph-mon[194833]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:32 smithi144 ceph-mon[194833]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:32 smithi144 ceph-mon[194833]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:32 smithi144 ceph-mon[194833]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.706 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/306501 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:32.706 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:32 smithi144 ceph-mon[194833]: osdmap e182: 8 total, 7 up, 7 in 2023-09-23T19:58:32.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:32.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[221113]: pgmap v480: 99 pgs: 3 remapped+peering, 2 active+undersized+degraded, 4 active+undersized, 90 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 5/306501 objects degraded (0.002%) 2023-09-23T19:58:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[221113]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[221113]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[221113]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[221113]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/306501 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:32.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[221113]: osdmap e182: 8 total, 7 up, 7 in 2023-09-23T19:58:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[213871]: pgmap v480: 99 pgs: 3 remapped+peering, 2 active+undersized+degraded, 4 active+undersized, 90 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 5/306501 objects degraded (0.002%) 2023-09-23T19:58:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[213871]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[213871]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[213871]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[213871]: failed to encode map e181 with expected crc 2023-09-23T19:58:32.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/306501 objects degraded (0.002%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:32.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:32 smithi135 ceph-mon[213871]: osdmap e182: 8 total, 7 up, 7 in 2023-09-23T19:58:32.808 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:32.860 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:32.992 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:33.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:33.108 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:33.176 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:33.277 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:33.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:33.385 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:33.464 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:33.615 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:58:33.658 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:58:33.688 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:33.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:33.706 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:58:33.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[221113]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[221113]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[221113]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[221113]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[221113]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.724 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:58:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:58:33] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T19:58:33.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[213871]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[213871]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[213871]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[213871]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:33 smithi135 ceph-mon[213871]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.751 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:58:33.795 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:33.796 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:58:33.828 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:58:33.858 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:33 smithi144 ceph-mon[194833]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.859 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:33 smithi144 ceph-mon[194833]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.859 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:33 smithi144 ceph-mon[194833]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.859 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:33 smithi144 ceph-mon[194833]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.859 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:33 smithi144 ceph-mon[194833]: failed to encode map e182 with expected crc 2023-09-23T19:58:33.862 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:33.879 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:58:33.879 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:58:33.880 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:58:33.880 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:58:33.880 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:58:33.880 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:58:33.880 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:58:33.880 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:58:33.880 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:58:33.948 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:33.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:34.063 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:34.064 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:58:34.064 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T19:58:34.064 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T19:58:34.065 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T19:58:34.065 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:58:34.118 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:34.197 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:34.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:34.287 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:58:34.311 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:34.363 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:34.409 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:58:34.409 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:58:34.410 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-09-23T19:58:34.410 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:58:34.450 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:34.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:34.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:34.558 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:34.624 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:34.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:34 smithi135 ceph-mon[213871]: pgmap v483: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 138 op/s; 6377/306759 objects degraded (2.079%); 0 B/s, 0 objects/s recovering 2023-09-23T19:58:34.628 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:34 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3346177400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:34.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:34 smithi135 ceph-mon[221113]: pgmap v483: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 138 op/s; 6377/306759 objects degraded (2.079%); 0 B/s, 0 objects/s recovering 2023-09-23T19:58:34.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:34 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3346177400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:34 smithi144 ceph-mon[194833]: pgmap v483: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 138 op/s; 6377/306759 objects degraded (2.079%); 0 B/s, 0 objects/s recovering 2023-09-23T19:58:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:34 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3346177400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:35 smithi135 ceph-mon[221113]: pgmap v484: 99 pgs: 1 active+clean+remapped, 79 active+clean, 4 active+undersized, 1 peering, 3 remapped+peering, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 KiB/s rd, 11 MiB/s wr, 414 op/s; 32238/304713 objects degraded (10.580%); 24890/304713 objects misplaced (8.168%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T19:58:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:35 smithi135 ceph-mon[213871]: pgmap v484: 99 pgs: 1 active+clean+remapped, 79 active+clean, 4 active+undersized, 1 peering, 3 remapped+peering, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 KiB/s rd, 11 MiB/s wr, 414 op/s; 32238/304713 objects degraded (10.580%); 24890/304713 objects misplaced (8.168%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T19:58:35.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:35 smithi144 ceph-mon[194833]: pgmap v484: 99 pgs: 1 active+clean+remapped, 79 active+clean, 4 active+undersized, 1 peering, 3 remapped+peering, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 KiB/s rd, 11 MiB/s wr, 414 op/s; 32238/304713 objects degraded (10.580%); 24890/304713 objects misplaced (8.168%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T19:58:35.924 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:58:35.925 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:58:35.961 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:35.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:35.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:35.997 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:58:36.073 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:36.133 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:36.179 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:36.213 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:36.372 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:36.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:36.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:36.485 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:36.548 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:36.646 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:36.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:36.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:36.756 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:36.825 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:37.342 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:37.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:37.661 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:37 smithi135 ceph-mon[221113]: pgmap v485: 99 pgs: 2 active+clean+remapped, 80 active+clean, 4 active+undersized, 1 peering, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 334 op/s; 32277/303834 objects degraded (10.623%); 31287/303834 objects misplaced (10.297%); 3.0 MiB/s, 49 objects/s recovering 2023-09-23T19:58:37.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:37.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:37 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32277/303834 objects degraded (10.623%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:37.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:58:37.662 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:37 smithi135 ceph-mon[213871]: pgmap v485: 99 pgs: 2 active+clean+remapped, 80 active+clean, 4 active+undersized, 1 peering, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 334 op/s; 32277/303834 objects degraded (10.623%); 31287/303834 objects misplaced (10.297%); 3.0 MiB/s, 49 objects/s recovering 2023-09-23T19:58:37.662 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:37.662 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:37 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32277/303834 objects degraded (10.623%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:37.662 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:58:37.762 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:37.838 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:37.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:37 smithi144 ceph-mon[194833]: pgmap v485: 99 pgs: 2 active+clean+remapped, 80 active+clean, 4 active+undersized, 1 peering, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 334 op/s; 32277/303834 objects degraded (10.623%); 31287/303834 objects misplaced (10.297%); 3.0 MiB/s, 49 objects/s recovering 2023-09-23T19:58:37.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:37.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:37 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32277/303834 objects degraded (10.623%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:37.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:58:37.926 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:58:39.308 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:39.348 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:39.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:39.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:39.460 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:39.533 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:39.681 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:39.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:39.689 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:39.789 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:39.859 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:39 smithi135 ceph-mon[213871]: pgmap v486: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 4.1 MiB/s wr, 569 op/s; 32295/296313 objects degraded (10.899%); 30591/296313 objects misplaced (10.324%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T19:58:39.859 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:39 smithi135 ceph-mon[221113]: pgmap v486: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 4.1 MiB/s wr, 569 op/s; 32295/296313 objects degraded (10.899%); 30591/296313 objects misplaced (10.324%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T19:58:39.871 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:39.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:39 smithi144 ceph-mon[194833]: pgmap v486: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 4.1 MiB/s wr, 569 op/s; 32295/296313 objects degraded (10.899%); 30591/296313 objects misplaced (10.324%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T19:58:40.247 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:40.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:40.361 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:40.466 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:40.756 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:40.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:40.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:40.792 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:58:40.795 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T19:58:40.862 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:40.913 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:41.088 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:41.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:41.192 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:41.253 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:41.633 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:41 smithi135 ceph-mon[221113]: pgmap v487: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 536 op/s; 32350/295290 objects degraded (10.955%); 30409/295290 objects misplaced (10.298%); 3.8 MiB/s, 62 objects/s recovering 2023-09-23T19:58:41.634 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1736494520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:41.634 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:41 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:41.634 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:41 smithi135 ceph-mon[221113]: from='client.35151 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:41.635 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:58:41.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:41 smithi135 ceph-mon[213871]: pgmap v487: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 536 op/s; 32350/295290 objects degraded (10.955%); 30409/295290 objects misplaced (10.298%); 3.8 MiB/s, 62 objects/s recovering 2023-09-23T19:58:41.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1736494520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:41 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:41 smithi135 ceph-mon[213871]: from='client.35151 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:41 smithi144 ceph-mon[194833]: pgmap v487: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 536 op/s; 32350/295290 objects degraded (10.955%); 30409/295290 objects misplaced (10.298%); 3.8 MiB/s, 62 objects/s recovering 2023-09-23T19:58:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1736494520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:41 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:58:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:41 smithi144 ceph-mon[194833]: from='client.35151 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:58:42.350 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:42.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:42.456 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:42.532 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:42.640 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:58:42.640 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[213871]: failed to encode map e183 with expected crc 2023-09-23T19:58:42.640 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:42.640 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[213871]: osdmap e183: 8 total, 7 up, 7 in 2023-09-23T19:58:42.640 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32350/295290 objects degraded (10.955%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:42.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:58:42.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[221113]: failed to encode map e183 with expected crc 2023-09-23T19:58:42.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:42.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[221113]: osdmap e183: 8 total, 7 up, 7 in 2023-09-23T19:58:42.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32350/295290 objects degraded (10.955%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:43.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:42 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:58:43.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:42 smithi144 ceph-mon[194833]: failed to encode map e183 with expected crc 2023-09-23T19:58:43.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:42 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:58:43.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:42 smithi144 ceph-mon[194833]: osdmap e183: 8 total, 7 up, 7 in 2023-09-23T19:58:43.137 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32350/295290 objects degraded (10.955%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:43.797 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:43.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:43.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:43.912 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:43.956 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:58:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:58:43] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-09-23T19:58:43.962 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:44.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:44 smithi135 ceph-mon[213871]: pgmap v489: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 17 KiB/s rd, 2.4 MiB/s wr, 507 op/s; 32350/294198 objects degraded (10.996%); 30409/294198 objects misplaced (10.336%); 3.5 MiB/s, 57 objects/s recovering 2023-09-23T19:58:44.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:44.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:44.624 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:44 smithi135 ceph-mon[221113]: pgmap v489: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 17 KiB/s rd, 2.4 MiB/s wr, 507 op/s; 32350/294198 objects degraded (10.996%); 30409/294198 objects misplaced (10.336%); 3.5 MiB/s, 57 objects/s recovering 2023-09-23T19:58:44.624 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:44.624 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:44 smithi144 ceph-mon[194833]: pgmap v489: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 17 KiB/s rd, 2.4 MiB/s wr, 507 op/s; 32350/294198 objects degraded (10.996%); 30409/294198 objects misplaced (10.336%); 3.5 MiB/s, 57 objects/s recovering 2023-09-23T19:58:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:44.983 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:44.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:44.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:45.087 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:45.153 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:45.285 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":487,"stamp":"2023-09-23T19:58:40.510160+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6453443351,"num_objects":98430,"num_object_clones":0,"num_object_copies":295290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32350,"num_objects_misplaced":30409,"num_objects_unfound":0,"num_objects_dirty":98430,"num_whiteouts":0,"num_read":6085,"num_read_kb":13380,"num_write":107856,"num_write_kb":6582958,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":600,"num_bytes_recovered":36568814,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102855,"ondisk_log_size":102855,"up":289,"acting":284,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":325,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17496532,"kb_used_data":16379228,"kb_used_omap":27,"kb_used_meta":1117028,"kb_avail":638719532,"statfs":{"total":671965249536,"available":654048800768,"internally_reserved":0,"allocated":16772329472,"data_stored":16767775756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28263,"internal_metadata":1143837081},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,10,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-245128348,"num_objects":-3737,"num_object_clones":0,"num_object_copies":-11211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32345,"num_objects_misplaced":30409,"num_objects_unfound":0,"num_objects_dirty":-3737,"num_whiteouts":0,"num_read":90,"num_read_kb":168,"num_write":4810,"num_write_kb":33337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":573,"num_bytes_recovered":36564061,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.125072"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":160,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.406725+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T19:58:31.406725+0000","last_peered":"2023-09-23T19:58:31.406725+0000","last_clean":"2023-09-23T19:58:31.406725+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T19:58:31.406725+0000","last_undegraded":"2023-09-23T19:58:31.406725+0000","last_fullsized":"2023-09-23T19:58:31.406725+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.403928+0000","last_change":"2023-09-23T19:58:29.389543+0000","last_active":"2023-09-23T19:58:32.403928+0000","last_peered":"2023-09-23T19:58:32.403928+0000","last_clean":"2023-09-23T19:58:32.403928+0000","last_became_active":"2023-09-23T19:58:29.389229+0000","last_became_peered":"2023-09-23T19:58:29.389229+0000","last_unstale":"2023-09-23T19:58:32.403928+0000","last_undegraded":"2023-09-23T19:58:32.403928+0000","last_fullsized":"2023-09-23T19:58:32.403928+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:49.510035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":130,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:58:31.483939+0000","last_change":"2023-09-23T19:58:30.390820+0000","last_active":"2023-09-23T19:58:31.483939+0000","last_peered":"2023-09-23T19:58:31.483939+0000","last_clean":"2023-09-23T19:58:31.483939+0000","last_became_active":"2023-09-23T19:58:30.390391+0000","last_became_peered":"2023-09-23T19:58:30.390391+0000","last_unstale":"2023-09-23T19:58:31.483939+0000","last_undegraded":"2023-09-23T19:58:31.483939+0000","last_fullsized":"2023-09-23T19:58:31.483939+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:14.854239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":137,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.406013+0000","last_change":"2023-09-23T19:58:29.422354+0000","last_active":"2023-09-23T19:58:32.406013+0000","last_peered":"2023-09-23T19:58:32.406013+0000","last_clean":"2023-09-23T19:58:32.406013+0000","last_became_active":"2023-09-23T19:58:29.422092+0000","last_became_peered":"2023-09-23T19:58:29.422092+0000","last_unstale":"2023-09-23T19:58:32.406013+0000","last_undegraded":"2023-09-23T19:58:32.406013+0000","last_fullsized":"2023-09-23T19:58:32.406013+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:14.228245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":96,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.670881+0000","last_change":"2023-09-23T19:58:29.403634+0000","last_active":"2023-09-23T19:58:31.670881+0000","last_peered":"2023-09-23T19:58:31.670881+0000","last_clean":"2023-09-23T19:58:31.670881+0000","last_became_active":"2023-09-23T19:58:29.403300+0000","last_became_peered":"2023-09-23T19:58:29.403300+0000","last_unstale":"2023-09-23T19:58:31.670881+0000","last_undegraded":"2023-09-23T19:58:31.670881+0000","last_fullsized":"2023-09-23T19:58:31.670881+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:57.067695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.395747+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T19:58:31.395747+0000","last_peered":"2023-09-23T19:58:31.395747+0000","last_clean":"2023-09-23T19:58:31.395747+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T19:58:31.395747+0000","last_undegraded":"2023-09-23T19:58:31.395747+0000","last_fullsized":"2023-09-23T19:58:31.395747+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:07.049200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":135,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.478675+0000","last_change":"2023-09-23T19:58:29.403726+0000","last_active":"2023-09-23T19:58:31.478675+0000","last_peered":"2023-09-23T19:58:31.478675+0000","last_clean":"2023-09-23T19:58:31.478675+0000","last_became_active":"2023-09-23T19:58:29.403144+0000","last_became_peered":"2023-09-23T19:58:29.403144+0000","last_unstale":"2023-09-23T19:58:31.478675+0000","last_undegraded":"2023-09-23T19:58:31.478675+0000","last_fullsized":"2023-09-23T19:58:31.478675+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:02.873885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.483623+0000","last_change":"2023-09-23T19:58:29.394940+0000","last_active":"2023-09-23T19:58:31.483623+0000","last_peered":"2023-09-23T19:58:31.483623+0000","last_clean":"2023-09-23T19:58:31.483623+0000","last_became_active":"2023-09-23T19:58:29.394626+0000","last_became_peered":"2023-09-23T19:58:29.394626+0000","last_unstale":"2023-09-23T19:58:31.483623+0000","last_undegraded":"2023-09-23T19:58:31.483623+0000","last_fullsized":"2023-09-23T19:58:31.483623+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:40.655915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":141,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.390895+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T19:58:31.390895+0000","last_peered":"2023-09-23T19:58:31.390895+0000","last_clean":"2023-09-23T19:58:31.390895+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T19:58:31.390895+0000","last_undegraded":"2023-09-23T19:58:31.390895+0000","last_fullsized":"2023-09-23T19:58:31.390895+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":94,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.402786+0000","last_change":"2023-09-23T19:58:29.403678+0000","last_active":"2023-09-23T19:58:31.402786+0000","last_peered":"2023-09-23T19:58:31.402786+0000","last_clean":"2023-09-23T19:58:31.402786+0000","last_became_active":"2023-09-23T19:58:29.403397+0000","last_became_peered":"2023-09-23T19:58:29.403397+0000","last_unstale":"2023-09-23T19:58:31.402786+0000","last_undegraded":"2023-09-23T19:58:31.402786+0000","last_fullsized":"2023-09-23T19:58:31.402786+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:35.267242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":161,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.412950+0000","last_change":"2023-09-23T19:58:29.414712+0000","last_active":"2023-09-23T19:58:31.412950+0000","last_peered":"2023-09-23T19:58:31.412950+0000","last_clean":"2023-09-23T19:58:31.412950+0000","last_became_active":"2023-09-23T19:58:29.403578+0000","last_became_peered":"2023-09-23T19:58:29.403578+0000","last_unstale":"2023-09-23T19:58:31.412950+0000","last_undegraded":"2023-09-23T19:58:31.412950+0000","last_fullsized":"2023-09-23T19:58:31.412950+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:46.132992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":136,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.482376+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T19:58:31.482376+0000","last_peered":"2023-09-23T19:58:31.482376+0000","last_clean":"2023-09-23T19:58:31.482376+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T19:58:31.482376+0000","last_undegraded":"2023-09-23T19:58:31.482376+0000","last_fullsized":"2023-09-23T19:58:31.482376+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:52.846135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":174,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.397584+0000","last_change":"2023-09-23T19:56:39.292849+0000","last_active":"2023-09-23T19:58:31.397584+0000","last_peered":"2023-09-23T19:58:31.397584+0000","last_clean":"2023-09-23T19:58:31.397584+0000","last_became_active":"2023-09-23T19:56:39.284622+0000","last_became_peered":"2023-09-23T19:56:39.284622+0000","last_unstale":"2023-09-23T19:58:31.397584+0000","last_undegraded":"2023-09-23T19:58:31.397584+0000","last_fullsized":"2023-09-23T19:58:31.397584+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:24.088523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":148,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T19:58:31.400377+0000","last_change":"2023-09-23T19:54:45.327308+0000","last_active":"2023-09-23T19:58:31.400377+0000","last_peered":"2023-09-23T19:58:31.400377+0000","last_clean":"2023-09-23T19:54:37.307838+0000","last_became_active":"2023-09-23T19:54:45.327308+0000","last_became_peered":"2023-09-23T19:54:45.327308+0000","last_unstale":"2023-09-23T19:58:31.400377+0000","last_undegraded":"2023-09-23T19:58:31.400377+0000","last_fullsized":"2023-09-23T19:54:45.254218+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:34.491557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":144,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.404502+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T19:58:31.404502+0000","last_peered":"2023-09-23T19:58:31.404502+0000","last_clean":"2023-09-23T19:58:31.404502+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T19:58:31.404502+0000","last_undegraded":"2023-09-23T19:58:31.404502+0000","last_fullsized":"2023-09-23T19:58:31.404502+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.487387+0000","last_change":"2023-09-23T19:58:29.414727+0000","last_active":"2023-09-23T19:58:31.487387+0000","last_peered":"2023-09-23T19:58:31.487387+0000","last_clean":"2023-09-23T19:58:31.487387+0000","last_became_active":"2023-09-23T19:58:29.414027+0000","last_became_peered":"2023-09-23T19:58:29.414027+0000","last_unstale":"2023-09-23T19:58:31.487387+0000","last_undegraded":"2023-09-23T19:58:31.487387+0000","last_fullsized":"2023-09-23T19:58:31.487387+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:46.779525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":135,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.391432+0000","last_change":"2023-09-23T19:58:29.420974+0000","last_active":"2023-09-23T19:58:31.391432+0000","last_peered":"2023-09-23T19:58:31.391432+0000","last_clean":"2023-09-23T19:58:31.391432+0000","last_became_active":"2023-09-23T19:58:29.420551+0000","last_became_peered":"2023-09-23T19:58:29.420551+0000","last_unstale":"2023-09-23T19:58:31.391432+0000","last_undegraded":"2023-09-23T19:58:31.391432+0000","last_fullsized":"2023-09-23T19:58:31.391432+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:06.781967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":150,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.392206+0000","last_change":"2023-09-23T19:58:29.404717+0000","last_active":"2023-09-23T19:58:31.392206+0000","last_peered":"2023-09-23T19:58:31.392206+0000","last_clean":"2023-09-23T19:58:31.392206+0000","last_became_active":"2023-09-23T19:58:29.404363+0000","last_became_peered":"2023-09-23T19:58:29.404363+0000","last_unstale":"2023-09-23T19:58:31.392206+0000","last_undegraded":"2023-09-23T19:58:31.392206+0000","last_fullsized":"2023-09-23T19:58:31.392206+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:17.171647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":132,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T19:58:31.402341+0000","last_change":"2023-09-23T19:54:45.330750+0000","last_active":"2023-09-23T19:58:31.402341+0000","last_peered":"2023-09-23T19:58:31.402341+0000","last_clean":"2023-09-23T19:54:37.324687+0000","last_became_active":"2023-09-23T19:54:45.330750+0000","last_became_peered":"2023-09-23T19:54:45.330750+0000","last_unstale":"2023-09-23T19:58:31.402341+0000","last_undegraded":"2023-09-23T19:58:31.402341+0000","last_fullsized":"2023-09-23T19:54:45.253851+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:18.890717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":136,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.479159+0000","last_change":"2023-09-23T19:58:29.409570+0000","last_active":"2023-09-23T19:58:31.479159+0000","last_peered":"2023-09-23T19:58:31.479159+0000","last_clean":"2023-09-23T19:58:31.479159+0000","last_became_active":"2023-09-23T19:58:29.409301+0000","last_became_peered":"2023-09-23T19:58:29.409301+0000","last_unstale":"2023-09-23T19:58:31.479159+0000","last_undegraded":"2023-09-23T19:58:31.479159+0000","last_fullsized":"2023-09-23T19:58:31.479159+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:37.913005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":58,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:58:31.485147+0000","last_change":"2023-09-23T19:54:45.331318+0000","last_active":"2023-09-23T19:58:31.485147+0000","last_peered":"2023-09-23T19:58:31.485147+0000","last_clean":"2023-09-23T19:41:41.886282+0000","last_became_active":"2023-09-23T19:54:45.331318+0000","last_became_peered":"2023-09-23T19:54:45.331318+0000","last_unstale":"2023-09-23T19:58:31.485147+0000","last_undegraded":"2023-09-23T19:54:45.255936+0000","last_fullsized":"2023-09-23T19:54:45.255813+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:16.760756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":79,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.409223+0000","last_change":"2023-09-23T19:58:29.403391+0000","last_active":"2023-09-23T19:58:31.409223+0000","last_peered":"2023-09-23T19:58:31.409223+0000","last_clean":"2023-09-23T19:58:31.409223+0000","last_became_active":"2023-09-23T19:58:29.402908+0000","last_became_peered":"2023-09-23T19:58:29.402908+0000","last_unstale":"2023-09-23T19:58:31.409223+0000","last_undegraded":"2023-09-23T19:58:31.409223+0000","last_fullsized":"2023-09-23T19:58:31.409223+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:27.502454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"182'6687","reported_seq":6713,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:38.188006+0000","last_change":"2023-09-23T19:56:51.336830+0000","last_active":"2023-09-23T19:58:38.188006+0000","last_peered":"2023-09-23T19:58:38.188006+0000","last_clean":"2023-09-23T19:58:38.188006+0000","last_became_active":"2023-09-23T19:56:51.336645+0000","last_became_peered":"2023-09-23T19:56:51.336645+0000","last_unstale":"2023-09-23T19:58:38.188006+0000","last_undegraded":"2023-09-23T19:58:38.188006+0000","last_fullsized":"2023-09-23T19:58:38.188006+0000","mapping_epoch":163,"log_start":"167'100","ondisk_log_start":"167'100","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:27.681999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400752640,"num_objects":6115,"num_object_clones":0,"num_object_copies":18345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6687,"num_write_kb":409664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.409320+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T19:58:31.409320+0000","last_peered":"2023-09-23T19:58:31.409320+0000","last_clean":"2023-09-23T19:58:31.409320+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T19:58:31.409320+0000","last_undegraded":"2023-09-23T19:58:31.409320+0000","last_fullsized":"2023-09-23T19:58:31.409320+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":140,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.390557+0000","last_change":"2023-09-23T19:56:39.275614+0000","last_active":"2023-09-23T19:58:31.390557+0000","last_peered":"2023-09-23T19:58:31.390557+0000","last_clean":"2023-09-23T19:58:31.390557+0000","last_became_active":"2023-09-23T19:56:39.275242+0000","last_became_peered":"2023-09-23T19:56:39.275242+0000","last_unstale":"2023-09-23T19:58:31.390557+0000","last_undegraded":"2023-09-23T19:58:31.390557+0000","last_fullsized":"2023-09-23T19:58:31.390557+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:55.343087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"182'6474","reported_seq":6584,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T19:58:38.210338+0000","last_change":"2023-09-23T19:58:30.699539+0000","last_active":"2023-09-23T19:58:38.210338+0000","last_peered":"2023-09-23T19:58:38.210338+0000","last_clean":"2023-09-23T19:58:28.328264+0000","last_became_active":"2023-09-23T19:58:30.570887+0000","last_became_peered":"2023-09-23T19:58:30.570887+0000","last_unstale":"2023-09-23T19:58:38.210338+0000","last_undegraded":"2023-09-23T19:58:38.210338+0000","last_fullsized":"2023-09-23T19:58:38.210338+0000","mapping_epoch":179,"log_start":"167'200","ondisk_log_start":"167'200","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6274,"log_dups_size":0,"ondisk_log_size":6274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:44.128962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390070272,"num_objects":5952,"num_object_clones":0,"num_object_copies":17856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5922,"num_objects_unfound":0,"num_objects_dirty":5952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6474,"num_write_kb":397632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":2162688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":5952}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":144,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.409409+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T19:58:31.409409+0000","last_peered":"2023-09-23T19:58:31.409409+0000","last_clean":"2023-09-23T19:58:31.409409+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T19:58:31.409409+0000","last_undegraded":"2023-09-23T19:58:31.409409+0000","last_fullsized":"2023-09-23T19:58:31.409409+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":168,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.397416+0000","last_change":"2023-09-23T19:58:29.403263+0000","last_active":"2023-09-23T19:58:31.397416+0000","last_peered":"2023-09-23T19:58:31.397416+0000","last_clean":"2023-09-23T19:58:31.397416+0000","last_became_active":"2023-09-23T19:58:29.402946+0000","last_became_peered":"2023-09-23T19:58:29.402946+0000","last_unstale":"2023-09-23T19:58:31.397416+0000","last_undegraded":"2023-09-23T19:58:31.397416+0000","last_fullsized":"2023-09-23T19:58:31.397416+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:09.811221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"182'6745","reported_seq":6779,"reported_epoch":182,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T19:58:38.214178+0000","last_change":"2023-09-23T19:58:30.716785+0000","last_active":"2023-09-23T19:58:38.214178+0000","last_peered":"2023-09-23T19:58:38.214178+0000","last_clean":"2023-09-23T19:58:28.376184+0000","last_became_active":"2023-09-23T19:58:30.697357+0000","last_became_peered":"2023-09-23T19:58:30.697357+0000","last_unstale":"2023-09-23T19:58:38.214178+0000","last_undegraded":"2023-09-23T19:58:30.470004+0000","last_fullsized":"2023-09-23T19:58:30.383763+0000","mapping_epoch":179,"log_start":"167'438","ondisk_log_start":"167'438","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.710390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408485888,"num_objects":6233,"num_object_clones":0,"num_object_copies":18699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6488,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6745,"num_write_kb":415296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":6231},{"shards":"2,7","objects":-255}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":133,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:58:31.412827+0000","last_change":"2023-09-23T19:54:45.328405+0000","last_active":"2023-09-23T19:58:31.412827+0000","last_peered":"2023-09-23T19:58:31.412827+0000","last_clean":"2023-09-23T19:54:37.326665+0000","last_became_active":"2023-09-23T19:54:45.328405+0000","last_became_peered":"2023-09-23T19:54:45.328405+0000","last_unstale":"2023-09-23T19:58:31.412827+0000","last_undegraded":"2023-09-23T19:54:45.254169+0000","last_fullsized":"2023-09-23T19:54:45.254056+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:10.168687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":106,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.482500+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T19:58:31.482500+0000","last_peered":"2023-09-23T19:58:31.482500+0000","last_clean":"2023-09-23T19:58:31.482500+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T19:58:31.482500+0000","last_undegraded":"2023-09-23T19:58:31.482500+0000","last_fullsized":"2023-09-23T19:58:31.482500+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:30.170186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"182'6645","reported_seq":6673,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:36.649119+0000","last_change":"2023-09-23T19:56:51.330015+0000","last_active":"2023-09-23T19:58:36.649119+0000","last_peered":"2023-09-23T19:58:36.649119+0000","last_clean":"2023-09-23T19:58:36.649119+0000","last_became_active":"2023-09-23T19:56:51.329727+0000","last_became_peered":"2023-09-23T19:56:51.329727+0000","last_unstale":"2023-09-23T19:58:36.649119+0000","last_undegraded":"2023-09-23T19:58:36.649119+0000","last_fullsized":"2023-09-23T19:58:36.649119+0000","mapping_epoch":163,"log_start":"167'100","ondisk_log_start":"167'100","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6545,"log_dups_size":0,"ondisk_log_size":6545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:59.494181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408616960,"num_objects":6235,"num_object_clones":0,"num_object_copies":18705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6645,"num_write_kb":412160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.391122+0000","last_change":"2023-09-23T19:58:29.415320+0000","last_active":"2023-09-23T19:58:31.391122+0000","last_peered":"2023-09-23T19:58:31.391122+0000","last_clean":"2023-09-23T19:58:31.391122+0000","last_became_active":"2023-09-23T19:58:29.415087+0000","last_became_peered":"2023-09-23T19:58:29.415087+0000","last_unstale":"2023-09-23T19:58:31.391122+0000","last_undegraded":"2023-09-23T19:58:31.391122+0000","last_fullsized":"2023-09-23T19:58:31.391122+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:51.610733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.391522+0000","last_change":"2023-09-23T19:49:51.383516+0000","last_active":"2023-09-23T19:58:31.391522+0000","last_peered":"2023-09-23T19:58:31.391522+0000","last_clean":"2023-09-23T19:58:31.391522+0000","last_became_active":"2023-09-23T19:49:51.383064+0000","last_became_peered":"2023-09-23T19:49:51.383064+0000","last_unstale":"2023-09-23T19:58:31.391522+0000","last_undegraded":"2023-09-23T19:58:31.391522+0000","last_fullsized":"2023-09-23T19:58:31.391522+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:04.812057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"182'6698","reported_seq":6724,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:38.243447+0000","last_change":"2023-09-23T19:56:51.335737+0000","last_active":"2023-09-23T19:58:38.243447+0000","last_peered":"2023-09-23T19:58:38.243447+0000","last_clean":"2023-09-23T19:58:38.243447+0000","last_became_active":"2023-09-23T19:56:51.335399+0000","last_became_peered":"2023-09-23T19:56:51.335399+0000","last_unstale":"2023-09-23T19:58:38.243447+0000","last_undegraded":"2023-09-23T19:58:38.243447+0000","last_fullsized":"2023-09-23T19:58:38.243447+0000","mapping_epoch":163,"log_start":"167'741","ondisk_log_start":"167'741","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:20.933908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6698,"num_write_kb":411072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":582,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.427554+0000","last_change":"2023-09-23T19:58:32.427554+0000","last_active":"2023-09-23T19:58:32.427554+0000","last_peered":"2023-09-23T19:58:32.427554+0000","last_clean":"2023-09-23T19:58:32.427554+0000","last_became_active":"2023-09-23T19:58:32.427224+0000","last_became_peered":"2023-09-23T19:58:32.427224+0000","last_unstale":"2023-09-23T19:58:32.427554+0000","last_undegraded":"2023-09-23T19:58:32.427554+0000","last_fullsized":"2023-09-23T19:58:32.427554+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:02.329436+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":39,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.670955+0000","last_change":"2023-09-23T19:58:29.422325+0000","last_active":"2023-09-23T19:58:31.670955+0000","last_peered":"2023-09-23T19:58:31.670955+0000","last_clean":"2023-09-23T19:58:31.670955+0000","last_became_active":"2023-09-23T19:58:29.403874+0000","last_became_peered":"2023-09-23T19:58:29.403874+0000","last_unstale":"2023-09-23T19:58:31.670955+0000","last_undegraded":"2023-09-23T19:58:31.670955+0000","last_fullsized":"2023-09-23T19:58:31.670955+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:38.165127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":168,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.397448+0000","last_change":"2023-09-23T19:58:29.415446+0000","last_active":"2023-09-23T19:58:31.397448+0000","last_peered":"2023-09-23T19:58:31.397448+0000","last_clean":"2023-09-23T19:58:31.397448+0000","last_became_active":"2023-09-23T19:58:29.415142+0000","last_became_peered":"2023-09-23T19:58:29.415142+0000","last_unstale":"2023-09-23T19:58:31.397448+0000","last_undegraded":"2023-09-23T19:58:31.397448+0000","last_fullsized":"2023-09-23T19:58:31.397448+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:27.085562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"182'6816","reported_seq":6851,"reported_epoch":182,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T19:58:37.878141+0000","last_change":"2023-09-23T19:58:30.617425+0000","last_active":"2023-09-23T19:58:37.878141+0000","last_peered":"2023-09-23T19:58:37.878141+0000","last_clean":"2023-09-23T19:58:28.376381+0000","last_became_active":"2023-09-23T19:58:30.604563+0000","last_became_peered":"2023-09-23T19:58:30.604563+0000","last_unstale":"2023-09-23T19:58:37.878141+0000","last_undegraded":"2023-09-23T19:58:30.419764+0000","last_fullsized":"2023-09-23T19:58:30.379500+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6816,"log_dups_size":0,"ondisk_log_size":6816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:11.864197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411435008,"num_objects":6278,"num_object_clones":0,"num_object_copies":18834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6547,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6816,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":6278},{"shards":"2,4","objects":-269}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":517,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.424610+0000","last_change":"2023-09-23T19:58:32.424610+0000","last_active":"2023-09-23T19:58:32.424610+0000","last_peered":"2023-09-23T19:58:32.424610+0000","last_clean":"2023-09-23T19:58:32.424610+0000","last_became_active":"2023-09-23T19:58:32.424237+0000","last_became_peered":"2023-09-23T19:58:32.424237+0000","last_unstale":"2023-09-23T19:58:32.424610+0000","last_undegraded":"2023-09-23T19:58:32.424610+0000","last_fullsized":"2023-09-23T19:58:32.424610+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:45.590103+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.406631+0000","last_change":"2023-09-23T19:58:29.394046+0000","last_active":"2023-09-23T19:58:31.406631+0000","last_peered":"2023-09-23T19:58:31.406631+0000","last_clean":"2023-09-23T19:58:31.406631+0000","last_became_active":"2023-09-23T19:58:29.393647+0000","last_became_peered":"2023-09-23T19:58:29.393647+0000","last_unstale":"2023-09-23T19:58:31.406631+0000","last_undegraded":"2023-09-23T19:58:31.406631+0000","last_fullsized":"2023-09-23T19:58:31.406631+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:15.264891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":158,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.397779+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T19:58:31.397779+0000","last_peered":"2023-09-23T19:58:31.397779+0000","last_clean":"2023-09-23T19:58:31.397779+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T19:58:31.397779+0000","last_undegraded":"2023-09-23T19:58:31.397779+0000","last_fullsized":"2023-09-23T19:58:31.397779+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"182'6624","reported_seq":6724,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T19:58:38.196219+0000","last_change":"2023-09-23T19:58:30.715882+0000","last_active":"2023-09-23T19:58:38.196219+0000","last_peered":"2023-09-23T19:58:38.196219+0000","last_clean":"2023-09-23T19:58:28.375382+0000","last_became_active":"2023-09-23T19:58:30.544381+0000","last_became_peered":"2023-09-23T19:58:30.544381+0000","last_unstale":"2023-09-23T19:58:38.196219+0000","last_undegraded":"2023-09-23T19:58:38.196219+0000","last_fullsized":"2023-09-23T19:58:38.196219+0000","mapping_epoch":179,"log_start":"167'300","ondisk_log_start":"167'300","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6324,"log_dups_size":0,"ondisk_log_size":6324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:21.652848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400162816,"num_objects":6106,"num_object_clones":0,"num_object_copies":18318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6079,"num_objects_unfound":0,"num_objects_dirty":6106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6624,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1835008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":6106}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":142,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.397160+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T19:58:31.397160+0000","last_peered":"2023-09-23T19:58:31.397160+0000","last_clean":"2023-09-23T19:58:31.397160+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T19:58:31.397160+0000","last_undegraded":"2023-09-23T19:58:31.397160+0000","last_fullsized":"2023-09-23T19:58:31.397160+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":178,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.397950+0000","last_change":"2023-09-23T19:58:29.396550+0000","last_active":"2023-09-23T19:58:31.397950+0000","last_peered":"2023-09-23T19:58:31.397950+0000","last_clean":"2023-09-23T19:58:31.397950+0000","last_became_active":"2023-09-23T19:58:29.396253+0000","last_became_peered":"2023-09-23T19:58:29.396253+0000","last_unstale":"2023-09-23T19:58:31.397950+0000","last_undegraded":"2023-09-23T19:58:31.397950+0000","last_fullsized":"2023-09-23T19:58:31.397950+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:21.284951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"182'6699","reported_seq":6726,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:38.282580+0000","last_change":"2023-09-23T19:56:51.339206+0000","last_active":"2023-09-23T19:58:38.282580+0000","last_peered":"2023-09-23T19:58:38.282580+0000","last_clean":"2023-09-23T19:58:38.282580+0000","last_became_active":"2023-09-23T19:56:51.338963+0000","last_became_peered":"2023-09-23T19:56:51.338963+0000","last_unstale":"2023-09-23T19:58:38.282580+0000","last_undegraded":"2023-09-23T19:58:38.282580+0000","last_fullsized":"2023-09-23T19:58:38.282580+0000","mapping_epoch":163,"log_start":"167'100","ondisk_log_start":"167'100","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6599,"log_dups_size":0,"ondisk_log_size":6599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:49.976592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403570712,"num_objects":6159,"num_object_clones":0,"num_object_copies":18477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6159,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6699,"num_write_kb":411393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":520,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.394041+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T19:58:31.394041+0000","last_peered":"2023-09-23T19:58:31.394041+0000","last_clean":"2023-09-23T19:58:31.394041+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T19:58:31.394041+0000","last_undegraded":"2023-09-23T19:58:31.394041+0000","last_fullsized":"2023-09-23T19:58:31.394041+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.320089+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.477909+0000","last_change":"2023-09-23T19:58:29.398381+0000","last_active":"2023-09-23T19:58:31.477909+0000","last_peered":"2023-09-23T19:58:31.477909+0000","last_clean":"2023-09-23T19:58:31.477909+0000","last_became_active":"2023-09-23T19:58:29.398058+0000","last_became_peered":"2023-09-23T19:58:29.398058+0000","last_unstale":"2023-09-23T19:58:31.477909+0000","last_undegraded":"2023-09-23T19:58:31.477909+0000","last_fullsized":"2023-09-23T19:58:31.477909+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:39.296975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":130,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.493652+0000","last_change":"2023-09-23T19:58:29.394715+0000","last_active":"2023-09-23T19:58:31.493652+0000","last_peered":"2023-09-23T19:58:31.493652+0000","last_clean":"2023-09-23T19:58:31.493652+0000","last_became_active":"2023-09-23T19:58:29.394414+0000","last_became_peered":"2023-09-23T19:58:29.394414+0000","last_unstale":"2023-09-23T19:58:31.493652+0000","last_undegraded":"2023-09-23T19:58:31.493652+0000","last_fullsized":"2023-09-23T19:58:31.493652+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:42.196222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"182'6562","reported_seq":6597,"reported_epoch":182,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T19:58:36.662053+0000","last_change":"2023-09-23T19:58:30.724466+0000","last_active":"2023-09-23T19:58:36.662053+0000","last_peered":"2023-09-23T19:58:36.662053+0000","last_clean":"2023-09-23T19:58:28.355860+0000","last_became_active":"2023-09-23T19:58:30.710321+0000","last_became_peered":"2023-09-23T19:58:30.710321+0000","last_unstale":"2023-09-23T19:58:36.662053+0000","last_undegraded":"2023-09-23T19:58:30.426464+0000","last_fullsized":"2023-09-23T19:58:30.388101+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:49.493451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405274624,"num_objects":6184,"num_object_clones":0,"num_object_copies":18552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6372,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6562,"num_write_kb":407872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2","objects":-188},{"shards":"0,2,3","objects":6183}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":495,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.427685+0000","last_change":"2023-09-23T19:58:32.427685+0000","last_active":"2023-09-23T19:58:32.427685+0000","last_peered":"2023-09-23T19:58:32.427685+0000","last_clean":"2023-09-23T19:58:32.427685+0000","last_became_active":"2023-09-23T19:58:32.427323+0000","last_became_peered":"2023-09-23T19:58:32.427323+0000","last_unstale":"2023-09-23T19:58:32.427685+0000","last_undegraded":"2023-09-23T19:58:32.427685+0000","last_fullsized":"2023-09-23T19:58:32.427685+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:56:53.618048+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T19:56:53.618048+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":518,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.487515+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T19:58:31.487515+0000","last_peered":"2023-09-23T19:58:31.487515+0000","last_clean":"2023-09-23T19:58:31.487515+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T19:58:31.487515+0000","last_undegraded":"2023-09-23T19:58:31.487515+0000","last_fullsized":"2023-09-23T19:58:31.487515+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":170,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:58:31.392098+0000","last_change":"2023-09-23T19:58:30.397761+0000","last_active":"2023-09-23T19:58:31.392098+0000","last_peered":"2023-09-23T19:58:31.392098+0000","last_clean":"2023-09-23T19:58:31.392098+0000","last_became_active":"2023-09-23T19:58:30.397194+0000","last_became_peered":"2023-09-23T19:58:30.397194+0000","last_unstale":"2023-09-23T19:58:31.392098+0000","last_undegraded":"2023-09-23T19:58:31.392098+0000","last_fullsized":"2023-09-23T19:58:31.392098+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:07.189303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2,3],"avail_no_missing":["5","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.484440+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T19:58:31.484440+0000","last_peered":"2023-09-23T19:58:31.484440+0000","last_clean":"2023-09-23T19:58:31.484440+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T19:58:31.484440+0000","last_undegraded":"2023-09-23T19:58:31.484440+0000","last_fullsized":"2023-09-23T19:58:31.484440+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"182'6701","reported_seq":6819,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T19:58:38.244624+0000","last_change":"2023-09-23T19:58:30.702039+0000","last_active":"2023-09-23T19:58:38.244624+0000","last_peered":"2023-09-23T19:58:38.244624+0000","last_clean":"2023-09-23T19:58:28.352680+0000","last_became_active":"2023-09-23T19:58:30.532221+0000","last_became_peered":"2023-09-23T19:58:30.532221+0000","last_unstale":"2023-09-23T19:58:38.244624+0000","last_undegraded":"2023-09-23T19:58:38.244624+0000","last_fullsized":"2023-09-23T19:58:38.244624+0000","mapping_epoch":179,"log_start":"167'733","ondisk_log_start":"167'733","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5968,"log_dups_size":0,"ondisk_log_size":5968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:18.501624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401932288,"num_objects":6133,"num_object_clones":0,"num_object_copies":18399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6097,"num_objects_unfound":0,"num_objects_dirty":6133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6701,"num_write_kb":410688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":6133}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":493,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.390319+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T19:58:31.390319+0000","last_peered":"2023-09-23T19:58:31.390319+0000","last_clean":"2023-09-23T19:58:31.390319+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T19:58:31.390319+0000","last_undegraded":"2023-09-23T19:58:31.390319+0000","last_fullsized":"2023-09-23T19:58:31.390319+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":742,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.393805+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T19:58:31.393805+0000","last_peered":"2023-09-23T19:58:31.393805+0000","last_clean":"2023-09-23T19:58:31.393805+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T19:58:31.393805+0000","last_undegraded":"2023-09-23T19:58:31.393805+0000","last_fullsized":"2023-09-23T19:58:31.393805+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:34.015520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":159,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.400481+0000","last_change":"2023-09-23T19:56:39.283954+0000","last_active":"2023-09-23T19:58:31.400481+0000","last_peered":"2023-09-23T19:58:31.400481+0000","last_clean":"2023-09-23T19:58:31.400481+0000","last_became_active":"2023-09-23T19:56:39.283711+0000","last_became_peered":"2023-09-23T19:56:39.283711+0000","last_unstale":"2023-09-23T19:58:31.400481+0000","last_undegraded":"2023-09-23T19:58:31.400481+0000","last_fullsized":"2023-09-23T19:58:31.400481+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:55.890028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":146,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.487450+0000","last_change":"2023-09-23T19:58:29.403057+0000","last_active":"2023-09-23T19:58:31.487450+0000","last_peered":"2023-09-23T19:58:31.487450+0000","last_clean":"2023-09-23T19:58:31.487450+0000","last_became_active":"2023-09-23T19:58:29.402799+0000","last_became_peered":"2023-09-23T19:58:29.402799+0000","last_unstale":"2023-09-23T19:58:31.487450+0000","last_undegraded":"2023-09-23T19:58:31.487450+0000","last_fullsized":"2023-09-23T19:58:31.487450+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:38.212007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"182'6646","reported_seq":6682,"reported_epoch":182,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T19:58:38.242058+0000","last_change":"2023-09-23T19:58:30.735309+0000","last_active":"2023-09-23T19:58:38.242058+0000","last_peered":"2023-09-23T19:58:38.242058+0000","last_clean":"2023-09-23T19:58:28.375338+0000","last_became_active":"2023-09-23T19:58:30.721191+0000","last_became_peered":"2023-09-23T19:58:30.721191+0000","last_unstale":"2023-09-23T19:58:38.242058+0000","last_undegraded":"2023-09-23T19:58:30.445805+0000","last_fullsized":"2023-09-23T19:58:30.383825+0000","mapping_epoch":179,"log_start":"167'325","ondisk_log_start":"167'325","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6321,"log_dups_size":0,"ondisk_log_size":6321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:49.165283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400031744,"num_objects":6104,"num_object_clones":0,"num_object_copies":18312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6375,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6646,"num_write_kb":408000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":6104},{"shards":"2,7","objects":-271}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":562,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.404094+0000","last_change":"2023-09-23T19:52:57.745972+0000","last_active":"2023-09-23T19:58:31.404094+0000","last_peered":"2023-09-23T19:58:31.404094+0000","last_clean":"2023-09-23T19:58:31.404094+0000","last_became_active":"2023-09-23T19:52:57.348142+0000","last_became_peered":"2023-09-23T19:52:57.348142+0000","last_unstale":"2023-09-23T19:58:31.404094+0000","last_undegraded":"2023-09-23T19:58:31.404094+0000","last_fullsized":"2023-09-23T19:58:31.404094+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.745894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.745894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.660195+0000","scrub_duration":0.0079262489999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":712,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.416867+0000","last_change":"2023-09-23T19:58:32.416867+0000","last_active":"2023-09-23T19:58:32.416867+0000","last_peered":"2023-09-23T19:58:32.416867+0000","last_clean":"2023-09-23T19:58:32.416867+0000","last_became_active":"2023-09-23T19:58:32.416359+0000","last_became_peered":"2023-09-23T19:58:32.416359+0000","last_unstale":"2023-09-23T19:58:32.416867+0000","last_undegraded":"2023-09-23T19:58:32.416867+0000","last_fullsized":"2023-09-23T19:58:32.416867+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:36.220378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.482161+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T19:58:31.482161+0000","last_peered":"2023-09-23T19:58:31.482161+0000","last_clean":"2023-09-23T19:58:31.482161+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T19:58:31.482161+0000","last_undegraded":"2023-09-23T19:58:31.482161+0000","last_fullsized":"2023-09-23T19:58:31.482161+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:46.217415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.493696+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T19:58:31.493696+0000","last_peered":"2023-09-23T19:58:31.493696+0000","last_clean":"2023-09-23T19:58:31.493696+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T19:58:31.493696+0000","last_undegraded":"2023-09-23T19:58:31.493696+0000","last_fullsized":"2023-09-23T19:58:31.493696+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"182'6678","reported_seq":6714,"reported_epoch":182,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T19:58:38.306086+0000","last_change":"2023-09-23T19:58:30.744519+0000","last_active":"2023-09-23T19:58:38.306086+0000","last_peered":"2023-09-23T19:58:38.306086+0000","last_clean":"2023-09-23T19:58:28.375952+0000","last_became_active":"2023-09-23T19:58:30.729142+0000","last_became_peered":"2023-09-23T19:58:30.729142+0000","last_unstale":"2023-09-23T19:58:38.306086+0000","last_undegraded":"2023-09-23T19:58:30.443350+0000","last_fullsized":"2023-09-23T19:58:30.386882+0000","mapping_epoch":179,"log_start":"167'100","ondisk_log_start":"167'100","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:51.795779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404488192,"num_objects":6172,"num_object_clones":0,"num_object_copies":18516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6425,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6678,"num_write_kb":411200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":6172},{"shards":"4,5","objects":-253}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":990,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.392535+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T19:58:31.392535+0000","last_peered":"2023-09-23T19:58:31.392535+0000","last_clean":"2023-09-23T19:58:31.392535+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T19:58:31.392535+0000","last_undegraded":"2023-09-23T19:58:31.392535+0000","last_fullsized":"2023-09-23T19:58:31.392535+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":570,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.402624+0000","last_change":"2023-09-23T19:56:42.408346+0000","last_active":"2023-09-23T19:58:31.402624+0000","last_peered":"2023-09-23T19:58:31.402624+0000","last_clean":"2023-09-23T19:58:31.402624+0000","last_became_active":"2023-09-23T19:56:42.407370+0000","last_became_peered":"2023-09-23T19:56:42.407370+0000","last_unstale":"2023-09-23T19:58:31.402624+0000","last_undegraded":"2023-09-23T19:58:31.402624+0000","last_fullsized":"2023-09-23T19:58:31.402624+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:38.144096+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":93,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.393956+0000","last_change":"2023-09-23T19:58:29.394816+0000","last_active":"2023-09-23T19:58:31.393956+0000","last_peered":"2023-09-23T19:58:31.393956+0000","last_clean":"2023-09-23T19:58:31.393956+0000","last_became_active":"2023-09-23T19:58:29.394560+0000","last_became_peered":"2023-09-23T19:58:29.394560+0000","last_unstale":"2023-09-23T19:58:31.393956+0000","last_undegraded":"2023-09-23T19:58:31.393956+0000","last_fullsized":"2023-09-23T19:58:31.393956+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:51.502725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.397948+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T19:58:31.397948+0000","last_peered":"2023-09-23T19:58:31.397948+0000","last_clean":"2023-09-23T19:58:31.397948+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T19:58:31.397948+0000","last_undegraded":"2023-09-23T19:58:31.397948+0000","last_fullsized":"2023-09-23T19:58:31.397948+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:31.068090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"182'6857","reported_seq":7350,"reported_epoch":182,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T19:58:39.850761+0000","last_change":"2023-09-23T19:58:32.457402+0000","last_active":"2023-09-23T19:58:39.850761+0000","last_peered":"2023-09-23T19:58:39.850761+0000","last_clean":"2023-09-23T19:58:28.364120+0000","last_became_active":"2023-09-23T19:58:30.666439+0000","last_became_peered":"2023-09-23T19:58:30.666439+0000","last_unstale":"2023-09-23T19:58:39.850761+0000","last_undegraded":"2023-09-23T19:58:32.452813+0000","last_fullsized":"2023-09-23T19:58:30.377565+0000","mapping_epoch":179,"log_start":"168'1384","ondisk_log_start":"168'1384","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:46.606935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407699456,"num_objects":6221,"num_object_clones":0,"num_object_copies":18663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":94,"num_objects_misplaced":6215,"num_objects_unfound":0,"num_objects_dirty":6221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6857,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":229,"num_bytes_recovered":15007744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":6215},{"shards":"1,5","objects":-88}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":519,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.485373+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T19:58:31.485373+0000","last_peered":"2023-09-23T19:58:31.485373+0000","last_clean":"2023-09-23T19:58:31.485373+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T19:58:31.485373+0000","last_undegraded":"2023-09-23T19:58:31.485373+0000","last_fullsized":"2023-09-23T19:58:31.485373+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:32.858007+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":900,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.402711+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T19:58:31.402711+0000","last_peered":"2023-09-23T19:58:31.402711+0000","last_clean":"2023-09-23T19:58:31.402711+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T19:58:31.402711+0000","last_undegraded":"2023-09-23T19:58:31.402711+0000","last_fullsized":"2023-09-23T19:58:31.402711+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":135,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.391489+0000","last_change":"2023-09-23T19:58:29.404687+0000","last_active":"2023-09-23T19:58:31.391489+0000","last_peered":"2023-09-23T19:58:31.391489+0000","last_clean":"2023-09-23T19:58:31.391489+0000","last_became_active":"2023-09-23T19:58:29.404376+0000","last_became_peered":"2023-09-23T19:58:29.404376+0000","last_unstale":"2023-09-23T19:58:31.391489+0000","last_undegraded":"2023-09-23T19:58:31.391489+0000","last_fullsized":"2023-09-23T19:58:31.391489+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:10.908961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":140,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.479383+0000","last_change":"2023-09-23T19:50:24.001852+0000","last_active":"2023-09-23T19:58:31.479383+0000","last_peered":"2023-09-23T19:58:31.479383+0000","last_clean":"2023-09-23T19:58:31.479383+0000","last_became_active":"2023-09-23T19:50:24.001504+0000","last_became_peered":"2023-09-23T19:50:24.001504+0000","last_unstale":"2023-09-23T19:58:31.479383+0000","last_undegraded":"2023-09-23T19:58:31.479383+0000","last_fullsized":"2023-09-23T19:58:31.479383+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:21.176109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"182'6722","reported_seq":6749,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:39.812646+0000","last_change":"2023-09-23T19:56:51.329836+0000","last_active":"2023-09-23T19:58:39.812646+0000","last_peered":"2023-09-23T19:58:39.812646+0000","last_clean":"2023-09-23T19:58:39.812646+0000","last_became_active":"2023-09-23T19:56:51.329518+0000","last_became_peered":"2023-09-23T19:56:51.329518+0000","last_unstale":"2023-09-23T19:58:39.812646+0000","last_undegraded":"2023-09-23T19:58:39.812646+0000","last_fullsized":"2023-09-23T19:58:39.812646+0000","mapping_epoch":163,"log_start":"168'1251","ondisk_log_start":"168'1251","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5471,"log_dups_size":0,"ondisk_log_size":5471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:38.536519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398983168,"num_objects":6088,"num_object_clones":0,"num_object_copies":18264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6722,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":504,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.418031+0000","last_change":"2023-09-23T19:58:32.418031+0000","last_active":"2023-09-23T19:58:32.418031+0000","last_peered":"2023-09-23T19:58:32.418031+0000","last_clean":"2023-09-23T19:58:32.418031+0000","last_became_active":"2023-09-23T19:58:32.417693+0000","last_became_peered":"2023-09-23T19:58:32.417693+0000","last_unstale":"2023-09-23T19:58:32.418031+0000","last_undegraded":"2023-09-23T19:58:32.418031+0000","last_fullsized":"2023-09-23T19:58:32.418031+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:56:54.487110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:56:54.487110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:09.889998+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":260,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.412851+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T19:58:31.412851+0000","last_peered":"2023-09-23T19:58:31.412851+0000","last_clean":"2023-09-23T19:58:31.412851+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T19:58:31.412851+0000","last_undegraded":"2023-09-23T19:58:31.412851+0000","last_fullsized":"2023-09-23T19:58:31.412851+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.487582+0000","last_change":"2023-09-23T19:50:23.998092+0000","last_active":"2023-09-23T19:58:31.487582+0000","last_peered":"2023-09-23T19:58:31.487582+0000","last_clean":"2023-09-23T19:58:31.487582+0000","last_became_active":"2023-09-23T19:50:23.997423+0000","last_became_peered":"2023-09-23T19:50:23.997423+0000","last_unstale":"2023-09-23T19:58:31.487582+0000","last_undegraded":"2023-09-23T19:58:31.487582+0000","last_fullsized":"2023-09-23T19:58:31.487582+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.750683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":112,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.477764+0000","last_change":"2023-09-23T19:48:37.573288+0000","last_active":"2023-09-23T19:58:31.477764+0000","last_peered":"2023-09-23T19:58:31.477764+0000","last_clean":"2023-09-23T19:58:31.477764+0000","last_became_active":"2023-09-23T19:48:37.571832+0000","last_became_peered":"2023-09-23T19:48:37.571832+0000","last_unstale":"2023-09-23T19:58:31.477764+0000","last_undegraded":"2023-09-23T19:58:31.477764+0000","last_fullsized":"2023-09-23T19:58:31.477764+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:05.772143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"182'6604","reported_seq":7060,"reported_epoch":182,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T19:58:38.282325+0000","last_change":"2023-09-23T19:58:31.837932+0000","last_active":"2023-09-23T19:58:38.282325+0000","last_peered":"2023-09-23T19:58:38.282325+0000","last_clean":"2023-09-23T19:58:28.369844+0000","last_became_active":"2023-09-23T19:58:30.688669+0000","last_became_peered":"2023-09-23T19:58:30.688669+0000","last_unstale":"2023-09-23T19:58:38.282325+0000","last_undegraded":"2023-09-23T19:58:31.811308+0000","last_fullsized":"2023-09-23T19:58:30.382029+0000","mapping_epoch":179,"log_start":"167'668","ondisk_log_start":"167'668","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":5936,"log_dups_size":0,"ondisk_log_size":5936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:11.957596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399507456,"num_objects":6096,"num_object_clones":0,"num_object_copies":18288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":44,"num_objects_misplaced":6096,"num_objects_unfound":0,"num_objects_dirty":6096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6604,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":210,"num_bytes_recovered":13762560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":6096},{"shards":"2,4","objects":-44}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":520,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.397669+0000","last_change":"2023-09-23T19:56:42.336491+0000","last_active":"2023-09-23T19:58:31.397669+0000","last_peered":"2023-09-23T19:58:31.397669+0000","last_clean":"2023-09-23T19:58:31.397669+0000","last_became_active":"2023-09-23T19:56:42.336151+0000","last_became_peered":"2023-09-23T19:56:42.336151+0000","last_unstale":"2023-09-23T19:58:31.397669+0000","last_undegraded":"2023-09-23T19:58:31.397669+0000","last_fullsized":"2023-09-23T19:58:31.397669+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:52.411762+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":41,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.409198+0000","last_change":"2023-09-23T19:58:29.407254+0000","last_active":"2023-09-23T19:58:31.409198+0000","last_peered":"2023-09-23T19:58:31.409198+0000","last_clean":"2023-09-23T19:58:31.409198+0000","last_became_active":"2023-09-23T19:58:29.398776+0000","last_became_peered":"2023-09-23T19:58:29.398776+0000","last_unstale":"2023-09-23T19:58:31.409198+0000","last_undegraded":"2023-09-23T19:58:31.409198+0000","last_fullsized":"2023-09-23T19:58:31.409198+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:34.254510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":540,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.417783+0000","last_change":"2023-09-23T19:58:32.417783+0000","last_active":"2023-09-23T19:58:32.417783+0000","last_peered":"2023-09-23T19:58:32.417783+0000","last_clean":"2023-09-23T19:58:32.417783+0000","last_became_active":"2023-09-23T19:58:32.417276+0000","last_became_peered":"2023-09-23T19:58:32.417276+0000","last_unstale":"2023-09-23T19:58:32.417783+0000","last_undegraded":"2023-09-23T19:58:32.417783+0000","last_fullsized":"2023-09-23T19:58:32.417783+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:35.161546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":74,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.402160+0000","last_change":"2023-09-23T19:58:29.403458+0000","last_active":"2023-09-23T19:58:31.402160+0000","last_peered":"2023-09-23T19:58:31.402160+0000","last_clean":"2023-09-23T19:58:31.402160+0000","last_became_active":"2023-09-23T19:58:29.403196+0000","last_became_peered":"2023-09-23T19:58:29.403196+0000","last_unstale":"2023-09-23T19:58:31.402160+0000","last_undegraded":"2023-09-23T19:58:31.402160+0000","last_fullsized":"2023-09-23T19:58:31.402160+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:30.409149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.3","version":"182'6747","reported_seq":6774,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:38.249339+0000","last_change":"2023-09-23T19:56:51.331630+0000","last_active":"2023-09-23T19:58:38.249339+0000","last_peered":"2023-09-23T19:58:38.249339+0000","last_clean":"2023-09-23T19:58:38.249339+0000","last_became_active":"2023-09-23T19:56:51.331338+0000","last_became_peered":"2023-09-23T19:56:51.331338+0000","last_unstale":"2023-09-23T19:58:38.249339+0000","last_undegraded":"2023-09-23T19:58:38.249339+0000","last_fullsized":"2023-09-23T19:58:38.249339+0000","mapping_epoch":163,"log_start":"167'413","ondisk_log_start":"167'413","created":163,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:56:50.306555+0000","last_clean_scrub_stamp":"2023-09-23T19:56:50.306555+0000","objects_scrubbed":0,"log_size":6334,"log_dups_size":0,"ondisk_log_size":6334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:45.394928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404946944,"num_objects":6179,"num_object_clones":0,"num_object_copies":18537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6747,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":460,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.428714+0000","last_change":"2023-09-23T19:58:32.428714+0000","last_active":"2023-09-23T19:58:32.428714+0000","last_peered":"2023-09-23T19:58:32.428714+0000","last_clean":"2023-09-23T19:58:32.428714+0000","last_became_active":"2023-09-23T19:58:32.428342+0000","last_became_peered":"2023-09-23T19:58:32.428342+0000","last_unstale":"2023-09-23T19:58:32.428714+0000","last_undegraded":"2023-09-23T19:58:32.428714+0000","last_fullsized":"2023-09-23T19:58:32.428714+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T19:53:00.704900+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":816370,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":344,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T19:58:31.409295+0000","last_change":"2023-09-23T19:54:45.328104+0000","last_active":"2023-09-23T19:58:31.409295+0000","last_peered":"2023-09-23T19:58:31.409295+0000","last_clean":"2023-09-23T19:43:08.441095+0000","last_became_active":"2023-09-23T19:54:45.328104+0000","last_became_peered":"2023-09-23T19:54:45.328104+0000","last_unstale":"2023-09-23T19:58:31.409295+0000","last_undegraded":"2023-09-23T19:58:31.409295+0000","last_fullsized":"2023-09-23T19:54:45.325453+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:40.212909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":130,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.479309+0000","last_change":"2023-09-23T19:49:51.383851+0000","last_active":"2023-09-23T19:58:31.479309+0000","last_peered":"2023-09-23T19:58:31.479309+0000","last_clean":"2023-09-23T19:58:31.479309+0000","last_became_active":"2023-09-23T19:49:51.383603+0000","last_became_peered":"2023-09-23T19:49:51.383603+0000","last_unstale":"2023-09-23T19:58:31.479309+0000","last_undegraded":"2023-09-23T19:58:31.479309+0000","last_fullsized":"2023-09-23T19:58:31.479309+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:57.664315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":131,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T19:58:31.397250+0000","last_change":"2023-09-23T19:54:45.330341+0000","last_active":"2023-09-23T19:58:31.397250+0000","last_peered":"2023-09-23T19:58:31.397250+0000","last_clean":"2023-09-23T19:54:37.305426+0000","last_became_active":"2023-09-23T19:54:45.330341+0000","last_became_peered":"2023-09-23T19:54:45.330341+0000","last_unstale":"2023-09-23T19:58:31.397250+0000","last_undegraded":"2023-09-23T19:58:31.397250+0000","last_fullsized":"2023-09-23T19:54:45.253699+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:22.534625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":115,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.405764+0000","last_change":"2023-09-23T19:48:37.572402+0000","last_active":"2023-09-23T19:58:32.405764+0000","last_peered":"2023-09-23T19:58:32.405764+0000","last_clean":"2023-09-23T19:58:32.405764+0000","last_became_active":"2023-09-23T19:48:37.571611+0000","last_became_peered":"2023-09-23T19:48:37.571611+0000","last_unstale":"2023-09-23T19:58:32.405764+0000","last_undegraded":"2023-09-23T19:58:32.405764+0000","last_fullsized":"2023-09-23T19:58:32.405764+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:23.968123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":174,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.503569+0000","last_change":"2023-09-23T19:58:29.405159+0000","last_active":"2023-09-23T19:58:31.503569+0000","last_peered":"2023-09-23T19:58:31.503569+0000","last_clean":"2023-09-23T19:58:31.503569+0000","last_became_active":"2023-09-23T19:58:29.404824+0000","last_became_peered":"2023-09-23T19:58:29.404824+0000","last_unstale":"2023-09-23T19:58:31.503569+0000","last_undegraded":"2023-09-23T19:58:31.503569+0000","last_fullsized":"2023-09-23T19:58:31.503569+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.615859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.391368+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T19:58:31.391368+0000","last_peered":"2023-09-23T19:58:31.391368+0000","last_clean":"2023-09-23T19:58:31.391368+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T19:58:31.391368+0000","last_undegraded":"2023-09-23T19:58:31.391368+0000","last_fullsized":"2023-09-23T19:58:31.391368+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":168,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.400537+0000","last_change":"2023-09-23T19:58:29.397024+0000","last_active":"2023-09-23T19:58:31.400537+0000","last_peered":"2023-09-23T19:58:31.400537+0000","last_clean":"2023-09-23T19:58:31.400537+0000","last_became_active":"2023-09-23T19:58:29.396674+0000","last_became_peered":"2023-09-23T19:58:29.396674+0000","last_unstale":"2023-09-23T19:58:31.400537+0000","last_undegraded":"2023-09-23T19:58:31.400537+0000","last_fullsized":"2023-09-23T19:58:31.400537+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:00.203288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":147,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.392019+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T19:58:31.392019+0000","last_peered":"2023-09-23T19:58:31.392019+0000","last_clean":"2023-09-23T19:58:31.392019+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T19:58:31.392019+0000","last_undegraded":"2023-09-23T19:58:31.392019+0000","last_fullsized":"2023-09-23T19:58:31.392019+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:03.354856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":168,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.406546+0000","last_change":"2023-09-23T19:58:29.403250+0000","last_active":"2023-09-23T19:58:31.406546+0000","last_peered":"2023-09-23T19:58:31.406546+0000","last_clean":"2023-09-23T19:58:31.406546+0000","last_became_active":"2023-09-23T19:58:29.402942+0000","last_became_peered":"2023-09-23T19:58:29.402942+0000","last_unstale":"2023-09-23T19:58:31.406546+0000","last_undegraded":"2023-09-23T19:58:31.406546+0000","last_fullsized":"2023-09-23T19:58:31.406546+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:32.039169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.404141+0000","last_change":"2023-09-23T19:58:29.396205+0000","last_active":"2023-09-23T19:58:32.404141+0000","last_peered":"2023-09-23T19:58:32.404141+0000","last_clean":"2023-09-23T19:58:32.404141+0000","last_became_active":"2023-09-23T19:58:29.395727+0000","last_became_peered":"2023-09-23T19:58:29.395727+0000","last_unstale":"2023-09-23T19:58:32.404141+0000","last_undegraded":"2023-09-23T19:58:32.404141+0000","last_fullsized":"2023-09-23T19:58:32.404141+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:21.519147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":99,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T19:58:32.408504+0000","last_change":"2023-09-23T19:58:29.393274+0000","last_active":"2023-09-23T19:58:32.408504+0000","last_peered":"2023-09-23T19:58:32.408504+0000","last_clean":"2023-09-23T19:58:32.408504+0000","last_became_active":"2023-09-23T19:58:29.392861+0000","last_became_peered":"2023-09-23T19:58:29.392861+0000","last_unstale":"2023-09-23T19:58:32.408504+0000","last_undegraded":"2023-09-23T19:58:32.408504+0000","last_fullsized":"2023-09-23T19:58:32.408504+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:28.826509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":130,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.478760+0000","last_change":"2023-09-23T19:50:24.001673+0000","last_active":"2023-09-23T19:58:31.478760+0000","last_peered":"2023-09-23T19:58:31.478760+0000","last_clean":"2023-09-23T19:58:31.478760+0000","last_became_active":"2023-09-23T19:50:24.001304+0000","last_became_peered":"2023-09-23T19:50:24.001304+0000","last_unstale":"2023-09-23T19:58:31.478760+0000","last_undegraded":"2023-09-23T19:58:31.478760+0000","last_fullsized":"2023-09-23T19:58:31.478760+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:23.934045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":168,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T19:58:31.399483+0000","last_change":"2023-09-23T19:58:29.396489+0000","last_active":"2023-09-23T19:58:31.399483+0000","last_peered":"2023-09-23T19:58:31.399483+0000","last_clean":"2023-09-23T19:58:31.399483+0000","last_became_active":"2023-09-23T19:58:29.396181+0000","last_became_peered":"2023-09-23T19:58:29.396181+0000","last_unstale":"2023-09-23T19:58:31.399483+0000","last_undegraded":"2023-09-23T19:58:31.399483+0000","last_fullsized":"2023-09-23T19:58:31.399483+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:52.650788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6448873496,"num_objects":98403,"num_object_clones":0,"num_object_copies":295209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32345,"num_objects_misplaced":30409,"num_objects_unfound":0,"num_objects_dirty":98403,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":106905,"num_write_kb":6569793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":539,"num_bytes_recovered":35323904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19483537408,"data_stored":19483525192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100052,"ondisk_log_size":100052,"up":48,"acting":41,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4824,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27103,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":91,"acting":93,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1240067,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234210,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3302860,"kb_used_data":3158340,"kb_used_omap":0,"kb_used_meta":144384,"kb_avail":90442292,"statfs":{"total":95995035648,"available":92612907008,"internally_reserved":0,"allocated":3234140160,"data_stored":3233570536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":147849216},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:58:00 2023","interfaces":[{"interface":"back","average":{"1min":3.8370000000000002,"5min":2.6179999999999999,"15min":1.282},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":62.061999999999998,"5min":62.061999999999998,"15min":62.061999999999998},"last":0.27300000000000002},{"interface":"front","average":{"1min":3.875,"5min":3.3079999999999998,"15min":1.528},"min":{"1min":0.216,"5min":0.215,"15min":0.215},"max":{"1min":62.131,"5min":62.131,"15min":62.131},"last":0.97999999999999998}]},{"osd":1,"last update":"Sat Sep 23 19:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":1.8919999999999999,"15min":1.0309999999999999},"min":{"1min":0.25900000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":15.491,"5min":15.491,"15min":15.491},"last":0.375},{"interface":"front","average":{"1min":2.2080000000000002,"5min":2.1080000000000001,"15min":1.103},"min":{"1min":0.28100000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.558,"5min":15.558,"15min":15.558},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":6.7619999999999996,"5min":4.7850000000000001,"15min":2.286},"min":{"1min":0.35099999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":94.629000000000005,"5min":130.673,"15min":130.673},"last":0.70699999999999996},{"interface":"front","average":{"1min":7.7149999999999999,"5min":5.1139999999999999,"15min":2.234},"min":{"1min":0.34000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":94.620000000000005,"5min":130.71100000000001,"15min":130.71100000000001},"last":0.92500000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:58:11 2023","interfaces":[{"interface":"back","average":{"1min":10.554,"5min":3.6930000000000001,"15min":1.7949999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":80.113,"5min":80.113,"15min":80.113},"last":0.67000000000000004},{"interface":"front","average":{"1min":9.9800000000000004,"5min":3.6120000000000001,"15min":1.657},"min":{"1min":0.36599999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":83.138000000000005,"5min":83.138000000000005,"15min":83.138000000000005},"last":0.45100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:57:39 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":4.008,"15min":1.661},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.48499999999999999,"5min":143.255,"15min":143.255},"last":1.226},{"interface":"front","average":{"1min":0.39600000000000002,"5min":3.806,"15min":1.6359999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.53000000000000003,"5min":143.41200000000001,"15min":143.41200000000001},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:57:39 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":2.411,"15min":1.1639999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.821,"5min":84.566999999999993,"15min":84.566999999999993},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.38500000000000001,"5min":2.194,"15min":1.1020000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.61399999999999999,"5min":81.244,"15min":81.244},"last":0.35899999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.0189999999999999,"15min":0.71499999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.6449999999999996,"5min":5.6449999999999996,"15min":5.6449999999999996},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.121,"15min":0.76500000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.7720000000000002,"5min":5.7720000000000002,"15min":5.7720000000000002},"last":0.38900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495800,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379500,"kb_used_data":1334692,"kb_used_omap":0,"kb_used_meta":44800,"kb_avail":92365652,"statfs":{"total":95995035648,"available":94582427648,"internally_reserved":0,"allocated":1366724608,"data_stored":1366198307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":45875200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:53:52 2023","interfaces":[{"interface":"back","average":{"1min":4.415,"5min":1.3500000000000001,"15min":0.85199999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":60.738,"5min":60.738,"15min":60.738},"last":1.476},{"interface":"front","average":{"1min":4.202,"5min":1.333,"15min":0.88700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":60.722999999999999,"5min":60.722999999999999,"15min":60.722999999999999},"last":1.377}]},{"osd":1,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.85799999999999998,"15min":0.69999999999999996},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6520000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.395,"5min":0.94299999999999995,"15min":0.69099999999999995},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 19:54:32 2023","interfaces":[{"interface":"back","average":{"1min":4.3029999999999999,"5min":2.7829999999999999,"15min":1.3149999999999999},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.245},"max":{"1min":53.667999999999999,"5min":61.155000000000001,"15min":61.155000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":4.024,"5min":2.7989999999999999,"15min":1.417},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":53.628,"5min":60.506,"15min":60.506},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":0.97899999999999998,"15min":0.76700000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":5.7619999999999996},"last":1.361},{"interface":"front","average":{"1min":2.1379999999999999,"5min":0.94299999999999995,"15min":0.69299999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":1.387}]},{"osd":4,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":1.0740000000000001,"15min":0.68600000000000005},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.4859999999999998,"5min":24.003,"15min":24.003},"last":0.377},{"interface":"front","average":{"1min":1.607,"5min":1.0920000000000001,"15min":0.70199999999999996},"min":{"1min":0.378,"5min":0.313,"15min":0.26500000000000001},"max":{"1min":6.6020000000000003,"5min":24.802,"15min":24.802},"last":1.3149999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.147,"15min":0.71099999999999997},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":14.378,"5min":24.065999999999999,"15min":24.065999999999999},"last":1.4079999999999999},{"interface":"front","average":{"1min":1.641,"5min":1.1359999999999999,"15min":0.74399999999999999},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":6.0529999999999999,"5min":24.841000000000001,"15min":24.841000000000001},"last":1.2769999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:54:39 2023","interfaces":[{"interface":"back","average":{"1min":6.1509999999999998,"5min":1.8149999999999999,"15min":0.98099999999999998},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":105.71299999999999,"5min":105.71299999999999,"15min":105.71299999999999},"last":2.1579999999999999},{"interface":"front","average":{"1min":6.5620000000000003,"5min":2.0259999999999998,"15min":1.075},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":103.206,"5min":103.206,"15min":103.206},"last":0.46100000000000002}]}]},{"osd":1,"up_from":99,"seq":425201762412,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2518064,"kb_used_data":2355944,"kb_used_omap":7,"kb_used_meta":162104,"kb_avail":91227088,"statfs":{"total":95995035648,"available":93416538112,"internally_reserved":0,"allocated":2412486656,"data_stored":2411710090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":165995322},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:58:08 2023","interfaces":[{"interface":"back","average":{"1min":5.577,"5min":8.4960000000000004,"15min":3.101},"min":{"1min":0.19600000000000001,"5min":0.151,"15min":0.151},"max":{"1min":61.033000000000001,"5min":202.917,"15min":202.917},"last":0.25800000000000001},{"interface":"front","average":{"1min":4.8680000000000003,"5min":8.6769999999999996,"15min":3.1819999999999999},"min":{"1min":0.21199999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":60.938000000000002,"5min":203.34899999999999,"15min":203.34899999999999},"last":0.28100000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:57:38 2023","interfaces":[{"interface":"back","average":{"1min":8.5419999999999998,"5min":9.9860000000000007,"15min":3.5640000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.192},"max":{"1min":62.697000000000003,"5min":231.57499999999999,"15min":231.57499999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":9.1219999999999999,"5min":9.2959999999999994,"15min":3.335},"min":{"1min":0.254,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":61.008000000000003,"5min":236.077,"15min":236.077},"last":0.30099999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:58:08 2023","interfaces":[{"interface":"back","average":{"1min":12.154999999999999,"5min":4.8499999999999996,"15min":1.9119999999999999},"min":{"1min":0.26400000000000001,"5min":0.217,"15min":0.217},"max":{"1min":68.180999999999997,"5min":68.180999999999997,"15min":68.180999999999997},"last":0.40000000000000002},{"interface":"front","average":{"1min":12.106999999999999,"5min":4.1539999999999999,"15min":1.6890000000000001},"min":{"1min":0.26300000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":66.509,"5min":66.509,"15min":66.509},"last":3.149}]},{"osd":4,"last update":"Sat Sep 23 19:58:08 2023","interfaces":[{"interface":"back","average":{"1min":4.4480000000000004,"5min":4.4029999999999996,"15min":1.919},"min":{"1min":0.29299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":60.951000000000001,"5min":161.68299999999999,"15min":161.68299999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":4.4809999999999999,"5min":4.2140000000000004,"15min":1.8300000000000001},"min":{"1min":0.32800000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":60.841000000000001,"5min":161.286,"15min":161.286},"last":1.7350000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:58:08 2023","interfaces":[{"interface":"back","average":{"1min":4.4210000000000003,"5min":6.5039999999999996,"15min":2.613},"min":{"1min":0.33200000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":60.904000000000003,"5min":219.048,"15min":219.048},"last":0.44700000000000001},{"interface":"front","average":{"1min":4.1100000000000003,"5min":6.9939999999999998,"15min":2.7970000000000002},"min":{"1min":0.33400000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":59.231999999999999,"5min":218.69499999999999,"15min":218.69499999999999},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.097,"15min":0.80500000000000005},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.9649999999999999,"5min":7.9649999999999999,"15min":7.9649999999999999},"last":14.897},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.0800000000000001,"15min":0.82399999999999995},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":14.904999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:58:08 2023","interfaces":[{"interface":"back","average":{"1min":4.1799999999999997,"5min":3.1970000000000001,"15min":1.548},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":60.213000000000001,"5min":60.213000000000001,"15min":60.213000000000001},"last":4.952},{"interface":"front","average":{"1min":4.6399999999999997,"5min":2.2949999999999999,"15min":1.2410000000000001},"min":{"1min":0.36599999999999999,"5min":0.308,"15min":0.308},"max":{"1min":61.298999999999999,"5min":61.298999999999999,"15min":61.298999999999999},"last":4.7039999999999997}]}]},{"osd":0,"up_from":94,"seq":403726925946,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1777648,"kb_used_data":1601576,"kb_used_omap":8,"kb_used_meta":176055,"kb_avail":91967504,"statfs":{"total":95995035648,"available":94174724096,"internally_reserved":0,"allocated":1640013824,"data_stored":1639236797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":180280929},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:58:13 2023","interfaces":[{"interface":"back","average":{"1min":3.5619999999999998,"5min":2.5030000000000001,"15min":1.071},"min":{"1min":0.19500000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":24.640000000000001,"5min":74.751000000000005,"15min":74.751000000000005},"last":0.433},{"interface":"front","average":{"1min":4.5830000000000002,"5min":3.3109999999999999,"15min":1.3540000000000001},"min":{"1min":0.22500000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":24.253,"5min":74.998000000000005,"15min":74.998000000000005},"last":2.6200000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:57:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":3.6070000000000002,"15min":1.48},"min":{"1min":0.214,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":24.611999999999998,"5min":74.795000000000002,"15min":74.795000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":3.5059999999999998,"5min":4.2030000000000003,"15min":1.649},"min":{"1min":0.24199999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":24.677,"5min":78.366,"15min":78.366},"last":8.3200000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:58:13 2023","interfaces":[{"interface":"back","average":{"1min":16.614000000000001,"5min":6.8570000000000002,"15min":2.548},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":145.72800000000001,"5min":145.72800000000001,"15min":145.72800000000001},"last":2.6669999999999998},{"interface":"front","average":{"1min":16.629000000000001,"5min":7.2460000000000004,"15min":2.6720000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":145.50700000000001,"5min":145.50700000000001,"15min":145.50700000000001},"last":5.1539999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:57:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":3.931,"15min":1.748},"min":{"1min":0.32100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7919999999999998,"5min":110.312,"15min":110.312},"last":7.7430000000000003},{"interface":"front","average":{"1min":1.1000000000000001,"5min":3.528,"15min":1.6240000000000001},"min":{"1min":0.29399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7669999999999999,"5min":110.256,"15min":110.256},"last":2.6930000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":3.7639999999999998,"15min":1.7210000000000001},"min":{"1min":0.35499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.7320000000000002,"5min":131.34399999999999,"15min":131.34399999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.8180000000000001,"5min":4.3200000000000003,"15min":1.9079999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":7.8280000000000003,"5min":131.386,"15min":131.386},"last":5.9160000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":0.89700000000000002,"15min":0.78300000000000003},"min":{"1min":0.40899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.9139999999999997,"5min":6.5179999999999998,"15min":6.5179999999999998},"last":2.4100000000000001},{"interface":"front","average":{"1min":2.3410000000000002,"5min":0.996,"15min":0.81000000000000005},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.6920000000000002,"5min":8.6920000000000002,"15min":8.6920000000000002},"last":1.1679999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:57:53 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":2.5840000000000001,"15min":1.3360000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.7709999999999999,"5min":77.132999999999996,"15min":77.132999999999996},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.978,"5min":3.0659999999999998,"15min":1.508},"min":{"1min":0.39000000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.8550000000000004,"5min":75.465999999999994,"15min":75.465999999999994},"last":7.694}]}]},{"osd":2,"up_from":104,"seq":446676598885,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4232684,"kb_used_data":3962340,"kb_used_omap":12,"kb_used_meta":270323,"kb_avail":89512468,"statfs":{"total":95995035648,"available":91660767232,"internally_reserved":0,"allocated":4057436160,"data_stored":4056666225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":276811774},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":2.8559999999999999,"15min":1.196},"min":{"1min":0.16900000000000001,"5min":0.154,"15min":0.154},"max":{"1min":4.4809999999999999,"5min":50.506999999999998,"15min":50.506999999999998},"last":3.98},{"interface":"front","average":{"1min":1.0429999999999999,"5min":2.9329999999999998,"15min":1.2350000000000001},"min":{"1min":0.192,"5min":0.187,"15min":0.187},"max":{"1min":6.0629999999999997,"5min":50.802999999999997,"15min":50.802999999999997},"last":5.0090000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":2.5249999999999999,"15min":1.0960000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":6.0759999999999996,"5min":50.061,"15min":50.061},"last":4.7450000000000001},{"interface":"front","average":{"1min":1.7589999999999999,"5min":2.6960000000000002,"15min":1.1559999999999999},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":6.4240000000000004,"5min":50.533999999999999,"15min":50.533999999999999},"last":4.1429999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":5.423,"5min":3.5870000000000002,"15min":1.5209999999999999},"min":{"1min":0.218,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":45.164000000000001,"5min":50.186999999999998,"15min":50.186999999999998},"last":3.7429999999999999},{"interface":"front","average":{"1min":5.2190000000000003,"5min":3.7719999999999998,"15min":1.546},"min":{"1min":0.26100000000000001,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":44.704999999999998,"5min":50.585000000000001,"15min":50.585000000000001},"last":6.9900000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":2.3969999999999998,"15min":1.216},"min":{"1min":0.38200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.3170000000000002,"5min":50.640000000000001,"15min":50.640000000000001},"last":1.03},{"interface":"front","average":{"1min":0.93200000000000005,"5min":2.4510000000000001,"15min":1.232},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.1240000000000001,"5min":50.154000000000003,"15min":50.154000000000003},"last":3.8050000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.7850000000000001,"15min":1.3500000000000001},"min":{"1min":0.38,"5min":0.308,"15min":0.308},"max":{"1min":5.2670000000000003,"5min":50.616,"15min":50.616},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.369,"5min":2.5059999999999998,"15min":1.2529999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.2169999999999996,"5min":50.762,"15min":50.762},"last":0.86399999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.827,"5min":1.343,"15min":0.84999999999999998},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4740000000000002,"5min":7.4740000000000002,"15min":7.4740000000000002},"last":1.0660000000000001},{"interface":"front","average":{"1min":2.899,"5min":1.3120000000000001,"15min":0.85699999999999998},"min":{"1min":0.42399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.3029999999999999,"5min":7.3029999999999999,"15min":7.3029999999999999},"last":1.008}]},{"osd":7,"last update":"Sat Sep 23 19:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":2.4980000000000002,"15min":1.274},"min":{"1min":0.30199999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.1070000000000002,"5min":50.118000000000002,"15min":50.118000000000002},"last":3.77},{"interface":"front","average":{"1min":1.2070000000000001,"5min":2.956,"15min":1.429},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.3799999999999999,"5min":50.664999999999999,"15min":50.664999999999999},"last":0.40100000000000002}]}]},{"osd":3,"up_from":0,"seq":468151435358,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019120,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2134476,"kb_used_data":1985668,"kb_used_omap":0,"kb_used_meta":148736,"kb_avail":91610676,"statfs":{"total":95995035648,"available":93809332224,"internally_reserved":0,"allocated":2033324032,"data_stored":2032754940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":152305664},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,5,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:57 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":5.1139999999999999,"15min":2.097},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.9800000000000004,"5min":104.604,"15min":104.604},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.4199999999999999,"5min":4.9889999999999999,"15min":2.0790000000000002},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.9640000000000004,"5min":104.687,"15min":104.687},"last":4.3710000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:58:05 2023","interfaces":[{"interface":"back","average":{"1min":6.2380000000000004,"5min":3.1240000000000001,"15min":1.452},"min":{"1min":0.252,"5min":0.224,"15min":0.216},"max":{"1min":77.807000000000002,"5min":77.807000000000002,"15min":77.807000000000002},"last":5.2160000000000002},{"interface":"front","average":{"1min":6.2039999999999997,"5min":2.5619999999999998,"15min":1.2549999999999999},"min":{"1min":0.23100000000000001,"5min":0.22,"15min":0.22},"max":{"1min":77.927000000000007,"5min":77.927000000000007,"15min":77.927000000000007},"last":7.7919999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:57:41 2023","interfaces":[{"interface":"back","average":{"1min":4.4029999999999996,"5min":4.6920000000000002,"15min":1.9379999999999999},"min":{"1min":0.26900000000000002,"5min":0.25,"15min":0.25},"max":{"1min":69.989000000000004,"5min":98.063999999999993,"15min":98.063999999999993},"last":0.30599999999999999},{"interface":"front","average":{"1min":4.077,"5min":4.5199999999999996,"15min":1.873},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":69.799000000000007,"5min":97.921000000000006,"15min":97.921000000000006},"last":5.0199999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:58:05 2023","interfaces":[{"interface":"back","average":{"1min":6.3209999999999997,"5min":3.8679999999999999,"15min":1.6559999999999999},"min":{"1min":0.26400000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":69.305999999999997,"5min":94.141000000000005,"15min":94.141000000000005},"last":5.1689999999999996},{"interface":"front","average":{"1min":6.4390000000000001,"5min":4.3250000000000002,"15min":1.8440000000000001},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":69.194000000000003,"5min":94.037000000000006,"15min":94.037000000000006},"last":2.6480000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:57:57 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.296,"15min":0.76600000000000001},"min":{"1min":0.253,"5min":0.246,"15min":0.246},"max":{"1min":0.64300000000000002,"5min":10.246,"15min":10.246},"last":3.1709999999999998},{"interface":"front","average":{"1min":0.434,"5min":1.6220000000000001,"15min":0.88400000000000001},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":1.2150000000000001,"5min":12.675000000000001,"15min":12.675000000000001},"last":3.1520000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":0.84699999999999998,"15min":0.60199999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.867,"5min":5.867,"15min":5.867},"last":8.0150000000000006},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.80300000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":3.931}]},{"osd":7,"last update":"Sat Sep 23 19:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":1.4770000000000001,"15min":0.88200000000000001},"min":{"1min":0.32600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.9670000000000001,"5min":11.59,"15min":11.59},"last":3.1139999999999999},{"interface":"front","average":{"1min":0.73999999999999999,"5min":1.3380000000000001,"15min":0.86899999999999999},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.2290000000000001,"5min":9.3089999999999993,"15min":9.3089999999999993},"last":0.27200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757484,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2151300,"kb_used_data":1980668,"kb_used_omap":0,"kb_used_meta":170624,"kb_avail":91593852,"statfs":{"total":95995035648,"available":93792104448,"internally_reserved":0,"allocated":2028204032,"data_stored":2027638861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":174718976},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:57:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":2.069,"15min":1.0740000000000001},"min":{"1min":0.219,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.5830000000000002,"5min":24.852,"15min":24.852},"last":2.867},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.917,"15min":1.0129999999999999},"min":{"1min":0.23899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":4.0289999999999999,"5min":24.638999999999999,"15min":24.638999999999999},"last":2.8820000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:58:07 2023","interfaces":[{"interface":"back","average":{"1min":8.8390000000000004,"5min":3.4790000000000001,"15min":1.532},"min":{"1min":0.28399999999999997,"5min":0.216,"15min":0.216},"max":{"1min":148.03800000000001,"5min":148.03800000000001,"15min":148.03800000000001},"last":2.3730000000000002},{"interface":"front","average":{"1min":8.3409999999999993,"5min":3.343,"15min":1.466},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":147.952,"5min":147.952,"15min":147.952},"last":2.1930000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:57:42 2023","interfaces":[{"interface":"back","average":{"1min":5.4960000000000004,"5min":7.641,"15min":2.9060000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":58.246000000000002,"5min":167.10400000000001,"15min":167.10400000000001},"last":1.1419999999999999},{"interface":"front","average":{"1min":5.5250000000000004,"5min":7.6699999999999999,"15min":2.9489999999999998},"min":{"1min":0.34499999999999997,"5min":0.31900000000000001,"15min":0.317},"max":{"1min":58.268999999999998,"5min":167.167,"15min":167.167},"last":1.169}]},{"osd":3,"last update":"Sat Sep 23 19:58:07 2023","interfaces":[{"interface":"back","average":{"1min":9.6319999999999997,"5min":3.8130000000000002,"15min":1.6539999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":63.127000000000002,"5min":63.127000000000002,"15min":63.127000000000002},"last":2.996},{"interface":"front","average":{"1min":9.5310000000000006,"5min":3.4590000000000001,"15min":1.5900000000000001},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":63.095999999999997,"5min":63.095999999999997,"15min":63.095999999999997},"last":2.169}]},{"osd":4,"last update":"Sat Sep 23 19:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.246,"15min":0.73499999999999999},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.23699999999999999},"max":{"1min":1.8169999999999999,"5min":10.077,"15min":10.077},"last":2.1269999999999998},{"interface":"front","average":{"1min":0.40600000000000003,"5min":1.339,"15min":0.79200000000000004},"min":{"1min":0.26900000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.91700000000000004,"5min":10.226000000000001,"15min":10.226000000000001},"last":0.33300000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:54:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":0.91600000000000004,"15min":0.63400000000000001},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.28699999999999998},"max":{"1min":10.180999999999999,"5min":10.180999999999999,"15min":10.180999999999999},"last":4.2949999999999999},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.046,"15min":0.71299999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":10.406000000000001},"last":3.879}]},{"osd":7,"last update":"Sat Sep 23 19:57:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.351,"15min":0.81899999999999995},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.105,"5min":13.734999999999999,"15min":13.734999999999999},"last":1.109},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.524,"15min":0.88},"min":{"1min":0.32200000000000001,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":2.0270000000000001,"5min":10.041,"15min":10.041},"last":2.1600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1625559040,"data_stored":1625554968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2393247744,"data_stored":2393243672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4045012992,"data_stored":4045012992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4163698688,"data_stored":4163698688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2021457920,"data_stored":2021457920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2021724160,"data_stored":2021720088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3212836864,"data_stored":3212836864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:58:45.291 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T19:58:45.291 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pg_num 26 2023-09-23T19:58:45.379 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:45.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:45.484 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:45.532 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:45.633 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:45.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:45.738 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:45.763 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:58:45.763 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:58:45.763 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:58:45.764 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:58:45.764 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.765 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:148380 5.0 5:05c78c8a:::benchmark_data_smithi135_284570_object32596:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:42.664204+0000", 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 243.091579728, 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22489629, 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148380 2023-09-23T19:58:45.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664204+0000", 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664204+0000", 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664203+0000", 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664211+0000", 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6340000000000007e-06 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664227+0000", 2023-09-23T19:58:45.768 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6154000000000002e-05 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664237+0000", 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.594e-06 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664286+0000", 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8508999999999999e-05 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664371+0000", 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5094999999999998e-05 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.664443+0000", 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2585999999999999e-05 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.668156+0000", 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003712618 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.669560+0000", 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001404609 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889003+0000", 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21944269599999999 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889057+0000", 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3801000000000002e-05 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:42.889101+0000", 2023-09-23T19:58:45.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3724999999999999e-05 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:57936 6.b 6:dd9ec801:::benchmark_data_smithi135_344736_object57935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.377982+0000", 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.377802399000004, 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.35059963399999999, 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.773 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 57936 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.377982+0000", 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.377982+0000", 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.377980+0000", 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.378004+0000", 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4278999999999999e-05 2023-09-23T19:58:45.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.378017+0000", 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.313e-05 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.378025+0000", 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8080000000000005e-06 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.378922+0000", 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00089668699999999996 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.379050+0000", 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000128097 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.379187+0000", 2023-09-23T19:58:45.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013662799999999999 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382229+0000", 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030427399999999999 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.389810+0000", 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0075806670000000001 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728484+0000", 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33867403800000001 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728539+0000", 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4755e-05 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728581+0000", 2023-09-23T19:58:45.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2475999999999998e-05 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:57943 6.b 6:da51ebd1:::benchmark_data_smithi135_344736_object57942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.382497+0000", 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.373287435999998, 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.34619983500000001, 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.780 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 57943 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382497+0000", 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382497+0000", 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382496+0000", 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382530+0000", 2023-09-23T19:58:45.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3679000000000001e-05 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382538+0000", 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3599999999999996e-06 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382544+0000", 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8529999999999997e-06 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382591+0000", 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6996999999999997e-05 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382722+0000", 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000131209 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:58:45.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382835+0000", 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000112366 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.389771+0000", 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0069359840000000001 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.393945+0000", 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041749140000000001 2023-09-23T19:58:45.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728626+0000", 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33468065699999999 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728664+0000", 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7706000000000001e-05 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728697+0000", 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2684000000000001e-05 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:57944 6.b 6:d6e57c88:::benchmark_data_smithi135_344736_object57943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.382558+0000", 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.373225886, 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.34624006699999998, 2023-09-23T19:58:45.787 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 57944 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382558+0000", 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382558+0000", 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382558+0000", 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382583+0000", 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5825000000000001e-05 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382590+0000", 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5560000000000004e-06 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382595+0000", 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3759999999999999e-06 2023-09-23T19:58:45.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.384952+0000", 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023563529999999998 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.385092+0000", 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014022000000000001 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.385243+0000", 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000150987 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.389779+0000", 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045362620000000001 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.393981+0000", 2023-09-23T19:58:45.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0042014660000000001 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728731+0000", 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33474991599999998 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728765+0000", 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3942000000000001e-05 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728798+0000", 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3764999999999999e-05 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:57948 6.b 6:dd8a1dfe:::benchmark_data_smithi135_344736_object57947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.382657+0000", 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.373127273999998, 2023-09-23T19:58:45.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.346240834, 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 57948 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382657+0000", 2023-09-23T19:58:45.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382657+0000", 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382656+0000", 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382678+0000", 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1691999999999998e-05 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382685+0000", 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.883e-06 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.382691+0000", 2023-09-23T19:58:45.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6790000000000002e-06 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.387490+0000", 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047991459999999998 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.387649+0000", 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015905800000000001 2023-09-23T19:58:45.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.387806+0000", 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015698200000000001 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.389891+0000", 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020850589999999998 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.799 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.394010+0000", 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041189369999999996 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728832+0000", 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33482231099999998 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728868+0000", 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6168000000000003e-05 2023-09-23T19:58:45.800 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728898+0000", 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9417999999999999e-05 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:57953 6.8 e171/163 6:11741188:::benchmark_data_smithi135_344736_object57952:head v 171'3649, mlcod=171'3646)", 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.392114+0000", 2023-09-23T19:58:45.801 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.363670084999995, 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33638102800000003, 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.392114+0000", 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.802 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.392114+0000", 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.392114+0000", 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967296 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.392148+0000", 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3816000000000003e-05 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.803 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.392154+0000", 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5490000000000003e-06 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.392159+0000", 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7990000000000001e-06 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.392220+0000", 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1057999999999997e-05 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.392238+0000", 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8185e-05 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728458+0000", 2023-09-23T19:58:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33621942300000002 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728495+0000", 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7431999999999997e-05 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44942.0:57936 6.b e171/163 ondisk, result = 0)", 2023-09-23T19:58:45.806 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.395388+0000", 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.360395972000006, 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33320546099999998, 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395388+0000", 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395388+0000", 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395387+0000", 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395393+0000", 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9460000000000003e-06 2023-09-23T19:58:45.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395404+0000", 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1199999999999999e-05 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395411+0000", 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9909999999999996e-06 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728410+0000", 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33299869399999998 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728476+0000", 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6389000000000004e-05 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728594+0000", 2023-09-23T19:58:45.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117197 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44942.0:57943 6.b e171/163 ondisk, result = 0)", 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.395439+0000", 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.360345413000005, 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33326206000000003, 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.811 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395439+0000", 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395439+0000", 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.812 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395438+0000", 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395441+0000", 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2500000000000001e-06 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395446+0000", 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4909999999999996e-06 2023-09-23T19:58:45.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395450+0000", 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8990000000000003e-06 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728615+0000", 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33316514400000002 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728621+0000", 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.178e-06 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728701+0000", 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9380000000000002e-05 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.815 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:57955 6.4 e171/163 6:2538de9a:::benchmark_data_smithi135_344736_object57954:head v 171'3635, mlcod=171'3633)", 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.395771+0000", 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.360013409000004, 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.339744461, 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395771+0000", 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.816 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395771+0000", 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395770+0000", 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.395795+0000", 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4485e-05 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728382+0000", 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33258739300000001 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728417+0000", 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4628000000000001e-05 2023-09-23T19:58:45.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728455+0000", 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8281000000000002e-05 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728476+0000", 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1447999999999999e-05 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.735488+0000", 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0070115009999999998 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.735515+0000", 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7251999999999999e-05 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.820 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:57956 6.f e171/163 6:fc8cc13b:::benchmark_data_smithi135_344736_object57955:head v 171'3707, mlcod=171'3705)", 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:57:35.396794+0000", 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 70.358990148999993, 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33584925799999998, 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.396794+0000", 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.821 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.396794+0000", 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.396793+0000", 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.822 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.396818+0000", 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4966000000000001e-05 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728374+0000", 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33155611299999999 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728387+0000", 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.328e-05 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.823 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728431+0000", 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3763000000000001e-05 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.728456+0000", 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5216e-05 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.732606+0000", 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041496780000000004 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:57:35.732643+0000", 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7070999999999998e-05 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.825 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:89542 6.6 6:617285df:::benchmark_data_smithi135_344736_object89541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.867119+0000", 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.888665267, 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.225729919, 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 89542 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.826 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.867119+0000", 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.867119+0000", 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.867117+0000", 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.867143+0000", 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5735999999999999e-05 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.867157+0000", 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3910999999999999e-05 2023-09-23T19:58:45.828 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.867165+0000", 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6659999999999997e-06 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.874530+0000", 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073642339999999999 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.829 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.874675+0000", 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000144795 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.874814+0000", 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013925600000000001 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877641+0000", 2023-09-23T19:58:45.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002826758 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089508+0000", 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21186728399999999 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092764+0000", 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032563890000000002 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092809+0000", 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4824000000000003e-05 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092849+0000", 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9739e-05 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.832 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:89546 6.8 e175/163 6:1ca86a7a:::benchmark_data_smithi135_344736_object89545:head v 175'5668, mlcod=175'5665)", 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.875097+0000", 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.880687250000001, 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21439355800000001, 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875097+0000", 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.833 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875097+0000", 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875097+0000", 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.834 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875137+0000", 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0633999999999998e-05 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875144+0000", 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4949999999999996e-06 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875148+0000", 2023-09-23T19:58:45.835 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5240000000000002e-06 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875288+0000", 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013956299999999999 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875312+0000", 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.389e-05 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089443+0000", 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21413174099999999 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089490+0000", 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7003999999999999e-05 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.837 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:89548 6.8 e175/163 6:18504062:::benchmark_data_smithi135_344736_object89547:head v 175'5669, mlcod=175'5665)", 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.875160+0000", 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.880624144000002, 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21443963899999999, 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875160+0000", 2023-09-23T19:58:45.838 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875160+0000", 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875160+0000", 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967296 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.839 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875198+0000", 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8405999999999998e-05 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875204+0000", 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.694e-06 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.875208+0000", 2023-09-23T19:58:45.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3989999999999997e-06 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878276+0000", 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030677830000000001 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878294+0000", 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8107000000000001e-05 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089577+0000", 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.211282473 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089600+0000", 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2994e-05 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.842 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:89549 6.b 6:d56c14df:::benchmark_data_smithi135_344736_object89548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.877143+0000", 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.878640924000003, 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21818918900000001, 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 89549 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.843 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877143+0000", 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877143+0000", 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.844 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877141+0000", 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877938+0000", 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00079659299999999995 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877951+0000", 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2904e-05 2023-09-23T19:58:45.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877959+0000", 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0679999999999997e-06 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878120+0000", 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016067499999999999 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.846 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878249+0000", 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000129479 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878380+0000", 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000131037 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.095236+0000", 2023-09-23T19:58:45.847 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21685534400000001 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.095240+0000", 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.707e-06 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.095261+0000", 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0964000000000001e-05 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.095293+0000", 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1843e-05 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.095332+0000", 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9307000000000002e-05 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.849 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:89553 6.6 6:638f2461:::benchmark_data_smithi135_344736_object89552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.877975+0000", 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.877809493999997, 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22366504500000001, 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 89553 2023-09-23T19:58:45.850 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877975+0000", 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877975+0000", 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877974+0000", 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.877995+0000", 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0971e-05 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878001+0000", 2023-09-23T19:58:45.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.657e-06 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878007+0000", 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4110000000000004e-06 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089566+0000", 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21155933299999999 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089736+0000", 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016963300000000001 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089894+0000", 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000157878 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.101473+0000", 2023-09-23T19:58:45.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011579262999999999 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.101515+0000", 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1573999999999999e-05 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.101556+0000", 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1607999999999999e-05 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.855 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.101603+0000", 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6752999999999998e-05 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.101640+0000", 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6732e-05 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.856 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44942.0:89542 6.6 e175/163 ondisk, result = 0)", 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.878030+0000", 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.877754381000003, 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.211484963, 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878030+0000", 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878030+0000", 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878029+0000", 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878033+0000", 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0300000000000004e-06 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878043+0000", 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9860000000000006e-06 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878047+0000", 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7439999999999998e-06 2023-09-23T19:58:45.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089489+0000", 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21144126099999999 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089500+0000", 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1755e-05 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089515+0000", 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4399000000000001e-05 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:89547 6.4 e175/163 6:2d888439:::benchmark_data_smithi135_344736_object89546:head v 175'5668, mlcod=175'5666)", 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.878051+0000", 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.877732821999999, 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21180033400000001, 2023-09-23T19:58:45.861 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878051+0000", 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878051+0000", 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878051+0000", 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878074+0000", 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3354999999999999e-05 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.863 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878085+0000", 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.082e-05 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878092+0000", 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1029999999999996e-06 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878154+0000", 2023-09-23T19:58:45.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1828000000000005e-05 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.878173+0000", 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9493000000000001e-05 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089543+0000", 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.211369744 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089852+0000", 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000308695 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44942.0:89542 6.6 e175/163 ondisk, result = 0)", 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.880018+0000", 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.875765676, 2023-09-23T19:58:45.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21283580799999999, 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.880018+0000", 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.880018+0000", 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.880017+0000", 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.880025+0000", 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0090000000000007e-06 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089444+0000", 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20941910799999999 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089475+0000", 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1220999999999998e-05 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092752+0000", 2023-09-23T19:58:45.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032761639999999998 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092759+0000", 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1760000000000004e-06 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092854+0000", 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5428999999999997e-05 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.870 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:89556 6.b 6:d4ca9a82:::benchmark_data_smithi135_344736_object89555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:10.886354+0000", 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.869429664000002, 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21580239800000001, 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.871 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 89556 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.886354+0000", 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.886354+0000", 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.872 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.886353+0000", 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:10.886391+0000", 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8807999999999998e-05 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089429+0000", 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.20303739300000001 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.089453+0000", 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4246000000000001e-05 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092162+0000", 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027089599999999998 2023-09-23T19:58:45.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092315+0000", 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015276499999999999 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.092455+0000", 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014033900000000001 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.098974+0000", 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065191720000000002 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.098982+0000", 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9370000000000004e-06 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.102089+0000", 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031064 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.102124+0000", 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5224e-05 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:11.102157+0000", 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3089000000000001e-05 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.877 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:96662 6.b 6:dfd3e477:::benchmark_data_smithi135_344736_object96661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:20.136973+0000", 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 25.618811397999998, 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25554191399999998, 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:58:45.878 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 96662 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.136973+0000", 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.136973+0000", 2023-09-23T19:58:45.879 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.136971+0000", 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.137008+0000", 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7132999999999998e-05 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.137021+0000", 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3026e-05 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.137029+0000", 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8129999999999996e-06 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.141368+0000", 2023-09-23T19:58:45.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043395370000000001 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.282719+0000", 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14135078300000001 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.282884+0000", 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000165112 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.292878+0000", 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0099933639999999994 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.392342+0000", 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.099464583999999995 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:45.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.392421+0000", 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8647999999999996e-05 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.392474+0000", 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2587999999999999e-05 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:20.392515+0000", 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1060000000000003e-05 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.884 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.885 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.885 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:58:45.885 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:58:45.885 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:58:45.885 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:46.322 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:46 smithi135 ceph-mon[213871]: pgmap v490: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 6.4 KiB/s wr, 549 op/s; 32376/288237 objects degraded (11.232%); 29776/288237 objects misplaced (10.330%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T19:58:46.520 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:58:46.541 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:58:46.582 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:58:46.613 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:58:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:46 smithi144 ceph-mon[194833]: pgmap v490: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 6.4 KiB/s wr, 549 op/s; 32376/288237 objects degraded (11.232%); 29776/288237 objects misplaced (10.330%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T19:58:46.637 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:46.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:46.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:46.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:46 smithi135 ceph-mon[221113]: pgmap v490: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 6.4 KiB/s wr, 549 op/s; 32376/288237 objects degraded (11.232%); 29776/288237 objects misplaced (10.330%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T19:58:46.658 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:58:46.745 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:46.771 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:46.798 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:58:46.798 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:58:46.798 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:58:46.798 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:58:46.799 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:58:46.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:58:47.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1875405862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:47.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32376/288237 objects degraded (11.232%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:47.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1875405862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:47.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32376/288237 objects degraded (11.232%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:47.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1875405862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:47.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32376/288237 objects degraded (11.232%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:47.961 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:47.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:47.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:48.066 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:58:48.072 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:48.135 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:58:48.176 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:58:48.264 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:48.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:48.342 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:48 smithi144 ceph-mon[194833]: pgmap v491: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 32427/287172 objects degraded (11.292%); 29601/287172 objects misplaced (10.308%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:48.391 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:48.412 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:48.456 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:48.659 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:58:48.688 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:48.692 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:48 smithi135 ceph-mon[213871]: pgmap v491: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 32427/287172 objects degraded (11.292%); 29601/287172 objects misplaced (10.308%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:48.693 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:48 smithi135 ceph-mon[221113]: pgmap v491: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 32427/287172 objects degraded (11.292%); 29601/287172 objects misplaced (10.308%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:48.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:48.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:48.794 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:48.857 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:49.255 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:49.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:49.361 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:49.417 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:49.662 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:49.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:49.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:49.766 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:49.830 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:49.903 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:49.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:49.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:50.012 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:50.038 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:50.083 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:50.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:50.177 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:58:50.184 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:50.266 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:50 smithi135 ceph-mon[221113]: pgmap v492: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 32468/279573 objects degraded (11.613%); 28862/279573 objects misplaced (10.324%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:50.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:50 smithi135 ceph-mon[213871]: pgmap v492: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 32468/279573 objects degraded (11.613%); 28862/279573 objects misplaced (10.324%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:50.604 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:50 smithi144 ceph-mon[194833]: pgmap v492: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 32468/279573 objects degraded (11.613%); 28862/279573 objects misplaced (10.324%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:50.765 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:50.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:50.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:50.869 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:50.942 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:51.053 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:51.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:51.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:51.161 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:51.192 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:51.285 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:51.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:51.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:51.314 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:58:51.315 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:58:51.390 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:58:51.443 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:51.714 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:58:51.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:51.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:58:51.829 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:51.897 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:51.937 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:52.191 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:52.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:52.298 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:52.340 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:52.476 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:52.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:52.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:52.562 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:58:52.580 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:52.603 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:52 smithi144 ceph-mon[194833]: pgmap v493: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 32510/278514 objects degraded (11.673%); 28692/278514 objects misplaced (10.302%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:52.603 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:52.603 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:52.603 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:52 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32510/278514 objects degraded (11.673%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:52.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:52 smithi135 ceph-mon[213871]: pgmap v493: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 32510/278514 objects degraded (11.673%); 28692/278514 objects misplaced (10.302%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:52.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:52.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:52.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:52 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32510/278514 objects degraded (11.673%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:52.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:52 smithi135 ceph-mon[221113]: pgmap v493: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 32510/278514 objects degraded (11.673%); 28692/278514 objects misplaced (10.302%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T19:58:52.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:52.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:58:52.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:52 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32510/278514 objects degraded (11.673%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:52.663 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:52.776 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:52.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:52.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:52.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:52.955 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:53.125 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:53.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:53.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:53.200 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:58:53.230 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:53.337 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:53.401 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:58:53.409 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 6 pg_num to 26 2023-09-23T19:58:53.512 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:53.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:53.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1718917195' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:58:53.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:58:53.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1042320992' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:53.621 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:53.660 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1718917195' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:58:53.661 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:58:53.661 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1042320992' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:53.661 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1718917195' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:58:53.661 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:58:53.661 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1042320992' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:58:53.663 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:53.975 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:58:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:58:53] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.43.0" 2023-09-23T19:58:54.079 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:54.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:54.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:54.200 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:54.274 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:54.379 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:54.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: pgmap v494: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 32510/277500 objects degraded (11.715%); 28692/277500 objects misplaced (10.339%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1042320992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: osdmap e184: 8 total, 7 up, 7 in 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[221113]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: pgmap v494: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 32510/277500 objects degraded (11.715%); 28692/277500 objects misplaced (10.339%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:58:54.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:54.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T19:58:54.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1042320992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:58:54.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: osdmap e184: 8 total, 7 up, 7 in 2023-09-23T19:58:54.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.390 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.391 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:54 smithi135 ceph-mon[213871]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.484 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:54.511 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:54.552 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:54.554 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: pgmap v494: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 32510/277500 objects degraded (11.715%); 28692/277500 objects misplaced (10.339%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:58:54.554 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:58:54.554 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T19:58:54.554 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1042320992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:58:54.555 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: osdmap e184: 8 total, 7 up, 7 in 2023-09-23T19:58:54.555 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.555 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.555 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.555 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:54 smithi144 ceph-mon[194833]: failed to encode map e184 with expected crc 2023-09-23T19:58:54.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:54.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:54.656 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:54.682 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:54.725 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:54.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:54.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:54.828 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:54.890 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:55.135 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:55.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:55.260 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:55.332 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:55.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:55 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T19:58:55.416 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:55 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T19:58:55.533 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:55.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:55.660 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:55.697 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:55 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T19:58:55.740 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:55.790 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:55.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:55.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:55.851 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:58:55.851 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:58:55.894 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:55.922 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:56.065 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:56.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:56.068 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:58:56.087 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:58:56.107 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:58:56.127 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:58:56.152 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:58:56.168 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:56.172 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:58:56.194 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:56.196 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:58:56.197 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:58:56.197 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:58:56.197 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:58:56.197 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:58:56.197 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:58:56.197 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:58:56.197 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:58:56.197 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:58:56.238 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:56.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:56.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:56.342 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:56.372 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[221113]: pgmap v496: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 32524/271698 objects degraded (11.971%); 28062/271698 objects misplaced (10.328%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T19:58:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[221113]: failed to encode map e185 with expected crc 2023-09-23T19:58:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T19:58:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[221113]: osdmap e185: 8 total, 7 up, 7 in 2023-09-23T19:58:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[221113]: failed to encode map e185 with expected crc 2023-09-23T19:58:56.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3276789077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[213871]: pgmap v496: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 32524/271698 objects degraded (11.971%); 28062/271698 objects misplaced (10.328%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[213871]: failed to encode map e185 with expected crc 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[213871]: osdmap e185: 8 total, 7 up, 7 in 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[213871]: failed to encode map e185 with expected crc 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3276789077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:56.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:56.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:56 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:56.509 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:56.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:56.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:56.619 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:56.650 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:56.770 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:56 smithi144 ceph-mon[194833]: pgmap v496: 99 pgs: 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 32524/271698 objects degraded (11.971%); 28062/271698 objects misplaced (10.328%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T19:58:56.770 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:56 smithi144 ceph-mon[194833]: failed to encode map e185 with expected crc 2023-09-23T19:58:56.770 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:56 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T19:58:56.770 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:56 smithi144 ceph-mon[194833]: osdmap e185: 8 total, 7 up, 7 in 2023-09-23T19:58:56.770 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:56 smithi144 ceph-mon[194833]: failed to encode map e185 with expected crc 2023-09-23T19:58:56.770 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3276789077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:58:56.770 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:56 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:56.770 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:56 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:56.781 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:58:57.142 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:58:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:57 smithi144 ceph-mon[194833]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:57 smithi144 ceph-mon[194833]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:57 smithi144 ceph-mon[194833]: osdmap e186: 8 total, 7 up, 7 in 2023-09-23T19:58:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:57 smithi144 ceph-mon[194833]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:57 smithi144 ceph-mon[194833]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:57 smithi144 ceph-mon[194833]: 6.6 continuing backfill to osd.5 from (168'1333,184'7298] MIN to 184'7298 2023-09-23T19:58:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32524/271698 objects degraded (11.971%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:57 smithi144 ceph-mon[194833]: osdmap e187: 8 total, 7 up, 7 in 2023-09-23T19:58:57.650 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:57.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:57.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:57.689 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:58:57.711 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[213871]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[213871]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[213871]: osdmap e186: 8 total, 7 up, 7 in 2023-09-23T19:58:57.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[213871]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[213871]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[213871]: 6.6 continuing backfill to osd.5 from (168'1333,184'7298] MIN to 184'7298 2023-09-23T19:58:57.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32524/271698 objects degraded (11.971%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:57.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[213871]: osdmap e187: 8 total, 7 up, 7 in 2023-09-23T19:58:57.713 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[221113]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.713 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[221113]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[221113]: osdmap e186: 8 total, 7 up, 7 in 2023-09-23T19:58:57.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[221113]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[221113]: failed to encode map e186 with expected crc 2023-09-23T19:58:57.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[221113]: 6.6 continuing backfill to osd.5 from (168'1333,184'7298] MIN to 184'7298 2023-09-23T19:58:57.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32524/271698 objects degraded (11.971%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:58:57.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:57 smithi135 ceph-mon[221113]: osdmap e187: 8 total, 7 up, 7 in 2023-09-23T19:58:57.734 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:58:57.735 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:58:57.760 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:58:57.804 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:57.855 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:58:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:57.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:58:57.963 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:57.992 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:58.039 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:58.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:58.132 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:58:58.145 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:58:58.174 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:58.261 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:58:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:58.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:58:58.365 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:58.410 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:58.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: pgmap v499: 109 pgs: 10 unknown, 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 32572/270690 objects degraded (12.033%); 27902/270690 objects misplaced (10.308%); 2.4 MiB/s, 39 objects/s recovering 2023-09-23T19:58:58.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:58.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[213871]: osdmap e188: 8 total, 7 up, 7 in 2023-09-23T19:58:58.706 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: pgmap v499: 109 pgs: 10 unknown, 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 32572/270690 objects degraded (12.033%); 27902/270690 objects misplaced (10.308%); 2.4 MiB/s, 39 objects/s recovering 2023-09-23T19:58:58.706 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.706 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.706 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:58:58.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:58:58.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:58.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:58 smithi135 ceph-mon[221113]: osdmap e188: 8 total, 7 up, 7 in 2023-09-23T19:58:58.744 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:58:58.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:58.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:58:58.868 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:58:58.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: pgmap v499: 109 pgs: 10 unknown, 2 active+clean+remapped, 81 active+clean, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 32572/270690 objects degraded (12.033%); 27902/270690 objects misplaced (10.308%); 2.4 MiB/s, 39 objects/s recovering 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: failed to encode map e187 with expected crc 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:58:58.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:58 smithi144 ceph-mon[194833]: osdmap e188: 8 total, 7 up, 7 in 2023-09-23T19:58:58.914 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:59.130 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:58:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:59.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:58:59.239 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:58:59.301 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:59.414 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:58:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:59.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:58:59.521 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:58:59.569 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:59 smithi144 ceph-mon[194833]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.570 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:59 smithi144 ceph-mon[194833]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.570 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:59 smithi144 ceph-mon[194833]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.570 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:58:59 smithi144 ceph-mon[194833]: 6.16 continuing backfill to osd.5 from (168'1333,184'7298] MIN to 184'7298 2023-09-23T19:58:59.580 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:58:59.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:59 smithi135 ceph-mon[213871]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:59 smithi135 ceph-mon[213871]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:59 smithi135 ceph-mon[213871]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:58:59 smithi135 ceph-mon[213871]: 6.16 continuing backfill to osd.5 from (168'1333,184'7298] MIN to 184'7298 2023-09-23T19:58:59.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:59 smithi135 ceph-mon[221113]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:59 smithi135 ceph-mon[221113]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:59 smithi135 ceph-mon[221113]: failed to encode map e188 with expected crc 2023-09-23T19:58:59.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:58:59 smithi135 ceph-mon[221113]: 6.16 continuing backfill to osd.5 from (168'1333,184'7298] MIN to 184'7298 2023-09-23T19:58:59.727 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:58:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:58:59.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:58:59.736 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:58:59.807 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:58:59.807 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T19:58:59.807 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T19:58:59.807 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T19:58:59.807 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:148379 5.e 5:772fcc4e:::benchmark_data_smithi135_284570_object32595:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:42.665408+0000", 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 257.13883745200002, 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.232853012, 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148379 2023-09-23T19:58:59.808 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665408+0000", 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665408+0000", 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.809 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665407+0000", 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665410+0000", 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.65e-06 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.810 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665415+0000", 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2299999999999999e-06 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.665419+0000", 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1540000000000004e-06 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.669141+0000", 2023-09-23T19:58:59.811 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037221590000000001 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.669250+0000", 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000108668 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.669358+0000", 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107997 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.812 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.675806+0000", 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0064474700000000003 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.684988+0000", 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0091826259999999993 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898190+0000", 2023-09-23T19:58:59.813 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.213201737 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898226+0000", 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6353000000000002e-05 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:42.898261+0000", 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4272e-05 2023-09-23T19:58:59.814 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:203736 5.e 5:769acb00:::benchmark_data_smithi135_284570_object87952:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:59.317474+0000", 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 180.48677132700001, 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.270062107, 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.815 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 203736 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317474+0000", 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.816 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317474+0000", 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317473+0000", 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317480+0000", 2023-09-23T19:58:59.817 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.593e-06 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317497+0000", 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6568000000000001e-05 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317502+0000", 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7379999999999999e-06 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.818 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317537+0000", 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4981999999999998e-05 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317662+0000", 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012491800000000001 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:59.819 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.317802+0000", 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000140109 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.320633+0000", 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028309450000000001 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.320859+0000", 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00022572599999999999 2023-09-23T19:58:59.820 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.587439+0000", 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.26657951000000002 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.587492+0000", 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3653999999999998e-05 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.821 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.587536+0000", 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3483999999999999e-05 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:203739 5.e 5:78001ac3:::benchmark_data_smithi135_284570_object87955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:59.322087+0000", 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 180.48215780199999, 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.26868331000000001, 2023-09-23T19:58:59.822 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 203739 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322087+0000", 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.823 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322087+0000", 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322087+0000", 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.824 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322092+0000", 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2299999999999999e-06 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322102+0000", 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0463e-05 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322107+0000", 2023-09-23T19:58:59.825 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8069999999999999e-06 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322142+0000", 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4669999999999998e-05 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322218+0000", 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5876000000000003e-05 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.826 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.322284+0000", 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6440999999999996e-05 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.325790+0000", 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035055339999999998 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.827 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.327103+0000", 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0013134450000000001 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.590673+0000", 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.26356996599999999 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.590725+0000", 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1783000000000002e-05 2023-09-23T19:58:59.828 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.590771+0000", 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5779999999999999e-05 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:203744 5.e 5:756207b1:::benchmark_data_smithi135_284570_object87960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:59.339381+0000", 2023-09-23T19:58:59.829 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 180.46486376600001, 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25202122599999999, 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 203744 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.830 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339381+0000", 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339381+0000", 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339380+0000", 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:59.831 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339386+0000", 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6880000000000002e-06 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339401+0000", 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4447000000000001e-05 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.832 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339408+0000", 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.109e-06 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339450+0000", 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2456999999999997e-05 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339539+0000", 2023-09-23T19:58:59.833 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9030000000000006e-05 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.339615+0000", 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5193999999999999e-05 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.342421+0000", 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028068070000000001 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.834 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.342614+0000", 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000192652 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.591327+0000", 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.24871296400000001 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.835 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.591370+0000", 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2577999999999998e-05 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:59.591402+0000", 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2910999999999997e-05 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.836 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:222016 5.e 5:7c7c053d:::benchmark_data_smithi135_284570_object106232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:24.285082+0000", 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 155.51916331499999, 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.242105458, 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 222016 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.837 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285082+0000", 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285082+0000", 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.838 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285080+0000", 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285087+0000", 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7819999999999996e-06 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285099+0000", 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2251e-05 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.839 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285105+0000", 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9390000000000002e-06 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285186+0000", 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0529000000000001e-05 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.840 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285291+0000", 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105424 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285396+0000", 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010489600000000001 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.292362+0000", 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0069656470000000002 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.841 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.305916+0000", 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013554372 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527116+0000", 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.22120009299999999 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.842 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527158+0000", 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1838999999999997e-05 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527187+0000", 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9204999999999999e-05 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.843 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:222020 5.e 5:7ff5b4db:::benchmark_data_smithi135_284570_object106236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:24.285119+0000", 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 155.519125836, 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.24215035400000001, 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 222020 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.844 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285119+0000", 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285119+0000", 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285119+0000", 2023-09-23T19:58:59.845 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285121+0000", 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3520000000000001e-06 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285127+0000", 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5720000000000002e-06 2023-09-23T19:58:59.846 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.285130+0000", 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.614e-06 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.288912+0000", 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037813339999999999 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.847 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.289029+0000", 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000117748 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.289144+0000", 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011466100000000001 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.292514+0000", 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033697620000000001 2023-09-23T19:58:59.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.310303+0000", 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.017789503000000002 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527222+0000", 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.216918746 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.849 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527248+0000", 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6203e-05 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:24.527270+0000", 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1217e-05 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.850 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116233 6.12 6:4c8078d8:::benchmark_data_smithi135_344736_object13577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:56.627662+0000", 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 3.1765834970000002, 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.751567815, 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116233 2023-09-23T19:58:59.851 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.627662+0000", 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.627662+0000", 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.852 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.627660+0000", 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.627671+0000", 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1299e-05 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.853 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.627686+0000", 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4082e-05 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.627691+0000", 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1189999999999996e-06 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.627727+0000", 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6127999999999997e-05 2023-09-23T19:58:59.854 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.627738+0000", 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0992000000000001e-05 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.315745+0000", 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.68800687000000005 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.855 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.315754+0000", 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6809999999999993e-06 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.341202+0000", 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.025447926 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.341332+0000", 2023-09-23T19:58:59.856 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000129405 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.341474+0000", 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000142428 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.377119+0000", 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.035644985999999997 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.857 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.377161+0000", 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1950000000000003e-05 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.377196+0000", 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5170999999999999e-05 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.858 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.377265+0000", 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8973000000000005e-05 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.379229+0000", 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0019641620000000002 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.859 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116238 6.12 6:49e60f51:::benchmark_data_smithi135_344736_object13582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.370673+0000", 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.433571514, 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.205216627, 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116238 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.860 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.370673+0000", 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.370673+0000", 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.861 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.370672+0000", 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.370683+0000", 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1554e-05 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.862 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.370703+0000", 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9584000000000001e-05 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.370714+0000", 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0621e-05 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.372572+0000", 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001858478 2023-09-23T19:58:59.863 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.372580+0000", 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4789999999999999e-06 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.379276+0000", 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0066960920000001 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.864 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.379375+0000", 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8918000000000004e-05 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.379563+0000", 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000188872 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.384618+0000", 2023-09-23T19:58:59.865 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0050545620000000003 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386536+0000", 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0019182870000000001 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.573551+0000", 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18701447800000001 2023-09-23T19:58:59.866 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.573619+0000", 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8559999999999994e-05 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.575890+0000", 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022708350000000001 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.867 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116239 6.12 6:48c2055c:::benchmark_data_smithi135_344736_object13583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.380536+0000", 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.4237086090000002, 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1974479040000001, 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.868 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116239 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.380536+0000", 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.380536+0000", 2023-09-23T19:58:59.869 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.380535+0000", 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.380544+0000", 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7110000000000007e-06 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.870 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.380567+0000", 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2589e-05 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.380576+0000", 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1810000000000007e-06 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.871 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.381740+0000", 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0011644769999999999 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.381754+0000", 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3295000000000001e-05 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.380382+0000", 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99862792899999997 2023-09-23T19:58:59.872 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.380470+0000", 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7960000000000005e-05 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.380608+0000", 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000138374 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.873 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.384627+0000", 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040191080000000004 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386578+0000", 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0019505659999999999 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.874 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.575954+0000", 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18937598899999999 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.575990+0000", 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6505999999999999e-05 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.577984+0000", 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0019940909999999999 2023-09-23T19:58:59.875 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116242 6.16 6:6c93e87a:::benchmark_data_smithi135_344736_object13586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.396114+0000", 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.4081307320000001, 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.96333833099999999, 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.876 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116242 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396114+0000", 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.877 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396114+0000", 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396112+0000", 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999976 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.878 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396122+0000", 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5429999999999999e-06 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396154+0000", 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2289999999999997e-05 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396162+0000", 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9510000000000006e-06 2023-09-23T19:58:59.879 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396594+0000", 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043217199999999998 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396604+0000", 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0444e-05 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.880 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.353542+0000", 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.95693777899999999 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.353622+0000", 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0143999999999997e-05 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,5,7", 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.353802+0000", 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017964399999999999 2023-09-23T19:58:59.881 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.356089+0000", 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022868490000000001 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.356121+0000", 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2419999999999998e-05 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.882 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359282+0000", 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031603159999999998 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359317+0000", 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5484000000000001e-05 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359351+0000", 2023-09-23T19:58:59.883 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4334e-05 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359453+0000", 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101215 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116243 6.12 6:4852d320:::benchmark_data_smithi135_344736_object13587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.396409+0000", 2023-09-23T19:58:59.884 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.4078356460000001, 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1840158940000001, 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116243 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396409+0000", 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396409+0000", 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396409+0000", 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:58:59.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396413+0000", 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5970000000000002e-06 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396422+0000", 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8550000000000008e-06 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.887 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.396427+0000", 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1599999999999997e-06 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.401076+0000", 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046489569999999996 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.401085+0000", 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7430000000000008e-06 2023-09-23T19:58:59.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.381433+0000", 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98034775100000004 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.381547+0000", 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114121 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.381686+0000", 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013930299999999999 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386486+0000", 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004799495 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.388578+0000", 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002091861 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.578051+0000", 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18947338799999999 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.578114+0000", 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3054999999999999e-05 2023-09-23T19:58:59.891 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.580425+0000", 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002311249 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116276 6.16 6:6f39d2be:::benchmark_data_smithi135_344736_object13620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.892 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.467843+0000", 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.3364022250000001, 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.89256164100000002, 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116276 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.893 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.467843+0000", 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.467843+0000", 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.467841+0000", 2023-09-23T19:58:59.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.467853+0000", 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2353e-05 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.467882+0000", 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8075000000000001e-05 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.467889+0000", 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3370000000000002e-06 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.467943+0000", 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3662999999999998e-05 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.467952+0000", 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1649999999999995e-06 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.354650+0000", 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.88669806600000001 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.354779+0000", 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012924200000000001 2023-09-23T19:58:59.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,5,7", 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.354986+0000", 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020715600000000001 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.356115+0000", 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001128743 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.899 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.356126+0000", 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1464000000000001e-05 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359535+0000", 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003408786 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.901 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360255+0000", 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00071958700000000005 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360309+0000", 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3973e-05 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360404+0000", 2023-09-23T19:58:59.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5727e-05 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116283 6.12 6:48687296:::benchmark_data_smithi135_344736_object13627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.480098+0000", 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.3241474530000001, 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1026167769999999, 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.903 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116283 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.480098+0000", 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.480098+0000", 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.480096+0000", 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.480102+0000", 2023-09-23T19:58:59.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0240000000000003e-06 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.480122+0000", 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0041e-05 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.480128+0000", 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.463e-06 2023-09-23T19:58:59.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.480158+0000", 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.04e-05 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.480165+0000", 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2339999999999997e-06 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.907 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.382441+0000", 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.90227569100000005 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.382554+0000", 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113022 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.382720+0000", 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016577399999999999 2023-09-23T19:58:59.908 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386587+0000", 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038669020000000002 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.389474+0000", 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028868349999999999 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.580487+0000", 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19101317200000001 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.580546+0000", 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8662000000000002e-05 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.582714+0000", 2023-09-23T19:58:59.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021688269999999999 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116360 6.16 6:6eb66079:::benchmark_data_smithi135_344736_object13704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.604377+0000", 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.1998683059999999, 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.75629612999999996, 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.911 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116360 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.604377+0000", 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.912 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.604377+0000", 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.604375+0000", 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.604383+0000", 2023-09-23T19:58:59.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3839999999999997e-06 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.604397+0000", 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4019000000000001e-05 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.604406+0000", 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8389999999999996e-06 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.604461+0000", 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4582e-05 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.604473+0000", 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1759999999999999e-05 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.356161+0000", 2023-09-23T19:58:59.915 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.75168840800000003 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.356297+0000", 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000136422 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,5,7", 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.356485+0000", 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018783100000000001 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.916 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359244+0000", 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027587520000000002 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359496+0000", 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00025150299999999998 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360221+0000", 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00072519000000000001 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360506+0000", 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00028557799999999998 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360555+0000", 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9076999999999998e-05 2023-09-23T19:58:59.918 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360673+0000", 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000117471 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116386 6.12 6:4a345c75:::benchmark_data_smithi135_344736_object13730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.664300+0000", 2023-09-23T19:58:59.919 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.1399448919999999, 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.92013510899999995, 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116386 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.920 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.664300+0000", 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.664300+0000", 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.664299+0000", 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:58:59.921 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.664304+0000", 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5049999999999998e-06 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.664315+0000", 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0190000000000001e-05 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.922 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.664319+0000", 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3590000000000001e-06 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.664352+0000", 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3318999999999997e-05 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.664360+0000", 2023-09-23T19:58:59.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9200000000000004e-06 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.383421+0000", 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.71906127200000003 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.383548+0000", 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000126802 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.383727+0000", 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000178794 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386590+0000", 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028633790000000001 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.391201+0000", 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046109710000000002 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.582774+0000", 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19157231 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.582820+0000", 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6595999999999999e-05 2023-09-23T19:58:59.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.584435+0000", 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001614877 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116404 6.16 6:689dd997:::benchmark_data_smithi135_344736_object13748:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.696278+0000", 2023-09-23T19:58:59.927 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.107967081, 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.66461694299999996, 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116404 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.696278+0000", 2023-09-23T19:58:59.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.696278+0000", 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.696276+0000", 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.696283+0000", 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4409999999999998e-06 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.696298+0000", 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5139e-05 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.696304+0000", 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8499999999999999e-06 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.696358+0000", 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4048999999999997e-05 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.696366+0000", 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2689999999999999e-06 2023-09-23T19:58:59.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.357444+0000", 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.66107738699999996 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.357554+0000", 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011017 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,5,7", 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.357743+0000", 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018900300000000001 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359252+0000", 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001509767 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.359779+0000", 2023-09-23T19:58:59.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00052635300000000002 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360463+0000", 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00068413000000000002 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360725+0000", 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000262188 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360782+0000", 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6413999999999999e-05 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360895+0000", 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113304 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116410 6.16 6:6adfaaed:::benchmark_data_smithi135_344736_object13754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.705180+0000", 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.099064861, 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.66285027900000004, 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116410 2023-09-23T19:58:59.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.705180+0000", 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.705180+0000", 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.705179+0000", 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.705184+0000", 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7990000000000001e-06 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.705193+0000", 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3579999999999992e-06 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.705198+0000", 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2020000000000003e-06 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.705252+0000", 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3943999999999997e-05 2023-09-23T19:58:59.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.705262+0000", 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4730000000000006e-06 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.360941+0000", 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.65567950200000003 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.361025+0000", 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3981999999999997e-05 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,5,7", 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.361226+0000", 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020032700000000001 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.364644+0000", 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0034184720000000001 2023-09-23T19:58:59.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.366379+0000", 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001735241 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.367043+0000", 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00066396300000000005 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.367881+0000", 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00083731300000000001 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.367931+0000", 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0024000000000001e-05 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.368030+0000", 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9844000000000002e-05 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116411 6.12 6:499faa02:::benchmark_data_smithi135_344736_object13755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.707287+0000", 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.0969575269999998, 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.878604724, 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.944 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116411 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.707287+0000", 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.945 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.707287+0000", 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.707286+0000", 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.707294+0000", 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3529999999999997e-06 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.707308+0000", 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4285e-05 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.707313+0000", 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2039999999999996e-06 2023-09-23T19:58:59.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.707356+0000", 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2760999999999999e-05 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.707366+0000", 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0267e-05 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.384662+0000", 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.67729614999999999 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.384748+0000", 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5787999999999999e-05 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.384896+0000", 2023-09-23T19:58:59.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014836800000000001 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386598+0000", 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001701401 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.391258+0000", 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046600030000000002 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.584481+0000", 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19322326500000001 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.584517+0000", 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5388000000000001e-05 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.585892+0000", 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001375696 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116425 6.12 6:4ec266a4:::benchmark_data_smithi135_344736_object13769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.782810+0000", 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.0214350620000001, 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.80475855200000002, 2023-09-23T19:58:59.952 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116425 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.782810+0000", 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.782810+0000", 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.782809+0000", 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.782815+0000", 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0650000000000004e-06 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.782828+0000", 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3342e-05 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.782833+0000", 2023-09-23T19:58:59.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8799999999999999e-06 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.782876+0000", 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3355000000000001e-05 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.782885+0000", 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5930000000000006e-06 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.385607+0000", 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.60272246399999996 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.385742+0000", 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013478200000000001 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.385897+0000", 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000155019 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.390120+0000", 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0042230499999999999 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.391295+0000", 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001174977 2023-09-23T19:58:59.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.585963+0000", 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.194668284 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.586029+0000", 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5506999999999995e-05 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.587568+0000", 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001539662 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116439 6.12 6:4eb21f1e:::benchmark_data_smithi135_344736_object13783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:57.899965+0000", 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 1.904280143, 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.68924390599999996, 2023-09-23T19:58:59.960 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116439 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.899965+0000", 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.899965+0000", 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.899964+0000", 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.899970+0000", 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9390000000000002e-06 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.899984+0000", 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4819e-05 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.899993+0000", 2023-09-23T19:58:59.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3319999999999992e-06 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.900028+0000", 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5271999999999997e-05 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.900034+0000", 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9179999999999999e-06 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386626+0000", 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.48659244099999999 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386722+0000", 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6185000000000002e-05 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:58:59.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.386852+0000", 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000129083 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.390129+0000", 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003277805 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.391332+0000", 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001202582 2023-09-23T19:58:59.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.587634+0000", 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.19630187499999999 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.587671+0000", 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7595999999999997e-05 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:58:59.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:58:59.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:58.589209+0000", 2023-09-23T19:58:59.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001537339 2023-09-23T19:58:59.968 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.968 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.968 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.968 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:58:59.968 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:58:59.968 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:58:59.968 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:58:59.969 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:58:59.969 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:58:59.969 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T19:58:59.969 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T19:58:59.999 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:00.407 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:00.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:00.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:00.523 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:00.632 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:00.717 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:00.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:00 smithi135 ceph-mon[213871]: pgmap v502: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+clean+remapped, 83 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 26333/267021 objects degraded (9.862%); 30156/267021 objects misplaced (11.293%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T19:59:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:00 smithi135 ceph-mon[221113]: pgmap v502: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+clean+remapped, 83 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 26333/267021 objects degraded (9.862%); 30156/267021 objects misplaced (11.293%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T19:59:00.806 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:00.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:00 smithi144 ceph-mon[194833]: pgmap v502: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+clean+remapped, 83 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 26333/267021 objects degraded (9.862%); 30156/267021 objects misplaced (11.293%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T19:59:00.913 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:59:00.925 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:00.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:00.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:01.042 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:01.091 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:01.168 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:01.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:01.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:01.276 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:01.369 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:01.426 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:02.212 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:02.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:02.325 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:02.386 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:02.477 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:02.485 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:59:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:02.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:02.596 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:02.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:02 smithi144 ceph-mon[194833]: pgmap v503: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 peering, 2 active+clean+remapped, 84 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 31951/266391 objects degraded (11.994%); 24446/266391 objects misplaced (9.177%); 6.1 MiB/s, 99 objects/s recovering 2023-09-23T19:59:02.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3323538569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:02.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:02 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:02.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:02 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 31951/266391 objects degraded (11.994%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:02.675 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:02.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:02 smithi135 ceph-mon[213871]: pgmap v503: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 peering, 2 active+clean+remapped, 84 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 31951/266391 objects degraded (11.994%); 24446/266391 objects misplaced (9.177%); 6.1 MiB/s, 99 objects/s recovering 2023-09-23T19:59:02.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3323538569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:02 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:02 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 31951/266391 objects degraded (11.994%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:02.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:02 smithi135 ceph-mon[221113]: pgmap v503: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 peering, 2 active+clean+remapped, 84 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 31951/266391 objects degraded (11.994%); 24446/266391 objects misplaced (9.177%); 6.1 MiB/s, 99 objects/s recovering 2023-09-23T19:59:02.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3323538569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:02.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:02 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:02.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:02 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 31951/266391 objects degraded (11.994%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:03.490 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44942.0:119803 6.15 e189/185 6:acba076d:::benchmark_data_smithi135_344736_object17147:head v 189'7313, mlcod=168'1034)", 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:03.478720+0000", 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.011294448, 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011324979000000001, 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.491 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.478720+0000", 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.478720+0000", 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:03.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.478719+0000", 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.478727+0000", 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8609999999999998e-06 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.478741+0000", 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4141000000000001e-05 2023-09-23T19:59:03.493 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.478747+0000", 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.446e-06 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.478784+0000", 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7370000000000003e-05 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.494 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.478807+0000", 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3034999999999999e-05 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44942.0:119800 6.5 e189/186 6:a6fc8d0f:::benchmark_data_smithi135_344736_object17144:head v 189'7312, mlcod=168'1032)", 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:03.484141+0000", 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0058740959999999997, 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0059732129999999998, 2023-09-23T19:59:03.495 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484141+0000", 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484141+0000", 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484140+0000", 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484146+0000", 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9030000000000001e-06 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484154+0000", 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7179999999999996e-06 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484159+0000", 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8829999999999997e-06 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484254+0000", 2023-09-23T19:59:03.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5798999999999995e-05 2023-09-23T19:59:03.499 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.499 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.499 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:03.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484281+0000", 2023-09-23T19:59:03.499 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6937e-05 2023-09-23T19:59:03.499 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.499 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:03.499 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44942.0:119798 6.5 e189/186 6:a7763a2f:::benchmark_data_smithi135_344736_object17142:head v 189'7313, mlcod=168'1032)", 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:03.484171+0000", 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0058434139999999999, 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.006003314, 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.500 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484171+0000", 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484171+0000", 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.501 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.502 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:03.502 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484171+0000", 2023-09-23T19:59:03.502 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:03.502 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.502 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.502 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:03.502 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484176+0000", 2023-09-23T19:59:03.502 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.78e-06 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484182+0000", 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8390000000000003e-06 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.484185+0000", 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9099999999999998e-06 2023-09-23T19:59:03.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.504 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:03.504 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.488022+0000", 2023-09-23T19:59:03.504 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038367570000000001 2023-09-23T19:59:03.504 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.504 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:03.504 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.488045+0000", 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3044000000000002e-05 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44942.0:119812 6.6 e189/185 6:66e22de6:::benchmark_data_smithi135_344736_object17156:head v 189'7406, mlcod=189'7403)", 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:03.488906+0000", 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0011087169999999999, 2023-09-23T19:59:03.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0013285630000000001, 2023-09-23T19:59:03.506 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:03.506 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T19:59:03.506 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:03.506 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.506 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:03.506 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.488906+0000", 2023-09-23T19:59:03.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.488906+0000", 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.488905+0000", 2023-09-23T19:59:03.507 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.488914+0000", 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7399999999999999e-06 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:03.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.488949+0000", 2023-09-23T19:59:03.509 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4686e-05 2023-09-23T19:59:03.509 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.509 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.509 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:03.509 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.488956+0000", 2023-09-23T19:59:03.509 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3780000000000003e-06 2023-09-23T19:59:03.509 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.510 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.510 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:03.510 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.489033+0000", 2023-09-23T19:59:03.510 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6979999999999998e-05 2023-09-23T19:59:03.510 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.510 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.510 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:03.511 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.489053+0000", 2023-09-23T19:59:03.511 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9837e-05 2023-09-23T19:59:03.511 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.511 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:03.511 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.511 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.512 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.512 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:119796 6.17 6:e99d26b4:::benchmark_data_smithi135_344736_object17140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-09-23T19:59:03.512 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:03.463664+0000", 2023-09-23T19:59:03.512 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.026350255999999999, 2023-09-23T19:59:03.512 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.026634628, 2023-09-23T19:59:03.512 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:03.512 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:59:03.512 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 119796 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.463664+0000", 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.513 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.463664+0000", 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.463663+0000", 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.463669+0000", 2023-09-23T19:59:03.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7060000000000001e-06 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.463679+0000", 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0128e-05 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.463684+0000", 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2240000000000003e-06 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:03.516 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.463727+0000", 2023-09-23T19:59:03.516 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2907000000000001e-05 2023-09-23T19:59:03.516 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.516 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.516 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:03.516 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.464330+0000", 2023-09-23T19:59:03.516 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00060322200000000005 2023-09-23T19:59:03.516 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.485788+0000", 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021457339999999998 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.485913+0000", 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012553900000000001 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:03.517 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,4", 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:03.486273+0000", 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00035963299999999999 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 5 2023-09-23T19:59:03.518 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:59:03.519 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:03 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:03.519 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:03 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:03.519 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:03 smithi144 ceph-mon[194833]: osdmap e189: 8 total, 7 up, 7 in 2023-09-23T19:59:03.519 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:03 smithi144 ceph-mon[194833]: failed to encode map e189 with expected crc 2023-09-23T19:59:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:59:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:59:03] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.43.0" 2023-09-23T19:59:03.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:03 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:03.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:03 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:03 smithi135 ceph-mon[213871]: osdmap e189: 8 total, 7 up, 7 in 2023-09-23T19:59:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:03 smithi135 ceph-mon[213871]: failed to encode map e189 with expected crc 2023-09-23T19:59:03.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:03 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:03.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:03 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:03.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:03 smithi135 ceph-mon[221113]: osdmap e189: 8 total, 7 up, 7 in 2023-09-23T19:59:03.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:03 smithi135 ceph-mon[221113]: failed to encode map e189 with expected crc 2023-09-23T19:59:04.589 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:59:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:04 smithi144 ceph-mon[194833]: pgmap v505: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 1 active+recovering, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 31933/265554 objects degraded (12.025%); 27011/265554 objects misplaced (10.172%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T19:59:04.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:04 smithi135 ceph-mon[221113]: pgmap v505: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 1 active+recovering, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 31933/265554 objects degraded (12.025%); 27011/265554 objects misplaced (10.172%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T19:59:04.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:04 smithi135 ceph-mon[213871]: pgmap v505: 109 pgs: 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 1 active+recovering, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 31933/265554 objects degraded (12.025%); 27011/265554 objects misplaced (10.172%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T19:59:05.159 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:05.286 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:05.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:05.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:05.408 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:05.493 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:06.167 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:06.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:06.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:06.271 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:06.306 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:06.529 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:06.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:06.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:06.643 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:06.722 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:06.782 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:59:06.831 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:06.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:06.836 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:06 smithi135 ceph-mon[213871]: pgmap v506: 109 pgs: 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 4 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 34046/258330 objects degraded (13.179%); 23671/258330 objects misplaced (9.163%); 13 MiB/s, 202 objects/s recovering 2023-09-23T19:59:06.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:06.838 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:06 smithi135 ceph-mon[221113]: pgmap v506: 109 pgs: 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 4 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 34046/258330 objects degraded (13.179%); 23671/258330 objects misplaced (9.163%); 13 MiB/s, 202 objects/s recovering 2023-09-23T19:59:06.840 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:59:06.874 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:59:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:06 smithi144 ceph-mon[194833]: pgmap v506: 109 pgs: 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 4 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 34046/258330 objects degraded (13.179%); 23671/258330 objects misplaced (9.163%); 13 MiB/s, 202 objects/s recovering 2023-09-23T19:59:06.925 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:59:06.938 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:06.960 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:59:06.988 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:07.008 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:59:07.044 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:59:07.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:59:07.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:59:07.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:59:07.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:59:07.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:59:07.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:59:07.045 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:59:07.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:59:07.116 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:07.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:07.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:07.221 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:07.282 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:07.351 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:07.351 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:07.407 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:07.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:07.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:07.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:07 smithi135 conmon[213845]: 2023-09-23T19:59:07.201+0000 7f9708921700 -1 mon.a@0(leader).osd e189 definitely_dead 0 2023-09-23T19:59:07.512 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:07.594 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:07.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:07 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/86920966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:07.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:59:07.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:07.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:07 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1453794379' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T19:59:07.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:07 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T19:59:07.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:07 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 34046/258330 objects degraded (13.179%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:07.669 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:07.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:07.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:07.773 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:07.840 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:07.888 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:07.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:07.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:07.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/86920966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:07.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:59:07.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:07.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1453794379' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T19:59:07.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T19:59:07.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 34046/258330 objects degraded (13.179%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/86920966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T19:59:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1453794379' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T19:59:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T19:59:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:07 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 34046/258330 objects degraded (13.179%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:07.991 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:08.049 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:08.168 INFO:teuthology.orchestra.run.smithi135.stderr:marked in osd.3. 2023-09-23T19:59:08.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:08.249 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:08.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:08.282 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:59:08.350 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:08.377 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:08.399 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:59:08.419 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:08.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:08.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:08.526 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:08.554 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:08 smithi144 ceph-mon[194833]: pgmap v507: 109 pgs: 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 4 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 33851/257160 objects degraded (13.163%); 23671/257160 objects misplaced (9.205%); 11 MiB/s, 183 objects/s recovering 2023-09-23T19:59:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:08 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T19:59:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:08 smithi144 ceph-mon[194833]: osdmap e190: 8 total, 7 up, 8 in 2023-09-23T19:59:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:08 smithi144 ceph-mon[194833]: failed to encode map e190 with expected crc 2023-09-23T19:59:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:08 smithi135 ceph-mon[221113]: pgmap v507: 109 pgs: 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 4 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 33851/257160 objects degraded (13.163%); 23671/257160 objects misplaced (9.205%); 11 MiB/s, 183 objects/s recovering 2023-09-23T19:59:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:08 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T19:59:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:08 smithi135 ceph-mon[221113]: osdmap e190: 8 total, 7 up, 8 in 2023-09-23T19:59:08.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:08 smithi135 ceph-mon[221113]: failed to encode map e190 with expected crc 2023-09-23T19:59:08.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:08 smithi135 ceph-mon[213871]: pgmap v507: 109 pgs: 9 active+recovery_wait+degraded, 2 active+clean+remapped, 84 active+clean, 4 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 33851/257160 objects degraded (13.163%); 23671/257160 objects misplaced (9.205%); 11 MiB/s, 183 objects/s recovering 2023-09-23T19:59:08.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:08 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T19:59:08.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:08 smithi135 ceph-mon[213871]: osdmap e190: 8 total, 7 up, 8 in 2023-09-23T19:59:08.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:08 smithi135 ceph-mon[213871]: failed to encode map e190 with expected crc 2023-09-23T19:59:08.761 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:09.048 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:09.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:09.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:09.151 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:09.210 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:09.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:09 smithi144 ceph-mon[194833]: failed to encode map e190 with expected crc 2023-09-23T19:59:09.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:09 smithi144 ceph-mon[194833]: pgmap v509: 109 pgs: 7 remapped+peering, 18 peering, 3 active+recovery_wait+degraded, 1 active+clean+remapped, 72 active+clean, 4 active+undersized, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 9590/249501 objects degraded (3.844%); 7529/249501 objects misplaced (3.018%); 17 MiB/s, 275 objects/s recovering 2023-09-23T19:59:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:09 smithi144 ceph-mon[194833]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:09 smithi144 ceph-mon[194833]: osdmap e191: 8 total, 7 up, 8 in 2023-09-23T19:59:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:09 smithi144 ceph-mon[194833]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:09 smithi144 ceph-mon[194833]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:09 smithi144 ceph-mon[194833]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.793 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[221113]: failed to encode map e190 with expected crc 2023-09-23T19:59:09.793 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[221113]: pgmap v509: 109 pgs: 7 remapped+peering, 18 peering, 3 active+recovery_wait+degraded, 1 active+clean+remapped, 72 active+clean, 4 active+undersized, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 9590/249501 objects degraded (3.844%); 7529/249501 objects misplaced (3.018%); 17 MiB/s, 275 objects/s recovering 2023-09-23T19:59:09.793 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[221113]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.793 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[221113]: osdmap e191: 8 total, 7 up, 8 in 2023-09-23T19:59:09.793 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[221113]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.793 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[221113]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.794 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[221113]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.794 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[213871]: failed to encode map e190 with expected crc 2023-09-23T19:59:09.794 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[213871]: pgmap v509: 109 pgs: 7 remapped+peering, 18 peering, 3 active+recovery_wait+degraded, 1 active+clean+remapped, 72 active+clean, 4 active+undersized, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 9590/249501 objects degraded (3.844%); 7529/249501 objects misplaced (3.018%); 17 MiB/s, 275 objects/s recovering 2023-09-23T19:59:09.794 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[213871]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.794 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[213871]: osdmap e191: 8 total, 7 up, 8 in 2023-09-23T19:59:09.794 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[213871]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.794 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[213871]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.794 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:09 smithi135 ceph-mon[213871]: failed to encode map e191 with expected crc 2023-09-23T19:59:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:59:09.795 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:59:09.978 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:09.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:10.067 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T19:59:10.081 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:10.109 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:10.223 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:10.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:10.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:10.238 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:10.328 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:10.419 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:10.489 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:10.489 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:10.575 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:10.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:10.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:10.662 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[213871]: failed to encode map e191 with expected crc 2023-09-23T19:59:10.662 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[213871]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[213871]: osdmap e192: 8 total, 7 up, 8 in 2023-09-23T19:59:10.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[213871]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[213871]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.664 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[221113]: failed to encode map e191 with expected crc 2023-09-23T19:59:10.664 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[221113]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.664 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[221113]: osdmap e192: 8 total, 7 up, 8 in 2023-09-23T19:59:10.664 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[221113]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.664 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:10 smithi135 ceph-mon[221113]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.679 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:10.748 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:10.865 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:10.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:10.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:10.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:10 smithi144 ceph-mon[194833]: failed to encode map e191 with expected crc 2023-09-23T19:59:10.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:10 smithi144 ceph-mon[194833]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:10 smithi144 ceph-mon[194833]: osdmap e192: 8 total, 7 up, 8 in 2023-09-23T19:59:10.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:10 smithi144 ceph-mon[194833]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:10 smithi144 ceph-mon[194833]: failed to encode map e192 with expected crc 2023-09-23T19:59:10.972 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:11.000 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:11.046 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:11.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:11.150 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:11.192 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:11.389 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:11.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:11.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:11.455 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:59:11.496 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:11.543 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:11.592 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:11.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:11.700 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:11.726 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:11.767 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:11.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:11.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:11.796 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:59:11.838 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:11.844 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: pgmap v512: 109 pgs: 10 remapped+peering, 19 peering, 1 active+recovery_wait+degraded, 71 active+clean, 4 active+undersized, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 3932/248787 objects degraded (1.580%); 7529/248787 objects misplaced (3.026%); 14 MiB/s, 231 objects/s recovering 2023-09-23T19:59:11.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: osdmap e193: 8 total, 7 up, 8 in 2023-09-23T19:59:11.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[221113]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: pgmap v512: 109 pgs: 10 remapped+peering, 19 peering, 1 active+recovery_wait+degraded, 71 active+clean, 4 active+undersized, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 3932/248787 objects degraded (1.580%); 7529/248787 objects misplaced (3.026%); 14 MiB/s, 231 objects/s recovering 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: osdmap e193: 8 total, 7 up, 8 in 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.846 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:11 smithi135 ceph-mon[213871]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.871 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: failed to encode map e192 with expected crc 2023-09-23T19:59:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: pgmap v512: 109 pgs: 10 remapped+peering, 19 peering, 1 active+recovery_wait+degraded, 71 active+clean, 4 active+undersized, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 3932/248787 objects degraded (1.580%); 7529/248787 objects misplaced (3.026%); 14 MiB/s, 231 objects/s recovering 2023-09-23T19:59:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: osdmap e193: 8 total, 7 up, 8 in 2023-09-23T19:59:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:11 smithi144 ceph-mon[194833]: failed to encode map e193 with expected crc 2023-09-23T19:59:11.957 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:12.270 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:12.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:12.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:12.383 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:12.458 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:12.550 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:12 smithi135 ceph-mon[213871]: failed to encode map e193 with expected crc 2023-09-23T19:59:12.708 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:12.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:12.812 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:12.817 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:12.871 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:12 smithi135 ceph-mon[213871]: osdmap e194: 8 total, 7 up, 8 in 2023-09-23T19:59:12.871 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 3932/248787 objects degraded (1.580%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:12.871 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:12 smithi135 ceph-mon[221113]: failed to encode map e193 with expected crc 2023-09-23T19:59:12.871 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:12 smithi135 ceph-mon[221113]: osdmap e194: 8 total, 7 up, 8 in 2023-09-23T19:59:12.872 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 3932/248787 objects degraded (1.580%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:12.875 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:12 smithi144 ceph-mon[194833]: failed to encode map e193 with expected crc 2023-09-23T19:59:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:12 smithi144 ceph-mon[194833]: osdmap e194: 8 total, 7 up, 8 in 2023-09-23T19:59:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 3932/248787 objects degraded (1.580%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:13.116 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:13.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:13.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:13.225 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:13.261 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:13.373 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:13.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:13.478 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:13.544 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:13.603 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:59:13.715 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:13.715 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:13.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:13 smithi135 ceph-mon[221113]: failed to encode map e194 with expected crc 2023-09-23T19:59:13.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:13 smithi135 ceph-mon[221113]: pgmap v515: 109 pgs: 6 remapped+peering, 21 peering, 73 active+clean, 4 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 2129/247542 objects degraded (0.860%); 5273/247542 objects misplaced (2.130%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T19:59:13.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2414062045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:13.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:13 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:13.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:13 smithi135 ceph-mon[213871]: failed to encode map e194 with expected crc 2023-09-23T19:59:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:13 smithi135 ceph-mon[213871]: pgmap v515: 109 pgs: 6 remapped+peering, 21 peering, 73 active+clean, 4 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 2129/247542 objects degraded (0.860%); 5273/247542 objects misplaced (2.130%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T19:59:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2414062045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:13.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:13 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:13.724 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:59:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:59:13] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-09-23T19:59:13.741 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:13 smithi144 ceph-mon[194833]: failed to encode map e194 with expected crc 2023-09-23T19:59:13.741 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:13 smithi144 ceph-mon[194833]: pgmap v515: 109 pgs: 6 remapped+peering, 21 peering, 73 active+clean, 4 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 2129/247542 objects degraded (0.860%); 5273/247542 objects misplaced (2.130%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T19:59:13.741 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2414062045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:13.741 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:13 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:14.496 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T19:59:14.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:14 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:14.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:14 smithi144 ceph-mon[194833]: failed to encode map e195 with expected crc 2023-09-23T19:59:14.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:14 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:14.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:14 smithi144 ceph-mon[194833]: osdmap e195: 8 total, 7 up, 8 in 2023-09-23T19:59:14.885 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:14 smithi144 ceph-mon[194833]: failed to encode map e195 with expected crc 2023-09-23T19:59:14.892 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:14.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:14.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[221113]: failed to encode map e195 with expected crc 2023-09-23T19:59:14.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[221113]: osdmap e195: 8 total, 7 up, 8 in 2023-09-23T19:59:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[221113]: failed to encode map e195 with expected crc 2023-09-23T19:59:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[213871]: failed to encode map e195 with expected crc 2023-09-23T19:59:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[213871]: osdmap e195: 8 total, 7 up, 8 in 2023-09-23T19:59:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:14 smithi135 ceph-mon[213871]: failed to encode map e195 with expected crc 2023-09-23T19:59:14.974 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:14.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:14.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:15.068 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T19:59:15.068 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:59:15.068 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T19:59:15.068 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T19:59:15.085 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:15.138 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:15.803 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:15.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:15.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:15.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:15 smithi144 ceph-mon[194833]: pgmap v517: 109 pgs: 1 active+recovery_wait+remapped, 3 remapped+peering, 3 peering, 83 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 6131/243348 objects degraded (2.519%); 1/243348 objects misplaced (0.000%); 1.4 MiB/s, 105 objects/s recovering 2023-09-23T19:59:15.890 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:15 smithi135 ceph-mon[221113]: pgmap v517: 109 pgs: 1 active+recovery_wait+remapped, 3 remapped+peering, 3 peering, 83 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 6131/243348 objects degraded (2.519%); 1/243348 objects misplaced (0.000%); 1.4 MiB/s, 105 objects/s recovering 2023-09-23T19:59:15.891 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:15 smithi135 ceph-mon[213871]: pgmap v517: 109 pgs: 1 active+recovery_wait+remapped, 3 remapped+peering, 3 peering, 83 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 6131/243348 objects degraded (2.519%); 1/243348 objects misplaced (0.000%); 1.4 MiB/s, 105 objects/s recovering 2023-09-23T19:59:15.909 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:15.942 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:16.149 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:16.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:16.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:16.258 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:16.336 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:16.406 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:16.454 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:16.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:16.544 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:59:16.564 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:16.589 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:59:16.630 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:59:16.668 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:16.676 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:59:16.714 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:59:16.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/275522162' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:16.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/275522162' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:16.720 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:16.720 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:16.750 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/275522162' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:16.751 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:59:16.787 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:16.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:16.804 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:59:16.804 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:59:16.805 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:59:16.805 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:59:16.805 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:59:16.805 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:59:16.805 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:59:16.805 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:59:16.805 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:59:16.898 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:16.933 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:16.995 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:16.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:16.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:17.099 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:17.136 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:17.200 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:17.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:17.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:17.304 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:17.341 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:17.397 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:17.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:17.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:17.503 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:17.586 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:17.692 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:17 smithi144 ceph-mon[194833]: pgmap v518: 109 pgs: 1 active+recovery_wait+remapped, 1 peering, 85 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 7401/239583 objects degraded (3.089%); 1/239583 objects misplaced (0.000%); 1013 KiB/s, 77 objects/s recovering 2023-09-23T19:59:17.692 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7401/239583 objects degraded (3.089%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:17.712 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T19:59:17.932 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:59:17.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:17 smithi135 ceph-mon[221113]: pgmap v518: 109 pgs: 1 active+recovery_wait+remapped, 1 peering, 85 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 7401/239583 objects degraded (3.089%); 1/239583 objects misplaced (0.000%); 1013 KiB/s, 77 objects/s recovering 2023-09-23T19:59:17.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7401/239583 objects degraded (3.089%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:17.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:17 smithi135 ceph-mon[213871]: pgmap v518: 109 pgs: 1 active+recovery_wait+remapped, 1 peering, 85 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 7401/239583 objects degraded (3.089%); 1/239583 objects misplaced (0.000%); 1013 KiB/s, 77 objects/s recovering 2023-09-23T19:59:17.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7401/239583 objects degraded (3.089%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:18.137 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:59:18.981 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:18.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:19.069 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:59:19.069 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:59:19.089 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:19.117 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:19.175 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:19.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:19.278 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:19.317 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:19.401 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:19.514 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:19.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:19.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:19.622 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:19 smithi135 ceph-mon[213871]: pgmap v519: 109 pgs: 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7401/230652 objects degraded (3.209%); 1/230652 objects misplaced (0.000%); 825 KiB/s, 115 objects/s recovering 2023-09-23T19:59:19.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:19 smithi135 ceph-mon[221113]: pgmap v519: 109 pgs: 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7401/230652 objects degraded (3.209%); 1/230652 objects misplaced (0.000%); 825 KiB/s, 115 objects/s recovering 2023-09-23T19:59:19.639 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:19.698 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:19.838 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:19 smithi144 ceph-mon[194833]: pgmap v519: 109 pgs: 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7401/230652 objects degraded (3.209%); 1/230652 objects misplaced (0.000%); 825 KiB/s, 115 objects/s recovering 2023-09-23T19:59:19.909 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:19.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:19.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:20.015 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:20.073 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T19:59:20.073 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T19:59:20.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T19:59:20.073 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:126108 5.4 5:21918726:::benchmark_data_smithi135_284570_object10324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:11.226095+0000", 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 308.84484928000001, 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28227993200000001, 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.074 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 126108 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226095+0000", 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226095+0000", 2023-09-23T19:59:20.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226094+0000", 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226101+0000", 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5599999999999996e-06 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226128+0000", 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6851000000000001e-05 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226136+0000", 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5539999999999998e-06 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226189+0000", 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2916e-05 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226304+0000", 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114191 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.226421+0000", 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000116996 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.234910+0000", 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0084897110000000005 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.241278+0000", 2023-09-23T19:59:20.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0063678600000000004 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.508288+0000", 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.26701033800000001 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.508332+0000", 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3680000000000002e-05 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:11.508375+0000", 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2895000000000002e-05 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149309 5.9 5:90e65886:::benchmark_data_smithi135_284570_object33525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.246164+0000", 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 275.82478042899999, 2023-09-23T19:59:20.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.106800241, 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149309 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246164+0000", 2023-09-23T19:59:20.082 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246164+0000", 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246162+0000", 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246172+0000", 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5470000000000002e-06 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246189+0000", 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6769000000000002e-05 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246197+0000", 2023-09-23T19:59:20.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7260000000000008e-06 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246251+0000", 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.325e-05 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246392+0000", 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014115199999999999 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246525+0000", 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013331399999999999 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.256018+0000", 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.009493352 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.265047+0000", 2023-09-23T19:59:20.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090289589999999996 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.265056+0000", 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6139999999999992e-06 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352889+0000", 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0878331240000001 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352964+0000", 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5154000000000006e-05 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149362 5.9 5:96e7d67a:::benchmark_data_smithi135_284570_object33578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.354858+0000", 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 275.71608686600001, 2023-09-23T19:59:20.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.005834238, 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149362 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354858+0000", 2023-09-23T19:59:20.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354858+0000", 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354857+0000", 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354866+0000", 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8339999999999994e-06 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354879+0000", 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2714000000000001e-05 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354887+0000", 2023-09-23T19:59:20.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0430000000000008e-06 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354939+0000", 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2045000000000001e-05 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354947+0000", 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3210000000000005e-06 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352991+0000", 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.998044025 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.353109+0000", 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011731999999999999 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T19:59:20.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.353230+0000", 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000121514 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.358433+0000", 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052028939999999996 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360549+0000", 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021155750000000002 2023-09-23T19:59:20.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360599+0000", 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9743000000000001e-05 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360692+0000", 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3247999999999999e-05 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149825 5.4 5:207530db:::benchmark_data_smithi135_284570_object34041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:45.852692+0000", 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 274.21825262900001, 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25976732600000002, 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149825 2023-09-23T19:59:20.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852692+0000", 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852692+0000", 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852690+0000", 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852698+0000", 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4900000000000003e-06 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852715+0000", 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6963e-05 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852721+0000", 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8590000000000001e-06 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852759+0000", 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8395000000000001e-05 2023-09-23T19:59:20.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.852898+0000", 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000139073 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.853026+0000", 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000127899 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.856882+0000", 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038562650000000002 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.856968+0000", 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5603e-05 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112369+0000", 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25540094400000002 2023-09-23T19:59:20.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112417+0000", 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8297999999999997e-05 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112459+0000", 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2116999999999997e-05 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.102 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149832 5.4 5:26481272:::benchmark_data_smithi135_284570_object34048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:45.858049+0000", 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 274.21289492900002, 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.254506177, 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149832 2023-09-23T19:59:20.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858049+0000", 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858049+0000", 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858048+0000", 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858054+0000", 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5260000000000001e-06 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858066+0000", 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2330000000000001e-05 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858071+0000", 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7029999999999997e-06 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858108+0000", 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7404999999999998e-05 2023-09-23T19:59:20.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858188+0000", 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9385e-05 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.858256+0000", 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8514000000000002e-05 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.861360+0000", 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003103401 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.862145+0000", 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00078522800000000001 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112495+0000", 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.25034988800000002 2023-09-23T19:59:20.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112527+0000", 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2110999999999998e-05 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:46.112556+0000", 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8765999999999999e-05 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.109 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:151030 5.4 5:2b25c154:::benchmark_data_smithi135_284570_object35246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:47.757786+0000", 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 272.31315824799998, 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28100605099999998, 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.110 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 151030 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757786+0000", 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757786+0000", 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757785+0000", 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757791+0000", 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8100000000000003e-06 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757800+0000", 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9309999999999997e-06 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757804+0000", 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1710000000000004e-06 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757843+0000", 2023-09-23T19:59:20.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9303e-05 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.757940+0000", 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6701999999999997e-05 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.758023+0000", 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3245999999999995e-05 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.760993+0000", 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002969377 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:47.767091+0000", 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0060981780000000001 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:48.038692+0000", 2023-09-23T19:59:20.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.271601384 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:48.038752+0000", 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9178999999999997e-05 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:48.038792+0000", 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0713000000000001e-05 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.116 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:195641 5.4 5:2f9c08d9:::benchmark_data_smithi135_284570_object79857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:48.122659+0000", 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 211.94828501999999, 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29659205, 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.117 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 195641 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122659+0000", 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122659+0000", 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122659+0000", 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122663+0000", 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.16e-06 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122673+0000", 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9799999999999993e-06 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122677+0000", 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8469999999999998e-06 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122717+0000", 2023-09-23T19:59:20.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9867999999999998e-05 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122810+0000", 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3264999999999996e-05 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.122896+0000", 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6045999999999996e-05 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.125913+0000", 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0030173159999999999 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.126349+0000", 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043560300000000001 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.122 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.419161+0000", 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29281219400000003 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.419206+0000", 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5012000000000002e-05 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.419251+0000", 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5460000000000002e-05 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:195643 5.4 5:227f4ff3:::benchmark_data_smithi135_284570_object79859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:48.127393+0000", 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 211.943551394, 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29418176800000001, 2023-09-23T19:59:20.124 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 195643 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127393+0000", 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127393+0000", 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127392+0000", 2023-09-23T19:59:20.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127398+0000", 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5559999999999998e-06 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127410+0000", 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1887e-05 2023-09-23T19:59:20.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127415+0000", 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0660000000000003e-06 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127447+0000", 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2240000000000003e-05 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127533+0000", 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6580000000000001e-05 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.127611+0000", 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7192000000000002e-05 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.131281+0000", 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003670151 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.131597+0000", 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000316621 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.421510+0000", 2023-09-23T19:59:20.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28991222900000002 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.421540+0000", 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0307999999999998e-05 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.421575+0000", 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4882000000000002e-05 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.131 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:195647 5.4 5:262c0f54:::benchmark_data_smithi135_284570_object79863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:55:48.132161+0000", 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 211.93878295799999, 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29250667499999999, 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.132 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 195647 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132161+0000", 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132161+0000", 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132160+0000", 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132170+0000", 2023-09-23T19:59:20.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0045e-05 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132183+0000", 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2731000000000001e-05 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132189+0000", 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5740000000000001e-06 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132234+0000", 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4857000000000001e-05 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132326+0000", 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2430999999999996e-05 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.132406+0000", 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9338000000000005e-05 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.135238+0000", 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028321750000000001 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.135418+0000", 2023-09-23T19:59:20.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017962699999999999 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.424495+0000", 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28907732000000003 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.424576+0000", 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1153e-05 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.138 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:55:48.424668+0000", 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2083000000000006e-05 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:220413 5.4 5:23b6015e:::benchmark_data_smithi135_284570_object104629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:21.650921+0000", 2023-09-23T19:59:20.139 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 178.42002381500001, 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.31297672500000001, 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 220413 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.140 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650921+0000", 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650921+0000", 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650919+0000", 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:59:20.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650926+0000", 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6760000000000001e-06 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650941+0000", 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5191000000000001e-05 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.650948+0000", 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3219999999999998e-06 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.653753+0000", 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002805714 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.653909+0000", 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000155725 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.654041+0000", 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013156800000000001 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.657951+0000", 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003910302 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.671727+0000", 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013775597000000001 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.963804+0000", 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.29207719100000001 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.145 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.963858+0000", 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3888000000000003e-05 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.963897+0000", 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9699000000000001e-05 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.146 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:220417 5.4 5:2b4f3313:::benchmark_data_smithi135_284570_object104633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:56:21.658119+0000", 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 178.412825545, 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.317104999, 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 220417 2023-09-23T19:59:20.147 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658119+0000", 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658119+0000", 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658117+0000", 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658125+0000", 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3760000000000002e-06 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658141+0000", 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6640999999999999e-05 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658149+0000", 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2060000000000001e-06 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.150 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658196+0000", 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7549000000000002e-05 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658334+0000", 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000137515 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.658476+0000", 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014275299999999999 2023-09-23T19:59:20.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.662437+0000", 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0039602600000000002 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.671760+0000", 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0093229220000000008 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.975140+0000", 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.30337997700000002 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.975186+0000", 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6607999999999998e-05 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:56:21.975224+0000", 2023-09-23T19:59:20.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7709000000000001e-05 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:57953 6.8 6:11741188:::benchmark_data_smithi135_344736_object57952:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:57:35.391430+0000", 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 104.67951467899999, 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.33742135499999998, 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.154 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 57953 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391430+0000", 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391430+0000", 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391429+0000", 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391462+0000", 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3142000000000002e-05 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391476+0000", 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4064e-05 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.157 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391481+0000", 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0189999999999997e-06 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391531+0000", 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9821000000000003e-05 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391647+0000", 2023-09-23T19:59:20.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011642999999999999 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.391751+0000", 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010374699999999999 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.393337+0000", 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001586395 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.396622+0000", 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003284361 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.728780+0000", 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.33215785199999998 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.728822+0000", 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2080999999999999e-05 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:57:35.728851+0000", 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9448e-05 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.161 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116222 6.8 6:15bca91d:::benchmark_data_smithi135_344736_object13566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:56.581514+0000", 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 23.489430786, 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.82753597599999995, 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116222 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.162 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.581514+0000", 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.581514+0000", 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.164 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.165 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.581512+0000", 2023-09-23T19:59:20.165 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:20.165 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.165 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.165 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.581520+0000", 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5970000000000003e-06 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.581536+0000", 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5747999999999999e-05 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.581541+0000", 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8540000000000002e-06 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.587420+0000", 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0058784600000000003 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.587428+0000", 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1799999999999996e-06 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.348097+0000", 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.76066925100000005 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.348753+0000", 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00065603099999999995 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.393078+0000", 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.044324491000000001 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.393178+0000", 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9630999999999995e-05 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2", 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.393573+0000", 2023-09-23T19:59:20.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00039591499999999999 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.395218+0000", 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0016448179999999999 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.399217+0000", 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0039983329999999997 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.170 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.406398+0000", 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0071818239999999998 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.406459+0000", 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0819999999999997e-05 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.171 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.409050+0000", 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025903900000000001 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116223 6.5 6:a635f6b1:::benchmark_data_smithi135_344736_object13567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:56.583860+0000", 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 23.487084210999999, 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.83640667199999996, 2023-09-23T19:59:20.172 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116223 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.583860+0000", 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.583860+0000", 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.583859+0000", 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.583875+0000", 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6330000000000001e-05 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.583891+0000", 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5747999999999999e-05 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.583898+0000", 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2830000000000004e-06 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.583971+0000", 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3034999999999994e-05 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.583984+0000", 2023-09-23T19:59:20.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2548000000000001e-05 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.384783+0000", 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.80079927299999998 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.385582+0000", 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00079902499999999997 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.406157+0000", 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.020574762999999999 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.406249+0000", 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1768000000000006e-05 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.406540+0000", 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00029084500000000002 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.410557+0000", 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040176960000000003 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.411113+0000", 2023-09-23T19:59:20.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00055551600000000004 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.418582+0000", 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074687950000000003 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.418634+0000", 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1925000000000002e-05 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.420267+0000", 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0016333599999999999 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:116237 6.5 6:a6e2ee26:::benchmark_data_smithi135_344736_object13581:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:56.644222+0000", 2023-09-23T19:59:20.181 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 23.426722173000002, 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.77431270699999999, 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116237 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.182 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.644222+0000", 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.644222+0000", 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.644220+0000", 2023-09-23T19:59:20.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.644228+0000", 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2640000000000002e-06 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.644243+0000", 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5790999999999998e-05 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.644250+0000", 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7089999999999996e-06 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.644305+0000", 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4695999999999999e-05 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:56.644314+0000", 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4299999999999995e-06 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.385616+0000", 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.74130210200000002 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.386117+0000", 2023-09-23T19:59:20.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00050026400000000005 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.404486+0000", 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.018369818999999999 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.404579+0000", 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2964e-05 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.404937+0000", 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00035803400000000002 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.409270+0000", 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043326759999999997 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.410551+0000", 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001280705 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.416523+0000", 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005972153 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.416571+0000", 2023-09-23T19:59:20.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7559999999999999e-05 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:57.418535+0000", 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001964412 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:117452 6.b 6:d2bb813c:::benchmark_data_smithi135_344736_object14796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:58:59.809376+0000", 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 20.261568388000001, 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.250159517, 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 117452 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.191 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809376+0000", 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809376+0000", 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809375+0000", 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809382+0000", 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1459999999999999e-06 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809393+0000", 2023-09-23T19:59:20.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1280999999999999e-05 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809405+0000", 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.207e-05 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809463+0000", 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7630999999999998e-05 2023-09-23T19:59:20.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809582+0000", 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000119771 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3,4", 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.809669+0000", 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6205000000000006e-05 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.812336+0000", 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0026673560000000001 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.813059+0000", 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00072338999999999997 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:58:59.813548+0000", 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00048862799999999996 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:00.059367+0000", 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.245819383 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:00.059431+0000", 2023-09-23T19:59:20.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3390000000000001e-05 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:00.059536+0000", 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000104721 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:120126 6.3 6:c50f7d96:::benchmark_data_smithi135_344736_object17470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-09-23T19:59:20.198 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:04.004740+0000", 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.066204380999999, 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.247550767, 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 120126 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004740+0000", 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004740+0000", 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004740+0000", 2023-09-23T19:59:20.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004742+0000", 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.915e-06 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004749+0000", 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.421e-06 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004754+0000", 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9490000000000004e-06 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004807+0000", 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3244e-05 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004928+0000", 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000121112 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.005271+0000", 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00034313399999999998 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.007498+0000", 2023-09-23T19:59:20.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022266260000000002 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.007732+0000", 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00023377000000000001 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.252198+0000", 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.24446625699999999 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.252247+0000", 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8792000000000003e-05 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.252291+0000", 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4002000000000003e-05 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.205 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:120127 6.5 6:a5d9ca4e:::benchmark_data_smithi135_344736_object17471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:04.004779+0000", 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.066165510000001, 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.24856832500000001, 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:20.206 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 120127 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004779+0000", 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004779+0000", 2023-09-23T19:59:20.207 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004779+0000", 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004781+0000", 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7810000000000001e-06 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004788+0000", 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3300000000000004e-06 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004792+0000", 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1010000000000002e-06 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.004841+0000", 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9257e-05 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.005497+0000", 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000655869 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.010486+0000", 2023-09-23T19:59:20.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0049886349999999999 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.010562+0000", 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6768999999999996e-05 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.010882+0000", 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00032013899999999999 2023-09-23T19:59:20.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.013239+0000", 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023562650000000002 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.014709+0000", 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014705740000000001 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.016331+0000", 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0016218389999999999 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.016378+0000", 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6504000000000001e-05 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.253347+0000", 2023-09-23T19:59:20.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.23696963600000001 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.44942.0:120134 6.12 e189/187 6:4e1b35e1:::benchmark_data_smithi135_344736_object17478:head v 189'7292, mlcod=0'0)", 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:04.014128+0000", 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.056815947, 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.246711974, 2023-09-23T19:59:20.214 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.014128+0000", 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.014128+0000", 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.014128+0000", 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.014136+0000", 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1769999999999998e-06 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.014147+0000", 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0784e-05 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.014152+0000", 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4389999999999999e-06 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.015969+0000", 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018172889999999999 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.016006+0000", 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6489999999999998e-05 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.218 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.260800+0000", 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.24479366 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.260840+0000", 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0912999999999999e-05 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:120139 6.8 6:14bb3e65:::benchmark_data_smithi135_344736_object17483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-09-23T19:59:20.219 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:04.018980+0000", 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.051964080000001, 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.271432436, 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 120139 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T19:59:20.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.018980+0000", 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.018980+0000", 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T19:59:20.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.018979+0000", 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.018986+0000", 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5809999999999999e-06 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.252196+0000", 2023-09-23T19:59:20.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.233210004 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.252207+0000", 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0806e-05 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.252309+0000", 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101746 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.253769+0000", 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014596609999999999 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.265317+0000", 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.01154868 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.265393+0000", 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5328999999999997e-05 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2", 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.265726+0000", 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000333092 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.277560+0000", 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011833791999999999 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.278712+0000", 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001152578 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.288135+0000", 2023-09-23T19:59:20.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0094229310000000007 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.288184+0000", 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8699999999999998e-05 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:04.290413+0000", 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022290370000000001 2023-09-23T19:59:20.227 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.228 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.228 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.228 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T19:59:20.228 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T19:59:20.228 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T19:59:20.228 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:20.301 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:20.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:20.413 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:20.469 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:20.514 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:20.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:20.618 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:20.635 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:20 smithi135 ceph-mon[213871]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:20.635 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','6':'15937','7':'62260'}"}]: dispatch 2023-09-23T19:59:20.641 INFO:teuthology.orchestra.run.smithi135.stdout:moved 3 / 327 (0.917431%) 2023-09-23T19:59:20.641 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T19:59:20.641 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.27922 -> 6.36957 (expected baseline 6.32778) 2023-09-23T19:59:20.641 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.5 with 37 -> 37 pgs (0.792049 -> 0.792049 * mean) 2023-09-23T19:59:20.641 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.3 with 57 -> 58 pgs (1.22018 -> 1.24159 * mean) 2023-09-23T19:59:20.641 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:59:20.641 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T19:59:20.641 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T19:59:20.642 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:59:20.642 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0245 2023-09-23T19:59:20.642 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0269 2023-09-23T19:59:20.642 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T19:59:20.642 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 weight 1.0000 -> 0.9500 2023-09-23T19:59:20.642 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 0.1932 -> 0.2432 2023-09-23T19:59:20.642 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 weight 1.0000 -> 0.9500 2023-09-23T19:59:20.652 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:20.694 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:20.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:20.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:20.797 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:20.823 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:20.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:20 smithi144 ceph-mon[194833]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:20.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','6':'15937','7':'62260'}"}]: dispatch 2023-09-23T19:59:20.887 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:20.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:20.894 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:20 smithi135 ceph-mon[221113]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:20.894 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','6':'15937','7':'62260'}"}]: dispatch 2023-09-23T19:59:20.977 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:59:20.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:21.071 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:59:21.099 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:21.476 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:21.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:21.598 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:21.629 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:21.693 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:21.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:21.802 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:21.834 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:21.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:21 smithi144 ceph-mon[194833]: pgmap v520: 109 pgs: 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 7401/229116 objects degraded (3.230%); 1/229116 objects misplaced (0.000%); 15 KiB/s, 89 objects/s recovering 2023-09-23T19:59:21.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','6':'15937','7':'62260'}"}]': finished 2023-09-23T19:59:21.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:21 smithi144 ceph-mon[194833]: osdmap e196: 8 total, 7 up, 8 in 2023-09-23T19:59:21.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:21 smithi144 ceph-mon[194833]: failed to encode map e196 with expected crc 2023-09-23T19:59:21.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:21 smithi144 ceph-mon[194833]: failed to encode map e196 with expected crc 2023-09-23T19:59:21.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:21 smithi144 ceph-mon[194833]: failed to encode map e196 with expected crc 2023-09-23T19:59:21.898 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:21.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:21.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:21.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[213871]: pgmap v520: 109 pgs: 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 7401/229116 objects degraded (3.230%); 1/229116 objects misplaced (0.000%); 15 KiB/s, 89 objects/s recovering 2023-09-23T19:59:21.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','6':'15937','7':'62260'}"}]': finished 2023-09-23T19:59:21.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[213871]: osdmap e196: 8 total, 7 up, 8 in 2023-09-23T19:59:21.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[213871]: failed to encode map e196 with expected crc 2023-09-23T19:59:21.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[213871]: failed to encode map e196 with expected crc 2023-09-23T19:59:21.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[213871]: failed to encode map e196 with expected crc 2023-09-23T19:59:21.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[221113]: pgmap v520: 109 pgs: 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 7401/229116 objects degraded (3.230%); 1/229116 objects misplaced (0.000%); 15 KiB/s, 89 objects/s recovering 2023-09-23T19:59:21.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260','6':'15937','7':'62260'}"}]': finished 2023-09-23T19:59:21.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[221113]: osdmap e196: 8 total, 7 up, 8 in 2023-09-23T19:59:21.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[221113]: failed to encode map e196 with expected crc 2023-09-23T19:59:21.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[221113]: failed to encode map e196 with expected crc 2023-09-23T19:59:21.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:21 smithi135 ceph-mon[221113]: failed to encode map e196 with expected crc 2023-09-23T19:59:22.006 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:22.090 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:22.395 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:59:22.396 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T19:59:22.450 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:22.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:22.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:22.557 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:22.593 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:22.646 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:22.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:22.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:22.670 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[213871]: failed to encode map e196 with expected crc 2023-09-23T19:59:22.671 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[213871]: failed to encode map e197 with expected crc 2023-09-23T19:59:22.671 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[213871]: osdmap e197: 8 total, 7 up, 8 in 2023-09-23T19:59:22.671 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:22.671 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7401/229116 objects degraded (3.230%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:22.752 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:22.783 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:22.833 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:22.841 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:22 smithi144 ceph-mon[194833]: failed to encode map e196 with expected crc 2023-09-23T19:59:22.841 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:22 smithi144 ceph-mon[194833]: failed to encode map e197 with expected crc 2023-09-23T19:59:22.841 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:22 smithi144 ceph-mon[194833]: osdmap e197: 8 total, 7 up, 8 in 2023-09-23T19:59:22.841 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:22.842 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7401/229116 objects degraded (3.230%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:22.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:22.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:22.883 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:22.933 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:22.943 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[221113]: failed to encode map e196 with expected crc 2023-09-23T19:59:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[221113]: failed to encode map e197 with expected crc 2023-09-23T19:59:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[221113]: osdmap e197: 8 total, 7 up, 8 in 2023-09-23T19:59:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7401/229116 objects degraded (3.230%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:23.036 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:23.257 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:23.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:23.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:23.361 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:23.460 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:23.521 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:23.723 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T19:59:23.725 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:23.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:23.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:23.852 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:23.882 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:23.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:59:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:59:23] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-09-23T19:59:23.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[221113]: pgmap v523: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 85 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 759 op/s; 7401/225309 objects degraded (3.285%); 1/225309 objects misplaced (0.000%); 20 KiB/s, 48 objects/s recovering 2023-09-23T19:59:23.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[221113]: osdmap e198: 8 total, 7 up, 8 in 2023-09-23T19:59:23.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[221113]: failed to encode map e198 with expected crc 2023-09-23T19:59:23.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2690742891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:23.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:59:23.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[213871]: pgmap v523: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 85 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 759 op/s; 7401/225309 objects degraded (3.285%); 1/225309 objects misplaced (0.000%); 20 KiB/s, 48 objects/s recovering 2023-09-23T19:59:23.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[213871]: osdmap e198: 8 total, 7 up, 8 in 2023-09-23T19:59:23.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[213871]: failed to encode map e198 with expected crc 2023-09-23T19:59:23.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2690742891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:23.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:23 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:59:24.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:23 smithi144 ceph-mon[194833]: pgmap v523: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 85 active+clean, 1 active+recovering+undersized+remapped, 4 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 759 op/s; 7401/225309 objects degraded (3.285%); 1/225309 objects misplaced (0.000%); 20 KiB/s, 48 objects/s recovering 2023-09-23T19:59:24.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:23 smithi144 ceph-mon[194833]: osdmap e198: 8 total, 7 up, 8 in 2023-09-23T19:59:24.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:23 smithi144 ceph-mon[194833]: failed to encode map e198 with expected crc 2023-09-23T19:59:24.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:23 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2690742891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:24.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:23 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:59:24.292 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:24.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:24.398 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:24.433 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:24.499 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:24.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:24.604 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:24.635 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:24.672 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:24 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:59:24.673 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:24 smithi135 ceph-mon[213871]: failed to encode map e199 with expected crc 2023-09-23T19:59:24.673 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2690742891' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:24.673 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:24 smithi135 ceph-mon[213871]: osdmap e199: 8 total, 7 up, 8 in 2023-09-23T19:59:24.698 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:24.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:24.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:24.706 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:24 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:59:24.706 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:24 smithi144 ceph-mon[194833]: failed to encode map e199 with expected crc 2023-09-23T19:59:24.706 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2690742891' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:24.707 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:24 smithi144 ceph-mon[194833]: osdmap e199: 8 total, 7 up, 8 in 2023-09-23T19:59:24.806 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:24.837 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:24.900 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:24.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:24.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:24 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:59:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:24 smithi135 ceph-mon[221113]: failed to encode map e199 with expected crc 2023-09-23T19:59:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2690742891' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:24 smithi135 ceph-mon[221113]: osdmap e199: 8 total, 7 up, 8 in 2023-09-23T19:59:25.009 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:25.108 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:25.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[213871]: pgmap v526: 109 pgs: 2 active+recovery_wait+remapped, 1 remapped+peering, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 904 op/s; 7401/218463 objects degraded (3.388%); 7/218463 objects misplaced (0.003%); 39 KiB/s, 64 objects/s recovering 2023-09-23T19:59:25.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[213871]: osdmap e200: 8 total, 7 up, 8 in 2023-09-23T19:59:25.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[213871]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[213871]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[213871]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[213871]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[213871]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[221113]: pgmap v526: 109 pgs: 2 active+recovery_wait+remapped, 1 remapped+peering, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 904 op/s; 7401/218463 objects degraded (3.388%); 7/218463 objects misplaced (0.003%); 39 KiB/s, 64 objects/s recovering 2023-09-23T19:59:25.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[221113]: osdmap e200: 8 total, 7 up, 8 in 2023-09-23T19:59:25.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[221113]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[221113]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[221113]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[221113]: failed to encode map e200 with expected crc 2023-09-23T19:59:25.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:25 smithi135 ceph-mon[221113]: failed to encode map e200 with expected crc 2023-09-23T19:59:26.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:25 smithi144 ceph-mon[194833]: pgmap v526: 109 pgs: 2 active+recovery_wait+remapped, 1 remapped+peering, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 904 op/s; 7401/218463 objects degraded (3.388%); 7/218463 objects misplaced (0.003%); 39 KiB/s, 64 objects/s recovering 2023-09-23T19:59:26.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:25 smithi144 ceph-mon[194833]: osdmap e200: 8 total, 7 up, 8 in 2023-09-23T19:59:26.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:25 smithi144 ceph-mon[194833]: failed to encode map e200 with expected crc 2023-09-23T19:59:26.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:25 smithi144 ceph-mon[194833]: failed to encode map e200 with expected crc 2023-09-23T19:59:26.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:25 smithi144 ceph-mon[194833]: failed to encode map e200 with expected crc 2023-09-23T19:59:26.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:25 smithi144 ceph-mon[194833]: failed to encode map e200 with expected crc 2023-09-23T19:59:26.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:25 smithi144 ceph-mon[194833]: failed to encode map e200 with expected crc 2023-09-23T19:59:26.262 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:26.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:26.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:26.372 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:26.401 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:26.461 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:26.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:26.569 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:26.607 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:26.671 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:26.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:26.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:26.778 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:26.806 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:27.248 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:27.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:27.364 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:27.400 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:27.457 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:27.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:27.564 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:27.641 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:27.783 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:27.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:27.810 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:59:27.810 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T19:59:27.810 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T19:59:27.810 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:59:27.904 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:27.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:27 smithi135 ceph-mon[213871]: pgmap v528: 109 pgs: 2 active+recovery_wait+remapped, 1 remapped+peering, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 45 KiB/s wr, 864 op/s; 7401/214368 objects degraded (3.452%); 7/214368 objects misplaced (0.003%); 32 KiB/s, 53 objects/s recovering 2023-09-23T19:59:27.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7401/214368 objects degraded (3.452%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:27.915 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:27 smithi135 ceph-mon[221113]: pgmap v528: 109 pgs: 2 active+recovery_wait+remapped, 1 remapped+peering, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 45 KiB/s wr, 864 op/s; 7401/214368 objects degraded (3.452%); 7/214368 objects misplaced (0.003%); 32 KiB/s, 53 objects/s recovering 2023-09-23T19:59:27.915 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7401/214368 objects degraded (3.452%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:27.985 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:28.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:27 smithi144 ceph-mon[194833]: pgmap v528: 109 pgs: 2 active+recovery_wait+remapped, 1 remapped+peering, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 45 KiB/s wr, 864 op/s; 7401/214368 objects degraded (3.452%); 7/214368 objects misplaced (0.003%); 32 KiB/s, 53 objects/s recovering 2023-09-23T19:59:28.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7401/214368 objects degraded (3.452%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:28.245 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:59:28.285 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:59:28.315 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:59:28.353 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:59:28.384 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:59:28.421 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:28.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:28.429 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:59:28.468 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:59:28.468 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:59:28.468 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:59:28.468 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:59:28.468 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:59:28.468 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:59:28.469 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:59:28.469 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:59:28.469 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:59:28.530 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:28.587 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:28.647 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:28.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:28.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:28.750 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:28.777 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:28.829 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:28.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:28.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:28.935 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:28.962 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:29.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3511300724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:29.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3511300724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:29.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3511300724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:29.296 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:59:29.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[221113]: pgmap v529: 109 pgs: 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 7401/204813 objects degraded (3.614%); 7/204813 objects misplaced (0.003%); 0 B/s, 87 objects/s recovering 2023-09-23T19:59:29.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:59:29.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[221113]: failed to encode map e201 with expected crc 2023-09-23T19:59:29.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[221113]: failed to encode map e201 with expected crc 2023-09-23T19:59:29.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[221113]: osdmap e201: 8 total, 7 up, 8 in 2023-09-23T19:59:29.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[213871]: pgmap v529: 109 pgs: 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 7401/204813 objects degraded (3.614%); 7/204813 objects misplaced (0.003%); 0 B/s, 87 objects/s recovering 2023-09-23T19:59:29.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:59:29.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[213871]: failed to encode map e201 with expected crc 2023-09-23T19:59:29.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[213871]: failed to encode map e201 with expected crc 2023-09-23T19:59:29.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:29 smithi135 ceph-mon[213871]: osdmap e201: 8 total, 7 up, 8 in 2023-09-23T19:59:30.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:29 smithi144 ceph-mon[194833]: pgmap v529: 109 pgs: 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 7401/204813 objects degraded (3.614%); 7/204813 objects misplaced (0.003%); 0 B/s, 87 objects/s recovering 2023-09-23T19:59:30.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:29 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T19:59:30.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:29 smithi144 ceph-mon[194833]: failed to encode map e201 with expected crc 2023-09-23T19:59:30.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:29 smithi144 ceph-mon[194833]: failed to encode map e201 with expected crc 2023-09-23T19:59:30.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:29 smithi144 ceph-mon[194833]: osdmap e201: 8 total, 7 up, 8 in 2023-09-23T19:59:30.327 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:30.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:30.433 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:30.497 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:30.556 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:59:30.556 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:59:30.603 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:59:30.630 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:30.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:30.745 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:30.812 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:30.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:30 smithi135 ceph-mon[213871]: osdmap e202: 8 total, 7 up, 8 in 2023-09-23T19:59:30.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:30 smithi135 ceph-mon[213871]: failed to encode map e202 with expected crc 2023-09-23T19:59:30.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:30 smithi135 ceph-mon[213871]: failed to encode map e202 with expected crc 2023-09-23T19:59:30.875 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:30 smithi135 ceph-mon[221113]: osdmap e202: 8 total, 7 up, 8 in 2023-09-23T19:59:30.875 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:30 smithi135 ceph-mon[221113]: failed to encode map e202 with expected crc 2023-09-23T19:59:30.875 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:30 smithi135 ceph-mon[221113]: failed to encode map e202 with expected crc 2023-09-23T19:59:30.893 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:31.105 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:31.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:31.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:31.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:30 smithi144 ceph-mon[194833]: osdmap e202: 8 total, 7 up, 8 in 2023-09-23T19:59:31.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:30 smithi144 ceph-mon[194833]: failed to encode map e202 with expected crc 2023-09-23T19:59:31.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:30 smithi144 ceph-mon[194833]: failed to encode map e202 with expected crc 2023-09-23T19:59:31.213 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:31.246 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:31.305 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:31.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:31.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:31.411 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:31.493 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:31.788 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:31 smithi135 ceph-mon[213871]: pgmap v532: 109 pgs: 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 7401/203373 objects degraded (3.639%); 7/203373 objects misplaced (0.003%); 0 B/s, 44 objects/s recovering 2023-09-23T19:59:31.788 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:31 smithi135 ceph-mon[221113]: pgmap v532: 109 pgs: 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 7401/203373 objects degraded (3.639%); 7/203373 objects misplaced (0.003%); 0 B/s, 44 objects/s recovering 2023-09-23T19:59:31.807 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:31.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:31.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:31.863 INFO:teuthology.orchestra.run.smithi135.stdout:moved 47 / 327 (14.3731%) 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.36957 -> 8.12655 (expected baseline 6.32778) 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.5 with 37 -> 48 pgs (0.792049 -> 1.02752 * mean) 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.3 with 58 -> 38 pgs (1.24159 -> 0.813456 * mean) 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 1 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0219 2023-09-23T19:59:31.864 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0241 2023-09-23T19:59:31.865 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 weight 0.9500 -> 0.5873 2023-09-23T19:59:31.865 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 weight 0.9500 -> 0.6361 2023-09-23T19:59:31.865 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 0.2432 -> 0.3616 2023-09-23T19:59:31.865 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 weight 0.9500 -> 0.8370 2023-09-23T19:59:31.878 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:59:31.912 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:31.938 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:31.982 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:31.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:32.091 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:32.115 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:31 smithi144 ceph-mon[194833]: pgmap v532: 109 pgs: 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 7401/203373 objects degraded (3.639%); 7/203373 objects misplaced (0.003%); 0 B/s, 44 objects/s recovering 2023-09-23T19:59:32.117 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:32.157 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:32.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:32.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:32.221 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:59:31 smithi135 conmon[201366]: 2023-09-23T19:59:31.860+0000 7fcf49f9e700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T19:59:32.266 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:32.307 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:32.557 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:59:32.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[221113]: osdmap e203: 8 total, 7 up, 8 in 2023-09-23T19:59:32.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[221113]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[221113]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[221113]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[221113]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[221113]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[221113]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7401/203373 objects degraded (3.639%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[213871]: osdmap e203: 8 total, 7 up, 8 in 2023-09-23T19:59:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[213871]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[213871]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[213871]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[213871]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[213871]: failed to encode map e203 with expected crc 2023-09-23T19:59:32.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[213871]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:32.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7401/203373 objects degraded (3.639%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:33.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:32 smithi144 ceph-mon[194833]: osdmap e203: 8 total, 7 up, 8 in 2023-09-23T19:59:33.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:32 smithi144 ceph-mon[194833]: failed to encode map e203 with expected crc 2023-09-23T19:59:33.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:32 smithi144 ceph-mon[194833]: failed to encode map e203 with expected crc 2023-09-23T19:59:33.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:32 smithi144 ceph-mon[194833]: failed to encode map e203 with expected crc 2023-09-23T19:59:33.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:32 smithi144 ceph-mon[194833]: failed to encode map e203 with expected crc 2023-09-23T19:59:33.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:32 smithi144 ceph-mon[194833]: failed to encode map e203 with expected crc 2023-09-23T19:59:33.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:32 smithi144 ceph-mon[194833]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:33.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7401/203373 objects degraded (3.639%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:33.395 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:33.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:33.508 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:33.534 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:33.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:33 smithi135 ceph-mon[221113]: pgmap v534: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 6816/199944 objects degraded (3.409%); 7/199944 objects misplaced (0.004%); 0 B/s, 142 objects/s recovering 2023-09-23T19:59:33.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:33 smithi135 ceph-mon[221113]: failed to encode map e204 with expected crc 2023-09-23T19:59:33.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:33 smithi135 ceph-mon[221113]: osdmap e204: 8 total, 7 up, 8 in 2023-09-23T19:59:33.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:33 smithi135 ceph-mon[221113]: failed to encode map e204 with expected crc 2023-09-23T19:59:33.976 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:59:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:59:33] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-09-23T19:59:33.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:33 smithi135 ceph-mon[213871]: pgmap v534: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 6816/199944 objects degraded (3.409%); 7/199944 objects misplaced (0.004%); 0 B/s, 142 objects/s recovering 2023-09-23T19:59:33.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:33 smithi135 ceph-mon[213871]: failed to encode map e204 with expected crc 2023-09-23T19:59:33.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:33 smithi135 ceph-mon[213871]: osdmap e204: 8 total, 7 up, 8 in 2023-09-23T19:59:33.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:33 smithi135 ceph-mon[213871]: failed to encode map e204 with expected crc 2023-09-23T19:59:34.077 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:34.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:34.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:33 smithi144 ceph-mon[194833]: pgmap v534: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 86 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 6816/199944 objects degraded (3.409%); 7/199944 objects misplaced (0.004%); 0 B/s, 142 objects/s recovering 2023-09-23T19:59:34.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:33 smithi144 ceph-mon[194833]: failed to encode map e204 with expected crc 2023-09-23T19:59:34.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:33 smithi144 ceph-mon[194833]: osdmap e204: 8 total, 7 up, 8 in 2023-09-23T19:59:34.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:33 smithi144 ceph-mon[194833]: failed to encode map e204 with expected crc 2023-09-23T19:59:34.187 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:34.215 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:34.267 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:34.275 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T19:59:34.275 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T19:59:34.472 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:34.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:34.591 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:34.622 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:34.676 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:34.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:34.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:34.781 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:34.813 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:34.896 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:35.036 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:35.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:35.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:35.144 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:35.245 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:35.846 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T19:59:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:35 smithi144 ceph-mon[194833]: pgmap v536: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 87 active+clean, 3 active+undersized, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6816/194667 objects degraded (3.501%); 7/194667 objects misplaced (0.004%); 0 B/s, 104 objects/s recovering 2023-09-23T19:59:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:35 smithi144 ceph-mon[194833]: osdmap e205: 8 total, 7 up, 8 in 2023-09-23T19:59:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:35 smithi144 ceph-mon[194833]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:35 smithi144 ceph-mon[194833]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:35 smithi144 ceph-mon[194833]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:35 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/713909394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:35 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[213871]: pgmap v536: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 87 active+clean, 3 active+undersized, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6816/194667 objects degraded (3.501%); 7/194667 objects misplaced (0.004%); 0 B/s, 104 objects/s recovering 2023-09-23T19:59:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[213871]: osdmap e205: 8 total, 7 up, 8 in 2023-09-23T19:59:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[213871]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[213871]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[213871]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/713909394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:36.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[221113]: pgmap v536: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 87 active+clean, 3 active+undersized, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6816/194667 objects degraded (3.501%); 7/194667 objects misplaced (0.004%); 0 B/s, 104 objects/s recovering 2023-09-23T19:59:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[221113]: osdmap e205: 8 total, 7 up, 8 in 2023-09-23T19:59:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[221113]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[221113]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[221113]: failed to encode map e205 with expected crc 2023-09-23T19:59:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/713909394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:35 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:59:36.409 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:36.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:36.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:36.516 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:36.546 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:36.595 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:36.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:36.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:36.701 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:36.740 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:37.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:36 smithi144 ceph-mon[194833]: failed to encode map e205 with expected crc 2023-09-23T19:59:37.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:36 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:37.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:59:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:36 smithi144 ceph-mon[194833]: osdmap e206: 8 total, 7 up, 8 in 2023-09-23T19:59:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:36 smithi144 ceph-mon[194833]: failed to encode map e206 with expected crc 2023-09-23T19:59:37.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[213871]: failed to encode map e205 with expected crc 2023-09-23T19:59:37.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:37.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:59:37.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[213871]: osdmap e206: 8 total, 7 up, 8 in 2023-09-23T19:59:37.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[213871]: failed to encode map e206 with expected crc 2023-09-23T19:59:37.227 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[221113]: failed to encode map e205 with expected crc 2023-09-23T19:59:37.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:37.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:59:37.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[221113]: osdmap e206: 8 total, 7 up, 8 in 2023-09-23T19:59:37.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:36 smithi135 ceph-mon[221113]: failed to encode map e206 with expected crc 2023-09-23T19:59:37.357 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:37.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:37.471 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:37.550 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:38.066 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:38.119 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:37 smithi135 ceph-mon[221113]: pgmap v539: 109 pgs: 2 peering, 2 active+recovery_wait+remapped, 87 active+clean, 3 active+undersized, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 6170/191490 objects degraded (3.222%); 7/191490 objects misplaced (0.004%); 0 B/s, 263 objects/s recovering 2023-09-23T19:59:38.119 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:38.119 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:37 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6170/191490 objects degraded (3.222%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:38.120 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:37 smithi135 ceph-mon[213871]: pgmap v539: 109 pgs: 2 peering, 2 active+recovery_wait+remapped, 87 active+clean, 3 active+undersized, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 6170/191490 objects degraded (3.222%); 7/191490 objects misplaced (0.004%); 0 B/s, 263 objects/s recovering 2023-09-23T19:59:38.120 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:38.121 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:37 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6170/191490 objects degraded (3.222%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:38.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:37 smithi144 ceph-mon[194833]: pgmap v539: 109 pgs: 2 peering, 2 active+recovery_wait+remapped, 87 active+clean, 3 active+undersized, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 6170/191490 objects degraded (3.222%); 7/191490 objects misplaced (0.004%); 0 B/s, 263 objects/s recovering 2023-09-23T19:59:38.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:38.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:37 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6170/191490 objects degraded (3.222%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:38.282 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:38.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:38.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:38.388 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:38.467 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:38.686 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:38.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:38.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:38.799 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:38.885 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:38.886 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:38.887 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:40.121 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:59:40.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:39 smithi144 ceph-mon[194833]: pgmap v540: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 5555/183021 objects degraded (3.035%); 7/183021 objects misplaced (0.004%); 0 B/s, 180 objects/s recovering 2023-09-23T19:59:40.166 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:39 smithi135 ceph-mon[221113]: pgmap v540: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 5555/183021 objects degraded (3.035%); 7/183021 objects misplaced (0.004%); 0 B/s, 180 objects/s recovering 2023-09-23T19:59:40.167 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:39 smithi135 ceph-mon[213871]: pgmap v540: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 5555/183021 objects degraded (3.035%); 7/183021 objects misplaced (0.004%); 0 B/s, 180 objects/s recovering 2023-09-23T19:59:40.964 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:40.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:40.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: osdmap e207: 8 total, 7 up, 8 in 2023-09-23T19:59:40.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[213871]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: osdmap e207: 8 total, 7 up, 8 in 2023-09-23T19:59:40.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:40.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:40 smithi135 ceph-mon[221113]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.066 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:59:41.072 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:41.179 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:41.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: osdmap e207: 8 total, 7 up, 8 in 2023-09-23T19:59:41.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:40 smithi144 ceph-mon[194833]: failed to encode map e207 with expected crc 2023-09-23T19:59:41.402 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:59:41.409 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:41.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:41.439 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:59:41.478 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:59:41.517 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:41.542 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:59:41.577 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:59:41.604 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:41.622 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:41.622 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:59:41.728 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:59:41.728 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:59:41.728 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:59:41.728 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:59:41.729 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:59:41.729 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:59:41.729 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:59:41.729 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:59:41.729 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:59:41.762 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:41.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:41.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:41.866 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:41.898 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:41.900 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[221113]: pgmap v542: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5542/181380 objects degraded (3.055%); 7/181380 objects misplaced (0.004%); 0 B/s, 179 objects/s recovering 2023-09-23T19:59:41.900 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[221113]: failed to encode map e208 with expected crc 2023-09-23T19:59:41.900 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[221113]: osdmap e208: 8 total, 7 up, 8 in 2023-09-23T19:59:41.900 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[221113]: failed to encode map e208 with expected crc 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[221113]: failed to encode map e208 with expected crc 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[221113]: failed to encode map e208 with expected crc 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/359581340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[213871]: pgmap v542: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5542/181380 objects degraded (3.055%); 7/181380 objects misplaced (0.004%); 0 B/s, 179 objects/s recovering 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[213871]: failed to encode map e208 with expected crc 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[213871]: osdmap e208: 8 total, 7 up, 8 in 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[213871]: failed to encode map e208 with expected crc 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[213871]: failed to encode map e208 with expected crc 2023-09-23T19:59:41.901 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[213871]: failed to encode map e208 with expected crc 2023-09-23T19:59:41.902 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/359581340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:41.950 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:41.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:41.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:42.059 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:42.091 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:42.152 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:42.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:42.161 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:41 smithi144 ceph-mon[194833]: pgmap v542: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5542/181380 objects degraded (3.055%); 7/181380 objects misplaced (0.004%); 0 B/s, 179 objects/s recovering 2023-09-23T19:59:42.161 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:41 smithi144 ceph-mon[194833]: failed to encode map e208 with expected crc 2023-09-23T19:59:42.162 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:41 smithi144 ceph-mon[194833]: osdmap e208: 8 total, 7 up, 8 in 2023-09-23T19:59:42.162 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:41 smithi144 ceph-mon[194833]: failed to encode map e208 with expected crc 2023-09-23T19:59:42.162 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:41 smithi144 ceph-mon[194833]: failed to encode map e208 with expected crc 2023-09-23T19:59:42.162 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:41 smithi144 ceph-mon[194833]: failed to encode map e208 with expected crc 2023-09-23T19:59:42.162 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/359581340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:42.262 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:42.337 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:42.456 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:42.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:42.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:42.562 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:42.597 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 19:59:42 smithi144 conmon[173765]: logger=sqlstore.transactions t=2023-09-23T19:59:42.328513771Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T19:59:42.600 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:42.661 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:42.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:42.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:42.768 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:42.802 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:42.862 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:42.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:42.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:42.921 INFO:teuthology.orchestra.run.smithi135.stdout:moved 43 / 327 (13.1498%) 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 6.36957 -> 7.90505 (expected baseline 6.32778) 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.5 with 37 -> 46 pgs (0.792049 -> 0.984709 * mean) 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.3 with 58 -> 41 pgs (1.24159 -> 0.877676 * mean) 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 1 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0198 2023-09-23T19:59:42.922 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0218 2023-09-23T19:59:42.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 weight 0.9500 -> 0.6044 2023-09-23T19:59:42.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 weight 0.9500 -> 0.6375 2023-09-23T19:59:42.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 0.2432 -> 0.3268 2023-09-23T19:59:42.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7 weight 0.9500 -> 0.8367 2023-09-23T19:59:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:59:42.960 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:59:42.969 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:42.994 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:43.032 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:43.037 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:43.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:43.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:42 smithi144 ceph-mon[194833]: failed to encode map e209 with expected crc 2023-09-23T19:59:43.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:42 smithi144 ceph-mon[194833]: osdmap e209: 8 total, 7 up, 8 in 2023-09-23T19:59:43.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:42 smithi144 ceph-mon[194833]: failed to encode map e209 with expected crc 2023-09-23T19:59:43.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:42 smithi144 ceph-mon[194833]: from='client.54925 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:43.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:42 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'41778','3':'39613','6':'21420','7':'54831'}"}]: dispatch 2023-09-23T19:59:43.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5542/181380 objects degraded (3.055%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:43.144 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:43.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[221113]: failed to encode map e209 with expected crc 2023-09-23T19:59:43.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[221113]: osdmap e209: 8 total, 7 up, 8 in 2023-09-23T19:59:43.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[221113]: failed to encode map e209 with expected crc 2023-09-23T19:59:43.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[221113]: from='client.54925 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:43.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'41778','3':'39613','6':'21420','7':'54831'}"}]: dispatch 2023-09-23T19:59:43.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5542/181380 objects degraded (3.055%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:43.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[213871]: failed to encode map e209 with expected crc 2023-09-23T19:59:43.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[213871]: osdmap e209: 8 total, 7 up, 8 in 2023-09-23T19:59:43.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[213871]: failed to encode map e209 with expected crc 2023-09-23T19:59:43.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[213871]: from='client.54925 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:43.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'41778','3':'39613','6':'21420','7':'54831'}"}]: dispatch 2023-09-23T19:59:43.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5542/181380 objects degraded (3.055%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:43.173 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:59:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:59:43] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-09-23T19:59:43.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:43 smithi135 ceph-mon[213871]: pgmap v545: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5542/177846 objects degraded (3.116%); 7/177846 objects misplaced (0.004%); 0 B/s, 72 objects/s recovering 2023-09-23T19:59:43.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:43 smithi135 ceph-mon[213871]: failed to encode map e210 with expected crc 2023-09-23T19:59:43.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:43 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'41778','3':'39613','6':'21420','7':'54831'}"}]': finished 2023-09-23T19:59:43.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:43 smithi135 ceph-mon[213871]: osdmap e210: 8 total, 7 up, 8 in 2023-09-23T19:59:43.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:43 smithi135 ceph-mon[221113]: pgmap v545: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5542/177846 objects degraded (3.116%); 7/177846 objects misplaced (0.004%); 0 B/s, 72 objects/s recovering 2023-09-23T19:59:43.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:43 smithi135 ceph-mon[221113]: failed to encode map e210 with expected crc 2023-09-23T19:59:43.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:43 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'41778','3':'39613','6':'21420','7':'54831'}"}]': finished 2023-09-23T19:59:43.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:43 smithi135 ceph-mon[221113]: osdmap e210: 8 total, 7 up, 8 in 2023-09-23T19:59:44.240 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:44.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:44.344 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:44.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:43 smithi144 ceph-mon[194833]: pgmap v545: 109 pgs: 1 peering, 2 active+recovery_wait+remapped, 88 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5542/177846 objects degraded (3.116%); 7/177846 objects misplaced (0.004%); 0 B/s, 72 objects/s recovering 2023-09-23T19:59:44.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:43 smithi144 ceph-mon[194833]: failed to encode map e210 with expected crc 2023-09-23T19:59:44.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:43 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'41778','3':'39613','6':'21420','7':'54831'}"}]': finished 2023-09-23T19:59:44.368 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:43 smithi144 ceph-mon[194833]: osdmap e210: 8 total, 7 up, 8 in 2023-09-23T19:59:44.369 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:44.412 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:44.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:44.421 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:59:44.421 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T19:59:44.520 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:59:44.652 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:44.696 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T19:59:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:44.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:59:44.803 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:44.830 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:44.874 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:44.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:44.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:44.934 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.935 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.935 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.935 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: osdmap e211: 8 total, 7 up, 8 in 2023-09-23T19:59:44.935 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.935 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.936 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.936 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.936 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.936 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.936 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[213871]: failed to encode map e211 with expected crc 2023-09-23T19:59:44.961 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:59:44.980 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:45.098 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: osdmap e211: 8 total, 7 up, 8 in 2023-09-23T19:59:45.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:44 smithi135 ceph-mon[221113]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.228 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:45.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:45.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:45.332 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:45.361 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: osdmap e211: 8 total, 7 up, 8 in 2023-09-23T19:59:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:44 smithi144 ceph-mon[194833]: failed to encode map e211 with expected crc 2023-09-23T19:59:45.886 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:45.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:45.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:45.997 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:46.026 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T19:59:46.029 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:46.081 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:46.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:46.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:46.093 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:46.188 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:46.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: pgmap v548: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 16 peering, 1 active+remapped, 75 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 2 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3771/172050 objects degraded (2.192%); 6/172050 objects misplaced (0.003%); 144 KiB/s, 215 objects/s recovering 2023-09-23T19:59:46.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: osdmap e212: 8 total, 7 up, 8 in 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 209'8282 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 209'8311 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 209'8321 2023-09-23T19:59:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 209'8308 2023-09-23T19:59:46.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 209'9263 2023-09-23T19:59:46.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 209'9333 2023-09-23T19:59:46.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1087147973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:46.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:46.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: pgmap v548: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 16 peering, 1 active+remapped, 75 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 2 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3771/172050 objects degraded (2.192%); 6/172050 objects misplaced (0.003%); 144 KiB/s, 215 objects/s recovering 2023-09-23T19:59:46.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: osdmap e212: 8 total, 7 up, 8 in 2023-09-23T19:59:46.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:59:46.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 209'8282 2023-09-23T19:59:46.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 209'8311 2023-09-23T19:59:46.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 209'8321 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 209'8308 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 209'9263 2023-09-23T19:59:46.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 209'9333 2023-09-23T19:59:46.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1087147973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:46.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:45 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:46.266 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: pgmap v548: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 16 peering, 1 active+remapped, 75 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 2 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3771/172050 objects degraded (2.192%); 6/172050 objects misplaced (0.003%); 144 KiB/s, 215 objects/s recovering 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: osdmap e212: 8 total, 7 up, 8 in 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 209'8282 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 209'8311 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 209'8321 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 209'8308 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: failed to encode map e212 with expected crc 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 209'9263 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 209'9333 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1087147973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:46.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:45 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:59:46.408 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:46.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:46.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:46.518 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:46.591 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:46.688 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:46.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:46.796 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:46.839 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:47.113 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:47.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:47.119 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:47.119 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:47.119 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[213871]: osdmap e213: 8 total, 7 up, 8 in 2023-09-23T19:59:47.119 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[213871]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.119 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[213871]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.119 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[213871]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.119 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:47.120 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:47.120 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[221113]: osdmap e213: 8 total, 7 up, 8 in 2023-09-23T19:59:47.120 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[221113]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.120 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[221113]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.120 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:46 smithi135 ceph-mon[221113]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.220 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:47.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:46 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:47.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:59:47.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:46 smithi144 ceph-mon[194833]: osdmap e213: 8 total, 7 up, 8 in 2023-09-23T19:59:47.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:46 smithi144 ceph-mon[194833]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:46 smithi144 ceph-mon[194833]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:46 smithi144 ceph-mon[194833]: failed to encode map e213 with expected crc 2023-09-23T19:59:47.254 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:47.313 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:47.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:47.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:47.424 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:47.468 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:47.528 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T19:59:47.528 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T19:59:47.528 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T19:59:47.528 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T19:59:47.529 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.529 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149320 5.3 5:cd92c7f8:::benchmark_data_smithi135_284570_object33536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:47.529 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.280804+0000", 2023-09-23T19:59:47.529 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 303.24585881299998, 2023-09-23T19:59:47.529 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0708736379999999, 2023-09-23T19:59:47.529 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.529 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149320 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280804+0000", 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.530 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280804+0000", 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280803+0000", 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:47.531 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280809+0000", 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.161e-06 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280822+0000", 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3189e-05 2023-09-23T19:59:47.532 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280830+0000", 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5580000000000001e-06 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.282819+0000", 2023-09-23T19:59:47.533 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019887889999999999 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.282828+0000", 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3950000000000001e-06 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.347419+0000", 2023-09-23T19:59:47.534 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.06459086 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.347570+0000", 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015181 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.535 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.347717+0000", 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000146434 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351560+0000", 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038436 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351566+0000", 2023-09-23T19:59:47.536 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7300000000000002e-06 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351603+0000", 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7171e-05 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.537 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351678+0000", 2023-09-23T19:59:47.538 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4312000000000003e-05 2023-09-23T19:59:47.538 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.538 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.538 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.538 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.538 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.538 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149323 5.2 5:47d02f96:::benchmark_data_smithi135_284570_object33539:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.280843+0000", 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 303.24581977700001, 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0859534829999999, 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.539 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149323 2023-09-23T19:59:47.540 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.540 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.540 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.540 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.540 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280843+0000", 2023-09-23T19:59:47.540 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.540 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.540 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.541 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.541 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280843+0000", 2023-09-23T19:59:47.541 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.541 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.541 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.541 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.541 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280842+0000", 2023-09-23T19:59:47.541 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280846+0000", 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.179e-06 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280852+0000", 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4479999999999999e-06 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.542 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280857+0000", 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.383e-06 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280898+0000", 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0759000000000001e-05 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.543 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280923+0000", 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4875000000000002e-05 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.352747+0000", 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.071823787 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.355785+0000", 2023-09-23T19:59:47.544 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030379999999999999 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.356129+0000", 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034367799999999998 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.365029+0000", 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0089008030000000005 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.545 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366669+0000", 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016391800000000001 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366715+0000", 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6289999999999999e-05 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.546 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366796+0000", 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1630000000000003e-05 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149324 5.3 5:ce00d33f:::benchmark_data_smithi135_284570_object33540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.280870+0000", 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 303.24579268399998, 2023-09-23T19:59:47.547 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.085749139, 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149324 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.548 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280870+0000", 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280870+0000", 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280870+0000", 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:47.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280872+0000", 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.384e-06 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280878+0000", 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9699999999999996e-06 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.550 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280883+0000", 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9520000000000002e-06 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.282849+0000", 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001965751 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.282854+0000", 2023-09-23T19:59:47.551 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3460000000000001e-06 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351709+0000", 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0688548769999999 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351843+0000", 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013402800000000001 2023-09-23T19:59:47.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.351994+0000", 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015095700000000001 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366391+0000", 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014397092 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.553 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366490+0000", 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9444000000000005e-05 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366537+0000", 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6789999999999998e-05 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.554 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366619+0000", 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1923999999999997e-05 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149332 5.3 5:c5fd6d95:::benchmark_data_smithi135_284570_object33548:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.280890+0000", 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 303.24577256600003, 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0858478300000001, 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149332 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280890+0000", 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280890+0000", 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280890+0000", 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280893+0000", 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0180000000000002e-06 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280899+0000", 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0610000000000001e-06 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.280921+0000", 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2518e-05 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.284960+0000", 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0040385450000000002 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.284971+0000", 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0883000000000001e-05 2023-09-23T19:59:47.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.355670+0000", 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.070698873 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.355783+0000", 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011296199999999999 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.560 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.355910+0000", 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000127749 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366479+0000", 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010569017999999999 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366629+0000", 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000149727 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366667+0000", 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7789e-05 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366738+0000", 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1017000000000001e-05 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149341 5.3 5:ce411052:::benchmark_data_smithi135_284570_object33557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.310999+0000", 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 303.21566379699999, 2023-09-23T19:59:47.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0559260479999999, 2023-09-23T19:59:47.564 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.564 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.564 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.564 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.564 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.564 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149341 2023-09-23T19:59:47.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.564 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.310999+0000", 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.310999+0000", 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.565 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.310998+0000", 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311003+0000", 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9860000000000002e-06 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311012+0000", 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9889999999999998e-06 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311017+0000", 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6710000000000004e-06 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311078+0000", 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0819000000000002e-05 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311090+0000", 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.167e-05 2023-09-23T19:59:47.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.359565+0000", 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0484753490000001 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.359659+0000", 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3467000000000005e-05 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.359738+0000", 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9252000000000006e-05 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366748+0000", 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0070098000000000001 2023-09-23T19:59:47.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366795+0000", 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7707999999999998e-05 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366838+0000", 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2372e-05 2023-09-23T19:59:47.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366925+0000", 2023-09-23T19:59:47.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7219999999999995e-05 2023-09-23T19:59:47.572 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.572 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.572 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149342 5.3 5:c85e3fd6:::benchmark_data_smithi135_284570_object33558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.311033+0000", 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 303.21562939199998, 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0636845029999999, 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.573 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149342 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311033+0000", 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311033+0000", 2023-09-23T19:59:47.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311033+0000", 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311036+0000", 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5859999999999999e-06 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311041+0000", 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7690000000000002e-06 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311047+0000", 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4960000000000004e-06 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311117+0000", 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0098999999999994e-05 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.311124+0000", 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.17e-06 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.362787+0000", 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0516628180000001 2023-09-23T19:59:47.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.362915+0000", 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000128133 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.363035+0000", 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011958199999999999 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366969+0000", 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003934033 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374590+0000", 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0076213009999999996 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374640+0000", 2023-09-23T19:59:47.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0034999999999998e-05 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374718+0000", 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7803e-05 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.581 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149346 5.2 5:4e06899e:::benchmark_data_smithi135_284570_object33562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.315466+0000", 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 303.21119660400001, 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0514979799999999, 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149346 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315466+0000", 2023-09-23T19:59:47.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315466+0000", 2023-09-23T19:59:47.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315465+0000", 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315473+0000", 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3769999999999996e-06 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315485+0000", 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1821000000000001e-05 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.315493+0000", 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7610000000000007e-06 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.318492+0000", 2023-09-23T19:59:47.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002999191 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.318504+0000", 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1802e-05 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.361234+0000", 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0427307020000001 2023-09-23T19:59:47.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.361347+0000", 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000112432 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.361477+0000", 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000129951 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.365236+0000", 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0037598369999999998 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366836+0000", 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015994659999999999 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366877+0000", 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0794999999999998e-05 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366964+0000", 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7371000000000001e-05 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:149353 5.3 5:c1a9dfba:::benchmark_data_smithi135_284570_object33569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:54:44.332016+0000", 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 303.19464700100002, 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.042929658, 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 149353 2023-09-23T19:59:47.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332016+0000", 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332016+0000", 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332014+0000", 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332022+0000", 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0299999999999994e-06 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332041+0000", 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9198000000000002e-05 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332051+0000", 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0376e-05 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332106+0000", 2023-09-23T19:59:47.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4449e-05 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:44.332119+0000", 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3286e-05 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.366991+0000", 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0348714619999999 2023-09-23T19:59:47.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.367054+0000", 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3359000000000001e-05 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.367121+0000", 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7280999999999995e-05 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374731+0000", 2023-09-23T19:59:47.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0076100459999999997 2023-09-23T19:59:47.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374817+0000", 2023-09-23T19:59:47.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6079e-05 2023-09-23T19:59:47.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374862+0000", 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4888000000000001e-05 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:54:45.374945+0000", 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3095000000000003e-05 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.599 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203721 5.3 5:cfbf513c:::benchmark_data_smithi135_284570_object87937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.301178+0000", 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 228.22548512200001, 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30152889100000002, 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.600 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203721 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301178+0000", 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301178+0000", 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301177+0000", 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:47.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301180+0000", 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3309999999999998e-06 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301186+0000", 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4790000000000001e-06 2023-09-23T19:59:47.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301191+0000", 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9529999999999998e-06 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.585593+0000", 2023-09-23T19:59:47.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28440167300000002 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.585719+0000", 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012650399999999999 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.585850+0000", 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013128200000000001 2023-09-23T19:59:47.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602545+0000", 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016694229000000001 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602592+0000", 2023-09-23T19:59:47.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7463000000000003e-05 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602630+0000", 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8025999999999998e-05 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602707+0000", 2023-09-23T19:59:47.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6346000000000004e-05 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203724 5.3 5:c7db1c20:::benchmark_data_smithi135_284570_object87940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.301224+0000", 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 228.22543911400001, 2023-09-23T19:59:47.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30165318699999999, 2023-09-23T19:59:47.609 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.609 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.609 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203724 2023-09-23T19:59:47.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.609 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301224+0000", 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301224+0000", 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301223+0000", 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301226+0000", 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7319999999999998e-06 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301232+0000", 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4150000000000004e-06 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301239+0000", 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4609999999999996e-06 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.590412+0000", 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28917298800000002 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.590557+0000", 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000145463 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.590676+0000", 2023-09-23T19:59:47.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000119067 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602550+0000", 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011873711 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602757+0000", 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020686599999999999 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602801+0000", 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4097999999999997e-05 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602877+0000", 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5827000000000004e-05 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203725 5.3 5:cd7d81d5:::benchmark_data_smithi135_284570_object87941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.301446+0000", 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 228.22521685999999, 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31410600700000002, 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203725 2023-09-23T19:59:47.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301446+0000", 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301446+0000", 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301445+0000", 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.301450+0000", 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9470000000000002e-06 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.579691+0000", 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27824077600000002 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.579711+0000", 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9721e-05 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.594999+0000", 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01528876 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.595109+0000", 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010924100000000001 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5", 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.595218+0000", 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000109893 2023-09-23T19:59:47.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.602555+0000", 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073367709999999997 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.615415+0000", 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012859628 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.615463+0000", 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8649999999999997e-05 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.615552+0000", 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8417000000000004e-05 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.622 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:116216 6.0 6:006ae4ef:::benchmark_data_smithi135_344736_object13560:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:56.519668+0000", 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 51.006995195000002, 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.87800207500000005, 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:47.623 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116216 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.519668+0000", 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.519668+0000", 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.519665+0000", 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.519671+0000", 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0939999999999996e-06 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.519681+0000", 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6139999999999998e-06 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.519689+0000", 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2549999999999997e-06 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.556359+0000", 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.036670029999999999 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.556373+0000", 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3361e-05 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.367410+0000", 2023-09-23T19:59:47.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.81103717600000003 2023-09-23T19:59:47.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:47.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.368022+0000", 2023-09-23T19:59:47.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00061179400000000003 2023-09-23T19:59:47.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.374969+0000", 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0069470010000000004 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.375041+0000", 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2113000000000006e-05 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.375348+0000", 2023-09-23T19:59:47.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00030678500000000001 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.377931+0000", 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025835189999999998 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.377947+0000", 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6062e-05 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.395921+0000", 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017973745999999999 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.395976+0000", 2023-09-23T19:59:47.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5529999999999999e-05 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.397670+0000", 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016932189999999999 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:116224 6.0 6:059778af:::benchmark_data_smithi135_344736_object13568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:56.584344+0000", 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 50.942319157, 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.81433079799999997, 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116224 2023-09-23T19:59:47.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.584344+0000", 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.584344+0000", 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.584342+0000", 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.584350+0000", 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0819999999999999e-06 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.584367+0000", 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6657e-05 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.584376+0000", 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8819999999999996e-06 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.584429+0000", 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2049000000000003e-05 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.584439+0000", 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0735e-05 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.370025+0000", 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.78558527300000003 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.370617+0000", 2023-09-23T19:59:47.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000592663 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.384037+0000", 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013420006 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.384142+0000", 2023-09-23T19:59:47.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010493300000000001 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.384501+0000", 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00035883200000000001 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.394013+0000", 2023-09-23T19:59:47.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0095118200000000007 2023-09-23T19:59:47.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.397729+0000", 2023-09-23T19:59:47.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0037164519999999999 2023-09-23T19:59:47.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.397759+0000", 2023-09-23T19:59:47.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0005000000000002e-05 2023-09-23T19:59:47.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.397794+0000", 2023-09-23T19:59:47.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4669000000000003e-05 2023-09-23T19:59:47.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.398674+0000", 2023-09-23T19:59:47.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00088043799999999999 2023-09-23T19:59:47.643 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.643 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.643 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.644 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:116227 6.0 6:02e55e3e:::benchmark_data_smithi135_344736_object13571:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T19:59:47.644 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:56.603078+0000", 2023-09-23T19:59:47.644 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 50.923584851999998, 2023-09-23T19:59:47.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.79642191600000001, 2023-09-23T19:59:47.644 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.644 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116227 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.603078+0000", 2023-09-23T19:59:47.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.603078+0000", 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.603076+0000", 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T19:59:47.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.603086+0000", 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8009999999999998e-06 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.603108+0000", 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2476999999999998e-05 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.603119+0000", 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0896999999999999e-05 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.603187+0000", 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8164000000000007e-05 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.603203+0000", 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5257e-05 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.370714+0000", 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.76751127399999997 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.371251+0000", 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00053709500000000002 2023-09-23T19:59:47.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.388898+0000", 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017647187000000002 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.388997+0000", 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9130999999999997e-05 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.389295+0000", 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00029783000000000001 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.397687+0000", 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083917090000000007 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.398723+0000", 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010362590000000001 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.398750+0000", 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7219000000000001e-05 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.398782+0000", 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2246000000000003e-05 2023-09-23T19:59:47.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.399500+0000", 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00071730999999999997 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:116236 6.0 6:02976725:::benchmark_data_smithi135_344736_object13580:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T19:59:47.653 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:56.644154+0000", 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 50.882508649999998, 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.76081703300000003, 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116236 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.654 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.644154+0000", 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.644154+0000", 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.644153+0000", 2023-09-23T19:59:47.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.644161+0000", 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3829999999999992e-06 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.644181+0000", 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0276999999999999e-05 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.644189+0000", 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0010000000000001e-06 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.644257+0000", 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8077000000000006e-05 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.644273+0000", 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5498e-05 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.371303+0000", 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.72703036600000004 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.371956+0000", 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00065329099999999996 2023-09-23T19:59:47.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.394044+0000", 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.022087814000000001 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.394124+0000", 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9688e-05 2023-09-23T19:59:47.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.394441+0000", 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00031681 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.398690+0000", 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0042487219999999999 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.399562+0000", 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000872141 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.402686+0000", 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031244419999999998 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.402746+0000", 2023-09-23T19:59:47.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0228e-05 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.404971+0000", 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022247859999999999 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.662 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148800 6.9 6:9622c4aa:::benchmark_data_smithi135_344736_object46144:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.921964+0000", 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 4.6046983849999998, 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2108138190000002, 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148800 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.921964+0000", 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.921964+0000", 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.921962+0000", 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.921971+0000", 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8419999999999994e-06 2023-09-23T19:59:47.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.921983+0000", 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1958e-05 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.921993+0000", 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8030000000000008e-06 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.922072+0000", 2023-09-23T19:59:47.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8862999999999994e-05 2023-09-23T19:59:47.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.922221+0000", 2023-09-23T19:59:47.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014947699999999999 2023-09-23T19:59:47.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.922696+0000", 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00047537300000000002 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.930498+0000", 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0078011870000000002 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.930510+0000", 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2689e-05 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.047086+0000", 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1165753629999999 2023-09-23T19:59:47.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.047101+0000", 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5557999999999999e-05 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.129427+0000", 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.082325771 2023-09-23T19:59:47.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.129573+0000", 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000146343 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.132702+0000", 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031291819999999999 2023-09-23T19:59:47.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.132778+0000", 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5709000000000003e-05 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.672 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148831 6.9 6:92c3823a:::benchmark_data_smithi135_344736_object46175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.185436+0000", 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.341226807, 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32977368800000001, 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T19:59:47.673 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148831 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185436+0000", 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185436+0000", 2023-09-23T19:59:47.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185434+0000", 2023-09-23T19:59:47.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T19:59:47.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185439+0000", 2023-09-23T19:59:47.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9130000000000002e-06 2023-09-23T19:59:47.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185449+0000", 2023-09-23T19:59:47.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6500000000000008e-06 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185455+0000", 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4960000000000001e-06 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185606+0000", 2023-09-23T19:59:47.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000150836 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185778+0000", 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000171719 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.186234+0000", 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045615899999999998 2023-09-23T19:59:47.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.188694+0000", 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00245956 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.197210+0000", 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085166029999999993 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206754+0000", 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0095434669999999999 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.515055+0000", 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30830142500000002 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.515104+0000", 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8616999999999999e-05 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.515210+0000", 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010583500000000001 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:148809 6.a e212/211 6:512c2dc0:::benchmark_data_smithi135_344736_object46153:head v 212'9264, mlcod=0'0)", 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.206248+0000", 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.320414456, 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33356148800000002, 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206248+0000", 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206248+0000", 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206248+0000", 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T19:59:47.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206253+0000", 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5450000000000003e-06 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206261+0000", 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7020000000000001e-06 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206266+0000", 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1710000000000001e-06 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206800+0000", 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00053379100000000004 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206821+0000", 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0737000000000002e-05 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.539772+0000", 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33295126600000002 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.539810+0000", 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7786e-05 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:148810 6.a e212/211 6:5d5fc8b8:::benchmark_data_smithi135_344736_object46154:head v 212'9265, mlcod=0'0)", 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.206277+0000", 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.3203852980000002, 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33355396999999998, 2023-09-23T19:59:47.687 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206277+0000", 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206277+0000", 2023-09-23T19:59:47.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206277+0000", 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967296 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206280+0000", 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1690000000000001e-06 2023-09-23T19:59:47.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206285+0000", 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9200000000000003e-06 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.206289+0000", 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7799999999999998e-06 2023-09-23T19:59:47.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.525017+0000", 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31872744400000003 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.525038+0000", 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1804e-05 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.691 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.539813+0000", 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014774964 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.539831+0000", 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8096000000000001e-05 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.692 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:148834 6.a e212/211 6:5db4081a:::benchmark_data_smithi135_344736_object46178:head v 212'9266, mlcod=0'0)", 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.208886+0000", 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.3177765780000001, 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33096303100000002, 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.208886+0000", 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.208886+0000", 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T19:59:47.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.208885+0000", 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.208892+0000", 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4289999999999998e-06 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.512694+0000", 2023-09-23T19:59:47.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30380244499999998 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.512706+0000", 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2425999999999999e-05 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527602+0000", 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014895036 2023-09-23T19:59:47.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527624+0000", 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2957000000000001e-05 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.539833+0000", 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012208841 2023-09-23T19:59:47.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.539849+0000", 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5858999999999998e-05 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T19:59:47.698 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T19:59:47.699 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T19:59:47.699 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:47.699 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:47.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:47.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:47.801 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:47.830 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:48.148 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[213871]: pgmap v551: 109 pgs: 3 active+clean+remapped, 1 remapped+peering, 1 active+recovery_wait+remapped, 15 peering, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3798/169677 objects degraded (2.238%); 13488/169677 objects misplaced (7.949%); 352 KiB/s, 219 objects/s recovering 2023-09-23T19:59:48.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[213871]: osdmap e214: 8 total, 7 up, 8 in 2023-09-23T19:59:48.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[213871]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[213871]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[213871]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[213871]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.149 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 3798/169677 objects degraded (2.238%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:48.149 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[221113]: pgmap v551: 109 pgs: 3 active+clean+remapped, 1 remapped+peering, 1 active+recovery_wait+remapped, 15 peering, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3798/169677 objects degraded (2.238%); 13488/169677 objects misplaced (7.949%); 352 KiB/s, 219 objects/s recovering 2023-09-23T19:59:48.149 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[221113]: osdmap e214: 8 total, 7 up, 8 in 2023-09-23T19:59:48.150 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[221113]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.150 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[221113]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.150 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[221113]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.150 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[221113]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.150 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 3798/169677 objects degraded (2.238%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:47 smithi144 ceph-mon[194833]: pgmap v551: 109 pgs: 3 active+clean+remapped, 1 remapped+peering, 1 active+recovery_wait+remapped, 15 peering, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3798/169677 objects degraded (2.238%); 13488/169677 objects misplaced (7.949%); 352 KiB/s, 219 objects/s recovering 2023-09-23T19:59:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:47 smithi144 ceph-mon[194833]: osdmap e214: 8 total, 7 up, 8 in 2023-09-23T19:59:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:47 smithi144 ceph-mon[194833]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:47 smithi144 ceph-mon[194833]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:47 smithi144 ceph-mon[194833]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:47 smithi144 ceph-mon[194833]: failed to encode map e214 with expected crc 2023-09-23T19:59:48.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 3798/169677 objects degraded (2.238%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:49.078 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:49.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:49.189 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:49.283 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:49.326 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:59:49.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:49 smithi144 ceph-mon[194833]: osdmap e215: 8 total, 7 up, 8 in 2023-09-23T19:59:49.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:49 smithi144 ceph-mon[194833]: failed to encode map e215 with expected crc 2023-09-23T19:59:49.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:49 smithi144 ceph-mon[194833]: failed to encode map e215 with expected crc 2023-09-23T19:59:49.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:49 smithi144 ceph-mon[194833]: failed to encode map e215 with expected crc 2023-09-23T19:59:49.407 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:49 smithi135 ceph-mon[213871]: osdmap e215: 8 total, 7 up, 8 in 2023-09-23T19:59:49.407 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:49 smithi135 ceph-mon[213871]: failed to encode map e215 with expected crc 2023-09-23T19:59:49.407 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:49 smithi135 ceph-mon[213871]: failed to encode map e215 with expected crc 2023-09-23T19:59:49.407 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:49 smithi135 ceph-mon[213871]: failed to encode map e215 with expected crc 2023-09-23T19:59:49.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:49 smithi135 ceph-mon[221113]: osdmap e215: 8 total, 7 up, 8 in 2023-09-23T19:59:49.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:49 smithi135 ceph-mon[221113]: failed to encode map e215 with expected crc 2023-09-23T19:59:49.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:49 smithi135 ceph-mon[221113]: failed to encode map e215 with expected crc 2023-09-23T19:59:49.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:49 smithi135 ceph-mon[221113]: failed to encode map e215 with expected crc 2023-09-23T19:59:50.127 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:59:50.207 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:50.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:50.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:50.315 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:50.349 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:50 smithi144 ceph-mon[194833]: pgmap v554: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 12152/165990 objects degraded (7.321%); 28715/165990 objects misplaced (17.299%); 2.8 MiB/s, 256 objects/s recovering 2023-09-23T19:59:50.349 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:50 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:59:50.354 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:50.424 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:50.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:50.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:50 smithi135 ceph-mon[221113]: pgmap v554: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 12152/165990 objects degraded (7.321%); 28715/165990 objects misplaced (17.299%); 2.8 MiB/s, 256 objects/s recovering 2023-09-23T19:59:50.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:50 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:59:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:50 smithi135 ceph-mon[213871]: pgmap v554: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 12152/165990 objects degraded (7.321%); 28715/165990 objects misplaced (17.299%); 2.8 MiB/s, 256 objects/s recovering 2023-09-23T19:59:50.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:50 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:59:50.531 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:50.558 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:50.994 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:51.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:51.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:51.103 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:51.198 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:51.390 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:51.391 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:51.521 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:51.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:51.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:51.627 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:51.698 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:51.839 INFO:teuthology.orchestra.run.smithi135.stdout:moved 5 / 327 (1.52905%) 2023-09-23T19:59:51.839 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T19:59:51.840 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.90505 -> 7.47922 (expected baseline 6.32778) 2023-09-23T19:59:51.840 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 29 -> 29 pgs (0.620795 -> 0.620795 * mean) 2023-09-23T19:59:51.840 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 55 -> 52 pgs (1.17737 -> 1.11315 * mean) 2023-09-23T19:59:51.840 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T19:59:51.840 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T19:59:51.840 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 1 2023-09-23T19:59:51.840 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T19:59:51.840 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0186 2023-09-23T19:59:51.841 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.1864 2023-09-23T19:59:51.841 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 0.3268 -> 0.4140 2023-09-23T19:59:51.878 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T19:59:51.889 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:51.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:51.961 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T19:59:51.983 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T19:59:52.005 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:59:52.034 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T19:59:52.072 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T19:59:52.096 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:52 smithi135 ceph-mon[213871]: pgmap v555: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 12152/164868 objects degraded (7.371%); 28341/164868 objects misplaced (17.190%); 2.0 MiB/s, 182 objects/s recovering 2023-09-23T19:59:52.096 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2238515884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:52.096 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:59:51 smithi135 conmon[201366]: 2023-09-23T19:59:51.833+0000 7fcf49f9e700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-23T19:59:52.097 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:52 smithi135 ceph-mon[221113]: pgmap v555: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 12152/164868 objects degraded (7.371%); 28341/164868 objects misplaced (17.190%); 2.0 MiB/s, 182 objects/s recovering 2023-09-23T19:59:52.097 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2238515884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:52.110 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:52.131 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T19:59:52.174 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T19:59:52.227 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T19:59:52.228 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:59:52.228 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:59:52.228 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:59:52.228 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:59:52.228 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:59:52.228 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:59:52.228 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T19:59:52.228 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:59:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:52 smithi144 ceph-mon[194833]: pgmap v555: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 12152/164868 objects degraded (7.371%); 28341/164868 objects misplaced (17.190%); 2.0 MiB/s, 182 objects/s recovering 2023-09-23T19:59:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2238515884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:59:53.079 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:53 smithi135 ceph-mon[213871]: from='client.54937 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:53.080 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:53.080 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12152/164868 objects degraded (7.371%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:53.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:53 smithi144 ceph-mon[194833]: from='client.54937 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:53.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:53.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12152/164868 objects degraded (7.371%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:53.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:53 smithi135 ceph-mon[221113]: from='client.54937 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:59:53.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:59:53.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12152/164868 objects degraded (7.371%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:53.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 19:59:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:19:59:53] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-09-23T19:59:54.549 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T19:59:54.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:54.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:59:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:54 smithi144 ceph-mon[194833]: pgmap v556: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 12229/162120 objects degraded (7.543%); 27791/162120 objects misplaced (17.142%); 2.9 MiB/s, 175 objects/s recovering 2023-09-23T19:59:54.667 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:54.709 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:54 smithi135 ceph-mon[213871]: pgmap v556: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 12229/162120 objects degraded (7.543%); 27791/162120 objects misplaced (17.142%); 2.9 MiB/s, 175 objects/s recovering 2023-09-23T19:59:54.709 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:54 smithi135 ceph-mon[221113]: pgmap v556: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 12229/162120 objects degraded (7.543%); 27791/162120 objects misplaced (17.142%); 2.9 MiB/s, 175 objects/s recovering 2023-09-23T19:59:54.713 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T19:59:54.714 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T19:59:54.770 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:55.059 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:55.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:55.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:59:55.106 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:59:55.168 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:55.260 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:55.907 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:55.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:56.015 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:56.051 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:56.122 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:56.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:56.227 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:56.253 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:56.299 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:56.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:56.406 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:59:56.460 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:56.523 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T19:59:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:56.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:59:56.631 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:59:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:56 smithi144 ceph-mon[194833]: pgmap v557: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 12229/156048 objects degraded (7.837%); 26558/156048 objects misplaced (17.019%); 6.7 MiB/s, 211 objects/s recovering 2023-09-23T19:59:56.667 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:56.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:56 smithi135 ceph-mon[213871]: pgmap v557: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 12229/156048 objects degraded (7.837%); 26558/156048 objects misplaced (17.019%); 6.7 MiB/s, 211 objects/s recovering 2023-09-23T19:59:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:56 smithi135 ceph-mon[221113]: pgmap v557: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 12229/156048 objects degraded (7.837%); 26558/156048 objects misplaced (17.019%); 6.7 MiB/s, 211 objects/s recovering 2023-09-23T19:59:56.740 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T19:59:56.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:56.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:59:56.848 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:59:56.959 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:57.107 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:59:57.201 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:59:57.289 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T19:59:57.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:57.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:59:57.400 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12229/156048 objects degraded (7.837%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12229/156048 objects degraded (7.837%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:57.529 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:57.700 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:57.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:57.711 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T19:59:57.711 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:59:57.807 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:59:57.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12229/156048 objects degraded (7.837%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T19:59:57.924 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:58.397 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T19:59:58.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:58.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:59:58.454 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T19:59:58.501 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:58.581 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T19:59:58.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:58 smithi135 ceph-mon[221113]: pgmap v558: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 12378/152556 objects degraded (8.114%); 25634/152556 objects misplaced (16.803%); 6.2 MiB/s, 187 objects/s recovering 2023-09-23T19:59:58.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1248895871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:58.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:59:58.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:58 smithi135 ceph-mon[213871]: pgmap v558: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 12378/152556 objects degraded (8.114%); 25634/152556 objects misplaced (16.803%); 6.2 MiB/s, 187 objects/s recovering 2023-09-23T19:59:58.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1248895871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:59:58.767 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:58 smithi144 ceph-mon[194833]: pgmap v558: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 12378/152556 objects degraded (8.114%); 25634/152556 objects misplaced (16.803%); 6.2 MiB/s, 187 objects/s recovering 2023-09-23T19:59:58.767 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1248895871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:59:58.768 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:59:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[213871]: failed to encode map e216 with expected crc 2023-09-23T19:59:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1248895871' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[213871]: osdmap e216: 8 total, 7 up, 8 in 2023-09-23T19:59:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[213871]: failed to encode map e216 with expected crc 2023-09-23T19:59:59.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:59.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[221113]: failed to encode map e216 with expected crc 2023-09-23T19:59:59.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1248895871' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:59.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[221113]: osdmap e216: 8 total, 7 up, 8 in 2023-09-23T19:59:59.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 19:59:59 smithi135 ceph-mon[221113]: failed to encode map e216 with expected crc 2023-09-23T19:59:59.785 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T19:59:59.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T19:59:59.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:59:59.885 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:59 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:59:59.886 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:59 smithi144 ceph-mon[194833]: failed to encode map e216 with expected crc 2023-09-23T19:59:59.886 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1248895871' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:59:59.886 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:59 smithi144 ceph-mon[194833]: osdmap e216: 8 total, 7 up, 8 in 2023-09-23T19:59:59.886 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 19:59:59 smithi144 ceph-mon[194833]: failed to encode map e216 with expected crc 2023-09-23T19:59:59.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:59:59.983 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:00.678 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:00:00.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pgmap v560: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12378/144942 objects degraded (8.540%); 24030/144942 objects misplaced (16.579%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T20:00:00.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 12378/144942 objects degraded (8.540%), 12 pgs degraded, 5 pgs undersized 2023-09-23T20:00:00.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: osd.6 (root=default,host=smithi144) is down 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: [WRN] PG_DEGRADED: Degraded data redundancy: 12378/144942 objects degraded (8.540%), 12 pgs degraded, 5 pgs undersized 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 3.d is stuck undersized for 5m, current state active+undersized+degraded, last acting [7,5] 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 3.10 is stuck undersized for 5m, current state active+undersized+degraded, last acting [0,5] 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 3.11 is stuck undersized for 5m, current state active+undersized, last acting [7,4] 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 4.1 is stuck undersized for 5m, current state active+undersized, last acting [4,5] 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 4.13 is stuck undersized for 5m, current state active+undersized, last acting [4,1] 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [4,2] 2023-09-23T20:00:00.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.4 is active+recovery_wait+degraded+remapped, acting [1,5,3] 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.8 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.f is active+recovering+undersized+degraded+remapped, acting [3,4] 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.10 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.12 is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.17 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[213871]: pg 6.18 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pgmap v560: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12378/144942 objects degraded (8.540%); 24030/144942 objects misplaced (16.579%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 12378/144942 objects degraded (8.540%), 12 pgs degraded, 5 pgs undersized 2023-09-23T20:00:00.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: osd.6 (root=default,host=smithi144) is down 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: [WRN] PG_DEGRADED: Degraded data redundancy: 12378/144942 objects degraded (8.540%), 12 pgs degraded, 5 pgs undersized 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 3.d is stuck undersized for 5m, current state active+undersized+degraded, last acting [7,5] 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 3.10 is stuck undersized for 5m, current state active+undersized+degraded, last acting [0,5] 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 3.11 is stuck undersized for 5m, current state active+undersized, last acting [7,4] 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 4.1 is stuck undersized for 5m, current state active+undersized, last acting [4,5] 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 4.13 is stuck undersized for 5m, current state active+undersized, last acting [4,1] 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [4,2] 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.4 is active+recovery_wait+degraded+remapped, acting [1,5,3] 2023-09-23T20:00:00.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T20:00:00.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.8 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-09-23T20:00:00.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.f is active+recovering+undersized+degraded+remapped, acting [3,4] 2023-09-23T20:00:00.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.10 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T20:00:00.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.12 is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-09-23T20:00:00.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.17 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T20:00:00.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:00 smithi135 ceph-mon[221113]: pg 6.18 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-09-23T20:00:00.811 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:00.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:00.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:00.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pgmap v560: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12378/144942 objects degraded (8.540%); 24030/144942 objects misplaced (16.579%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 12378/144942 objects degraded (8.540%), 12 pgs degraded, 5 pgs undersized 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: osd.6 (root=default,host=smithi144) is down 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: [WRN] PG_DEGRADED: Degraded data redundancy: 12378/144942 objects degraded (8.540%), 12 pgs degraded, 5 pgs undersized 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 3.d is stuck undersized for 5m, current state active+undersized+degraded, last acting [7,5] 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 3.10 is stuck undersized for 5m, current state active+undersized+degraded, last acting [0,5] 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 3.11 is stuck undersized for 5m, current state active+undersized, last acting [7,4] 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 4.1 is stuck undersized for 5m, current state active+undersized, last acting [4,5] 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 4.13 is stuck undersized for 5m, current state active+undersized, last acting [4,1] 2023-09-23T20:00:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [4,2] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.4 is active+recovery_wait+degraded+remapped, acting [1,5,3] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.8 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.f is active+recovering+undersized+degraded+remapped, acting [3,4] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.10 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.12 is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.17 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T20:00:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:00 smithi144 ceph-mon[194833]: pg 6.18 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-09-23T20:00:00.910 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:00.939 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:00.980 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:01.104 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:01.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:01.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:01.210 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:01.293 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:01.483 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:01.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:01.593 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:01.644 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:01.811 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:01.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:01.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:01.917 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:01.951 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:02.024 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:02.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:02.135 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:02.220 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:02.466 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[213871]: pgmap v561: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 12378/143658 objects degraded (8.616%); 23602/143658 objects misplaced (16.429%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T20:00:02.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'24696'}"}]: dispatch 2023-09-23T20:00:02.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:02.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:02.468 INFO:teuthology.orchestra.run.smithi135.stdout:moved 3 / 327 (0.917431%) 2023-09-23T20:00:02.469 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T20:00:02.469 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.90505 -> 7.65986 (expected baseline 6.32778) 2023-09-23T20:00:02.469 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 29 -> 29 pgs (0.620795 -> 0.620795 * mean) 2023-09-23T20:00:02.469 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 55 -> 53 pgs (1.17737 -> 1.13456 * mean) 2023-09-23T20:00:02.469 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:00:02.469 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T20:00:02.469 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T20:00:02.469 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:00:02.470 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0171 2023-09-23T20:00:02.470 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.1707 2023-09-23T20:00:02.470 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6 weight 0.3268 -> 0.3768 2023-09-23T20:00:02.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12378/143658 objects degraded (8.616%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[221113]: pgmap v561: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 12378/143658 objects degraded (8.616%); 23602/143658 objects misplaced (16.429%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T20:00:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'24696'}"}]: dispatch 2023-09-23T20:00:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:02 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12378/143658 objects degraded (8.616%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:02.750 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:02.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:02.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:02.855 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:02.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:02 smithi144 ceph-mon[194833]: pgmap v561: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 12378/143658 objects degraded (8.616%); 23602/143658 objects misplaced (16.429%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T20:00:02.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'24696'}"}]: dispatch 2023-09-23T20:00:02.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:02.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:02.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:02 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12378/143658 objects degraded (8.616%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:02.886 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:02.941 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:02.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:02.965 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:00:03.046 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:03.075 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:03.481 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:03.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:03.586 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:03.628 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:03.697 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:03.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:03.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:03.805 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:03.844 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:03 smithi144 ceph-mon[194833]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:00:03.844 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:03 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'24696'}"}]': finished 2023-09-23T20:00:03.844 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:03 smithi144 ceph-mon[194833]: osdmap e217: 8 total, 7 up, 8 in 2023-09-23T20:00:03.844 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:03 smithi144 ceph-mon[194833]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.844 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:03 smithi144 ceph-mon[194833]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.844 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:03 smithi144 ceph-mon[194833]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.844 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:03 smithi144 ceph-mon[194833]: pgmap v563: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 12524/140298 objects degraded (8.927%); 22948/140298 objects misplaced (16.357%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:00:03.846 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:03.917 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:03.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:03.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:03.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:00:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:00:03] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-09-23T20:00:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[213871]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:00:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'24696'}"}]': finished 2023-09-23T20:00:03.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[213871]: osdmap e217: 8 total, 7 up, 8 in 2023-09-23T20:00:03.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[213871]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[213871]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[213871]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[213871]: pgmap v563: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 12524/140298 objects degraded (8.927%); 22948/140298 objects misplaced (16.357%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:00:03.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[221113]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:00:03.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'24696'}"}]': finished 2023-09-23T20:00:03.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[221113]: osdmap e217: 8 total, 7 up, 8 in 2023-09-23T20:00:03.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[221113]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[221113]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[221113]: failed to encode map e217 with expected crc 2023-09-23T20:00:03.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:03 smithi135 ceph-mon[221113]: pgmap v563: 109 pgs: 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 12524/140298 objects degraded (8.927%); 22948/140298 objects misplaced (16.357%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:00:04.023 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:04.074 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:04.670 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:04.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:04.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:04.782 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:04.810 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:04 smithi144 ceph-mon[194833]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.810 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:04 smithi144 ceph-mon[194833]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.810 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:04 smithi144 ceph-mon[194833]: osdmap e218: 8 total, 7 up, 8 in 2023-09-23T20:00:04.810 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:04 smithi144 ceph-mon[194833]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.810 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:04 smithi144 ceph-mon[194833]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.810 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:04 smithi144 ceph-mon[194833]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.813 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:04.872 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:04.877 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[221113]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.877 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[221113]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.878 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[221113]: osdmap e218: 8 total, 7 up, 8 in 2023-09-23T20:00:04.878 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[221113]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.878 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[221113]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.878 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[221113]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.879 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:04.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:04.881 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[213871]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.881 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[213871]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.881 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[213871]: osdmap e218: 8 total, 7 up, 8 in 2023-09-23T20:00:04.881 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[213871]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.881 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[213871]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.881 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:04 smithi135 ceph-mon[213871]: failed to encode map e218 with expected crc 2023-09-23T20:00:04.981 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:05.042 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:05.092 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:05.122 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:05.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:05.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:05.230 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:05.265 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:05.457 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:05.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:05.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:05.563 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:05.593 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:00:05.594 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:05.645 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:00:05.647 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:05.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:05.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:05.681 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:00:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[221113]: osdmap e219: 8 total, 7 up, 8 in 2023-09-23T20:00:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[221113]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[221113]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[221113]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[221113]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[221113]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[221113]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[221113]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[213871]: osdmap e219: 8 total, 7 up, 8 in 2023-09-23T20:00:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[213871]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[213871]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[213871]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[213871]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[213871]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[213871]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:05 smithi135 ceph-mon[213871]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.728 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:00:05.759 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:05.773 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:00:05.816 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:00:05.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:00:05.873 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:05 smithi144 ceph-mon[194833]: osdmap e219: 8 total, 7 up, 8 in 2023-09-23T20:00:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:05 smithi144 ceph-mon[194833]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:05 smithi144 ceph-mon[194833]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:05 smithi144 ceph-mon[194833]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:05 smithi144 ceph-mon[194833]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:05 smithi144 ceph-mon[194833]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:05 smithi144 ceph-mon[194833]: failed to encode map e219 with expected crc 2023-09-23T20:00:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:05 smithi144 ceph-mon[194833]: failed to encode map e219 with expected crc 2023-09-23T20:00:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:06 smithi144 ceph-mon[194833]: pgmap v566: 109 pgs: 1 remapped+peering, 1 peering, 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 12524/134811 objects degraded (9.290%); 19252/134811 objects misplaced (14.281%); 4.2 MiB/s, 190 objects/s recovering 2023-09-23T20:00:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:06 smithi144 ceph-mon[194833]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:06 smithi144 ceph-mon[194833]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:06 smithi144 ceph-mon[194833]: osdmap e220: 8 total, 7 up, 8 in 2023-09-23T20:00:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:06 smithi144 ceph-mon[194833]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1928849010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:06.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[221113]: pgmap v566: 109 pgs: 1 remapped+peering, 1 peering, 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 12524/134811 objects degraded (9.290%); 19252/134811 objects misplaced (14.281%); 4.2 MiB/s, 190 objects/s recovering 2023-09-23T20:00:06.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[221113]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[221113]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[221113]: osdmap e220: 8 total, 7 up, 8 in 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[221113]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1928849010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[213871]: pgmap v566: 109 pgs: 1 remapped+peering, 1 peering, 8 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 12524/134811 objects degraded (9.290%); 19252/134811 objects misplaced (14.281%); 4.2 MiB/s, 190 objects/s recovering 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[213871]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[213871]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[213871]: osdmap e220: 8 total, 7 up, 8 in 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[213871]: failed to encode map e220 with expected crc 2023-09-23T20:00:06.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1928849010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:07 smithi144 ceph-mon[194833]: pgmap v568: 109 pgs: 1 remapped+peering, 1 peering, 9 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 12559/131820 objects degraded (9.527%); 18436/131820 objects misplaced (13.986%); 7.2 MiB/s, 295 objects/s recovering 2023-09-23T20:00:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:07 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12524/134811 objects degraded (9.290%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:07.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:07 smithi135 ceph-mon[213871]: pgmap v568: 109 pgs: 1 remapped+peering, 1 peering, 9 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 12559/131820 objects degraded (9.527%); 18436/131820 objects misplaced (13.986%); 7.2 MiB/s, 295 objects/s recovering 2023-09-23T20:00:07.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:07.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:07 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12524/134811 objects degraded (9.290%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:07 smithi135 ceph-mon[221113]: pgmap v568: 109 pgs: 1 remapped+peering, 1 peering, 9 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 12559/131820 objects degraded (9.527%); 18436/131820 objects misplaced (13.986%); 7.2 MiB/s, 295 objects/s recovering 2023-09-23T20:00:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:07 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12524/134811 objects degraded (9.290%), 12 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:08.100 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:08.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:08.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:08.213 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:08.298 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:08.491 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:08.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:08.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:08.546 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:00:08.547 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:00:08.598 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:08.657 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:08.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:08 smithi135 ceph-mon[213871]: pgmap v569: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 12521/125160 objects degraded (10.004%); 20432/125160 objects misplaced (16.325%); 4.9 MiB/s, 262 objects/s recovering 2023-09-23T20:00:08.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:08 smithi135 ceph-mon[221113]: pgmap v569: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 12521/125160 objects degraded (10.004%); 20432/125160 objects misplaced (16.325%); 4.9 MiB/s, 262 objects/s recovering 2023-09-23T20:00:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:08 smithi144 ceph-mon[194833]: pgmap v569: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 12521/125160 objects degraded (10.004%); 20432/125160 objects misplaced (16.325%); 4.9 MiB/s, 262 objects/s recovering 2023-09-23T20:00:09.393 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:09.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:09.505 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:09.592 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:09.717 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:09.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:09.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:09.822 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:09.912 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:09.918 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:00:09.919 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:00:09.919 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T20:00:09.919 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:00:09.919 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:00:09.919 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T20:00:10.032 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:00:10.032 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:00:10.032 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:00:10.033 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:00:10.033 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:00:10.052 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:10.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:10.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:10.166 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:10.309 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:10.548 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:00:10.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:10 smithi144 ceph-mon[194833]: pgmap v570: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 12521/123861 objects degraded (10.109%); 19999/123861 objects misplaced (16.146%); 1.5 MiB/s, 75 objects/s recovering 2023-09-23T20:00:10.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:10 smithi135 ceph-mon[221113]: pgmap v570: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 12521/123861 objects degraded (10.109%); 19999/123861 objects misplaced (16.146%); 1.5 MiB/s, 75 objects/s recovering 2023-09-23T20:00:10.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:10 smithi135 ceph-mon[213871]: pgmap v570: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 12521/123861 objects degraded (10.109%); 19999/123861 objects misplaced (16.146%); 1.5 MiB/s, 75 objects/s recovering 2023-09-23T20:00:11.352 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:11.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:11.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:11.464 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:11.497 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:11.555 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:11.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:11.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:11.638 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:11 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4004957528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:11.639 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:11 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4004957528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:11.663 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:11.669 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:00:11.693 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:11.696 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:00:11.749 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:11.755 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:11 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4004957528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:11.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:11.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:11.857 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:11.985 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:12.418 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:12.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:12.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:12.530 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:12.600 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:12.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:00:12.981 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4004957528' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:12.981 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[221113]: osdmap e221: 8 total, 7 up, 8 in 2023-09-23T20:00:12.981 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[221113]: failed to encode map e221 with expected crc 2023-09-23T20:00:12.981 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12521/123861 objects degraded (10.109%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:12.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:00:12.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4004957528' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:12.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[213871]: osdmap e221: 8 total, 7 up, 8 in 2023-09-23T20:00:12.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[213871]: failed to encode map e221 with expected crc 2023-09-23T20:00:12.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12521/123861 objects degraded (10.109%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:13.089 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:13.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:13.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:13.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:12 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:00:13.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4004957528' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:13.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:12 smithi144 ceph-mon[194833]: osdmap e221: 8 total, 7 up, 8 in 2023-09-23T20:00:13.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:12 smithi144 ceph-mon[194833]: failed to encode map e221 with expected crc 2023-09-23T20:00:13.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12521/123861 objects degraded (10.109%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:13.194 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:13.224 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:13.278 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:13.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:13.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:13.387 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:13.418 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:13.468 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:13.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:13.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:13.573 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:13.649 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:13.742 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:13.744 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:13.872 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:13.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:13.946 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:13.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:00:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:00:13] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.43.0" 2023-09-23T20:00:13.979 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:14.015 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:14.079 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:14.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:14.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:14.186 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:14.250 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:14.351 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:14 smithi144 ceph-mon[194833]: pgmap v572: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 12628/120357 objects degraded (10.492%); 19338/120357 objects misplaced (16.067%); 2.2 MiB/s, 80 objects/s recovering 2023-09-23T20:00:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:14 smithi135 ceph-mon[213871]: pgmap v572: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 12628/120357 objects degraded (10.492%); 19338/120357 objects misplaced (16.067%); 2.2 MiB/s, 80 objects/s recovering 2023-09-23T20:00:14.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:14 smithi135 ceph-mon[221113]: pgmap v572: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 12628/120357 objects degraded (10.492%); 19338/120357 objects misplaced (16.067%); 2.2 MiB/s, 80 objects/s recovering 2023-09-23T20:00:15.198 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:15.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:15.305 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:15.308 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:00:15.415 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:16.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:16 smithi144 ceph-mon[194833]: pgmap v573: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 12628/113730 objects degraded (11.103%); 18633/113730 objects misplaced (16.384%); 1.9 MiB/s, 71 objects/s recovering 2023-09-23T20:00:16.676 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:16 smithi135 ceph-mon[221113]: pgmap v573: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 12628/113730 objects degraded (11.103%); 18633/113730 objects misplaced (16.384%); 1.9 MiB/s, 71 objects/s recovering 2023-09-23T20:00:16.677 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:16 smithi135 ceph-mon[213871]: pgmap v573: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 12628/113730 objects degraded (11.103%); 18633/113730 objects misplaced (16.384%); 1.9 MiB/s, 71 objects/s recovering 2023-09-23T20:00:16.714 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:16.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:16.825 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:16.902 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:17.905 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:17.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:17.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:18.015 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:18.085 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:17 smithi135 ceph-mon[221113]: pgmap v574: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 12737/110226 objects degraded (11.555%); 17706/110226 objects misplaced (16.063%); 1.4 MiB/s, 59 objects/s recovering 2023-09-23T20:00:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:17 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:17 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12628/113730 objects degraded (11.103%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:17 smithi135 ceph-mon[213871]: pgmap v574: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 12737/110226 objects degraded (11.555%); 17706/110226 objects misplaced (16.063%); 1.4 MiB/s, 59 objects/s recovering 2023-09-23T20:00:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:17 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:17 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12628/113730 objects degraded (11.103%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:17 smithi144 ceph-mon[194833]: pgmap v574: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 12737/110226 objects degraded (11.555%); 17706/110226 objects misplaced (16.063%); 1.4 MiB/s, 59 objects/s recovering 2023-09-23T20:00:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:17 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:17 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12628/113730 objects degraded (11.103%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:18.821 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:18.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:18.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:18.937 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:19.081 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:19.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:00:19.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:00:19.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:19.228 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:00:19.228 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:00:19.228 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:19.259 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:19.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:19.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:19.322 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:19.377 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:00:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:00:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:19.480 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:00:19.481 INFO:teuthology.orchestra.run.smithi135.stdout: "success": "" 2023-09-23T20:00:19.481 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:00:19.488 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:19.966 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:00:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:00:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:00:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:00:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:116217 6.4 6:2512e97b:::benchmark_data_smithi135_344736_object13561:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:56.523222+0000", 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 83.317542814000007, 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.87162493500000005, 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116217 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523222+0000", 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523222+0000", 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523220+0000", 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523229+0000", 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4630000000000001e-06 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523245+0000", 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.66e-05 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523255+0000", 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9490000000000008e-06 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523441+0000", 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018609799999999999 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523449+0000", 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7079999999999997e-06 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.372733+0000", 2023-09-23T20:00:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.84928379899999995 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.373252+0000", 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00051908200000000003 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.387567+0000", 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014315068 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.387671+0000", 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010451 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.387943+0000", 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00027156499999999998 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.391458+0000", 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00351498 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.391486+0000", 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8178e-05 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.392778+0000", 2023-09-23T20:00:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001291857 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.392837+0000", 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9407000000000002e-05 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.394847+0000", 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002009547 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:117441 6.4 6:2478ca8e:::benchmark_data_smithi135_344736_object14785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:59.791079+0000", 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 80.049685385999993, 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.27037266399999998, 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117441 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.791079+0000", 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.791079+0000", 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.791078+0000", 2023-09-23T20:00:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:00:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.791085+0000", 2023-09-23T20:00:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.224e-06 2023-09-23T20:00:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.791096+0000", 2023-09-23T20:00:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0957e-05 2023-09-23T20:00:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.791102+0000", 2023-09-23T20:00:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8079999999999999e-06 2023-09-23T20:00:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.791160+0000", 2023-09-23T20:00:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8114e-05 2023-09-23T20:00:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:00:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.791799+0000", 2023-09-23T20:00:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000639053 2023-09-23T20:00:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.802120+0000", 2023-09-23T20:00:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010320566999999999 2023-09-23T20:00:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.802227+0000", 2023-09-23T20:00:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107284 2023-09-23T20:00:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:00:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.049160+0000", 2023-09-23T20:00:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24693259300000001 2023-09-23T20:00:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.055496+0000", 2023-09-23T20:00:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0063357550000000002 2023-09-23T20:00:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.055528+0000", 2023-09-23T20:00:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1955000000000002e-05 2023-09-23T20:00:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.059105+0000", 2023-09-23T20:00:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0035770260000000001 2023-09-23T20:00:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.059166+0000", 2023-09-23T20:00:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1098000000000004e-05 2023-09-23T20:00:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.061452+0000", 2023-09-23T20:00:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022863139999999998 2023-09-23T20:00:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:117426 6.17 e188/185 6:ebe71941:::benchmark_data_smithi135_344736_object14770:head v 188'7311, mlcod=167'919)", 2023-09-23T20:00:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:59.797648+0000", 2023-09-23T20:00:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 80.043116436999995, 2023-09-23T20:00:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.261301069, 2023-09-23T20:00:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797648+0000", 2023-09-23T20:00:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797648+0000", 2023-09-23T20:00:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797648+0000", 2023-09-23T20:00:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967296 2023-09-23T20:00:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797652+0000", 2023-09-23T20:00:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4209999999999999e-06 2023-09-23T20:00:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797657+0000", 2023-09-23T20:00:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3120000000000001e-06 2023-09-23T20:00:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:19.992 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797661+0000", 2023-09-23T20:00:19.992 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0609999999999997e-06 2023-09-23T20:00:19.992 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.992 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.992 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:19.992 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797797+0000", 2023-09-23T20:00:19.992 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000136465 2023-09-23T20:00:19.992 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.993 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.993 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:19.993 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797820+0000", 2023-09-23T20:00:19.993 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2323e-05 2023-09-23T20:00:19.993 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.993 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.993 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:19.994 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.058911+0000", 2023-09-23T20:00:19.994 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26109116100000002 2023-09-23T20:00:19.994 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.994 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.994 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:19.994 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.058949+0000", 2023-09-23T20:00:19.994 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8526999999999999e-05 2023-09-23T20:00:19.994 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:19.995 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:19.995 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:19.995 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.995 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.995 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:117436 6.f e188/179 6:fb9c674c:::benchmark_data_smithi135_344736_object14780:head v 188'7491, mlcod=167'910)", 2023-09-23T20:00:19.995 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:59.797914+0000", 2023-09-23T20:00:19.995 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 80.042851022999997, 2023-09-23T20:00:19.995 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26107795499999997, 2023-09-23T20:00:19.996 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:19.996 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:19.996 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:19.996 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.996 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:19.996 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797914+0000", 2023-09-23T20:00:19.996 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:19.997 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.997 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.997 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:19.997 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797914+0000", 2023-09-23T20:00:19.997 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:19.997 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.998 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.998 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:19.998 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797913+0000", 2023-09-23T20:00:19.998 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:19.998 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.998 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.998 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:19.999 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797918+0000", 2023-09-23T20:00:19.999 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9049999999999996e-06 2023-09-23T20:00:19.999 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:19.999 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:19.999 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:19.999 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797928+0000", 2023-09-23T20:00:19.999 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6269999999999995e-06 2023-09-23T20:00:20.000 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.000 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.000 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.000 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.797934+0000", 2023-09-23T20:00:20.000 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8919999999999997e-06 2023-09-23T20:00:20.001 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.001 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.001 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.001 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.048871+0000", 2023-09-23T20:00:20.001 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25093731699999999 2023-09-23T20:00:20.001 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.001 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.002 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.002 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.048902+0000", 2023-09-23T20:00:20.002 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1310999999999999e-05 2023-09-23T20:00:20.002 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.002 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.002 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.002 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.058972+0000", 2023-09-23T20:00:20.003 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010070012999999999 2023-09-23T20:00:20.003 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.003 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.003 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.003 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.058992+0000", 2023-09-23T20:00:20.003 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9341999999999999e-05 2023-09-23T20:00:20.003 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.004 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.004 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.004 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.004 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.004 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:117448 6.11 6:8de85bd6:::benchmark_data_smithi135_344736_object14792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-23T20:00:20.004 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:59.798154+0000", 2023-09-23T20:00:20.005 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 80.042610513, 2023-09-23T20:00:20.005 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25309367399999999, 2023-09-23T20:00:20.005 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.005 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.005 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.005 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.005 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.006 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117448 2023-09-23T20:00:20.006 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.006 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.006 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.006 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.006 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798154+0000", 2023-09-23T20:00:20.006 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.007 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.007 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.007 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.007 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798154+0000", 2023-09-23T20:00:20.007 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.007 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.007 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.008 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.008 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798153+0000", 2023-09-23T20:00:20.008 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:20.008 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.008 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.008 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.008 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798159+0000", 2023-09-23T20:00:20.009 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5610000000000004e-06 2023-09-23T20:00:20.009 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.009 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.009 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.009 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798170+0000", 2023-09-23T20:00:20.009 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.11e-05 2023-09-23T20:00:20.010 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.010 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.010 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.010 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798178+0000", 2023-09-23T20:00:20.010 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5650000000000004e-06 2023-09-23T20:00:20.010 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.010 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.011 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.011 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798219+0000", 2023-09-23T20:00:20.011 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1353999999999997e-05 2023-09-23T20:00:20.011 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.011 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.011 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.011 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798321+0000", 2023-09-23T20:00:20.012 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101344 2023-09-23T20:00:20.012 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.012 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.012 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:00:20.012 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.798532+0000", 2023-09-23T20:00:20.012 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00021087800000000001 2023-09-23T20:00:20.012 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.013 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.013 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.013 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.048823+0000", 2023-09-23T20:00:20.013 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25029111300000001 2023-09-23T20:00:20.013 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.013 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.013 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.014 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.050586+0000", 2023-09-23T20:00:20.014 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001763442 2023-09-23T20:00:20.014 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.014 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.014 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.014 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.051151+0000", 2023-09-23T20:00:20.015 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00056463900000000003 2023-09-23T20:00:20.015 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.015 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.015 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.015 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.051200+0000", 2023-09-23T20:00:20.015 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9608999999999999e-05 2023-09-23T20:00:20.015 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.016 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.016 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.016 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.051248+0000", 2023-09-23T20:00:20.016 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7580000000000002e-05 2023-09-23T20:00:20.016 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.016 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.016 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.017 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.017 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.017 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:117458 6.14 6:297333cb:::benchmark_data_smithi135_344736_object14802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-09-23T20:00:20.017 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:59.817638+0000", 2023-09-23T20:00:20.017 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 80.023126958999995, 2023-09-23T20:00:20.017 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25346008199999998, 2023-09-23T20:00:20.018 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.018 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.018 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.018 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.018 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.018 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117458 2023-09-23T20:00:20.018 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.019 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.019 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.019 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.019 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.817638+0000", 2023-09-23T20:00:20.019 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.019 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.019 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.020 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.020 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.817638+0000", 2023-09-23T20:00:20.020 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.020 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.020 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.020 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.020 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.817636+0000", 2023-09-23T20:00:20.020 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:00:20.021 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.021 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.021 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.021 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:59.817646+0000", 2023-09-23T20:00:20.021 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0560000000000001e-05 2023-09-23T20:00:20.021 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.022 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.022 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.022 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.048837+0000", 2023-09-23T20:00:20.022 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.231190374 2023-09-23T20:00:20.022 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.022 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.022 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.023 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.048852+0000", 2023-09-23T20:00:20.023 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5811000000000002e-05 2023-09-23T20:00:20.023 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.023 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.023 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.023 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.050658+0000", 2023-09-23T20:00:20.023 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001805526 2023-09-23T20:00:20.024 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.024 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.024 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:00:20.024 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.051333+0000", 2023-09-23T20:00:20.024 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00067462900000000005 2023-09-23T20:00:20.024 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.024 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.025 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.025 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.060188+0000", 2023-09-23T20:00:20.025 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088553480000000007 2023-09-23T20:00:20.025 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.025 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.025 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.025 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.060278+0000", 2023-09-23T20:00:20.026 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9893000000000001e-05 2023-09-23T20:00:20.026 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.026 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.026 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:00:20.026 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.060519+0000", 2023-09-23T20:00:20.026 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00024077899999999999 2023-09-23T20:00:20.026 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.027 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.027 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.027 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.065379+0000", 2023-09-23T20:00:20.027 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048599419999999999 2023-09-23T20:00:20.027 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.027 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.027 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.028 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.065425+0000", 2023-09-23T20:00:20.028 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6008999999999999e-05 2023-09-23T20:00:20.028 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.028 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.028 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.028 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.068284+0000", 2023-09-23T20:00:20.028 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028597029999999999 2023-09-23T20:00:20.029 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.029 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.029 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.029 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.068336+0000", 2023-09-23T20:00:20.029 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1350000000000001e-05 2023-09-23T20:00:20.029 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.029 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.030 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.030 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:00.071098+0000", 2023-09-23T20:00:20.030 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027623180000000002 2023-09-23T20:00:20.030 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.030 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.030 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.031 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.031 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.031 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148803 6.14 6:2da19cbe:::benchmark_data_smithi135_344736_object46147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:00:20.031 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:00:20.031 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.913433736999998, 2023-09-23T20:00:20.031 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2584235440000002, 2023-09-23T20:00:20.031 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.032 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.032 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.032 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.032 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.032 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148803 2023-09-23T20:00:20.032 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.033 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.033 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.033 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.033 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:00:20.033 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.033 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.034 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.034 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.034 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:00:20.034 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.034 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.035 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927329+0000", 2023-09-23T20:00:20.035 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:00:20.035 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.035 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927340+0000", 2023-09-23T20:00:20.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0981e-05 2023-09-23T20:00:20.036 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.036 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927360+0000", 2023-09-23T20:00:20.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0913000000000001e-05 2023-09-23T20:00:20.036 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.037 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.037 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927371+0000", 2023-09-23T20:00:20.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0467e-05 2023-09-23T20:00:20.037 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.037 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.037 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933494+0000", 2023-09-23T20:00:20.038 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061233260000000001 2023-09-23T20:00:20.038 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.038 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.038 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:00:20.038 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933501+0000", 2023-09-23T20:00:20.038 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0530000000000001e-06 2023-09-23T20:00:20.038 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.038 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.039 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.039 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.951971+0000", 2023-09-23T20:00:20.039 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018469316999999999 2023-09-23T20:00:20.039 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.039 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.039 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:00:20.039 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.951983+0000", 2023-09-23T20:00:20.039 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2286000000000001e-05 2023-09-23T20:00:20.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.040 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.040 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050605+0000", 2023-09-23T20:00:20.040 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0986220259999999 2023-09-23T20:00:20.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:00:20.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050613+0000", 2023-09-23T20:00:20.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9969999999999998e-06 2023-09-23T20:00:20.041 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.041 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.171636+0000", 2023-09-23T20:00:20.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.121022937 2023-09-23T20:00:20.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.042 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.042 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.042 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.171779+0000", 2023-09-23T20:00:20.042 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000142738 2023-09-23T20:00:20.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.042 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.042 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:00:20.043 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.172136+0000", 2023-09-23T20:00:20.043 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00035740899999999999 2023-09-23T20:00:20.043 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.043 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174779+0000", 2023-09-23T20:00:20.043 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002642574 2023-09-23T20:00:20.043 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.044 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.044 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.180158+0000", 2023-09-23T20:00:20.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053795379999999997 2023-09-23T20:00:20.044 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.044 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.044 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185597+0000", 2023-09-23T20:00:20.045 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0054389099999999999 2023-09-23T20:00:20.045 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.045 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.045 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.045 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185657+0000", 2023-09-23T20:00:20.045 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0288999999999998e-05 2023-09-23T20:00:20.045 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.046 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.046 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.046 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185755+0000", 2023-09-23T20:00:20.046 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7291000000000001e-05 2023-09-23T20:00:20.046 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.046 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.046 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.046 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.047 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148808 6.4 6:27155b23:::benchmark_data_smithi135_344736_object46152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:00:20.047 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:00:20.047 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.907289814000002, 2023-09-23T20:00:20.047 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2790047449999999, 2023-09-23T20:00:20.047 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.047 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.047 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.048 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.048 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.048 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148808 2023-09-23T20:00:20.048 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.048 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.048 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.048 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:00:20.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.049 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.049 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:00:20.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.050 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.050 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933474+0000", 2023-09-23T20:00:20.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:00:20.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.050 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.050 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933479+0000", 2023-09-23T20:00:20.051 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3739999999999997e-06 2023-09-23T20:00:20.051 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.051 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.051 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933490+0000", 2023-09-23T20:00:20.051 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1051e-05 2023-09-23T20:00:20.051 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.051 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.052 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.052 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933497+0000", 2023-09-23T20:00:20.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.601e-06 2023-09-23T20:00:20.052 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.052 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.052 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.052 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933549+0000", 2023-09-23T20:00:20.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1745e-05 2023-09-23T20:00:20.053 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.053 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:00:20.053 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933555+0000", 2023-09-23T20:00:20.053 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5440000000000004e-06 2023-09-23T20:00:20.053 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.053 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.952509+0000", 2023-09-23T20:00:20.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018953932 2023-09-23T20:00:20.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:00:20.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.952518+0000", 2023-09-23T20:00:20.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.081e-06 2023-09-23T20:00:20.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050570+0000", 2023-09-23T20:00:20.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.098051485 2023-09-23T20:00:20.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050581+0000", 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0878e-05 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174805+0000", 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1242246469999999 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174952+0000", 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014714599999999999 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.175367+0000", 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000414855 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.180166+0000", 2023-09-23T20:00:20.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047984739999999996 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185798+0000", 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00563285 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212221+0000", 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.026422386999999999 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212281+0000", 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9735999999999999e-05 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212480+0000", 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000199125 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.059 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148812 6.1 6:81ec3361:::benchmark_data_smithi135_344736_object46156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.886866738000002, 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.216623062, 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148812 2023-09-23T20:00:20.060 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953897+0000", 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953966+0000", 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9086999999999998e-05 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953978+0000", 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1731e-05 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953989+0000", 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1902000000000001e-05 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.954238+0000", 2023-09-23T20:00:20.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000248216 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.954249+0000", 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.114e-05 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049700+0000", 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.095451519 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049712+0000", 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1780000000000001e-05 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.146621+0000", 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.096909012 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.146767+0000", 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014557700000000001 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,2,7", 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.147202+0000", 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043543599999999998 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.152224+0000", 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050223309999999997 2023-09-23T20:00:20.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.152512+0000", 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000287421 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.163594+0000", 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011081946000000001 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170347+0000", 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067530960000000001 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170399+0000", 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2540000000000002e-05 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170521+0000", 2023-09-23T20:00:20.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012170700000000001 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:148836 6.c e212/211 6:308e2f95:::benchmark_data_smithi135_344736_object46180:head v 212'9245, mlcod=0'0)", 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.212120+0000", 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.628644383999998, 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26585123399999999, 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.069 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212120+0000", 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212120+0000", 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.070 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212120+0000", 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212124+0000", 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1280000000000001e-06 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212130+0000", 2023-09-23T20:00:20.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9479999999999996e-06 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212134+0000", 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1520000000000002e-06 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212547+0000", 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00041262900000000002 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212567+0000", 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0441999999999998e-05 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.220920+0000", 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083529139999999995 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.477972+0000", 2023-09-23T20:00:20.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25705129700000001 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:148837 6.d e212/163 6:b8c39994:::benchmark_data_smithi135_344736_object46181:head v 212'9360, mlcod=209'9358)", 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.216922+0000", 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.623842836000001, 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26878671599999998, 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.074 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.216922+0000", 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.216922+0000", 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.075 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.216921+0000", 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.216930+0000", 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8200000000000003e-06 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.216943+0000", 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2615e-05 2023-09-23T20:00:20.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.216951+0000", 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1240000000000005e-06 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.217008+0000", 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7519999999999998e-05 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.217029+0000", 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0369000000000001e-05 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.485675+0000", 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26864628600000001 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.485709+0000", 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3878999999999999e-05 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:148840 6.d e212/163 6:bd6b718b:::benchmark_data_smithi135_344736_object46184:head v 212'9361, mlcod=209'9358)", 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.224700+0000", 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 34.616064719000001, 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.26114007299999997, 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.079 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.224700+0000", 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.224700+0000", 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.224698+0000", 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.224714+0000", 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5843000000000001e-05 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.477945+0000", 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25323101999999997 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.477957+0000", 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1428e-05 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.483003+0000", 2023-09-23T20:00:20.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050462809999999997 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.483031+0000", 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7962000000000001e-05 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.485815+0000", 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027835350000000002 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.485840+0000", 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.563e-05 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:150528 6.4 6:24244236:::benchmark_data_smithi135_344736_object47872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:48.376343+0000", 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 31.464421890000001, 2023-09-23T20:00:20.084 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.344640687, 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 150528 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376343+0000", 2023-09-23T20:00:20.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376343+0000", 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376342+0000", 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376345+0000", 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5670000000000002e-06 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376352+0000", 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5300000000000002e-06 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376357+0000", 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0579999999999997e-06 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.380794+0000", 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0044376499999999996 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.380916+0000", 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000121981 2023-09-23T20:00:20.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.381307+0000", 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000390314 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.385954+0000", 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046478209999999999 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.389124+0000", 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003169926 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720774+0000", 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33164994599999997 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720811+0000", 2023-09-23T20:00:20.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6789000000000003e-05 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720984+0000", 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017255499999999999 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:150529 6.14 6:297273f9:::benchmark_data_smithi135_344736_object47873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T20:00:20.091 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:48.376487+0000", 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 31.464278051000001, 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.344246259, 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 150529 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.092 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376487+0000", 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376487+0000", 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376486+0000", 2023-09-23T20:00:20.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376490+0000", 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2640000000000002e-06 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376499+0000", 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9979999999999992e-06 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.376505+0000", 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9800000000000003e-06 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.378795+0000", 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022897970000000001 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.378899+0000", 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000104412 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.379235+0000", 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00033600399999999999 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.383214+0000", 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003978522 2023-09-23T20:00:20.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.385948+0000", 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027343269999999999 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720589+0000", 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33464029099999998 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720639+0000", 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0528999999999997e-05 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720733+0000", 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3888999999999995e-05 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:150538 6.4 6:209d24ae:::benchmark_data_smithi135_344736_object47882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:48.382960+0000", 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 31.457804851999999, 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33818959700000001, 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 150538 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.099 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.382960+0000", 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.382960+0000", 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.382958+0000", 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.382967+0000", 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2989999999999997e-06 2023-09-23T20:00:20.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.382982+0000", 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5537999999999999e-05 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.382991+0000", 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0250000000000008e-06 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.383263+0000", 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00027167499999999998 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.383355+0000", 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2157999999999994e-05 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.383676+0000", 2023-09-23T20:00:20.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00032139499999999998 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.385962+0000", 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022858930000000002 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.392015+0000", 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.006052333 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.721015+0000", 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.329000399 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.721049+0000", 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.43e-05 2023-09-23T20:00:20.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.721150+0000", 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100182 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:150522 6.f e215/211 6:f5153914:::benchmark_data_smithi135_344736_object47866:head v 215'9591, mlcod=169'2924)", 2023-09-23T20:00:20.106 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:48.384395+0000", 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 31.456370068999998, 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.36230486099999998, 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.384395+0000", 2023-09-23T20:00:20.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.384395+0000", 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.384394+0000", 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:20.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.384400+0000", 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8119999999999996e-06 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.384408+0000", 2023-09-23T20:00:20.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4290000000000001e-06 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.384414+0000", 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0410000000000003e-06 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.386760+0000", 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023458250000000002 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.386785+0000", 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4907999999999999e-05 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.746674+0000", 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.35988928199999998 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.746700+0000", 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5177e-05 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:150531 6.f e215/211 6:fc84a426:::benchmark_data_smithi135_344736_object47875:head v 215'9592, mlcod=169'2924)", 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:48.391321+0000", 2023-09-23T20:00:20.112 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 31.449443894000002, 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.35540194600000002, 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.391321+0000", 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.391321+0000", 2023-09-23T20:00:20.113 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.391320+0000", 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.391329+0000", 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4859999999999997e-06 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.391340+0000", 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1024e-05 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.391347+0000", 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8889999999999996e-06 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720525+0000", 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.329178521 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720558+0000", 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2444000000000003e-05 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.746704+0000", 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.026146901 2023-09-23T20:00:20.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.746723+0000", 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8369999999999999e-05 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44942.0:150529 6.14 e215/211 ondisk, result = 0)", 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:48.394221+0000", 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 31.44654358, 2023-09-23T20:00:20.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32651749299999999, 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.394221+0000", 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.394221+0000", 2023-09-23T20:00:20.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.394220+0000", 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.394226+0000", 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6229999999999999e-06 2023-09-23T20:00:20.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720459+0000", 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.326232191 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720474+0000", 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5226e-05 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720573+0000", 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9047000000000002e-05 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720581+0000", 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7800000000000001e-06 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720739+0000", 2023-09-23T20:00:20.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000157971 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:150543 6.a e215/211 6:51653794:::benchmark_data_smithi135_344736_object47887:head v 215'9371, mlcod=215'9369)", 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:48.396721+0000", 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 31.444044241, 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.328425413, 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.122 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.396721+0000", 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.396721+0000", 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.396719+0000", 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.396729+0000", 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0018e-05 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720474+0000", 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32374492500000002 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720484+0000", 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5699999999999999e-06 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720565+0000", 2023-09-23T20:00:20.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0611000000000005e-05 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720587+0000", 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2849000000000001e-05 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.725115+0000", 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045277390000000002 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.725146+0000", 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0808e-05 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:150557 6.11 6:8d1962ea:::benchmark_data_smithi135_344736_object47901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T20:00:20.127 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:48.403353+0000", 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 31.437412257999998, 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32318336399999997, 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 150557 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.403353+0000", 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.403353+0000", 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.403350+0000", 2023-09-23T20:00:20.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.403360+0000", 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9499999999999996e-06 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720460+0000", 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31709915500000002 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:20.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720476+0000", 2023-09-23T20:00:20.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6806000000000001e-05 2023-09-23T20:00:20.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:20.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720566+0000", 2023-09-23T20:00:20.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0050999999999996e-05 2023-09-23T20:00:20.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.720727+0000", 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000160414 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,2,7", 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.721146+0000", 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00041907100000000003 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.723229+0000", 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020834600000000001 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.725631+0000", 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024015170000000001 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.726102+0000", 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000470848 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.726353+0000", 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00025122499999999999 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.726406+0000", 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3029000000000003e-05 2023-09-23T20:00:20.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:48.726536+0000", 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012989699999999999 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:20.137 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:00:20.141 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:19 smithi135 ceph-mon[221113]: pgmap v575: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12737/102516 objects degraded (12.424%); 16739/102516 objects misplaced (16.328%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:20.143 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:19 smithi135 ceph-mon[213871]: pgmap v575: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12737/102516 objects degraded (12.424%); 16739/102516 objects misplaced (16.328%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:19 smithi144 ceph-mon[194833]: pgmap v575: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12737/102516 objects degraded (12.424%); 16739/102516 objects misplaced (16.328%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:20.789 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:00:20.789 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:00:20.789 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:00:20.789 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:00:20.827 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:00:20.857 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:00:20.898 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:00:20.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:00:20.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:00:20.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:00:20.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:00:20.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:00:20.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:00:20.899 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:00:20.899 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:00:21.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1738585706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:21.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1738585706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1738585706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:21.436 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:21.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:21.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:21.545 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:21.623 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:22.254 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:22 smithi135 ceph-mon[213871]: pgmap v576: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 12737/101097 objects degraded (12.599%); 16266/101097 objects misplaced (16.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:22.256 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:22 smithi135 ceph-mon[221113]: pgmap v576: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 12737/101097 objects degraded (12.599%); 16266/101097 objects misplaced (16.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:22 smithi144 ceph-mon[194833]: pgmap v576: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 12737/101097 objects degraded (12.599%); 16266/101097 objects misplaced (16.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:23.096 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:00:23.140 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:00:23.141 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:00:23.325 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:23.325 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:23.325 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12737/101097 objects degraded (12.599%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:23.326 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:23.326 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:23.326 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12737/101097 objects degraded (12.599%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:23.365 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:23.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:23.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12737/101097 objects degraded (12.599%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:23.474 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:23.577 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:23.686 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:00:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:00:23] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-09-23T20:00:24.034 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:24.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:24.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:24.145 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:24.182 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:24.241 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:24.248 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:24.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:24.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:24.353 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:24.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:24 smithi135 ceph-mon[213871]: pgmap v577: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 12885/97803 objects degraded (13.174%); 15575/97803 objects misplaced (15.925%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:00:24.386 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:24.451 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:24.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:24 smithi144 ceph-mon[194833]: pgmap v577: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 12885/97803 objects degraded (13.174%); 15575/97803 objects misplaced (15.925%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:00:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:24.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:24.564 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:24.606 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:24 smithi135 ceph-mon[221113]: pgmap v577: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 12885/97803 objects degraded (13.174%); 15575/97803 objects misplaced (15.925%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:00:24.658 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:24.845 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:24.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:24.952 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:25.041 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:25.142 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:00:25.281 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:25.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:25.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:25.389 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:25.438 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:00:25.460 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:25.629 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/392677360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:25.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:25 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:25.635 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/392677360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:25.635 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:25 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:25.635 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/392677360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:25.635 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:25 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:00:25.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:25.737 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:25.805 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:26.483 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:00:26.483 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:00:26.483 INFO:tasks.thrashosds.thrasher:Reweighting osd 6 to 0.17256271896567704 2023-09-23T20:00:26.484 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 0.17256271896567704 2023-09-23T20:00:26.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[213871]: pgmap v578: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 12885/92139 objects degraded (13.984%); 14946/92139 objects misplaced (16.221%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:00:26.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:26.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[213871]: failed to encode map e222 with expected crc 2023-09-23T20:00:26.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:00:26.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[213871]: osdmap e222: 8 total, 7 up, 8 in 2023-09-23T20:00:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[221113]: pgmap v578: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 12885/92139 objects degraded (13.984%); 14946/92139 objects misplaced (16.221%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:00:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[221113]: failed to encode map e222 with expected crc 2023-09-23T20:00:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:00:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:26 smithi135 ceph-mon[221113]: osdmap e222: 8 total, 7 up, 8 in 2023-09-23T20:00:26.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:26 smithi144 ceph-mon[194833]: pgmap v578: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 12885/92139 objects degraded (13.984%); 14946/92139 objects misplaced (16.221%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:00:26.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:26 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:26.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:26 smithi144 ceph-mon[194833]: failed to encode map e222 with expected crc 2023-09-23T20:00:26.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:00:26.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:26 smithi144 ceph-mon[194833]: osdmap e222: 8 total, 7 up, 8 in 2023-09-23T20:00:27.054 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:27.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:27.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:27.161 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:27.291 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:27.392 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:27.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12885/92139 objects degraded (13.984%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:27.440 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:27.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:27.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:27.449 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:27.550 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:27.582 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:27.639 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:27.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:27.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:27.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12885/92139 objects degraded (13.984%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:27.747 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:27.780 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12885/92139 objects degraded (13.984%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:27.785 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:27.841 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:27.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:27.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:27.952 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:28.041 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:28.213 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:00:28.213 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:00:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:28 smithi135 ceph-mon[221113]: pgmap v580: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 12985/88821 objects degraded (14.619%); 14094/88821 objects misplaced (15.868%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:28 smithi135 ceph-mon[213871]: pgmap v580: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 12985/88821 objects degraded (14.619%); 14094/88821 objects misplaced (15.868%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:28.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:28 smithi144 ceph-mon[194833]: pgmap v580: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 12985/88821 objects degraded (14.619%); 14094/88821 objects misplaced (15.868%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:29.163 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:00:29.443 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:29.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:29.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:29.548 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:29.583 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:29.653 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:29.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:29.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:29.761 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:29.798 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:29.839 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:29.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:29.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:29.946 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:29.982 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:30.043 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:30.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:30.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:30.150 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:30.183 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:30.247 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:30.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:30.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:30.352 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:30.386 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:30.447 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:30.451 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:30 smithi144 ceph-mon[194833]: pgmap v581: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 12985/81588 objects degraded (15.915%); 13188/81588 objects misplaced (16.164%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:30.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:30.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:30.553 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:30.627 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:30.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:30 smithi135 ceph-mon[213871]: pgmap v581: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 12985/81588 objects degraded (15.915%); 13188/81588 objects misplaced (16.164%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:30.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:30 smithi135 ceph-mon[221113]: pgmap v581: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 12985/81588 objects degraded (15.915%); 13188/81588 objects misplaced (16.164%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:30.738 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:00:30.882 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:30.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:31.005 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:31.089 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:31.676 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:31.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:31.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:31.788 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:31.894 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:32.133 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:32.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:32.182 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:32.241 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:32.283 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:32.315 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:32.379 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:00:32.416 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:00:32.462 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:00:32.513 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:00:32.571 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:00:32.603 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:00:32.636 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:32.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:32.644 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:32 smithi135 ceph-mon[213871]: pgmap v582: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 12985/80253 objects degraded (16.180%); 12743/80253 objects misplaced (15.879%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:32.645 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2269627806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:32.645 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12985/80253 objects degraded (16.180%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:32.645 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:32 smithi135 ceph-mon[221113]: pgmap v582: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 12985/80253 objects degraded (16.180%); 12743/80253 objects misplaced (15.879%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:32.645 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2269627806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:32.645 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12985/80253 objects degraded (16.180%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:32.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:32.657 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:00:32.657 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:00:32.657 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:00:32.657 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:00:32.657 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:00:32.657 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:00:32.658 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:00:32.658 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:00:32.658 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:00:32.746 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:32.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:32 smithi144 ceph-mon[194833]: pgmap v582: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 12985/80253 objects degraded (16.180%); 12743/80253 objects misplaced (15.879%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:32.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2269627806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:32.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12985/80253 objects degraded (16.180%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:32.779 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:32.833 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:32.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:32.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:32.940 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:32.972 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:33.031 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:33.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:33.138 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:33.222 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:33.465 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.6 to 0.172563 (2c2d) 2023-09-23T20:00:33.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:33 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/734444430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]: dispatch 2023-09-23T20:00:33.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:33 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]: dispatch 2023-09-23T20:00:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:33 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/734444430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]: dispatch 2023-09-23T20:00:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:33 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]: dispatch 2023-09-23T20:00:33.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:33 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/734444430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]: dispatch 2023-09-23T20:00:33.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:33 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]: dispatch 2023-09-23T20:00:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:00:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:00:33] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-09-23T20:00:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[221113]: pgmap v583: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 13104/76881 objects degraded (17.045%); 12067/76881 objects misplaced (15.696%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]': finished 2023-09-23T20:00:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[221113]: osdmap e223: 8 total, 7 up, 8 in 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[221113]: failed to encode map e223 with expected crc 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[221113]: failed to encode map e223 with expected crc 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[221113]: osdmap e224: 8 total, 7 up, 8 in 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[213871]: pgmap v583: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 13104/76881 objects degraded (17.045%); 12067/76881 objects misplaced (15.696%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]': finished 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[213871]: osdmap e223: 8 total, 7 up, 8 in 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[213871]: failed to encode map e223 with expected crc 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[213871]: failed to encode map e223 with expected crc 2023-09-23T20:00:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:34 smithi135 ceph-mon[213871]: osdmap e224: 8 total, 7 up, 8 in 2023-09-23T20:00:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:34 smithi144 ceph-mon[194833]: pgmap v583: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 13104/76881 objects degraded (17.045%); 12067/76881 objects misplaced (15.696%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:00:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:34 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.17256271896567704}]': finished 2023-09-23T20:00:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:34 smithi144 ceph-mon[194833]: osdmap e223: 8 total, 7 up, 8 in 2023-09-23T20:00:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:34 smithi144 ceph-mon[194833]: failed to encode map e223 with expected crc 2023-09-23T20:00:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:34 smithi144 ceph-mon[194833]: failed to encode map e223 with expected crc 2023-09-23T20:00:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:34 smithi144 ceph-mon[194833]: osdmap e224: 8 total, 7 up, 8 in 2023-09-23T20:00:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[221113]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[221113]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[221113]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[221113]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[221113]: 6.a starting backfill to osd.1 from (171'3713,216'9977] MIN to 222'11343 2023-09-23T20:00:35.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[221113]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[221113]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[221113]: osdmap e225: 8 total, 7 up, 8 in 2023-09-23T20:00:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[213871]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[213871]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[213871]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[213871]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[213871]: 6.a starting backfill to osd.1 from (171'3713,216'9977] MIN to 222'11343 2023-09-23T20:00:35.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[213871]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[213871]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:35 smithi135 ceph-mon[213871]: osdmap e225: 8 total, 7 up, 8 in 2023-09-23T20:00:35.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:35 smithi144 ceph-mon[194833]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:35 smithi144 ceph-mon[194833]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:35 smithi144 ceph-mon[194833]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:35 smithi144 ceph-mon[194833]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:35 smithi144 ceph-mon[194833]: 6.a starting backfill to osd.1 from (171'3713,216'9977] MIN to 222'11343 2023-09-23T20:00:35.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:35 smithi144 ceph-mon[194833]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:35 smithi144 ceph-mon[194833]: failed to encode map e224 with expected crc 2023-09-23T20:00:35.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:35 smithi144 ceph-mon[194833]: osdmap e225: 8 total, 7 up, 8 in 2023-09-23T20:00:36.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[221113]: pgmap v586: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 13104/70959 objects degraded (18.467%); 11423/70959 objects misplaced (16.098%); 895 KiB/s, 13 objects/s recovering 2023-09-23T20:00:36.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[221113]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[221113]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[221113]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[221113]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[221113]: osdmap e226: 8 total, 7 up, 8 in 2023-09-23T20:00:36.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[213871]: pgmap v586: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 13104/70959 objects degraded (18.467%); 11423/70959 objects misplaced (16.098%); 895 KiB/s, 13 objects/s recovering 2023-09-23T20:00:36.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[213871]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[213871]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[213871]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[213871]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:36 smithi135 ceph-mon[213871]: osdmap e226: 8 total, 7 up, 8 in 2023-09-23T20:00:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:36 smithi144 ceph-mon[194833]: pgmap v586: 109 pgs: 11 active+clean+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 13104/70959 objects degraded (18.467%); 11423/70959 objects misplaced (16.098%); 895 KiB/s, 13 objects/s recovering 2023-09-23T20:00:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:36 smithi144 ceph-mon[194833]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:36 smithi144 ceph-mon[194833]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:36 smithi144 ceph-mon[194833]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:36 smithi144 ceph-mon[194833]: failed to encode map e225 with expected crc 2023-09-23T20:00:36.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:36 smithi144 ceph-mon[194833]: osdmap e226: 8 total, 7 up, 8 in 2023-09-23T20:00:37.045 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:37.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:37.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:37.150 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:37.221 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:37.274 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:00:37.274 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:00:37.411 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:37.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:37.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:37.521 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:37.611 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[221113]: failed to encode map e226 with expected crc 2023-09-23T20:00:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[221113]: failed to encode map e226 with expected crc 2023-09-23T20:00:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:37.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:37.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 13104/70959 objects degraded (18.467%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[213871]: failed to encode map e226 with expected crc 2023-09-23T20:00:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[213871]: failed to encode map e226 with expected crc 2023-09-23T20:00:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:37.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:37 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 13104/70959 objects degraded (18.467%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:37.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:37 smithi144 ceph-mon[194833]: failed to encode map e226 with expected crc 2023-09-23T20:00:37.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:37 smithi144 ceph-mon[194833]: failed to encode map e226 with expected crc 2023-09-23T20:00:37.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:00:37.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:37.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:37 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 13104/70959 objects degraded (18.467%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:38.591 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:38.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:38.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:38.703 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:38.747 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:38 smithi144 ceph-mon[194833]: pgmap v589: 109 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 1 peering, 8 active+clean+remapped, 3 active+recovery_wait+remapped, 79 active+clean, 3 active+undersized, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 16451/69225 objects degraded (23.765%); 7744/69225 objects misplaced (11.187%); 16 MiB/s, 249 objects/s recovering 2023-09-23T20:00:38.751 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:38.771 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:38 smithi135 ceph-mon[213871]: pgmap v589: 109 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 1 peering, 8 active+clean+remapped, 3 active+recovery_wait+remapped, 79 active+clean, 3 active+undersized, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 16451/69225 objects degraded (23.765%); 7744/69225 objects misplaced (11.187%); 16 MiB/s, 249 objects/s recovering 2023-09-23T20:00:38.772 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:38 smithi135 ceph-mon[221113]: pgmap v589: 109 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 1 peering, 8 active+clean+remapped, 3 active+recovery_wait+remapped, 79 active+clean, 3 active+undersized, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 16451/69225 objects degraded (23.765%); 7744/69225 objects misplaced (11.187%); 16 MiB/s, 249 objects/s recovering 2023-09-23T20:00:38.811 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:38.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:38.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:38.920 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:39.017 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:39.275 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:00:39.299 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:39.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:39.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:39.413 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:39.500 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:39.647 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:39.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:39.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:39.684 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:00:39.684 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:00:39.684 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:00:39.684 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:00:39.684 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:00:39.776 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:39.810 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:39.870 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:39.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:39.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:39.979 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:40.010 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:40.079 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:40.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:40.190 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:40.224 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:40.287 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:40.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:40.397 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:40.423 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:40.465 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:40.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:40.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:40.509 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:00:40.569 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:40.609 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:40.663 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:40.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:40.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:40.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[213871]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 1 peering, 2 active+clean+remapped, 2 active+recovery_wait+remapped, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 13259/65676 objects degraded (20.189%); 7980/65676 objects misplaced (12.151%); 12 MiB/s, 350 objects/s recovering 2023-09-23T20:00:40.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[213871]: osdmap e227: 8 total, 7 up, 8 in 2023-09-23T20:00:40.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[213871]: failed to encode map e227 with expected crc 2023-09-23T20:00:40.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[213871]: failed to encode map e227 with expected crc 2023-09-23T20:00:40.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2570900462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:40.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[221113]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 1 peering, 2 active+clean+remapped, 2 active+recovery_wait+remapped, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 13259/65676 objects degraded (20.189%); 7980/65676 objects misplaced (12.151%); 12 MiB/s, 350 objects/s recovering 2023-09-23T20:00:40.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[221113]: osdmap e227: 8 total, 7 up, 8 in 2023-09-23T20:00:40.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[221113]: failed to encode map e227 with expected crc 2023-09-23T20:00:40.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[221113]: failed to encode map e227 with expected crc 2023-09-23T20:00:40.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:40 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2570900462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:40.769 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:40.808 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:40 smithi144 ceph-mon[194833]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 1 peering, 2 active+clean+remapped, 2 active+recovery_wait+remapped, 85 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 13259/65676 objects degraded (20.189%); 7980/65676 objects misplaced (12.151%); 12 MiB/s, 350 objects/s recovering 2023-09-23T20:00:40.808 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:40 smithi144 ceph-mon[194833]: osdmap e227: 8 total, 7 up, 8 in 2023-09-23T20:00:40.808 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:40 smithi144 ceph-mon[194833]: failed to encode map e227 with expected crc 2023-09-23T20:00:40.808 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:40 smithi144 ceph-mon[194833]: failed to encode map e227 with expected crc 2023-09-23T20:00:40.808 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:40 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2570900462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:00:40.811 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:40.887 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:40.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:40.993 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:41.034 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:41.100 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:41.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:41.209 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:41.266 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:41.860 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:41.860 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[221113]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.860 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[221113]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.860 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2570900462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:41.860 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[221113]: osdmap e228: 8 total, 7 up, 8 in 2023-09-23T20:00:41.860 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[221113]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.861 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:41.861 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[213871]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.861 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[213871]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.861 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2570900462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:41.861 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[213871]: osdmap e228: 8 total, 7 up, 8 in 2023-09-23T20:00:41.861 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:41 smithi135 ceph-mon[213871]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:41 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:41.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:41 smithi144 ceph-mon[194833]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:41 smithi144 ceph-mon[194833]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2570900462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:00:41.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:41 smithi144 ceph-mon[194833]: osdmap e228: 8 total, 7 up, 8 in 2023-09-23T20:00:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:41 smithi144 ceph-mon[194833]: failed to encode map e228 with expected crc 2023-09-23T20:00:41.890 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:00:42.114 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:42.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:42.187 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:00:42.187 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:00:42.188 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T20:00:42.188 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:00:42.188 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:00:42.188 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T20:00:42.222 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:42.300 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:42.371 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:42.499 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:42.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:42.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:42.618 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:42.652 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:00:42.706 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:42.867 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:42.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:42.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:42.880 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:42 smithi135 ceph-mon[213871]: pgmap v593: 109 pgs: 1 active+clean+remapped, 2 active+recovery_wait+remapped, 87 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 354 op/s; 18777/64674 objects degraded (29.033%); 7597/64674 objects misplaced (11.747%); 6.0 MiB/s, 251 objects/s recovering 2023-09-23T20:00:42.880 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 18777/64674 objects degraded (29.033%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:42.880 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:42 smithi135 ceph-mon[221113]: pgmap v593: 109 pgs: 1 active+clean+remapped, 2 active+recovery_wait+remapped, 87 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 354 op/s; 18777/64674 objects degraded (29.033%); 7597/64674 objects misplaced (11.747%); 6.0 MiB/s, 251 objects/s recovering 2023-09-23T20:00:42.881 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 18777/64674 objects degraded (29.033%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:42.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:42 smithi144 ceph-mon[194833]: pgmap v593: 109 pgs: 1 active+clean+remapped, 2 active+recovery_wait+remapped, 87 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 354 op/s; 18777/64674 objects degraded (29.033%); 7597/64674 objects misplaced (11.747%); 6.0 MiB/s, 251 objects/s recovering 2023-09-23T20:00:42.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 18777/64674 objects degraded (29.033%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:42.974 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:43.070 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:43.210 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:43.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:43.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:43.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:43.379 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:43.421 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:43.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:43.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:43.523 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:43.576 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:43.669 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:43.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:43.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:43.776 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:43.838 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:43.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:00:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:00:43] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-09-23T20:00:44.189 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:44.204 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:44.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:44.269 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:00:44.307 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:44.412 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:44.748 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:44.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:44.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:44.837 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:44 smithi135 ceph-mon[221113]: pgmap v594: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 87 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 18897/61458 objects degraded (30.748%); 7063/61458 objects misplaced (11.492%); 5.0 MiB/s, 244 objects/s recovering 2023-09-23T20:00:44.837 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:44 smithi135 ceph-mon[213871]: pgmap v594: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 87 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 18897/61458 objects degraded (30.748%); 7063/61458 objects misplaced (11.492%); 5.0 MiB/s, 244 objects/s recovering 2023-09-23T20:00:44.866 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:44.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:44 smithi144 ceph-mon[194833]: pgmap v594: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 87 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 1.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 18897/61458 objects degraded (30.748%); 7063/61458 objects misplaced (11.492%); 5.0 MiB/s, 244 objects/s recovering 2023-09-23T20:00:44.911 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:44.977 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:44.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:44.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:45.081 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:45.155 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:00:45.204 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:45.210 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:00:45.292 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:45.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:45.300 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:00:45.303 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:00:45.342 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:00:45.382 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:00:45.396 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:45.418 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:00:45.418 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:00:45.419 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:00:45.419 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:00:45.419 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:00:45.419 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:00:45.419 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:00:45.419 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:00:45.419 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:00:45.488 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:45.563 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:45.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:45.667 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:45.708 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:45.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3539624820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:45.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3539624820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:45.788 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:45.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:45.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:45.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3539624820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:45.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:45.924 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:46.814 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:46 smithi144 ceph-mon[194833]: pgmap v595: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 18897/55587 objects degraded (33.995%); 6445/55587 objects misplaced (11.594%); 2.9 MiB/s, 176 objects/s recovering 2023-09-23T20:00:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:46 smithi135 ceph-mon[221113]: pgmap v595: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 18897/55587 objects degraded (33.995%); 6445/55587 objects misplaced (11.594%); 2.9 MiB/s, 176 objects/s recovering 2023-09-23T20:00:46.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:46 smithi135 ceph-mon[213871]: pgmap v595: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 18897/55587 objects degraded (33.995%); 6445/55587 objects misplaced (11.594%); 2.9 MiB/s, 176 objects/s recovering 2023-09-23T20:00:47.089 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:00:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:00:47.089 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:00:47.306 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:47.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:47.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:47.389 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:00:47.389 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:00:47.422 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:47.464 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:47.500 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:47.512 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:47.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:47.615 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:47.641 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:47.707 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 18897/55587 objects degraded (33.995%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:47.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 18897/55587 objects degraded (33.995%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:47.711 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:47.714 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 18897/55587 objects degraded (33.995%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:47.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:47.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:47.815 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:47.918 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:48.033 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:48.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:48.118 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:00:48.118 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:00:48.140 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:48.180 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:48.238 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:48.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:48.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:48.346 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:48.385 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:48.468 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:48.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:48.583 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:48 smithi144 ceph-mon[194833]: pgmap v596: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 18897/54144 objects degraded (34.901%); 6407/54144 objects misplaced (11.833%); 2.9 MiB/s, 75 objects/s recovering 2023-09-23T20:00:48.639 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:48.716 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:48.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:48.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:48.822 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:48.902 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:48 smithi135 ceph-mon[221113]: pgmap v596: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 18897/54144 objects degraded (34.901%); 6407/54144 objects misplaced (11.833%); 2.9 MiB/s, 75 objects/s recovering 2023-09-23T20:00:48.906 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:48.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:48 smithi135 ceph-mon[213871]: pgmap v596: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 18897/54144 objects degraded (34.901%); 6407/54144 objects misplaced (11.833%); 2.9 MiB/s, 75 objects/s recovering 2023-09-23T20:00:48.999 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:49.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:49.103 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:49.128 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:49.169 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:49.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:49.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:49.274 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:49.300 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:49.367 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:49.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:49.390 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:00:49.479 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:49.512 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:49 smithi144 ceph-mon[194833]: pgmap v597: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 940 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 18990/44979 objects degraded (42.220%); 5214/44979 objects misplaced (11.592%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T20:00:49.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:49 smithi135 ceph-mon[221113]: pgmap v597: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 940 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 18990/44979 objects degraded (42.220%); 5214/44979 objects misplaced (11.592%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T20:00:49.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:49 smithi135 ceph-mon[213871]: pgmap v597: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 940 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 18990/44979 objects degraded (42.220%); 5214/44979 objects misplaced (11.592%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T20:00:50.155 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:50.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:50.182 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:00:50.283 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:50.384 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:50.535 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:50.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:50.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:50.649 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:50.688 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:50.754 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:50.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:50.861 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:00:50.929 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:50.989 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:51.107 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:00:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:51.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:00:51.220 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:51.302 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:51.426 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:51.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:51.537 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:51.576 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:51.642 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:51.647 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:51 smithi135 ceph-mon[221113]: pgmap v598: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 913 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 18990/43722 objects degraded (43.434%); 5196/43722 objects misplaced (11.884%); 0 B/s, 41 objects/s recovering 2023-09-23T20:00:51.647 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3479137042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:00:51.647 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:00:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:51.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:51.690 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:00:51.740 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:51.752 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:51.792 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:51.868 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:51.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:51.875 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:51 smithi144 ceph-mon[194833]: pgmap v598: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 913 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 18990/43722 objects degraded (43.434%); 5196/43722 objects misplaced (11.884%); 0 B/s, 41 objects/s recovering 2023-09-23T20:00:51.875 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3479137042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:00:51.875 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:00:51.923 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:51 smithi135 ceph-mon[213871]: pgmap v598: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 913 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 18990/43722 objects degraded (43.434%); 5196/43722 objects misplaced (11.884%); 0 B/s, 41 objects/s recovering 2023-09-23T20:00:51.923 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3479137042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:00:51.924 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:00:51.975 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:52.026 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:52.106 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:52.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:52.213 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:52.292 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:00:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[221113]: osdmap e229: 8 total, 7 up, 8 in 2023-09-23T20:00:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[221113]: failed to encode map e229 with expected crc 2023-09-23T20:00:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[221113]: failed to encode map e229 with expected crc 2023-09-23T20:00:52.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:52.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 18990/43722 objects degraded (43.434%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:52.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:52.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:00:52.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[213871]: osdmap e229: 8 total, 7 up, 8 in 2023-09-23T20:00:52.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[213871]: failed to encode map e229 with expected crc 2023-09-23T20:00:52.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[213871]: failed to encode map e229 with expected crc 2023-09-23T20:00:52.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:52.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:52 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 18990/43722 objects degraded (43.434%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:53.014 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:53.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:53.025 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:00:53.025 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:00:53.025 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:00:53.025 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2023-09-23T20:00:53.025 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0 2023-09-23T20:00:53.123 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:53.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:52 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:00:53.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:00:53.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:52 smithi144 ceph-mon[194833]: osdmap e229: 8 total, 7 up, 8 in 2023-09-23T20:00:53.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:52 smithi144 ceph-mon[194833]: failed to encode map e229 with expected crc 2023-09-23T20:00:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:52 smithi144 ceph-mon[194833]: failed to encode map e229 with expected crc 2023-09-23T20:00:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:00:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:52 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 18990/43722 objects degraded (43.434%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:53.247 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:53.979 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:00:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:00:53] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-09-23T20:00:54.240 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:54.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:54.344 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:00:54.392 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:54.453 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:00:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:54.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:00:54.558 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:54.585 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:54 smithi135 ceph-mon[221113]: pgmap v600: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 847 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 19011/40524 objects degraded (46.913%); 4540/40524 objects misplaced (11.203%); 1.3 MiB/s, 48 objects/s recovering 2023-09-23T20:00:54.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:54 smithi135 ceph-mon[213871]: pgmap v600: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 847 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 19011/40524 objects degraded (46.913%); 4540/40524 objects misplaced (11.203%); 1.3 MiB/s, 48 objects/s recovering 2023-09-23T20:00:54.604 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:54.663 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:54.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:54.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:54.672 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:54 smithi144 ceph-mon[194833]: pgmap v600: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 847 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 19011/40524 objects degraded (46.913%); 4540/40524 objects misplaced (11.203%); 1.3 MiB/s, 48 objects/s recovering 2023-09-23T20:00:54.771 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:54.806 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:54.874 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:54.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:54.984 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:55.028 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:55.092 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:55.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:55.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:55.167 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:00:55.198 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:55.327 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:55.466 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:55.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:55.473 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:00:55.547 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:00:55.573 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:55.587 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:00:55.612 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:55.628 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:00:55.663 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:00:55.676 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:55.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:55.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:55.701 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:00:55.743 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:00:55.747 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:00:55.748 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:00:55.748 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:00:55.748 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:00:55.748 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:00:55.748 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:00:55.748 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:00:55.748 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:00:55.787 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:55.855 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:56.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:56 smithi135 ceph-mon[213871]: pgmap v601: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 723 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 19011/34596 objects degraded (54.951%); 3878/34596 objects misplaced (11.209%); 1.3 MiB/s, 45 objects/s recovering 2023-09-23T20:00:56.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2294482200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:56 smithi135 ceph-mon[221113]: pgmap v601: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 723 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 19011/34596 objects degraded (54.951%); 3878/34596 objects misplaced (11.209%); 1.3 MiB/s, 45 objects/s recovering 2023-09-23T20:00:56.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2294482200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:56.794 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:00:56.820 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:56.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:00:56.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:56 smithi144 ceph-mon[194833]: pgmap v601: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 723 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 19011/34596 objects degraded (54.951%); 3878/34596 objects misplaced (11.209%); 1.3 MiB/s, 45 objects/s recovering 2023-09-23T20:00:56.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2294482200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:00:56.931 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:56.963 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:57.025 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:57.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:57.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:57.131 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:57.228 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 19011/34596 objects degraded (54.951%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:57.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 19011/34596 objects degraded (54.951%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:57.864 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:57.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 19011/34596 objects degraded (54.951%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:00:57.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:57.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:57.985 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:00:58.022 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:00:58.078 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:00:58.104 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:00:58.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:58.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:00:58.210 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:58.251 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:58.319 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:58.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:58.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:58.425 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:00:58.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:00:58 smithi135 ceph-mon[213871]: pgmap v602: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 693 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 19011/33120 objects degraded (57.400%); 3852/33120 objects misplaced (11.630%); 1.3 MiB/s, 45 objects/s recovering 2023-09-23T20:00:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:00:58 smithi135 ceph-mon[221113]: pgmap v602: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 693 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 19011/33120 objects degraded (57.400%); 3852/33120 objects misplaced (11.630%); 1.3 MiB/s, 45 objects/s recovering 2023-09-23T20:00:58.500 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:58.593 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:00:58.593 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:00:58.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.593 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:198258 6.10 e229/211 6:0cb8d375:::benchmark_data_smithi135_344736_object95602:head v 229'9729, mlcod=169'2872)", 2023-09-23T20:00:58.593 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:58.577473+0000", 2023-09-23T20:00:58.593 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0072003739999999998, 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007224439, 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577473+0000", 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577473+0000", 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577472+0000", 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:58.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577477+0000", 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3059999999999999e-06 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577483+0000", 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6780000000000003e-06 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577495+0000", 2023-09-23T20:00:58.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1926999999999999e-05 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578982+0000", 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014866230000000001 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:58.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.579011+0000", 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9522e-05 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:198260 6.6 6:66c88b4d:::benchmark_data_smithi135_344736_object95604:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:58.577814+0000", 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0068592940000000002, 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0069259619999999999, 2023-09-23T20:00:58.598 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 198260 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577814+0000", 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577814+0000", 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577813+0000", 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577817+0000", 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9579999999999998e-06 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577826+0000", 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0450000000000001e-06 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577832+0000", 2023-09-23T20:00:58.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8449999999999997e-06 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577914+0000", 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2025000000000002e-05 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578039+0000", 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000124664 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578524+0000", 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00048508900000000002 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.584653+0000", 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061288740000000003 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:198265 6.6 6:650aea49:::benchmark_data_smithi135_344736_object95609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:58.578081+0000", 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0065919120000000001, 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.006706928, 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 198265 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578081+0000", 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578081+0000", 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578080+0000", 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578086+0000", 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1979999999999998e-06 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578100+0000", 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3396e-05 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.578109+0000", 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1640000000000007e-06 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.581318+0000", 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00320922 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.581434+0000", 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011585400000000001 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.581882+0000", 2023-09-23T20:00:58.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00044857399999999998 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44942.0:198262 6.17 e229/227 6:ef1bd099:::benchmark_data_smithi135_344736_object95606:head v 229'9812, mlcod=229'9810)", 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:58.579469+0000", 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0052044279999999997, 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053629740000000004, 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:00:58.609 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.579469+0000", 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.579469+0000", 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.579468+0000", 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.579475+0000", 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7209999999999997e-06 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:58.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.579484+0000", 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2909999999999996e-06 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.579490+0000", 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4590000000000003e-06 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.583297+0000", 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003807024 2023-09-23T20:00:58.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.583324+0000", 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6880999999999999e-05 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:198235 6.f 6:f27ed8cc:::benchmark_data_smithi135_344736_object95579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T20:00:58.613 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:58.550551+0000", 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.034121971000000001, 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.034322905000000001, 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 198235 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:58.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550551+0000", 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550551+0000", 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550551+0000", 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550553+0000", 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1229999999999998e-06 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:58.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550558+0000", 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1880000000000001e-06 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550562+0000", 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1989999999999999e-06 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.558751+0000", 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081884339999999996 2023-09-23T20:00:58.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.558914+0000", 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016337099999999999 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.559078+0000", 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016366199999999999 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.567037+0000", 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0079593830000000004 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577447+0000", 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010409948 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:58.619 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:198239 6.f 6:fefe06bf:::benchmark_data_smithi135_344736_object95583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:58.550566+0000", 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.034106989999999997, 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.034403732999999999, 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:00:58.620 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 198239 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550566+0000", 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550566+0000", 2023-09-23T20:00:58.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550566+0000", 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550568+0000", 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8870000000000001e-06 2023-09-23T20:00:58.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550572+0000", 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7210000000000002e-06 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.550576+0000", 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1790000000000001e-06 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.562289+0000", 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011712620999999999 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.562487+0000", 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000197915 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:00:58.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.562694+0000", 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000207102 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.567231+0000", 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045373000000000002 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:58.577484+0000", 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010252717 2023-09-23T20:00:58.625 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.626 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:00:58.626 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.626 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:00:58.626 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:00:58.626 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 6 2023-09-23T20:00:58.626 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:00:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:00:58 smithi144 ceph-mon[194833]: pgmap v602: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 693 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 19011/33120 objects degraded (57.400%); 3852/33120 objects misplaced (11.630%); 1.3 MiB/s, 45 objects/s recovering 2023-09-23T20:00:58.786 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:00:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:58.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:00:58.901 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:00:58.979 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:59.490 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:00:59.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:59.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:00:59.605 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:00:59.639 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:59.712 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:00:59.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:59.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:00:59.818 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:00:59.878 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:00:59.921 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:00:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:00:59.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:00.024 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:00.078 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:01:00.090 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:00.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:00 smithi135 ceph-mon[213871]: pgmap v603: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 497 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 19011/23733 objects degraded (80.104%); 2488/23733 objects misplaced (10.483%); 2.7 MiB/s, 66 objects/s recovering 2023-09-23T20:01:00.707 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:00.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:00.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:00 smithi135 ceph-mon[221113]: pgmap v603: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 497 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 19011/23733 objects degraded (80.104%); 2488/23733 objects misplaced (10.483%); 2.7 MiB/s, 66 objects/s recovering 2023-09-23T20:01:00.814 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:00.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:00 smithi144 ceph-mon[194833]: pgmap v603: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 497 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 19011/23733 objects degraded (80.104%); 2488/23733 objects misplaced (10.483%); 2.7 MiB/s, 66 objects/s recovering 2023-09-23T20:01:00.888 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:01.079 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:01.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:01.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:01.193 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:01:01.198 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:01.216 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:01.299 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:01.452 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:01.603 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:01.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:01.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:01.715 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:01.819 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:02.410 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:02.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:02.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:02.516 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:02.591 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:02 smithi135 ceph-mon[221113]: pgmap v604: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 469 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 19011/22371 objects degraded (84.981%); 2456/22371 objects misplaced (10.978%); 2.7 MiB/s, 66 objects/s recovering 2023-09-23T20:01:02.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:02 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 19011/22371 objects degraded (84.981%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:02.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3070267830' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T20:01:02.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:02 smithi135 ceph-mon[213871]: pgmap v604: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 469 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 19011/22371 objects degraded (84.981%); 2456/22371 objects misplaced (10.978%); 2.7 MiB/s, 66 objects/s recovering 2023-09-23T20:01:02.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:02 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 19011/22371 objects degraded (84.981%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:02.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3070267830' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T20:01:02.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:02 smithi144 ceph-mon[194833]: pgmap v604: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 469 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 19011/22371 objects degraded (84.981%); 2456/22371 objects misplaced (10.978%); 2.7 MiB/s, 66 objects/s recovering 2023-09-23T20:01:02.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:02 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 19011/22371 objects degraded (84.981%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:02.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3070267830' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T20:01:03.461 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.0 primary-affinity to 0 (802) 2023-09-23T20:01:03.794 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:03.815 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:03.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:03.919 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:03.945 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:01:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:01:03] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-09-23T20:01:03.948 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[221113]: pgmap v605: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 399 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 19011/19014 objects degraded (99.984%); 2007/19014 objects misplaced (10.555%); 2.5 MiB/s, 100 objects/s recovering 2023-09-23T20:01:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3070267830' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T20:01:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[221113]: osdmap e230: 8 total, 7 up, 8 in 2023-09-23T20:01:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[221113]: failed to encode map e230 with expected crc 2023-09-23T20:01:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[221113]: failed to encode map e230 with expected crc 2023-09-23T20:01:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[213871]: pgmap v605: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 399 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 19011/19014 objects degraded (99.984%); 2007/19014 objects misplaced (10.555%); 2.5 MiB/s, 100 objects/s recovering 2023-09-23T20:01:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3070267830' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T20:01:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[213871]: osdmap e230: 8 total, 7 up, 8 in 2023-09-23T20:01:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[213871]: failed to encode map e230 with expected crc 2023-09-23T20:01:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:04 smithi135 ceph-mon[213871]: failed to encode map e230 with expected crc 2023-09-23T20:01:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:04 smithi144 ceph-mon[194833]: pgmap v605: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 399 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 19011/19014 objects degraded (99.984%); 2007/19014 objects misplaced (10.555%); 2.5 MiB/s, 100 objects/s recovering 2023-09-23T20:01:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3070267830' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T20:01:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:04 smithi144 ceph-mon[194833]: osdmap e230: 8 total, 7 up, 8 in 2023-09-23T20:01:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:04 smithi144 ceph-mon[194833]: failed to encode map e230 with expected crc 2023-09-23T20:01:04.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:04 smithi144 ceph-mon[194833]: failed to encode map e230 with expected crc 2023-09-23T20:01:05.156 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:05.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:05.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:05.260 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:05.288 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:05.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[213871]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[213871]: osdmap e231: 8 total, 7 up, 8 in 2023-09-23T20:01:05.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[213871]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[213871]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[213871]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[213871]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[221113]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[221113]: osdmap e231: 8 total, 7 up, 8 in 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[221113]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[221113]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[221113]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:05 smithi135 ceph-mon[221113]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:05 smithi144 ceph-mon[194833]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:05 smithi144 ceph-mon[194833]: osdmap e231: 8 total, 7 up, 8 in 2023-09-23T20:01:05.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:05 smithi144 ceph-mon[194833]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:05 smithi144 ceph-mon[194833]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:05 smithi144 ceph-mon[194833]: failed to encode map e231 with expected crc 2023-09-23T20:01:05.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:05 smithi144 ceph-mon[194833]: failed to encode map e231 with expected crc 2023-09-23T20:01:06.446 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:06.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:06.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:06.550 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:06.637 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: pgmap v608: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 274 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 19011/13029 objects degraded (145.913%); 1344/13029 objects misplaced (10.315%); 1.8 MiB/s, 102 objects/s recovering 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: osdmap e232: 8 total, 7 up, 8 in 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: 6.9 continuing backfill to osd.3 from (167'700,229'9964] MIN to 229'9964 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[213871]: 6.19 continuing backfill to osd.3 from (167'700,229'9997] MIN to 229'9997 2023-09-23T20:01:06.638 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: pgmap v608: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 274 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 19011/13029 objects degraded (145.913%); 1344/13029 objects misplaced (10.315%); 1.8 MiB/s, 102 objects/s recovering 2023-09-23T20:01:06.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: osdmap e232: 8 total, 7 up, 8 in 2023-09-23T20:01:06.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: 6.9 continuing backfill to osd.3 from (167'700,229'9964] MIN to 229'9964 2023-09-23T20:01:06.639 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:06 smithi135 ceph-mon[221113]: 6.19 continuing backfill to osd.3 from (167'700,229'9997] MIN to 229'9997 2023-09-23T20:01:06.671 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:06.874 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:06.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:06.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:06.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: pgmap v608: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 274 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 19011/13029 objects degraded (145.913%); 1344/13029 objects misplaced (10.315%); 1.8 MiB/s, 102 objects/s recovering 2023-09-23T20:01:06.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: osdmap e232: 8 total, 7 up, 8 in 2023-09-23T20:01:06.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: failed to encode map e232 with expected crc 2023-09-23T20:01:06.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: 6.9 continuing backfill to osd.3 from (167'700,229'9964] MIN to 229'9964 2023-09-23T20:01:06.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:06 smithi144 ceph-mon[194833]: 6.19 continuing backfill to osd.3 from (167'700,229'9997] MIN to 229'9997 2023-09-23T20:01:06.981 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:07.031 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:07.117 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:07.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:07.224 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:07.358 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:07.454 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:01:07.501 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:01:07.547 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:01:07.604 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:01:07.633 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:07.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:07.641 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[213871]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.641 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[213871]: osdmap e233: 8 total, 7 up, 8 in 2023-09-23T20:01:07.641 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[213871]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.641 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[213871]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.641 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:07.641 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 19011/13029 objects degraded (145.913%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:07.641 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/127360209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:07.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:07.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[221113]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[221113]: osdmap e233: 8 total, 7 up, 8 in 2023-09-23T20:01:07.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[221113]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[221113]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:07.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 19011/13029 objects degraded (145.913%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:07.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:07 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/127360209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:07.649 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:01:07.694 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:01:07.743 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:01:07.743 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:01:07.743 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:01:07.743 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:01:07.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:01:07.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:01:07.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:01:07.744 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:01:07.744 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:01:07.744 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:07.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:07 smithi144 ceph-mon[194833]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:07 smithi144 ceph-mon[194833]: osdmap e233: 8 total, 7 up, 8 in 2023-09-23T20:01:07.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:07 smithi144 ceph-mon[194833]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:07 smithi144 ceph-mon[194833]: failed to encode map e233 with expected crc 2023-09-23T20:01:07.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:07.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:07 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 19011/13029 objects degraded (145.913%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:07.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:07 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/127360209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:07.795 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:07.876 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:07.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:07.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:07.982 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:08.070 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:08.683 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:01:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:08 smithi144 ceph-mon[194833]: pgmap v611: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 255 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 16893/12105 objects degraded (139.554%); 1322/12105 objects misplaced (10.921%); 1.8 MiB/s, 84 objects/s recovering 2023-09-23T20:01:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:08 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1236192082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:08 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:08.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:08 smithi135 ceph-mon[213871]: pgmap v611: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 255 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 16893/12105 objects degraded (139.554%); 1322/12105 objects misplaced (10.921%); 1.8 MiB/s, 84 objects/s recovering 2023-09-23T20:01:08.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:08 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1236192082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:08.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:08 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:08.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:08 smithi135 ceph-mon[221113]: pgmap v611: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 88 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 255 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 16893/12105 objects degraded (139.554%); 1322/12105 objects misplaced (10.921%); 1.8 MiB/s, 84 objects/s recovering 2023-09-23T20:01:08.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:08 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1236192082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:08.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:08 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:09.515 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:09.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:09.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:09.626 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:09.661 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:09.761 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:09.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:09.866 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:09.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:09 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:09.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:09 smithi144 ceph-mon[194833]: failed to encode map e234 with expected crc 2023-09-23T20:01:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:09 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:09 smithi144 ceph-mon[194833]: osdmap e234: 8 total, 7 up, 8 in 2023-09-23T20:01:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:09 smithi144 ceph-mon[194833]: failed to encode map e234 with expected crc 2023-09-23T20:01:09.898 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:09.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:09.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[221113]: failed to encode map e234 with expected crc 2023-09-23T20:01:09.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[221113]: osdmap e234: 8 total, 7 up, 8 in 2023-09-23T20:01:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[221113]: failed to encode map e234 with expected crc 2023-09-23T20:01:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[213871]: failed to encode map e234 with expected crc 2023-09-23T20:01:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[213871]: osdmap e234: 8 total, 7 up, 8 in 2023-09-23T20:01:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:09 smithi135 ceph-mon[213871]: failed to encode map e234 with expected crc 2023-09-23T20:01:09.988 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:10.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:10.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:10.103 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:10.129 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:10.174 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:10.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:10.279 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:10.351 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:10.539 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:10 smithi144 ceph-mon[194833]: pgmap v613: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 140 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 535 op/s; 15379/6570 objects degraded (234.079%); 630/6570 objects misplaced (9.589%); 3.5 MiB/s, 353 objects/s recovering 2023-09-23T20:01:10.645 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:10 smithi135 ceph-mon[213871]: pgmap v613: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 140 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 535 op/s; 15379/6570 objects degraded (234.079%); 630/6570 objects misplaced (9.589%); 3.5 MiB/s, 353 objects/s recovering 2023-09-23T20:01:10.645 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:10 smithi135 ceph-mon[221113]: pgmap v613: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 140 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 535 op/s; 15379/6570 objects degraded (234.079%); 630/6570 objects misplaced (9.589%); 3.5 MiB/s, 353 objects/s recovering 2023-09-23T20:01:10.736 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:10.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:01:10.780 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:01:10.807 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Removed 102653 objects 2023-09-23T20:01:10.807 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Clean up completed and total clean up time :159.883 2023-09-23T20:01:10.847 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:10.881 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:10.981 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:10.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:10.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:11.085 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:11.112 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:11.212 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:11.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:11.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:11.316 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:11.345 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:11.392 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:11.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:11.411 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:01:11.411 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:01:11.497 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:11.523 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:11.569 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:11.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:11.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:11.675 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:11.680 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:01:11.680 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:01:11.680 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:01:11.681 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-09-23T20:01:11.681 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T20:01:11.705 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:11.752 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:11.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:11.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:11.856 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:11.882 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:11.930 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:11.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:12.034 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:12.061 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:12.781 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:01:12.806 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:12 smithi135 ceph-mon[221113]: pgmap v614: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 116 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 433 op/s; 15379/5466 objects degraded (281.357%); 606/5466 objects misplaced (11.087%); 2.4 MiB/s, 317 objects/s recovering 2023-09-23T20:01:12.806 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 15379/5466 objects degraded (281.357%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:12.807 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:12 smithi135 ceph-mon[213871]: pgmap v614: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 116 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 433 op/s; 15379/5466 objects degraded (281.357%); 606/5466 objects misplaced (11.087%); 2.4 MiB/s, 317 objects/s recovering 2023-09-23T20:01:12.807 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 15379/5466 objects degraded (281.357%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:12 smithi144 ceph-mon[194833]: pgmap v614: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 116 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 433 op/s; 15379/5466 objects degraded (281.357%); 606/5466 objects misplaced (11.087%); 2.4 MiB/s, 317 objects/s recovering 2023-09-23T20:01:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 15379/5466 objects degraded (281.357%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:12.958 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:01:13.131 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:13.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:13.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:13.212 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:01:13.235 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:13.286 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:13.300 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:13.369 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:13.371 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:13.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:13.474 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:13.485 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:13.501 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:13.554 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:13.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:13.660 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:13.690 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:13.935 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:01:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:01:13] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-09-23T20:01:14.791 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:14.792 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:14 smithi135 ceph-mon[221113]: pgmap v615: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 74 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 31 KiB/s wr, 419 op/s; 15292/3426 objects degraded (446.351%); 433/3426 objects misplaced (12.639%); 1.8 MiB/s, 231 objects/s recovering 2023-09-23T20:01:14.793 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:14 smithi135 ceph-mon[213871]: pgmap v615: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 74 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 31 KiB/s wr, 419 op/s; 15292/3426 objects degraded (446.351%); 433/3426 objects misplaced (12.639%); 1.8 MiB/s, 231 objects/s recovering 2023-09-23T20:01:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:14.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:14.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:14 smithi144 ceph-mon[194833]: pgmap v615: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+clean+remapped, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 74 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 31 KiB/s wr, 419 op/s; 15292/3426 objects degraded (446.351%); 433/3426 objects misplaced (12.639%); 1.8 MiB/s, 231 objects/s recovering 2023-09-23T20:01:14.895 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:14.925 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:15.065 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T20:01:15.065 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T20:01:16.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:16 smithi144 ceph-mon[194833]: pgmap v616: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 17 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 493 op/s; 14869/702 objects degraded (2118.091%); 46/702 objects misplaced (6.553%); 1.6 MiB/s, 328 objects/s recovering 2023-09-23T20:01:16.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:16 smithi135 ceph-mon[221113]: pgmap v616: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 17 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 493 op/s; 14869/702 objects degraded (2118.091%); 46/702 objects misplaced (6.553%); 1.6 MiB/s, 328 objects/s recovering 2023-09-23T20:01:16.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:16 smithi135 ceph-mon[213871]: pgmap v616: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 17 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 493 op/s; 14869/702 objects degraded (2118.091%); 46/702 objects misplaced (6.553%); 1.6 MiB/s, 328 objects/s recovering 2023-09-23T20:01:17.119 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:17.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:17.223 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:17.252 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:17.352 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:17.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:17.456 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:17.471 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:17.484 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:17.527 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:17.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:17.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:17.634 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:17.664 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:17.698 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 14869/702 objects degraded (2118.091%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:17.698 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:17 smithi135 ceph-mon[221113]: osdmap e235: 8 total, 7 up, 8 in 2023-09-23T20:01:17.698 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 14869/702 objects degraded (2118.091%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:17.698 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:17 smithi135 ceph-mon[213871]: osdmap e235: 8 total, 7 up, 8 in 2023-09-23T20:01:17.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 14869/702 objects degraded (2118.091%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:17.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:17 smithi144 ceph-mon[194833]: osdmap e235: 8 total, 7 up, 8 in 2023-09-23T20:01:17.925 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:17.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:17.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:18.029 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:18.078 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:18.274 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:18.380 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:18.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:18.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:18.431 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:01:18.444 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:01:18.464 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:01:18.487 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:18.488 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:01:18.512 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:01:18.523 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:18.542 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:01:18.568 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:01:18.591 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:01:18.591 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:01:18.591 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:01:18.591 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:01:18.591 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:01:18.591 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:01:18.592 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:01:18.592 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:01:18.592 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:01:18.651 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: pgmap v617: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 406 op/s; 14869/585 objects degraded (2541.709%); 44/585 objects misplaced (7.521%); 1.3 MiB/s, 389 objects/s recovering 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/33642388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/974279135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:18.652 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:18.653 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:01:18.653 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: osdmap e236: 8 total, 7 up, 8 in 2023-09-23T20:01:18.653 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2474916260' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:01:18.653 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: pgmap v617: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 406 op/s; 14869/585 objects degraded (2541.709%); 44/585 objects misplaced (7.521%); 1.3 MiB/s, 389 objects/s recovering 2023-09-23T20:01:18.653 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.653 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.653 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/33642388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/974279135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:01:18.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: osdmap e236: 8 total, 7 up, 8 in 2023-09-23T20:01:18.655 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2474916260' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:01:18.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: pgmap v617: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 406 op/s; 14869/585 objects degraded (2541.709%); 44/585 objects misplaced (7.521%); 1.3 MiB/s, 389 objects/s recovering 2023-09-23T20:01:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: failed to encode map e235 with expected crc 2023-09-23T20:01:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:01:18.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/33642388' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:01:18.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:01:18.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/974279135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:18.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:18.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:01:18.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: osdmap e236: 8 total, 7 up, 8 in 2023-09-23T20:01:18.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2474916260' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:01:19.406 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T20:01:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[221113]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[221113]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[221113]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[221113]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2474916260' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:01:19.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[221113]: osdmap e237: 8 total, 7 up, 8 in 2023-09-23T20:01:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[213871]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[213871]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[213871]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[213871]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2474916260' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:01:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:19 smithi135 ceph-mon[213871]: osdmap e237: 8 total, 7 up, 8 in 2023-09-23T20:01:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:19 smithi144 ceph-mon[194833]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:19 smithi144 ceph-mon[194833]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:19 smithi144 ceph-mon[194833]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:19 smithi144 ceph-mon[194833]: failed to encode map e236 with expected crc 2023-09-23T20:01:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2474916260' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:01:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:19 smithi144 ceph-mon[194833]: osdmap e237: 8 total, 7 up, 8 in 2023-09-23T20:01:20.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:20 smithi144 ceph-mon[194833]: pgmap v620: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 13615/585 objects degraded (2327.350%); 44/585 objects misplaced (7.521%); 696 KiB/s, 447 objects/s recovering 2023-09-23T20:01:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:20 smithi144 ceph-mon[194833]: failed to encode map e237 with expected crc 2023-09-23T20:01:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:20 smithi144 ceph-mon[194833]: failed to encode map e237 with expected crc 2023-09-23T20:01:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:20 smithi144 ceph-mon[194833]: failed to encode map e237 with expected crc 2023-09-23T20:01:20.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:20 smithi135 ceph-mon[221113]: pgmap v620: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 13615/585 objects degraded (2327.350%); 44/585 objects misplaced (7.521%); 696 KiB/s, 447 objects/s recovering 2023-09-23T20:01:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:20 smithi135 ceph-mon[221113]: failed to encode map e237 with expected crc 2023-09-23T20:01:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:20 smithi135 ceph-mon[221113]: failed to encode map e237 with expected crc 2023-09-23T20:01:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:20 smithi135 ceph-mon[221113]: failed to encode map e237 with expected crc 2023-09-23T20:01:20.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:20 smithi135 ceph-mon[213871]: pgmap v620: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 89 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 13615/585 objects degraded (2327.350%); 44/585 objects misplaced (7.521%); 696 KiB/s, 447 objects/s recovering 2023-09-23T20:01:20.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:20 smithi135 ceph-mon[213871]: failed to encode map e237 with expected crc 2023-09-23T20:01:20.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:20 smithi135 ceph-mon[213871]: failed to encode map e237 with expected crc 2023-09-23T20:01:20.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:20 smithi135 ceph-mon[213871]: failed to encode map e237 with expected crc 2023-09-23T20:01:22.000 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:22.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:22.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:22.104 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:22.134 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:22.181 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:22.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:22.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:22.286 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:22.316 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:22.566 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[213871]: pgmap v622: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 90 active+clean, 3 active+undersized, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 9 op/s; 12966/585 objects degraded (2216.410%); 44/585 objects misplaced (7.521%); 0 B/s, 540 objects/s recovering 2023-09-23T20:01:22.567 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:22.567 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:22.567 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:22.567 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12966/585 objects degraded (2216.410%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:22 smithi144 ceph-mon[194833]: pgmap v622: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 90 active+clean, 3 active+undersized, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 9 op/s; 12966/585 objects degraded (2216.410%); 44/585 objects misplaced (7.521%); 0 B/s, 540 objects/s recovering 2023-09-23T20:01:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12966/585 objects degraded (2216.410%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:22.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[221113]: pgmap v622: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 90 active+clean, 3 active+undersized, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 9 op/s; 12966/585 objects degraded (2216.410%); 44/585 objects misplaced (7.521%); 0 B/s, 540 objects/s recovering 2023-09-23T20:01:22.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:22.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:22.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:22.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12966/585 objects degraded (2216.410%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:23.945 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:01:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:01:23] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-09-23T20:01:23.970 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:01:23.970 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:01:24.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:24 smithi144 ceph-mon[194833]: pgmap v623: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 90 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 428 objects/s recovering 2023-09-23T20:01:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:24 smithi135 ceph-mon[221113]: pgmap v623: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 90 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 428 objects/s recovering 2023-09-23T20:01:24.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:24 smithi135 ceph-mon[213871]: pgmap v623: 109 pgs: 2 peering, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 90 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 428 objects/s recovering 2023-09-23T20:01:25.972 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:01:26.210 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:26.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:26.314 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:26.340 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:26.543 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:26.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:26.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:26.599 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:26 smithi135 ceph-mon[221113]: pgmap v624: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 92 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 356 objects/s recovering 2023-09-23T20:01:26.599 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:26 smithi135 ceph-mon[213871]: pgmap v624: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 92 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 356 objects/s recovering 2023-09-23T20:01:26.646 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:26.680 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:26.730 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:26.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:26.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:26.837 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:26.864 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:26.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:26 smithi144 ceph-mon[194833]: pgmap v624: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 92 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 356 objects/s recovering 2023-09-23T20:01:27.847 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:27.848 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 10422/585 objects degraded (1781.538%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:27.849 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 10422/585 objects degraded (1781.538%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:27.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:27.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:27.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 10422/585 objects degraded (1781.538%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:27.951 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:27.988 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:28.092 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:28.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:28.195 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:28.226 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:28.352 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:01:28.352 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:01:28.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:01:28.352 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:01:28.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.352 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148788 6.f 6:f326eaaa:::benchmark_data_smithi135_344736_object46132:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.420214945, 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5928274390000001, 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148788 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.353 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928731+0000", 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928740+0000", 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4009999999999997e-06 2023-09-23T20:01:28.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928755+0000", 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5481999999999998e-05 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928765+0000", 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0239e-05 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928822+0000", 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6518000000000003e-05 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928829+0000", 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3819999999999998e-06 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944149+0000", 2023-09-23T20:01:28.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015319442000000001 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944158+0000", 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3000000000000007e-06 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049126+0000", 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.104968422 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049138+0000", 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1644999999999999e-05 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.265679+0000", 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2165408280000001 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.265834+0000", 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015490799999999999 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.521437+0000", 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25560343699999999 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.521561+0000", 2023-09-23T20:01:28.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123342 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148791 6.f 6:fce301f6:::benchmark_data_smithi135_344736_object46135:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.42015708, 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5984101370000001, 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:01:28.361 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148791 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928790+0000", 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928794+0000", 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8800000000000001e-06 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928802+0000", 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0499999999999992e-06 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928809+0000", 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1800000000000001e-06 2023-09-23T20:01:28.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928882+0000", 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3524999999999998e-05 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928887+0000", 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0010000000000001e-06 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944176+0000", 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015289166 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944182+0000", 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.105e-06 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049162+0000", 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1049793050000001 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049170+0000", 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3869999999999995e-06 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268117+0000", 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2189466419999999 2023-09-23T20:01:28.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268204+0000", 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7319999999999997e-05 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527103+0000", 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25889852899999999 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527201+0000", 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8562999999999998e-05 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148804 6.b 6:d3899bd3:::benchmark_data_smithi135_344736_object46148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.41333124499999, 2023-09-23T20:01:28.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1034934710000002, 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148804 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:01:28.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935615+0000", 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935620+0000", 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.775e-06 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935630+0000", 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9310000000000003e-06 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935637+0000", 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3440000000000003e-06 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.936071+0000", 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043330099999999998 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.936077+0000", 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6900000000000003e-06 2023-09-23T20:01:28.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.941346+0000", 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052686859999999999 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.941360+0000", 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3464e-05 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050328+0000", 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.108968468 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050338+0000", 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0088999999999999e-05 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.003742+0000", 2023-09-23T20:01:28.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.95340369300000005 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.003918+0000", 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017656799999999999 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.004058+0000", 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000139682 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.007338+0000", 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003280303 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.021476+0000", 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014137442 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.038947+0000", 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017471477999999999 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.039000+0000", 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3022000000000002e-05 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.039110+0000", 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000109966 2023-09-23T20:01:28.378 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148813 6.f 6:f77e0ce5:::benchmark_data_smithi135_344736_object46157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.38800511300001, 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.545090514, 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.379 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148813 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960941+0000", 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960950+0000", 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6949999999999995e-06 2023-09-23T20:01:28.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960969+0000", 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8785999999999999e-05 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960980+0000", 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0703000000000001e-05 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.961043+0000", 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2600000000000004e-05 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.961057+0000", 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4535999999999999e-05 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049192+0000", 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0881350000000001 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049200+0000", 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9049999999999997e-06 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268242+0000", 2023-09-23T20:01:28.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.219042129 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268348+0000", 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010592 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268839+0000", 2023-09-23T20:01:28.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00049129899999999995 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279896+0000", 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011056725 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279913+0000", 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6673000000000001e-05 2023-09-23T20:01:28.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.502819+0000", 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22290622900000001 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.502884+0000", 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4616000000000006e-05 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506033+0000", 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031498979999999999 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148815 6.f 6:f7552838:::benchmark_data_smithi135_344736_object46159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 104.22595400100001, 2023-09-23T20:01:28.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.385714227, 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148815 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:01:28.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122993+0000", 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:01:28.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123000+0000", 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9290000000000001e-06 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123009+0000", 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5689999999999994e-06 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123015+0000", 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8189999999999997e-06 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.152030+0000", 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029014671999999998 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.152042+0000", 2023-09-23T20:01:28.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1756000000000001e-05 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276241+0000", 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.124199189 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276452+0000", 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000210934 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276679+0000", 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00022696700000000001 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279921+0000", 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032419129999999999 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279990+0000", 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9807000000000004e-05 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506095+0000", 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22610454399999999 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506139+0000", 2023-09-23T20:01:28.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3659000000000003e-05 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.508708+0000", 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002569661 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148816 6.b 6:dde61f8d:::benchmark_data_smithi135_344736_object46160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:01:28.396 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 104.225754275, 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.001808705, 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148816 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123193+0000", 2023-09-23T20:01:28.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123196+0000", 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9679999999999998e-06 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123202+0000", 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9349999999999999e-06 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123206+0000", 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9960000000000004e-06 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123247+0000", 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1238000000000001e-05 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123256+0000", 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1999999999999994e-06 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018207+0000", 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.89495181199999996 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018364+0000", 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000156269 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018501+0000", 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013728199999999999 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.033263+0000", 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014761793 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.045761+0000", 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012498489 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.124820+0000", 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.079059044999999994 2023-09-23T20:01:28.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.124879+0000", 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9138999999999997e-05 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.125003+0000", 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012307900000000001 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.404 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148820 6.f 6:f8320d95:::benchmark_data_smithi135_344736_object46164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 103.29737046299999, 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.478686642, 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.405 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148820 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.406 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051576+0000", 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051584+0000", 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6049999999999998e-06 2023-09-23T20:01:28.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051600+0000", 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5387999999999999e-05 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051616+0000", 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6228000000000001e-05 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.100356+0000", 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.048739598000000002 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.100371+0000", 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5068999999999999e-05 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280046+0000", 2023-09-23T20:01:28.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17967550800000001 2023-09-23T20:01:28.410 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.410 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.410 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280245+0000", 2023-09-23T20:01:28.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019827899999999999 2023-09-23T20:01:28.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:01:28.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280450+0000", 2023-09-23T20:01:28.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000205689 2023-09-23T20:01:28.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.287010+0000", 2023-09-23T20:01:28.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065595749999999998 2023-09-23T20:01:28.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.287104+0000", 2023-09-23T20:01:28.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4639e-05 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527260+0000", 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24015514399999999 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527323+0000", 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3256000000000006e-05 2023-09-23T20:01:28.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.530264+0000", 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029414630000000001 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182372 6.b 6:d373db44:::benchmark_data_smithi135_344736_object79716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2023-09-23T20:01:28.414 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 54.899161008, 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1349242150000001, 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182372 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449786+0000", 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449790+0000", 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4469999999999999e-06 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449800+0000", 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6909999999999992e-06 2023-09-23T20:01:28.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449809+0000", 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3009999999999995e-06 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449948+0000", 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013898400000000001 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.450078+0000", 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012999600000000001 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.450185+0000", 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010655299999999999 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.419 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.457033+0000", 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0068485000000000004 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.457047+0000", 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.376e-05 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.567645+0000", 2023-09-23T20:01:28.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1105975640000001 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.567815+0000", 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000170104 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584606+0000", 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016791006000000001 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584711+0000", 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000105554 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182400 6.b 6:ded8ad1e:::benchmark_data_smithi135_344736_object79744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:01:28.422 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 54.845058825000002, 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0838019940000001, 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182400 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503888+0000", 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:28.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503895+0000", 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3239999999999997e-06 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503912+0000", 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7125999999999998e-05 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503921+0000", 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8990000000000007e-06 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503982+0000", 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0566999999999998e-05 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503995+0000", 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3181e-05 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568139+0000", 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0641444200000001 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568266+0000", 2023-09-23T20:01:28.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012720799999999999 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568776+0000", 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050917599999999996 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580439+0000", 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011663902 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580591+0000", 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000151371 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584754+0000", 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041630570000000004 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587511+0000", 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027568660000000002 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587567+0000", 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6041000000000001e-05 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587691+0000", 2023-09-23T20:01:28.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012448400000000001 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182410 6.b 6:d95be848:::benchmark_data_smithi135_344736_object79754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 54.827244444000002, 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0661552510000001, 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.431 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182410 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521702+0000", 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521709+0000", 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.64e-06 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521725+0000", 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6078000000000001e-05 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521733+0000", 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6809999999999998e-06 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521799+0000", 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5988999999999994e-05 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521812+0000", 2023-09-23T20:01:28.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3307e-05 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573040+0000", 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.051228013 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573162+0000", 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012158 2023-09-23T20:01:28.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573254+0000", 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2051000000000003e-05 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580447+0000", 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0071931850000000004 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580639+0000", 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000191631 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584823+0000", 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041845129999999999 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587735+0000", 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029121139999999999 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587767+0000", 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1823999999999999e-05 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587859+0000", 2023-09-23T20:01:28.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1924999999999998e-05 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182448 6.b 6:d9f3c59c:::benchmark_data_smithi135_344736_object79792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 54.741904009000002, 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98102556500000004, 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.440 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182448 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607042+0000", 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607050+0000", 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6240000000000002e-06 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607066+0000", 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5667000000000001e-05 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607074+0000", 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4460000000000001e-06 2023-09-23T20:01:28.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607133+0000", 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8736999999999997e-05 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607146+0000", 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2845e-05 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.576979+0000", 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96983302800000004 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.577095+0000", 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011613899999999999 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:01:28.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.577196+0000", 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101648 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580685+0000", 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00348869 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580699+0000", 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3509999999999999e-05 2023-09-23T20:01:28.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584873+0000", 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041742089999999999 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587899+0000", 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003026543 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587941+0000", 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1307000000000003e-05 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588070+0000", 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012892900000000001 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182450 6.b 6:d37fd40c:::benchmark_data_smithi135_344736_object79794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 54.735691719999998, 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97506342899999998, 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182450 2023-09-23T20:01:28.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613255+0000", 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613262+0000", 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8210000000000002e-06 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613278+0000", 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5359999999999999e-05 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613286+0000", 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613340+0000", 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3922999999999999e-05 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613349+0000", 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0340000000000002e-06 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580746+0000", 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96739630700000001 2023-09-23T20:01:28.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580902+0000", 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015640900000000001 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.581089+0000", 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018731100000000001 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584767+0000", 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036773520000000001 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.585362+0000", 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00059505599999999995 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587366+0000", 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020038109999999999 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588137+0000", 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00077110600000000003 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588191+0000", 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4521999999999997e-05 2023-09-23T20:01:28.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588320+0000", 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012857600000000001 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201805 6.0 6:04364655:::benchmark_data_smithi135_344736_object99149:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:01:28.457 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 23.882473041000001, 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.08500007, 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201805 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.458 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466479+0000", 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3380000000000004e-06 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466490+0000", 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.129e-05 2023-09-23T20:01:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466500+0000", 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0305999999999999e-05 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468241+0000", 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001740143 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468253+0000", 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2268e-05 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542090+0000", 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.073837172 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542227+0000", 2023-09-23T20:01:28.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013645199999999999 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542739+0000", 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00051271599999999997 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551138+0000", 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083988829999999993 2023-09-23T20:01:28.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551197+0000", 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9363000000000001e-05 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551211+0000", 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3148e-05 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.464 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551266+0000", 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5229000000000003e-05 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551475+0000", 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020925699999999999 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.465 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201830 6.10 6:08e5b611:::benchmark_data_smithi135_344736_object99174:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 23.882437392, 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1094386300000001, 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201830 2023-09-23T20:01:28.466 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466510+0000", 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466513+0000", 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4109999999999999e-06 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.468 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466519+0000", 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2500000000000003e-06 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466524+0000", 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5350000000000004e-06 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.469 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468278+0000", 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00175377 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468286+0000", 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9920000000000007e-06 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551513+0000", 2023-09-23T20:01:28.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0832270980000001 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.552197+0000", 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00068353800000000003 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556569+0000", 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043722819999999999 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556637+0000", 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8348000000000004e-05 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.557117+0000", 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00047992400000000001 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.472 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560978+0000", 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038601740000000001 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561510+0000", 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00053195499999999997 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575543+0000", 2023-09-23T20:01:28.473 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014033406 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575578+0000", 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5404999999999997e-05 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575949+0000", 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037097900000000001 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.474 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201836 6.10 6:0c24ea53:::benchmark_data_smithi135_344736_object99180:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 23.882418534999999, 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1159160830000001, 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.475 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201836 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466529+0000", 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466532+0000", 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.43e-06 2023-09-23T20:01:28.477 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466549+0000", 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7680999999999999e-05 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466556+0000", 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0380000000000002e-06 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468309+0000", 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001753075 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468317+0000", 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8699999999999992e-06 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554470+0000", 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.086153084 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555087+0000", 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000616151 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.569798+0000", 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014711695 2023-09-23T20:01:28.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.569892+0000", 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3364999999999998e-05 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.570445+0000", 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00055320300000000005 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.481 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576273+0000", 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058281779999999998 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582033+0000", 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0057600059999999998 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582060+0000", 2023-09-23T20:01:28.482 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7141999999999999e-05 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582101+0000", 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1272000000000001e-05 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582446+0000", 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034424599999999998 2023-09-23T20:01:28.483 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201842 6.0 6:03218c77:::benchmark_data_smithi135_344736_object99186:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 23.882386024999999, 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.088181898, 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.484 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201842 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.485 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466565+0000", 2023-09-23T20:01:28.486 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2610000000000002e-06 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466571+0000", 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6059999999999999e-06 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466576+0000", 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9280000000000001e-06 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.487 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468350+0000", 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017738249999999999 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468359+0000", 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4190000000000002e-06 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.488 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548143+0000", 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.079783959 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548255+0000", 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011229400000000001 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548588+0000", 2023-09-23T20:01:28.489 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00033316999999999998 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554427+0000", 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058389430000000001 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554444+0000", 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7081000000000001e-05 2023-09-23T20:01:28.490 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554491+0000", 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6495999999999997e-05 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554542+0000", 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1561999999999999e-05 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.491 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554744+0000", 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000201883 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201845 6.10 6:0a5d128c:::benchmark_data_smithi135_344736_object99189:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 23.882367435999999, 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1274511190000001, 2023-09-23T20:01:28.492 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201845 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466580+0000", 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.494 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466583+0000", 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4509999999999999e-06 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466588+0000", 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2159999999999997e-06 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.495 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466592+0000", 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4220000000000002e-06 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468381+0000", 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001788778 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468389+0000", 2023-09-23T20:01:28.496 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9689999999999994e-06 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555129+0000", 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.086740209 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555800+0000", 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00067015300000000005 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.497 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.572929+0000", 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017129485999999999 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.572997+0000", 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7753e-05 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:28.498 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.573486+0000", 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00048920300000000001 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582000+0000", 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085139759999999995 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585622+0000", 2023-09-23T20:01:28.499 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003622489 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.593594+0000", 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0079714260000000002 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.593638+0000", 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4351000000000003e-05 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.500 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594032+0000", 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00039355800000000002 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201858 6.10 6:0e829f63:::benchmark_data_smithi135_344736_object99202:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:01:28.501 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 23.882352089000001, 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.12779443, 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201858 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.502 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:01:28.503 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466598+0000", 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8920000000000001e-06 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466601+0000", 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0869999999999998e-06 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.504 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466604+0000", 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.596e-06 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468607+0000", 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020026620000000001 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.505 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468616+0000", 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7739999999999993e-06 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560293+0000", 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0916775329999999 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560929+0000", 2023-09-23T20:01:28.506 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00063605599999999997 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576296+0000", 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01536671 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576359+0000", 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3100000000000002e-05 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.507 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576772+0000", 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00041272599999999998 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582458+0000", 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056866119999999997 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.508 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585662+0000", 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032034730000000001 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594087+0000", 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084249849999999994 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594137+0000", 2023-09-23T20:01:28.509 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9614000000000003e-05 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594390+0000", 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00025390100000000001 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201894 6.0 6:00f5be23:::benchmark_data_smithi135_344736_object99238:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:01:28.510 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 23.882339834, 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0976265000000001, 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201894 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.511 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.512 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466610+0000", 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3309999999999998e-06 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466616+0000", 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3630000000000002e-06 2023-09-23T20:01:28.513 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466621+0000", 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1719999999999998e-06 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468641+0000", 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020203489999999998 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.514 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468650+0000", 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3739999999999998e-06 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551524+0000", 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0828746149999999 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.515 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551624+0000", 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100259 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551973+0000", 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034831199999999998 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556557+0000", 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045842579999999999 2023-09-23T20:01:28.516 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556597+0000", 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0182000000000003e-05 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.563996+0000", 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073992540000000001 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.517 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.564035+0000", 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8602000000000001e-05 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.564235+0000", 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019976 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.518 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201912 6.10 6:0a9a0b94:::benchmark_data_smithi135_344736_object99256:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 23.882322915, 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1498471859999999, 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201912 2023-09-23T20:01:28.519 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.520 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466628+0000", 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6249999999999999e-06 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:01:28.521 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466633+0000", 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6210000000000001e-06 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466638+0000", 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6809999999999997e-06 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468671+0000", 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020332229999999998 2023-09-23T20:01:28.522 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468677+0000", 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8409999999999996e-06 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561014+0000", 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0923374370000001 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.523 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561497+0000", 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00048270000000000002 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579372+0000", 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017874660000000001 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.524 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579464+0000", 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2050000000000001e-05 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579912+0000", 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00044868700000000002 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582554+0000", 2023-09-23T20:01:28.525 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00264197 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585730+0000", 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031761210000000001 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.615988+0000", 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030257373000000001 2023-09-23T20:01:28.526 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.616043+0000", 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4648000000000003e-05 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.616472+0000", 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00042985000000000001 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.527 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.528 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:01:28.528 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:01:28.528 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:01:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:28.529 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:28.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:28.630 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:28.649 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:01:28.667 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:28 smithi135 ceph-mon[221113]: pgmap v625: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 92 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 160 objects/s recovering 2023-09-23T20:01:28.668 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:28 smithi135 ceph-mon[213871]: pgmap v625: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 92 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 160 objects/s recovering 2023-09-23T20:01:28.671 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:28.777 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:28.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:28.880 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:28.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:28 smithi144 ceph-mon[194833]: pgmap v625: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 92 active+clean, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 160 objects/s recovering 2023-09-23T20:01:28.907 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:29.624 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_1' removed 2023-09-23T20:01:29.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4076879747' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:01:29.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4076879747' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:01:29.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4076879747' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:01:30.627 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:30.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:30 smithi144 ceph-mon[194833]: pgmap v626: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 92 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 198 objects/s recovering 2023-09-23T20:01:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4076879747' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:01:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:30 smithi144 ceph-mon[194833]: osdmap e238: 8 total, 7 up, 8 in 2023-09-23T20:01:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:30 smithi144 ceph-mon[194833]: failed to encode map e238 with expected crc 2023-09-23T20:01:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:30 smithi144 ceph-mon[194833]: failed to encode map e238 with expected crc 2023-09-23T20:01:30.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[221113]: pgmap v626: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 92 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 198 objects/s recovering 2023-09-23T20:01:30.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4076879747' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:01:30.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[221113]: osdmap e238: 8 total, 7 up, 8 in 2023-09-23T20:01:30.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[221113]: failed to encode map e238 with expected crc 2023-09-23T20:01:30.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[221113]: failed to encode map e238 with expected crc 2023-09-23T20:01:30.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[213871]: pgmap v626: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 92 active+clean, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10422/585 objects degraded (1781.538%); 44/585 objects misplaced (7.521%); 0 B/s, 198 objects/s recovering 2023-09-23T20:01:30.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4076879747' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:01:30.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[213871]: osdmap e238: 8 total, 7 up, 8 in 2023-09-23T20:01:30.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[213871]: failed to encode map e238 with expected crc 2023-09-23T20:01:30.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:30 smithi135 ceph-mon[213871]: failed to encode map e238 with expected crc 2023-09-23T20:01:31.348 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:01:31.478 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:31.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:31.582 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:31.588 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:01:31.588 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:01:31.588 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T20:01:31.589 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T20:01:31.612 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:31.631 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:01:31.631 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:01:31.631 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:01:31.631 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-09-23T20:01:31.631 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T20:01:31.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:31 smithi144 ceph-mon[194833]: failed to encode map e238 with expected crc 2023-09-23T20:01:31.657 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:31.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:31.706 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:31.765 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:31.791 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:31.833 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:31.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:31.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:31.936 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:31.962 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:31.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:31 smithi135 ceph-mon[221113]: failed to encode map e238 with expected crc 2023-09-23T20:01:31.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:31 smithi135 ceph-mon[213871]: failed to encode map e238 with expected crc 2023-09-23T20:01:32.006 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:32.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:32.110 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:32.138 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:32.428 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:32.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:32.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:32.532 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:32.559 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:32.627 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:32.657 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:32.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:32.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:32 smithi135 ceph-mon[221113]: pgmap v628: 83 pgs: 1 active+clean+remapped, 78 active+clean, 3 active+undersized, 1 active+undersized+degraded; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:32.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:32 smithi135 ceph-mon[213871]: pgmap v628: 83 pgs: 1 active+clean+remapped, 78 active+clean, 3 active+undersized, 1 active+undersized+degraded; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:32.762 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:32.789 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:32.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:32 smithi144 ceph-mon[194833]: pgmap v628: 83 pgs: 1 active+clean+remapped, 78 active+clean, 3 active+undersized, 1 active+undersized+degraded; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:32.938 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:32.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:32.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:33.043 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:33.077 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:33.091 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:33.128 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:33.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:33.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:33.231 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:33.259 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:33.427 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:33.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:33.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:33.531 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:33.564 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:33.774 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:33 smithi144 ceph-mon[194833]: pgmap v629: 83 pgs: 1 active+clean+remapped, 78 active+clean, 3 active+undersized, 1 active+undersized+degraded; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:33.897 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:01:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:01:33] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-09-23T20:01:33.898 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:33 smithi135 ceph-mon[221113]: pgmap v629: 83 pgs: 1 active+clean+remapped, 78 active+clean, 3 active+undersized, 1 active+undersized+degraded; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:33.898 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:33 smithi135 ceph-mon[213871]: pgmap v629: 83 pgs: 1 active+clean+remapped, 78 active+clean, 3 active+undersized, 1 active+undersized+degraded; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:33.965 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:01:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:01:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:01:33.966 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:01:34.016 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:34.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:34.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:34.120 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:34.152 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:34.157 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:01:34.183 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:01:34.202 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:34.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:34.206 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:01:34.228 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:01:34.246 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:34.247 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:01:34.273 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:01:34.295 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:01:34.296 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:01:34.296 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:01:34.296 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:01:34.296 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:01:34.296 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:01:34.296 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:01:34.296 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:01:34.296 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:01:34.307 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:34.337 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:34.627 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:01:34.627 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:34.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:34 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/366797055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:34.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:34 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/366797055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:34.732 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:34.760 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:34 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/366797055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:35.011 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:35.376 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:01:35.376 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:01:35.434 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:35.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:35.539 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:35.569 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:35.645 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:35 smithi135 ceph-mon[213871]: pgmap v630: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:35.645 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:35 smithi135 ceph-mon[221113]: pgmap v630: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:35.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:35 smithi144 ceph-mon[194833]: pgmap v630: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:36.461 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:36.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:36.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:36.523 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:01:36.523 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:01:36.566 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:36.597 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:36.623 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:01:36.647 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1581515484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:36.647 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:36.647 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1581515484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:36.647 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:36 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:36.751 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:36.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:36.855 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1581515484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:36.885 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:37.144 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:01:37.378 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:01:37.488 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:37.631 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:37.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:37.735 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:37.761 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:37.872 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:37 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:37.873 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:37 smithi144 ceph-mon[194833]: pgmap v631: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:37.873 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:37 smithi144 ceph-mon[194833]: failed to encode map e239 with expected crc 2023-09-23T20:01:37.873 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:37 smithi144 ceph-mon[194833]: failed to encode map e239 with expected crc 2023-09-23T20:01:37.873 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:37 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:37.873 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:37 smithi144 ceph-mon[194833]: osdmap e239: 8 total, 7 up, 8 in 2023-09-23T20:01:37.873 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:37.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:37.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[221113]: pgmap v631: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:37.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[221113]: failed to encode map e239 with expected crc 2023-09-23T20:01:37.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[221113]: failed to encode map e239 with expected crc 2023-09-23T20:01:37.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:37.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[221113]: osdmap e239: 8 total, 7 up, 8 in 2023-09-23T20:01:37.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:37.922 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:37.922 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[213871]: pgmap v631: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:37.923 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[213871]: failed to encode map e239 with expected crc 2023-09-23T20:01:37.923 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[213871]: failed to encode map e239 with expected crc 2023-09-23T20:01:37.923 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:37.923 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[213871]: osdmap e239: 8 total, 7 up, 8 in 2023-09-23T20:01:37.923 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:38.158 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:38.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:38.262 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:38.291 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:38.398 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:38.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:38.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:38.503 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:38.537 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:38.549 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:38.863 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:38.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:38.913 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_2' created 2023-09-23T20:01:38.972 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:38.982 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:01:38.982 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:01:38.999 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2989563204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:01:38.999 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:38 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:01:38.999 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:39.000 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:39.003 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:39.150 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:39.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:39.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:39.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2989563204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:01:39.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:38 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:01:39.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:39.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:39.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2989563204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:01:39.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:38 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:01:39.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:39.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:39.253 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:39.279 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:39.322 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:39.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:39.426 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:39.454 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:39.642 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:01:39.914 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T20:01:39.924 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:01:39.979 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:39.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:39.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:39.987 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:40.050 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T20:01:40.083 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:40.122 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:40.177 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: pgmap v633: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: failed to encode map e240 with expected crc 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: failed to encode map e240 with expected crc 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: osdmap e240: 8 total, 7 up, 8 in 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3705389316' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:01:40.178 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3705389316' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:01:40.179 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:39 smithi144 ceph-mon[194833]: osdmap e241: 8 total, 7 up, 8 in 2023-09-23T20:01:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: pgmap v633: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: failed to encode map e240 with expected crc 2023-09-23T20:01:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: failed to encode map e240 with expected crc 2023-09-23T20:01:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: osdmap e240: 8 total, 7 up, 8 in 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3705389316' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3705389316' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[221113]: osdmap e241: 8 total, 7 up, 8 in 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: pgmap v633: 83 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 78 active+clean, 3 active+undersized; 4.5 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:40.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: failed to encode map e240 with expected crc 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: failed to encode map e240 with expected crc 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: osdmap e240: 8 total, 7 up, 8 in 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3705389316' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3705389316' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:01:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:39 smithi135 ceph-mon[213871]: osdmap e241: 8 total, 7 up, 8 in 2023-09-23T20:01:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[213871]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[213871]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[213871]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[213871]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[213871]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[213871]: osdmap e242: 8 total, 7 up, 8 in 2023-09-23T20:01:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[221113]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[221113]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[221113]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[221113]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[221113]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:40 smithi135 ceph-mon[221113]: osdmap e242: 8 total, 7 up, 8 in 2023-09-23T20:01:41.223 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:41.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:41.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:41.270 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:40 smithi144 ceph-mon[194833]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.270 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:40 smithi144 ceph-mon[194833]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.271 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:40 smithi144 ceph-mon[194833]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.271 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:40 smithi144 ceph-mon[194833]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.271 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:40 smithi144 ceph-mon[194833]: failed to encode map e241 with expected crc 2023-09-23T20:01:41.271 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:40 smithi144 ceph-mon[194833]: osdmap e242: 8 total, 7 up, 8 in 2023-09-23T20:01:41.327 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:41.358 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:41.410 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:41.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:41.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:41.490 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:01:41.491 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:01:41.514 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:41.517 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:41.551 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:41.647 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:41.654 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:41.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:41.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:41.758 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:41.788 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:41.948 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:41.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:41.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:42.054 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:42.083 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:41 smithi144 ceph-mon[194833]: pgmap v636: 99 pgs: 12 unknown, 1 active+undersized+degraded, 1 active+clean+remapped, 82 active+clean, 3 active+undersized; 4.5 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:41 smithi144 ceph-mon[194833]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:41 smithi144 ceph-mon[194833]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:41 smithi144 ceph-mon[194833]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.137 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:01:42.137 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:01:42.137 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:01:42.156 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:01:42.174 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:01:42.177 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:01:42.194 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:01:42.215 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:01:42.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:01:42.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:01:42.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:01:42.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:01:42.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:01:42.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:01:42.216 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:01:42.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:01:42.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:41 smithi135 ceph-mon[221113]: pgmap v636: 99 pgs: 12 unknown, 1 active+undersized+degraded, 1 active+clean+remapped, 82 active+clean, 3 active+undersized; 4.5 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:42.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:41 smithi135 ceph-mon[221113]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:41 smithi135 ceph-mon[221113]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:41 smithi135 ceph-mon[221113]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:41 smithi135 ceph-mon[213871]: pgmap v636: 99 pgs: 12 unknown, 1 active+undersized+degraded, 1 active+clean+remapped, 82 active+clean, 3 active+undersized; 4.5 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:41 smithi135 ceph-mon[213871]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:41 smithi135 ceph-mon[213871]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:41 smithi135 ceph-mon[213871]: failed to encode map e242 with expected crc 2023-09-23T20:01:42.231 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:42.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:42.335 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:42.361 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:42.406 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:42.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:42.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:42.511 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:42.540 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:42.580 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:42.587 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:42.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:42.692 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:42.719 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:42.813 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:42.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:42.916 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:42.943 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:42.988 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:42.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:43.091 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:43.120 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:43.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1762216406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:43.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1762216406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:43.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1762216406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:43.958 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:01:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:01:43] "GET /metrics HTTP/1.1" 200 34109 "" "Prometheus/2.43.0" 2023-09-23T20:01:44.088 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:01:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:01:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:01:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:01:44.089 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:01:44.269 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:44.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:44.373 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:44.404 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:44.446 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:44.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:44.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:44.550 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:44.575 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:44 smithi135 ceph-mon[221113]: pgmap v638: 99 pgs: 8 unknown, 1 active+undersized+degraded, 1 active+clean+remapped, 86 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:44.575 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:44 smithi135 ceph-mon[213871]: pgmap v638: 99 pgs: 8 unknown, 1 active+undersized+degraded, 1 active+clean+remapped, 86 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:44.578 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:44.687 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:44 smithi144 ceph-mon[194833]: pgmap v638: 99 pgs: 8 unknown, 1 active+undersized+degraded, 1 active+clean+remapped, 86 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:44.699 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:01:44.712 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:01:44.713 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:01:44.850 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:44.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:44.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:44.954 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:44.983 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:45.040 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:45.079 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:45.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:45.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:45.182 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:45.209 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:45.253 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:45.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:45.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:45.356 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:01:45.383 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:45.408 INFO:teuthology.orchestra.run.smithi135.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T20:01:45.408 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:01:45.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3861591428' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:45.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1900073894' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:01:45.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:45 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:01:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3861591428' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1900073894' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:01:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:45 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:01:45.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3861591428' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:01:45.585 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1900073894' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:01:45.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:45 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:01:45.630 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:01:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:45.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:01:45.734 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:45.761 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:46.409 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:01:46.410 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:01:46.410 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T20:01:46.410 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T20:01:46.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: pgmap v639: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:46.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:46.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3861591428' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: osdmap e243: 8 total, 7 up, 8 in 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: 1.4 scrub starts 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[213871]: 1.4 scrub ok 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: pgmap v639: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3861591428' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: osdmap e243: 8 total, 7 up, 8 in 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: 1.4 scrub starts 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:46 smithi135 ceph-mon[221113]: 1.4 scrub ok 2023-09-23T20:01:46.554 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:01:46.555 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:01:46.555 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:01:46.555 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:01:46.555 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.555 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149325 5.f 5:fde8a226:::benchmark_data_smithi135_284570_object33541:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:01:46.555 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.280934+0000", 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 422.26955158999999, 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1011945750000001, 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149325 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.556 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280934+0000", 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280934+0000", 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.557 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.558 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.558 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280933+0000", 2023-09-23T20:01:46.558 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:46.558 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.558 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.558 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.558 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280936+0000", 2023-09-23T20:01:46.559 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0869999999999998e-06 2023-09-23T20:01:46.559 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.559 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.559 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.559 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280942+0000", 2023-09-23T20:01:46.559 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2829999999999998e-06 2023-09-23T20:01:46.559 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280947+0000", 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3780000000000003e-06 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280992+0000", 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5649000000000003e-05 2023-09-23T20:01:46.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.281004+0000", 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1311999999999999e-05 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.355856+0000", 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0748525609999999 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.561 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.562 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.562 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.355989+0000", 2023-09-23T20:01:46.562 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000133029 2023-09-23T20:01:46.562 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.562 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.562 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:01:46.562 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.356119+0000", 2023-09-23T20:01:46.562 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013006199999999999 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.373040+0000", 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016920418 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.381970+0000", 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0089305089999999997 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382036+0000", 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5178000000000005e-05 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382128+0000", 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2517999999999997e-05 2023-09-23T20:01:46.564 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.565 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.565 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.565 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.565 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.565 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149335 5.f 5:f870a179:::benchmark_data_smithi135_284570_object33551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:01:46.565 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.296109+0000", 2023-09-23T20:01:46.565 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 422.25437589500001, 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0862026, 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149335 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.566 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.567 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.567 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.567 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296109+0000", 2023-09-23T20:01:46.567 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.567 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.567 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.567 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.567 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296109+0000", 2023-09-23T20:01:46.568 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.568 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.568 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.568 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.568 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296108+0000", 2023-09-23T20:01:46.568 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:46.568 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.569 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.569 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.569 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296115+0000", 2023-09-23T20:01:46.569 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6409999999999996e-06 2023-09-23T20:01:46.569 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.569 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.569 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.569 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296127+0000", 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2825e-05 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296133+0000", 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6659999999999996e-06 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.570 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296179+0000", 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5927999999999998e-05 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296187+0000", 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1319999999999994e-06 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360611+0000", 2023-09-23T20:01:46.571 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0644241830000001 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360726+0000", 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114379 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360849+0000", 2023-09-23T20:01:46.572 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012293100000000001 2023-09-23T20:01:46.573 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.573 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.573 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.573 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.376986+0000", 2023-09-23T20:01:46.573 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016137459999999999 2023-09-23T20:01:46.573 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.574 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.574 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.574 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382195+0000", 2023-09-23T20:01:46.574 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052092550000000003 2023-09-23T20:01:46.574 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.574 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.574 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382238+0000", 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2998999999999999e-05 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382312+0000", 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3454000000000005e-05 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.575 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.576 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.576 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.576 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.576 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149337 5.f 5:f4eccdf5:::benchmark_data_smithi135_284570_object33553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:01:46.576 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.300192+0000", 2023-09-23T20:01:46.576 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 422.25029358099999, 2023-09-23T20:01:46.577 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0823250200000001, 2023-09-23T20:01:46.577 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.577 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.577 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.577 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:01:46.577 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:01:46.577 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149337 2023-09-23T20:01:46.578 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.578 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.578 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.578 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.578 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300192+0000", 2023-09-23T20:01:46.578 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.578 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.578 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.579 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.579 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300192+0000", 2023-09-23T20:01:46.579 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.579 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.579 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.579 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.579 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300191+0000", 2023-09-23T20:01:46.580 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:46.580 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.580 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.580 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.580 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300196+0000", 2023-09-23T20:01:46.581 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9629999999999997e-06 2023-09-23T20:01:46.581 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.581 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.581 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.581 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300205+0000", 2023-09-23T20:01:46.582 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1470000000000007e-06 2023-09-23T20:01:46.582 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.582 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.582 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.582 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300211+0000", 2023-09-23T20:01:46.583 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9739999999999999e-06 2023-09-23T20:01:46.583 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.583 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.583 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.583 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300251+0000", 2023-09-23T20:01:46.584 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0595e-05 2023-09-23T20:01:46.584 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.584 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.584 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.584 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300259+0000", 2023-09-23T20:01:46.585 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6510000000000001e-06 2023-09-23T20:01:46.585 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.585 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.585 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.585 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.365192+0000", 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0649325730000001 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.365303+0000", 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111457 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.586 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:01:46.587 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.365381+0000", 2023-09-23T20:01:46.587 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8209999999999998e-05 2023-09-23T20:01:46.587 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.587 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.587 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.587 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.376996+0000", 2023-09-23T20:01:46.587 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011615138000000001 2023-09-23T20:01:46.587 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382367+0000", 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005370934 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382424+0000", 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6459999999999998e-05 2023-09-23T20:01:46.588 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.589 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.589 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.589 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382517+0000", 2023-09-23T20:01:46.589 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2748e-05 2023-09-23T20:01:46.589 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.589 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.589 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.589 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.590 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.590 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149354 5.f 5:f2a5a837:::benchmark_data_smithi135_284570_object33570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:01:46.590 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.332074+0000", 2023-09-23T20:01:46.590 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 422.218411016, 2023-09-23T20:01:46.590 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.050649921, 2023-09-23T20:01:46.590 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.590 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.591 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.591 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:01:46.591 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:01:46.591 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149354 2023-09-23T20:01:46.591 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.591 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.591 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.591 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.592 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332074+0000", 2023-09-23T20:01:46.592 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.592 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.592 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.592 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.592 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332074+0000", 2023-09-23T20:01:46.592 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.593 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.593 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.593 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.593 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332073+0000", 2023-09-23T20:01:46.593 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:46.593 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.593 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.593 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.594 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332079+0000", 2023-09-23T20:01:46.594 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3609999999999997e-06 2023-09-23T20:01:46.594 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.594 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.594 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.594 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332095+0000", 2023-09-23T20:01:46.594 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5537e-05 2023-09-23T20:01:46.595 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.595 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.595 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.595 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332103+0000", 2023-09-23T20:01:46.595 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2630000000000003e-06 2023-09-23T20:01:46.595 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.595 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.595 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.596 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332158+0000", 2023-09-23T20:01:46.596 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5228e-05 2023-09-23T20:01:46.596 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.596 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.596 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.596 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332167+0000", 2023-09-23T20:01:46.596 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1810000000000007e-06 2023-09-23T20:01:46.596 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.368671+0000", 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0365039060000001 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.368789+0000", 2023-09-23T20:01:46.597 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000117879 2023-09-23T20:01:46.598 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.598 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.598 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:01:46.598 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.368928+0000", 2023-09-23T20:01:46.598 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013848199999999999 2023-09-23T20:01:46.598 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.598 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.598 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.599 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.381710+0000", 2023-09-23T20:01:46.599 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012782188999999999 2023-09-23T20:01:46.599 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.599 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.599 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.599 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382576+0000", 2023-09-23T20:01:46.599 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00086563600000000003 2023-09-23T20:01:46.599 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382631+0000", 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5569999999999998e-05 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382724+0000", 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2875999999999996e-05 2023-09-23T20:01:46.600 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149356 5.f 5:ff7f40ec:::benchmark_data_smithi135_284570_object33572:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.335103+0000", 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 422.215382487, 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0478277620000001, 2023-09-23T20:01:46.601 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.602 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.602 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.602 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:01:46.602 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:01:46.602 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149356 2023-09-23T20:01:46.602 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335103+0000", 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.603 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335103+0000", 2023-09-23T20:01:46.604 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.604 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.604 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.604 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.604 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335101+0000", 2023-09-23T20:01:46.604 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:46.604 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335111+0000", 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6360000000000006e-06 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335124+0000", 2023-09-23T20:01:46.605 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3264e-05 2023-09-23T20:01:46.606 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.606 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.606 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.606 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335129+0000", 2023-09-23T20:01:46.606 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0250000000000002e-06 2023-09-23T20:01:46.606 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.606 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335165+0000", 2023-09-23T20:01:46.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5111000000000003e-05 2023-09-23T20:01:46.607 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.607 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335170+0000", 2023-09-23T20:01:46.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5910000000000003e-06 2023-09-23T20:01:46.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.608 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.608 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.373070+0000", 2023-09-23T20:01:46.608 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0378999600000001 2023-09-23T20:01:46.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.609 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.373192+0000", 2023-09-23T20:01:46.609 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000121709 2023-09-23T20:01:46.609 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.609 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:01:46.610 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.373325+0000", 2023-09-23T20:01:46.610 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013317599999999999 2023-09-23T20:01:46.610 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.610 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.610 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.610 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.381895+0000", 2023-09-23T20:01:46.610 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0085699360000000002 2023-09-23T20:01:46.610 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.611 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.611 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.611 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382785+0000", 2023-09-23T20:01:46.611 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00089052200000000004 2023-09-23T20:01:46.611 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.611 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.611 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382834+0000", 2023-09-23T20:01:46.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8702000000000002e-05 2023-09-23T20:01:46.612 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.612 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.612 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382930+0000", 2023-09-23T20:01:46.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6261000000000006e-05 2023-09-23T20:01:46.613 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.613 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.613 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.613 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.613 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.613 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149360 5.f 5:fbe4ed6a:::benchmark_data_smithi135_284570_object33576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:01:46.613 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.346857+0000", 2023-09-23T20:01:46.614 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 422.20362820399998, 2023-09-23T20:01:46.614 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0362635920000001, 2023-09-23T20:01:46.614 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.614 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.614 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.615 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:01:46.615 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:01:46.615 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149360 2023-09-23T20:01:46.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.615 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.615 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.615 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346857+0000", 2023-09-23T20:01:46.616 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.616 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346857+0000", 2023-09-23T20:01:46.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.617 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.617 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.617 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.617 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346856+0000", 2023-09-23T20:01:46.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:46.617 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.618 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.618 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.618 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346871+0000", 2023-09-23T20:01:46.618 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4905e-05 2023-09-23T20:01:46.618 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.618 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.618 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346883+0000", 2023-09-23T20:01:46.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2498e-05 2023-09-23T20:01:46.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.619 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346890+0000", 2023-09-23T20:01:46.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.477e-06 2023-09-23T20:01:46.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.620 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.620 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346942+0000", 2023-09-23T20:01:46.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2201999999999999e-05 2023-09-23T20:01:46.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.621 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346953+0000", 2023-09-23T20:01:46.621 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0879999999999999e-05 2023-09-23T20:01:46.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.621 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.622 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.377037+0000", 2023-09-23T20:01:46.622 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0300842379999999 2023-09-23T20:01:46.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.622 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.622 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.622 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.377177+0000", 2023-09-23T20:01:46.622 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013932300000000001 2023-09-23T20:01:46.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.623 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.623 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:01:46.623 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.377326+0000", 2023-09-23T20:01:46.623 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000149615 2023-09-23T20:01:46.623 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.623 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.623 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382944+0000", 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0056173849999999999 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382985+0000", 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1128000000000003e-05 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.624 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.625 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.625 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.383034+0000", 2023-09-23T20:01:46.625 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9357999999999997e-05 2023-09-23T20:01:46.625 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.625 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.625 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.625 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.383120+0000", 2023-09-23T20:01:46.626 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6485999999999998e-05 2023-09-23T20:01:46.626 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.626 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.626 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.626 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.626 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.626 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148787 6.a 6:54a43917:::benchmark_data_smithi135_344736_object46131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:01:46.626 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.901403+0000", 2023-09-23T20:01:46.627 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 123.649081854, 2023-09-23T20:01:46.627 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2834841199999998, 2023-09-23T20:01:46.627 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.627 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:01:46.627 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.627 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.627 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.627 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148787 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901403+0000", 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.628 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901403+0000", 2023-09-23T20:01:46.629 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.629 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.629 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.629 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.629 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901402+0000", 2023-09-23T20:01:46.629 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:46.629 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.630 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.630 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.630 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901407+0000", 2023-09-23T20:01:46.630 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6229999999999996e-06 2023-09-23T20:01:46.630 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.630 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.630 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.630 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901422+0000", 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5262999999999999e-05 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901433+0000", 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0353000000000001e-05 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.631 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.632 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.905928+0000", 2023-09-23T20:01:46.632 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0044957510000000001 2023-09-23T20:01:46.632 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.632 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.632 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.632 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.906049+0000", 2023-09-23T20:01:46.632 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012040900000000001 2023-09-23T20:01:46.633 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.633 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.633 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:01:46.633 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.906125+0000", 2023-09-23T20:01:46.633 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6507000000000005e-05 2023-09-23T20:01:46.633 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.633 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.909603+0000", 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0034781069999999998 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.911131+0000", 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001527608 2023-09-23T20:01:46.634 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.635 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.635 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.635 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924795+0000", 2023-09-23T20:01:46.635 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.01366392 2023-09-23T20:01:46.635 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.635 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.635 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924807+0000", 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1746000000000001e-05 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925187+0000", 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0003801859999999 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.636 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.637 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.637 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925201+0000", 2023-09-23T20:01:46.637 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3718e-05 2023-09-23T20:01:46.637 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.637 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.637 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.637 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.184795+0000", 2023-09-23T20:01:46.637 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.25959485 2023-09-23T20:01:46.638 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.638 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.638 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.638 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.184887+0000", 2023-09-23T20:01:46.638 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1875999999999999e-05 2023-09-23T20:01:46.638 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.638 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.639 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.639 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.639 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.639 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148806 6.c 6:36c6f17c:::benchmark_data_smithi135_344736_object46150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:01:46.639 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.937119+0000", 2023-09-23T20:01:46.640 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 123.61336616299999, 2023-09-23T20:01:46.640 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2790684319999999, 2023-09-23T20:01:46.640 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.640 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.640 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.640 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.640 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.640 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148806 2023-09-23T20:01:46.641 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.641 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.641 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.641 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.641 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937119+0000", 2023-09-23T20:01:46.641 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.642 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.642 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.642 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.642 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937119+0000", 2023-09-23T20:01:46.642 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.643 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.643 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.643 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937119+0000", 2023-09-23T20:01:46.643 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:46.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.643 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.643 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.644 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937121+0000", 2023-09-23T20:01:46.644 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.773e-06 2023-09-23T20:01:46.646 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.646 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.646 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.646 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937128+0000", 2023-09-23T20:01:46.646 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3579999999999999e-06 2023-09-23T20:01:46.647 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.647 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.647 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.647 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937132+0000", 2023-09-23T20:01:46.648 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0319999999999997e-06 2023-09-23T20:01:46.648 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.648 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.648 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.648 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937188+0000", 2023-09-23T20:01:46.648 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5967000000000001e-05 2023-09-23T20:01:46.648 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.649 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.649 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.649 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937195+0000", 2023-09-23T20:01:46.649 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1380000000000001e-06 2023-09-23T20:01:46.649 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.650 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.650 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.650 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925897+0000", 2023-09-23T20:01:46.650 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98870237500000002 2023-09-23T20:01:46.650 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.650 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.651 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.651 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925908+0000", 2023-09-23T20:01:46.651 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.064e-05 2023-09-23T20:01:46.651 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.651 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.651 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.652 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.190584+0000", 2023-09-23T20:01:46.652 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2646758870000001 2023-09-23T20:01:46.652 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.652 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.652 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.652 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.190670+0000", 2023-09-23T20:01:46.652 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6321000000000002e-05 2023-09-23T20:01:46.652 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.190748+0000", 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7558000000000005e-05 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.195309+0000", 2023-09-23T20:01:46.653 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045612430000000004 2023-09-23T20:01:46.654 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.654 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.654 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.654 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.195781+0000", 2023-09-23T20:01:46.654 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00047211300000000002 2023-09-23T20:01:46.654 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.654 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.199253+0000", 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0034722970000000001 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.215988+0000", 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016734480999999999 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.655 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.216074+0000", 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6388e-05 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.216187+0000", 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113292 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.656 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148809 6.a 6:512c2dc0:::benchmark_data_smithi135_344736_object46153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.937144+0000", 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 123.613341362, 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6061435479999999, 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.657 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148809 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937144+0000", 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937144+0000", 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.658 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937143+0000", 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937146+0000", 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0800000000000002e-06 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.659 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937152+0000", 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6200000000000004e-06 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937156+0000", 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4999999999999999e-06 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937211+0000", 2023-09-23T20:01:46.660 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.533e-05 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937221+0000", 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.013e-05 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925294+0000", 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98807346500000004 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.661 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925306+0000", 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1430000000000001e-05 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.184916+0000", 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2596101420000001 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.662 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.185002+0000", 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5881e-05 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.185443+0000", 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00044055800000000002 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.196356+0000", 2023-09-23T20:01:46.663 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010913440999999999 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.209973+0000", 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013617402000000001 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.219759+0000", 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0097860410000000005 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.664 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543097+0000", 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.32333798800000002 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543161+0000", 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3901999999999999e-05 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543287+0000", 2023-09-23T20:01:46.665 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012600900000000001 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148810 6.a 6:5d5fc8b8:::benchmark_data_smithi135_344736_object46154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.937169+0000", 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 123.61331639399999, 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6063484140000002, 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.666 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148810 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937169+0000", 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.667 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937169+0000", 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937168+0000", 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937171+0000", 2023-09-23T20:01:46.668 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9000000000000002e-06 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937178+0000", 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9020000000000002e-06 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937182+0000", 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.631e-06 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.669 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937246+0000", 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3986999999999996e-05 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937252+0000", 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.993e-06 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925335+0000", 2023-09-23T20:01:46.670 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98808339700000003 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925342+0000", 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4300000000000003e-06 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.193209+0000", 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.267867882 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.671 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.193313+0000", 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000104054 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.193429+0000", 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115164 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.672 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.196466+0000", 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003037363 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.219303+0000", 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022836921 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.219784+0000", 2023-09-23T20:01:46.673 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000481552 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543348+0000", 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.323563989 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543403+0000", 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4934e-05 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.674 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543517+0000", 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113706 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182362 6.a 6:587b6d41:::benchmark_data_smithi135_344736_object79706:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.437025+0000", 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 73.113460430999993, 2023-09-23T20:01:46.675 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1483356659999999, 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182362 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.676 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437025+0000", 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437025+0000", 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.677 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437024+0000", 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437029+0000", 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8280000000000002e-06 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.678 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437040+0000", 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1742999999999999e-05 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437046+0000", 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5149999999999997e-06 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.679 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437099+0000", 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3625999999999998e-05 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437203+0000", 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103088 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.680 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437285+0000", 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2295999999999999e-05 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.440032+0000", 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027470379999999998 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.681 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.682 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.682 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.445210+0000", 2023-09-23T20:01:46.682 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005178177 2023-09-23T20:01:46.682 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.682 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.682 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.682 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.445223+0000", 2023-09-23T20:01:46.682 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2746e-05 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585252+0000", 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1400295599999999 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585360+0000", 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107982 2023-09-23T20:01:46.683 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.684 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.684 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.684 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.684 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.684 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182380 6.a 6:5ee9a171:::benchmark_data_smithi135_344736_object79724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:46.684 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.471936+0000", 2023-09-23T20:01:46.684 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 73.078549303000003, 2023-09-23T20:01:46.684 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.141209084, 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182380 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.685 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471936+0000", 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471936+0000", 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.686 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471934+0000", 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471940+0000", 2023-09-23T20:01:46.687 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5969999999999999e-06 2023-09-23T20:01:46.688 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.688 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.688 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.688 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471957+0000", 2023-09-23T20:01:46.688 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7572999999999999e-05 2023-09-23T20:01:46.688 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.688 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.688 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471963+0000", 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0950000000000001e-06 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.472016+0000", 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2750999999999997e-05 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.472027+0000", 2023-09-23T20:01:46.689 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0507e-05 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585386+0000", 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.11335973 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585520+0000", 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013361199999999999 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.690 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585677+0000", 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015681199999999999 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.597609+0000", 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011932096 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.691 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599956+0000", 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023472800000000002 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599990+0000", 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3587000000000002e-05 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.612967+0000", 2023-09-23T20:01:46.692 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012977572999999999 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613024+0000", 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6620999999999999e-05 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613145+0000", 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000120937 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.693 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182383 6.a 6:52d5eae7:::benchmark_data_smithi135_344736_object79727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.477344+0000", 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 73.073140823000003, 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.136035479, 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.694 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182383 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477344+0000", 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477344+0000", 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477343+0000", 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477350+0000", 2023-09-23T20:01:46.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477365+0000", 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5081e-05 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477371+0000", 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5199999999999997e-06 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477423+0000", 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2051e-05 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477432+0000", 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7399999999999999e-06 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.588349+0000", 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1109166269999999 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.588439+0000", 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9716000000000006e-05 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.588516+0000", 2023-09-23T20:01:46.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7050000000000003e-05 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599746+0000", 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011229784 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600017+0000", 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00027105100000000002 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.700 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600043+0000", 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6009000000000001e-05 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613199+0000", 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013156407 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.701 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613258+0000", 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9326000000000001e-05 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613380+0000", 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012138000000000001 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.702 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182389 6.a 6:57ff510e:::benchmark_data_smithi135_344736_object79733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.485261+0000", 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 73.065223864000004, 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1283127610000001, 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182389 2023-09-23T20:01:46.703 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485261+0000", 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485261+0000", 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485261+0000", 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485263+0000", 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5150000000000001e-06 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.705 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485271+0000", 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.96e-06 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485275+0000", 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9639999999999997e-06 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485337+0000", 2023-09-23T20:01:46.706 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2015999999999997e-05 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485348+0000", 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1174e-05 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.591401+0000", 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1060528839999999 2023-09-23T20:01:46.707 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.591542+0000", 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000140307 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.591625+0000", 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3741999999999999e-05 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599753+0000", 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0081273339999999999 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600068+0000", 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00031554800000000002 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600143+0000", 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4919999999999994e-05 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613424+0000", 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013280973 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.710 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613469+0000", 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4427e-05 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613574+0000", 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105371 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.711 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182397 6.a 6:5da29aad:::benchmark_data_smithi135_344736_object79741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.500485+0000", 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 73.050000134000001, 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.113300274, 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.712 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182397 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500485+0000", 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.713 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.714 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500485+0000", 2023-09-23T20:01:46.714 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.714 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.714 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.714 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.714 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500484+0000", 2023-09-23T20:01:46.714 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:01:46.714 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500489+0000", 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0440000000000003e-06 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500502+0000", 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3233e-05 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.715 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500523+0000", 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0930000000000001e-05 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500591+0000", 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8384000000000001e-05 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.716 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500604+0000", 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.295e-05 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.594382+0000", 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0937782300000001 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.717 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.594572+0000", 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000189526 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.594707+0000", 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013493399999999999 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.718 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599919+0000", 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052121670000000002 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600093+0000", 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017360800000000001 2023-09-23T20:01:46.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.612698+0000", 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012605016 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613628+0000", 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00093064799999999996 2023-09-23T20:01:46.720 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.721 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.721 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.721 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613677+0000", 2023-09-23T20:01:46.721 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8222e-05 2023-09-23T20:01:46.721 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.721 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.721 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.721 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613785+0000", 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000108701 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182416 6.a 6:5629a20a:::benchmark_data_smithi135_344736_object79760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.530109+0000", 2023-09-23T20:01:46.722 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 73.020376065999997, 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0838080189999999, 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182416 2023-09-23T20:01:46.723 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530109+0000", 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530109+0000", 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530108+0000", 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530114+0000", 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5289999999999997e-06 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.725 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530126+0000", 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1975e-05 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530131+0000", 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0470000000000001e-06 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530180+0000", 2023-09-23T20:01:46.726 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8770999999999998e-05 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530187+0000", 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1779999999999997e-06 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.597643+0000", 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0674553579999999 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.597758+0000", 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114963 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.597842+0000", 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4295999999999994e-05 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599996+0000", 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002154319 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.601872+0000", 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001875959 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.612762+0000", 2023-09-23T20:01:46.729 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010889598 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613821+0000", 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001059459 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613850+0000", 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8738000000000002e-05 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.730 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613917+0000", 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7012000000000001e-05 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:184941 6.17 6:eb7ca16f:::benchmark_data_smithi135_344736_object82285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:39.639104+0000", 2023-09-23T20:01:46.731 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 66.911381253000002, 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.94413358700000005, 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 184941 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.732 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639104+0000", 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639104+0000", 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639104+0000", 2023-09-23T20:01:46.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639105+0000", 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.751e-06 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639109+0000", 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5260000000000002e-06 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639111+0000", 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.26e-06 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639176+0000", 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4807000000000005e-05 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.735 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639187+0000", 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0727000000000001e-05 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.552727+0000", 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.91354033999999995 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.552834+0000", 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000106863 2023-09-23T20:01:46.736 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.553245+0000", 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000410744 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.567487+0000", 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.01424215 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.737 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583067+0000", 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015580112 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583143+0000", 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6168000000000007e-05 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.738 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583192+0000", 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9024999999999999e-05 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583237+0000", 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5379e-05 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.739 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:184942 6.17 6:ef4b34a6:::benchmark_data_smithi135_344736_object82286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:39.644227+0000", 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 66.906257948999993, 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.93935562299999997, 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 184942 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.740 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644227+0000", 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644227+0000", 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.741 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644226+0000", 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644231+0000", 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1479999999999997e-06 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644244+0000", 2023-09-23T20:01:46.742 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2921e-05 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644249+0000", 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8590000000000004e-06 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644298+0000", 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9360000000000002e-05 2023-09-23T20:01:46.743 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.744 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.744 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.744 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644307+0000", 2023-09-23T20:01:46.744 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2479999999999996e-06 2023-09-23T20:01:46.744 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.744 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.744 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.744 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.555315+0000", 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.91100864500000001 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.555438+0000", 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012302999999999999 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.555806+0000", 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000368024 2023-09-23T20:01:46.745 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.567497+0000", 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011690196999999999 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583113+0000", 2023-09-23T20:01:46.746 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015616849 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583505+0000", 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00039157500000000002 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583550+0000", 2023-09-23T20:01:46.747 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5334999999999999e-05 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583583+0000", 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2434000000000002e-05 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.748 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:185076 6.17 6:e9aef155:::benchmark_data_smithi135_344736_object82420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:39.844967+0000", 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 66.705518479999995, 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.73883129000000003, 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.749 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 185076 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844967+0000", 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.750 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844967+0000", 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844966+0000", 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.751 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.752 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844969+0000", 2023-09-23T20:01:46.752 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.847e-06 2023-09-23T20:01:46.752 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.752 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.752 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.752 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844982+0000", 2023-09-23T20:01:46.752 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2542e-05 2023-09-23T20:01:46.752 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844986+0000", 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5589999999999999e-06 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845037+0000", 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0881000000000003e-05 2023-09-23T20:01:46.753 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.754 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.754 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.754 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845041+0000", 2023-09-23T20:01:46.754 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.754e-06 2023-09-23T20:01:46.754 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.754 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.754 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.754 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.561449+0000", 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.71640810200000005 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.561559+0000", 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011042799999999999 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.755 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:01:46.756 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.561854+0000", 2023-09-23T20:01:46.756 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00029497200000000001 2023-09-23T20:01:46.756 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.756 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.756 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.756 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.571424+0000", 2023-09-23T20:01:46.756 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0095692190000000003 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583338+0000", 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011914401 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583726+0000", 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000387969 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.757 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583766+0000", 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0232999999999999e-05 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583798+0000", 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1764000000000003e-05 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.758 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:185077 6.17 6:ea269bbc:::benchmark_data_smithi135_344736_object82421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:39.844996+0000", 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 66.705489585999999, 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.73895094699999997, 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:01:46.759 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 185077 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844996+0000", 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844996+0000", 2023-09-23T20:01:46.760 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844995+0000", 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844998+0000", 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6759999999999999e-06 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.761 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845003+0000", 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1699999999999996e-06 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845006+0000", 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.484e-06 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.762 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845056+0000", 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9446000000000001e-05 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845059+0000", 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5530000000000001e-06 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.563857+0000", 2023-09-23T20:01:46.763 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.71879795999999996 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.563981+0000", 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012405599999999999 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.564269+0000", 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00028795600000000002 2023-09-23T20:01:46.764 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.571432+0000", 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0071621860000000001 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583461+0000", 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012029257 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.765 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583851+0000", 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00038999 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583907+0000", 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6047000000000001e-05 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:01:46.766 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583946+0000", 2023-09-23T20:01:46.767 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9582999999999997e-05 2023-09-23T20:01:46.767 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.767 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.767 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.767 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:46.767 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:46.767 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:01:46.767 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:01:46.768 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: pgmap v639: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:46.768 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:01:46.768 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3861591428' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: osdmap e243: 8 total, 7 up, 8 in 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: 1.4 scrub starts 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: failed to encode map e243 with expected crc 2023-09-23T20:01:46.769 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:46 smithi144 ceph-mon[194833]: 1.4 scrub ok 2023-09-23T20:01:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:46.770 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:46.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:46.871 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:46.901 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:46.911 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:46.945 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:47.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:47.147 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:47.177 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:47.219 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:47.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:47.233 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:01:47.322 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:47.348 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:47.393 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:47.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:47.498 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:47.522 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:47 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:01:47.522 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:47 smithi144 ceph-mon[194833]: 1.6 scrub ok 2023-09-23T20:01:47.525 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:47.543 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:47.570 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:47.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:47.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:47.590 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:47.676 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:47.702 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:47.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:47 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:01:47.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:47 smithi135 ceph-mon[221113]: 1.6 scrub ok 2023-09-23T20:01:47.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:47 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:01:47.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:47 smithi135 ceph-mon[213871]: 1.6 scrub ok 2023-09-23T20:01:47.796 DEBUG:teuthology.orchestra.run.smithi135:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T20:01:47.797 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:47.799 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:01:47.799 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:01:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:47.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:47.846 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:hints = 1 2023-09-23T20:01:47.846 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:01:47.846 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object prefix: benchmark_data_smithi135_404353 2023-09-23T20:01:47.846 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:01:47.846 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:01:47.902 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:47.941 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:48.071 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:48.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:48.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:48.175 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:48.208 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:48.374 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:48.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:48 smithi135 ceph-mon[213871]: pgmap v641: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:48 smithi135 ceph-mon[221113]: pgmap v641: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:48.479 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:48.547 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:48.746 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:48.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:48.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:48.763 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:48 smithi144 ceph-mon[194833]: pgmap v641: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 4.5 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:01:48.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:01:48.852 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:48.853 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 1 16 2614 2598 161.213 162.375 0.00278428 0.0061899 2023-09-23T20:01:48.896 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:49.001 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:49.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:49.105 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:49.156 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:49.367 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:01:49.367 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:01:49.367 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.367 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:4118 7.6 7:674fcc1e:::benchmark_data_smithi135_404353_object4117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T20:01:49.367 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:49.363196+0000", 2023-09-23T20:01:49.367 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.00088815499999999996, 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00092249300000000003, 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 4118 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.368 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363196+0000", 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363196+0000", 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.369 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:49.370 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363195+0000", 2023-09-23T20:01:49.370 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:49.370 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.371 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.371 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:49.371 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363287+0000", 2023-09-23T20:01:49.371 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2452000000000002e-05 2023-09-23T20:01:49.371 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.371 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.371 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:49.372 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363302+0000", 2023-09-23T20:01:49.372 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4287000000000001e-05 2023-09-23T20:01:49.372 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.372 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.372 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:49.372 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363308+0000", 2023-09-23T20:01:49.372 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8320000000000002e-06 2023-09-23T20:01:49.372 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.373 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.373 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:49.373 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363360+0000", 2023-09-23T20:01:49.373 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2450000000000001e-05 2023-09-23T20:01:49.373 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.373 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.373 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:49.374 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363492+0000", 2023-09-23T20:01:49.374 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013198800000000001 2023-09-23T20:01:49.374 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.374 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.374 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:01:49.374 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363629+0000", 2023-09-23T20:01:49.375 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013701699999999999 2023-09-23T20:01:49.375 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.375 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:49.375 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.375 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.375 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.375 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:4120 7.6 7:6d7f4fee:::benchmark_data_smithi135_404353_object4119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T20:01:49.376 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:49.363318+0000", 2023-09-23T20:01:49.376 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.00076581200000000005, 2023-09-23T20:01:49.376 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00089943900000000003, 2023-09-23T20:01:49.376 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:49.376 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:01:49.376 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:49.376 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:01:49.376 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:01:49.377 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 4120 2023-09-23T20:01:49.377 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.377 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:49.377 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.377 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:49.377 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363318+0000", 2023-09-23T20:01:49.377 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.378 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.378 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.378 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:49.378 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363318+0000", 2023-09-23T20:01:49.378 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.378 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.378 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.378 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363318+0000", 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363375+0000", 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6727000000000001e-05 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363383+0000", 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2740000000000007e-06 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363388+0000", 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5489999999999997e-06 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.380 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364001+0000", 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00061284100000000004 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364133+0000", 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000131645 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.381 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:4125 7.6 7.76a041c6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:49.363398+0000", 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.00068646699999999996, 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00089295599999999998, 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "queued for pg", 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:01:49.382 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 4125 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363398+0000", 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363398+0000", 2023-09-23T20:01:49.383 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363397+0000", 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363453+0000", 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5640000000000003e-05 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.384 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363464+0000", 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1772e-05 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363471+0000", 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1340000000000001e-06 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.385 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:4126 7.6 7.12b8f176 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:49.363478+0000", 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.00060654100000000005, 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00088717400000000001, 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "queued for pg", 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 4126 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.386 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363478+0000", 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363478+0000", 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.387 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363477+0000", 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363507+0000", 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0328000000000002e-05 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363515+0000", 2023-09-23T20:01:49.388 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3679999999999996e-06 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363519+0000", 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6269999999999999e-06 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35358.0:4119 7.8 e243/240 7:1615aeb1:::benchmark_data_smithi135_404353_object4118:head v 243'281, mlcod=243'279)", 2023-09-23T20:01:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:49.363491+0000", 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.00059259700000000005, 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000961396, 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363491+0000", 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.390 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363491+0000", 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363491+0000", 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363597+0000", 2023-09-23T20:01:49.391 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000106221 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363607+0000", 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4099999999999997e-06 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363612+0000", 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3589999999999999e-06 2023-09-23T20:01:49.392 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363687+0000", 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5294000000000001e-05 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.363705+0000", 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7725999999999999e-05 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:49.393 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35358.0:4121 7.8 e243/240 7:1470d356:::benchmark_data_smithi135_404353_object4120:head v 243'282, mlcod=243'279)", 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:49.364320+0000", 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 4294967295.999764, 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00021221099999999999, 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:01:49.394 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364320+0000", 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364320+0000", 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364320+0000", 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:01:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364368+0000", 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8415000000000003e-05 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364379+0000", 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0603e-05 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.396 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364384+0000", 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7400000000000001e-06 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364426+0000", 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1876999999999999e-05 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:49.364446+0000", 2023-09-23T20:01:49.397 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9320000000000001e-05 2023-09-23T20:01:49.398 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.398 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:01:49.398 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.398 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:01:49.398 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2023-09-23T20:01:49.398 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 6 2023-09-23T20:01:49.398 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:01:49.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:49 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1/1638 objects degraded (0.061%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:49.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:49 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1/1638 objects degraded (0.061%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:49.853 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 2 16 5756 5740 178.723 196.375 0.0044805 0.00558431 2023-09-23T20:01:49.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:49 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1/1638 objects degraded (0.061%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:01:50.247 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:50.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:50.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:50.351 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:50.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:50.526 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:50 smithi135 ceph-mon[213871]: pgmap v642: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 37 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.0 MiB/s wr, 59 op/s; 1/1638 objects degraded (0.061%) 2023-09-23T20:01:50.526 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:50 smithi135 ceph-mon[221113]: pgmap v642: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 37 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.0 MiB/s wr, 59 op/s; 1/1638 objects degraded (0.061%) 2023-09-23T20:01:50.616 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:01:50.661 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:50.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:50.767 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:50.836 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:50.854 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 3 16 7484 7468 155.168 108 0.0146895 0.00643249 2023-09-23T20:01:50.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:50 smithi144 ceph-mon[194833]: pgmap v642: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 37 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.0 MiB/s wr, 59 op/s; 1/1638 objects degraded (0.061%) 2023-09-23T20:01:50.943 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:50.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:51.053 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:51.076 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:51.097 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:01:51.166 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:51.244 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:51.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:51.276 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:01:51.318 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:01:51.349 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:01:51.358 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:01:51.389 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:51.397 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:01:51.438 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:01:51.448 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:01:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:51.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:01:51.469 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:01:51.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:51 smithi135 conmon[213845]: 2023-09-23T20:01:51.202+0000 7f9708921700 -1 mon.a@0(leader).osd e243 definitely_dead 0 2023-09-23T20:01:51.511 INFO:teuthology.orchestra.run.smithi135.stderr:marked out osd.6. 2023-09-23T20:01:51.513 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:01:51.516 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:01:51.516 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:01:51.516 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:01:51.516 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:01:51.516 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:01:51.516 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:01:51.516 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:01:51.516 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:01:51.552 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:51.622 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/492974188' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:01:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:01:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:51 smithi144 ceph-mon[194833]: Client client.admin marked osd.6 out, after it was down for 426 seconds 2023-09-23T20:01:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/759673176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:51.756 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/492974188' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:01:51.756 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:01:51.756 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:51 smithi135 ceph-mon[221113]: Client client.admin marked osd.6 out, after it was down for 426 seconds 2023-09-23T20:01:51.756 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/759673176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:51.761 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/492974188' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:01:51.761 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:01:51.762 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:51 smithi135 ceph-mon[213871]: Client client.admin marked osd.6 out, after it was down for 426 seconds 2023-09-23T20:01:51.762 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/759673176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:01:51.854 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 4 16 9733 9717 151.52 140.562 0.00295496 0.00659161 2023-09-23T20:01:51.920 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:51.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:51.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:52.025 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:52.052 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:52.854 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: pgmap v643: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 140 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s wr, 218 op/s; 1/6591 objects degraded (0.015%) 2023-09-23T20:01:52.854 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:01:52.855 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: failed to encode map e244 with expected crc 2023-09-23T20:01:52.855 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: failed to encode map e244 with expected crc 2023-09-23T20:01:52.855 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:01:52.855 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: osdmap e244: 8 total, 7 up, 7 in 2023-09-23T20:01:52.855 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:52.855 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:52.855 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:52 smithi144 ceph-mon[194833]: osdmap e245: 8 total, 7 up, 7 in 2023-09-23T20:01:52.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 5 16 11717 11701 145.967 124 0.00504603 0.00684091 2023-09-23T20:01:52.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: pgmap v643: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 140 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s wr, 218 op/s; 1/6591 objects degraded (0.015%) 2023-09-23T20:01:52.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:01:52.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: failed to encode map e244 with expected crc 2023-09-23T20:01:52.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: failed to encode map e244 with expected crc 2023-09-23T20:01:52.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:01:52.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: osdmap e244: 8 total, 7 up, 7 in 2023-09-23T20:01:52.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:52.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:52.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[221113]: osdmap e245: 8 total, 7 up, 7 in 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: pgmap v643: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 140 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s wr, 218 op/s; 1/6591 objects degraded (0.015%) 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: failed to encode map e244 with expected crc 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: failed to encode map e244 with expected crc 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: osdmap e244: 8 total, 7 up, 7 in 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:01:52.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:52 smithi135 ceph-mon[213871]: osdmap e245: 8 total, 7 up, 7 in 2023-09-23T20:01:53.459 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:01:53.459 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:01:53.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 6 16 14350 14334 149.059 164.562 0.00560209 0.00670154 2023-09-23T20:01:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:53 smithi144 ceph-mon[194833]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:53 smithi144 ceph-mon[194833]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:53 smithi144 ceph-mon[194833]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:53 smithi144 ceph-mon[194833]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:53 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:01:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:53 smithi144 ceph-mon[194833]: 1.2 scrub ok 2023-09-23T20:01:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:53 smithi144 ceph-mon[194833]: osdmap e246: 8 total, 7 up, 7 in 2023-09-23T20:01:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:01:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:01:53] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2023-09-23T20:01:53.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[213871]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[213871]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[213871]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[213871]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[213871]: 1.2 scrub ok 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[213871]: osdmap e246: 8 total, 7 up, 7 in 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[221113]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[221113]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[221113]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[221113]: failed to encode map e245 with expected crc 2023-09-23T20:01:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:01:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[221113]: 1.2 scrub ok 2023-09-23T20:01:53.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:53 smithi135 ceph-mon[221113]: osdmap e246: 8 total, 7 up, 7 in 2023-09-23T20:01:54.677 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:01:54.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 7 16 16631 16615 148.13 142.562 0.0080362 0.00674267 2023-09-23T20:01:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:54 smithi144 ceph-mon[194833]: pgmap v646: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 288 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 1/13683 objects degraded (0.007%) 2023-09-23T20:01:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:54 smithi144 ceph-mon[194833]: failed to encode map e246 with expected crc 2023-09-23T20:01:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:54 smithi144 ceph-mon[194833]: failed to encode map e246 with expected crc 2023-09-23T20:01:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:54 smithi144 ceph-mon[194833]: failed to encode map e246 with expected crc 2023-09-23T20:01:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:54 smithi135 ceph-mon[221113]: pgmap v646: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 288 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 1/13683 objects degraded (0.007%) 2023-09-23T20:01:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:54 smithi135 ceph-mon[221113]: failed to encode map e246 with expected crc 2023-09-23T20:01:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:54 smithi135 ceph-mon[221113]: failed to encode map e246 with expected crc 2023-09-23T20:01:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:54 smithi135 ceph-mon[221113]: failed to encode map e246 with expected crc 2023-09-23T20:01:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:54 smithi135 ceph-mon[213871]: pgmap v646: 99 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 94 active+clean, 3 active+undersized; 288 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 1/13683 objects degraded (0.007%) 2023-09-23T20:01:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:54 smithi135 ceph-mon[213871]: failed to encode map e246 with expected crc 2023-09-23T20:01:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:54 smithi135 ceph-mon[213871]: failed to encode map e246 with expected crc 2023-09-23T20:01:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:54 smithi135 ceph-mon[213871]: failed to encode map e246 with expected crc 2023-09-23T20:01:55.127 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:55.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:55.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:55.172 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:01:55.237 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:01:55.292 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:55.834 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:55 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/13683 objects degraded (0.007%), 1 pg degraded, 4 pgs undersized) 2023-09-23T20:01:55.856 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 8 16 18296 18280 142.627 104.062 0.00149949 0.00699677 2023-09-23T20:01:55.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:55 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/13683 objects degraded (0.007%), 1 pg degraded, 4 pgs undersized) 2023-09-23T20:01:55.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:55 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/13683 objects degraded (0.007%), 1 pg degraded, 4 pgs undersized) 2023-09-23T20:01:56.444 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:01:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:56.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:01:56.548 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:56.575 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:56.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:56 smithi135 ceph-mon[221113]: pgmap v648: 99 pgs: 1 peering, 98 active+clean; 650 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T20:01:56.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:56 smithi135 ceph-mon[213871]: pgmap v648: 99 pgs: 1 peering, 98 active+clean; 650 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T20:01:56.817 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:01:56.818 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:01:56.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 9 16 19438 19422 134.688 71.375 0.00421791 0.0074169 2023-09-23T20:01:56.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:56 smithi144 ceph-mon[194833]: pgmap v648: 99 pgs: 1 peering, 98 active+clean; 650 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T20:01:56.972 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:56.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:56.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:57.083 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:57.139 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:57.298 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:01:57.298 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:01:57.358 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:01:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:57.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:01:57.465 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:57.504 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:57.569 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:57.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:57.674 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:01:57.716 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:57.766 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:01:57.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:57.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:01:57.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 10 16 21149 21133 131.915 106.938 0.00158108 0.00752783 2023-09-23T20:01:57.871 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:57 smithi144 ceph-mon[194833]: 1.7 scrub starts 2023-09-23T20:01:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:57 smithi144 ceph-mon[194833]: 1.7 scrub ok 2023-09-23T20:01:57.897 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:57.938 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:57.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:57.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:57.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:57 smithi135 ceph-mon[213871]: 1.7 scrub starts 2023-09-23T20:01:57.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:57 smithi135 ceph-mon[213871]: 1.7 scrub ok 2023-09-23T20:01:57.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:57 smithi135 ceph-mon[221113]: 1.7 scrub starts 2023-09-23T20:01:57.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:57 smithi135 ceph-mon[221113]: 1.7 scrub ok 2023-09-23T20:01:58.042 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:01:58.076 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:58.362 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:01:58.366 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:01:58.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:58.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:01:58.471 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:58.579 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:58.607 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:01:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:58 smithi135 ceph-mon[213871]: pgmap v649: 99 pgs: 1 peering, 98 active+clean; 822 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 114 MiB/s wr, 1.81k op/s 2023-09-23T20:01:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/135834293' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:58 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:58 smithi135 ceph-mon[221113]: pgmap v649: 99 pgs: 1 peering, 98 active+clean; 822 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 114 MiB/s wr, 1.81k op/s 2023-09-23T20:01:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/135834293' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:58 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:58.819 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:01:58.822 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:58.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:58.858 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 11 16 22878 22862 129.748 108.062 0.00618894 0.0077005 2023-09-23T20:01:58.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:58 smithi144 ceph-mon[194833]: pgmap v649: 99 pgs: 1 peering, 98 active+clean; 822 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 114 MiB/s wr, 1.81k op/s 2023-09-23T20:01:58.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/135834293' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:58.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:58 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:01:58.929 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:01:59.009 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:59.100 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:01:59.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:01:59.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:01:59.203 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:01:59.268 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:01:59.860 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 12 16 24770 24754 128.775 118.25 0.00770022 0.0077606 2023-09-23T20:01:59.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:59 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:59.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:59 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:01:59.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:59 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:59.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:59 smithi144 ceph-mon[194833]: osdmap e247: 8 total, 7 up, 7 in 2023-09-23T20:01:59.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:59 smithi144 ceph-mon[194833]: failed to encode map e247 with expected crc 2023-09-23T20:01:59.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:01:59 smithi144 ceph-mon[194833]: pgmap v651: 99 pgs: 99 active+clean; 1.2 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 156 MiB/s wr, 2.49k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:01:59.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:59.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:01:59.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:59.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[221113]: osdmap e247: 8 total, 7 up, 7 in 2023-09-23T20:01:59.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[221113]: failed to encode map e247 with expected crc 2023-09-23T20:01:59.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[221113]: pgmap v651: 99 pgs: 99 active+clean; 1.2 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 156 MiB/s wr, 2.49k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:01:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:01:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:01:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:01:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[213871]: osdmap e247: 8 total, 7 up, 7 in 2023-09-23T20:01:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[213871]: failed to encode map e247 with expected crc 2023-09-23T20:01:59.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:01:59 smithi135 ceph-mon[213871]: pgmap v651: 99 pgs: 99 active+clean; 1.2 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 156 MiB/s wr, 2.49k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:00.863 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 13 16 25937 25921 124.456 72.9375 0.0078806 0.00802831 2023-09-23T20:02:00.895 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:00.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:01.002 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:01.095 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:01.150 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:01.200 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:01.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:01.219 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:01.308 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:01.375 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:01.538 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:01.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:01.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:01.648 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:01.684 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:01.746 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:01.750 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:01 smithi135 ceph-mon[221113]: pgmap v652: 99 pgs: 99 active+clean; 1.3 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:01.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:01.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:01.753 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:01 smithi135 ceph-mon[213871]: pgmap v652: 99 pgs: 99 active+clean; 1.3 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:01.853 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:01.865 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 14 16 27627 27611 123.111 105.625 0.00773086 0.00811635 2023-09-23T20:02:01.913 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:01.925 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 3] out_osds: [6] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:02:01.926 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:02:01.926 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T20:02:01.926 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:02:01.926 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:02:01.926 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T20:02:02.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:01 smithi144 ceph-mon[194833]: pgmap v652: 99 pgs: 99 active+clean; 1.3 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:02.473 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:02.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:02.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:02.581 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:02.637 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:02.863 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 15 16 29264 29248 121.726 102.312 0.00581745 0.00820869 2023-09-23T20:02:03.037 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:02:03.863 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 16 16 30720 30704 119.807 91 0.00647097 0.00834048 2023-09-23T20:02:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:02:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:02:03] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2023-09-23T20:02:04.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:04 smithi135 ceph-mon[213871]: pgmap v653: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:04.478 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:04 smithi135 ceph-mon[221113]: pgmap v653: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:04.629 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:04.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:04.737 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:04.845 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:04.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 17 16 31829 31813 116.834 69.3125 0.0155265 0.00855338 2023-09-23T20:02:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:04 smithi144 ceph-mon[194833]: pgmap v653: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:05.091 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:05.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:05.205 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:05.240 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:05.247 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:05.300 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:05.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:05.400 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:02:05.400 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:02:05.405 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:05.488 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:05.548 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:05.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 18 16 33248 33232 115.271 88.6875 0.0157849 0.00866991 2023-09-23T20:02:06.144 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:02:06.145 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:02:06.145 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:02:06.145 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:02:06.145 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:02:06.146 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:02:06.146 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:02:06.146 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:02:06.146 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:02:06.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:02:06.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:02:06.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:02:06.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:02:06.148 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:02:06.148 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:02:06.214 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:06.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:06.320 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:06.354 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:06.410 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:06.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:06 smithi135 ceph-mon[221113]: pgmap v654: 99 pgs: 99 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:06.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/406341666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:06.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:06 smithi135 ceph-mon[213871]: pgmap v654: 99 pgs: 99 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:06.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/406341666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:06.516 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:06.557 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:06 smithi144 ceph-mon[194833]: pgmap v654: 99 pgs: 99 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/406341666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:06.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 19 16 34602 34586 113.659 84.625 0.00152183 0.0087757 2023-09-23T20:02:07.531 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:02:07.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:02:07.865339+0000 min lat: 0.0011489 max lat: 0.453318 avg lat: 0.00892766 2023-09-23T20:02:07.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:02:07.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 20 16 35871 35855 111.943 79.3125 0.024506 0.00892766 2023-09-23T20:02:08.014 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:08.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:08.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:08.118 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:08.147 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:08.296 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:08 smithi135 ceph-mon[221113]: pgmap v655: 99 pgs: 99 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:08.296 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:08.296 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:08.296 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:08.296 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:08 smithi135 ceph-mon[213871]: pgmap v655: 99 pgs: 99 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:08.297 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:08.297 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:08.297 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:08 smithi144 ceph-mon[194833]: pgmap v655: 99 pgs: 99 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:02:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:08.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 21 16 37067 37051 110.173 74.75 0.00203633 0.00906082 2023-09-23T20:02:09.164 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:09.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:09.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:09.267 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:09.339 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:02:09.373 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:02:09.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 22 16 37956 37940 107.692 55.5625 0.00202465 0.00921479 2023-09-23T20:02:10.198 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:10.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:10.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:10.308 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:10.390 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:10 smithi135 ceph-mon[221113]: pgmap v656: 99 pgs: 99 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T20:02:10.394 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:10 smithi135 ceph-mon[213871]: pgmap v656: 99 pgs: 99 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T20:02:10.409 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:10.614 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:10.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:10.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:10 smithi144 ceph-mon[194833]: pgmap v656: 99 pgs: 99 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T20:02:10.720 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:10.749 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:10.771 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:10.808 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:10.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:10.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:10.864 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 23 16 39203 39187 106.399 77.9375 0.00503087 0.00939195 2023-09-23T20:02:10.883 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:02:10.883 INFO:teuthology.orchestra.run.smithi135.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:02:10.883 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:02:10.913 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:10.983 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:11.281 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:11.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:11.375 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:02:11.387 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:11.508 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:11.865 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 24 16 40476 40460 105.281 79.5625 0.00732065 0.00949272 2023-09-23T20:02:12.211 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:02:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:12 smithi135 ceph-mon[213871]: pgmap v657: 99 pgs: 99 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:02:12.503 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1177530643' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:12.504 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:12 smithi135 ceph-mon[221113]: pgmap v657: 99 pgs: 99 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:02:12.504 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1177530643' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:12.559 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:12.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:12 smithi144 ceph-mon[194833]: pgmap v657: 99 pgs: 99 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:02:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1177530643' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:12.663 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:12.697 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:12.750 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:12.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:12.856 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:12.866 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 25 16 41707 41691 104.141 76.9375 0.00655033 0.00959718 2023-09-23T20:02:12.927 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:13.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:13.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[221113]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1177530643' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[221113]: osdmap e248: 8 total, 7 up, 7 in 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[221113]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[221113]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[221113]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[213871]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1177530643' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[213871]: osdmap e248: 8 total, 7 up, 7 in 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[213871]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[213871]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:13 smithi135 ceph-mon[213871]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:13 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:13 smithi144 ceph-mon[194833]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1177530643' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:13 smithi144 ceph-mon[194833]: osdmap e248: 8 total, 7 up, 7 in 2023-09-23T20:02:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:13 smithi144 ceph-mon[194833]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:13 smithi144 ceph-mon[194833]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:13 smithi144 ceph-mon[194833]: failed to encode map e248 with expected crc 2023-09-23T20:02:13.694 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:13.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:13.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:13.804 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:13.830 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:02:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:02:13] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.43.0" 2023-09-23T20:02:13.866 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 26 16 42926 42910 103.066 76.1875 0.0120724 0.00969848 2023-09-23T20:02:13.875 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:13.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:13.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:14.073 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:14.155 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:14.215 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:14.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:14.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:14.319 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:14.357 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:14.406 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:02:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:14.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:14.515 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:14.540 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:14.581 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:14.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:14 smithi144 ceph-mon[194833]: pgmap v659: 99 pgs: 99 active+clean; 2.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T20:02:14.685 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:14.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:14 smithi135 ceph-mon[213871]: pgmap v659: 99 pgs: 99 active+clean; 2.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T20:02:14.722 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:14.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:14 smithi135 ceph-mon[221113]: pgmap v659: 99 pgs: 99 active+clean; 2.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T20:02:14.779 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:14.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:14.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:14.868 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 27 16 43910 43894 101.522 61.5 0.00415982 0.0098445 2023-09-23T20:02:14.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:14.948 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:15.032 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:15.071 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:15.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:15.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:15.185 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:15.216 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:15.276 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:15.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:15.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:15.380 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:15.449 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:15.715 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:15.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:15.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:15.825 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:15.846 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:02:15.873 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 28 16 45191 45175 100.754 80.0625 0.00699334 0.00991969 2023-09-23T20:02:15.890 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:16.147 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:02:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:02:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:02:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:02:16.147 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:02:16.257 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:16.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:16.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:16.362 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:16.414 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:16.478 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:16.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:16.584 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:16.626 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:16 smithi135 ceph-mon[221113]: pgmap v660: 99 pgs: 99 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:02:16.626 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:16 smithi135 ceph-mon[213871]: pgmap v660: 99 pgs: 99 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:02:16.631 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:16.876 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 29 16 46234 46218 99.5248 65.1875 0.0121727 0.0100429 2023-09-23T20:02:16.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:16 smithi144 ceph-mon[194833]: pgmap v660: 99 pgs: 99 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:02:17.664 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:17.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:17.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:17.799 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:17.870 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 30 16 47257 47241 98.3392 63.9375 0.00761396 0.0101633 2023-09-23T20:02:18.105 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:18.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:18.118 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:02:18.214 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:18.309 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:18.320 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:18.515 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:18.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:18.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:18.555 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 3] out_osds: [6] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:02:18.555 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-09-23T20:02:18.555 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-09-23T20:02:18.555 DEBUG:teuthology.orchestra.run.smithi144:osd.6> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T20:02:18.557 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl start ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T20:02:18.589 INFO:journalctl@ceph.osd.6.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T20:02:18.623 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:18.672 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:18.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:18 smithi135 ceph-mon[213871]: pgmap v661: 99 pgs: 99 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T20:02:18.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:18 smithi135 ceph-mon[221113]: pgmap v661: 99 pgs: 99 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T20:02:18.744 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:18.749 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:18 smithi144 systemd[1]: Starting Ceph osd.6 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:02:18.749 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:18 smithi144 ceph-mon[194833]: pgmap v661: 99 pgs: 99 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T20:02:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:18.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:18.851 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:18.855 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:02:18.872 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 31 16 48364 48348 97.3934 69.1875 0.0104472 0.0102608 2023-09-23T20:02:18.900 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:02:18.937 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:18.959 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:02:18.995 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:02:19.034 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:02:19.066 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:02:19.095 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:19.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:19.115 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:02:19.116 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:02:19.116 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:02:19.116 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:02:19.116 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:02:19.116 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:02:19.116 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:02:19.116 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:02:19.116 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:02:19.207 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:19.300 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:19.721 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2381546625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:19.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2381546625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:19.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2381546625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:19.816 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:19.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:19.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:19.873 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 32 16 49528 49512 96.6216 72.75 0.0103228 0.0103442 2023-09-23T20:02:19.930 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:19.984 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:20.095 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:20.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:20.132 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:19 smithi144 bash[261483]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:02:20.132 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:19 smithi144 bash[261483]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T20:02:20.132 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:19 smithi144 bash[261483]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2023-09-23T20:02:20.132 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:19 smithi144 bash[261483]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2023-09-23T20:02:20.133 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:19 smithi144 bash[261483]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2023-09-23T20:02:20.133 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:19 smithi144 bash[261483]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T20:02:20.133 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:19 smithi144 bash[261483]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T20:02:20.133 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:19 smithi144 bash[261483]: --> ceph-volume lvm activate successful for osd ID: 6 2023-09-23T20:02:20.199 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:20.266 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:20.332 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:20.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:20.435 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:20.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:20 smithi135 ceph-mon[213871]: pgmap v662: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:20.452 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:20 smithi135 ceph-mon[221113]: pgmap v662: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:20.478 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:20.536 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:02:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:20.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:20.639 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:20.666 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:20.711 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:20.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:20.715 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:20 smithi144 ceph-mon[194833]: pgmap v662: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:20.816 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:20.873 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 33 16 50397 50381 95.3382 54.3125 0.00688056 0.010483 2023-09-23T20:02:20.887 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:21.236 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:21.263 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:21.266 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:02:21.267 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:02:21.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:21.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:21.369 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:21.435 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:21.615 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:21.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:21.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:21.723 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:21.796 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:21.875 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 34 16 51600 51584 94.7417 75.1875 0.0134913 0.0105488 2023-09-23T20:02:21.950 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:21.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:21.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:22.060 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:22.142 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:22.651 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:22.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:22.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:22 smithi135 ceph-mon[213871]: pgmap v663: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:22.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:22 smithi135 ceph-mon[221113]: pgmap v663: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:22.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:22.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:22.761 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:22.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:22 smithi144 ceph-mon[194833]: pgmap v663: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:22.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:22.791 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:22.793 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:22.865 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:22.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:22.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:22.875 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 35 16 52817 52801 94.2082 76.0625 0.016665 0.0106094 2023-09-23T20:02:22.968 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:23.001 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:23.054 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:23.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:23.159 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:23.185 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:23.243 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:23.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:23.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:23.268 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:02:23.347 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:23.380 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:23.491 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:02:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:23 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3374649243' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:23 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:23.641 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:02:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:23.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:23.689 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:02:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:02:23] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2023-09-23T20:02:23.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:23 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3374649243' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:23.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:23 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:23.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:23 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3374649243' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:23.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:23 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:23.747 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:23.818 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:23.879 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 36 16 53868 53852 93.4049 65.6875 0.0268617 0.0106996 2023-09-23T20:02:24.301 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:02:24.557 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:24.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:24.631 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:24 smithi144 bash[262232]: ae2b44eb17fb4872f26a1d0d7711c6eca9ddbb6d9031ec1eccf52c89b82e6fde 2023-09-23T20:02:24.631 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:24 smithi144 systemd[1]: Started Ceph osd.6 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: pgmap v664: 99 pgs: 99 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: osdmap e249: 8 total, 7 up, 7 in 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:24 smithi144 ceph-mon[194833]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.664 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:24.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: pgmap v664: 99 pgs: 99 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T20:02:24.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:24.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:24.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: osdmap e249: 8 total, 7 up, 7 in 2023-09-23T20:02:24.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[221113]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: pgmap v664: 99 pgs: 99 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T20:02:24.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:24.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:24.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: osdmap e249: 8 total, 7 up, 7 in 2023-09-23T20:02:24.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:24 smithi135 ceph-mon[213871]: failed to encode map e249 with expected crc 2023-09-23T20:02:24.745 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:24.766 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:02:24.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 37 16 54859 54843 92.5295 61.9375 0.0121729 0.0108011 2023-09-23T20:02:24.931 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:24.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:25.044 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:25.075 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:24 smithi144 conmon[262273]: 2023-09-23T20:02:24.819+0000 7f3c59ed4540 -1 Falling back to public interface 2023-09-23T20:02:25.091 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:25.125 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:25.247 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:25.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:25.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:25.277 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:02:25.277 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:02:25.351 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:25.421 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:25.674 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:25.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:25.782 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:25.855 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:25.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 38 16 55989 55973 91.9535 70.625 0.033062 0.0108644 2023-09-23T20:02:25.908 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:25.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:25.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:26.012 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:26.046 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:26.281 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:26.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:26.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:26.387 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:26.427 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:26.597 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:26.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:26.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:26.704 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:26.718 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:26 smithi135 ceph-mon[221113]: pgmap v666: 99 pgs: 99 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:02:26.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:26 smithi135 ceph-mon[213871]: pgmap v666: 99 pgs: 99 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:02:26.778 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:26.782 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:02:26.805 INFO:teuthology.orchestra.run.smithi144.stderr:no valid command found; 10 closest matches: 2023-09-23T20:02:26.805 INFO:teuthology.orchestra.run.smithi144.stderr:0 2023-09-23T20:02:26.805 INFO:teuthology.orchestra.run.smithi144.stderr:1 2023-09-23T20:02:26.805 INFO:teuthology.orchestra.run.smithi144.stderr:2 2023-09-23T20:02:26.805 INFO:teuthology.orchestra.run.smithi144.stderr:abort 2023-09-23T20:02:26.806 INFO:teuthology.orchestra.run.smithi144.stderr:assert 2023-09-23T20:02:26.806 INFO:teuthology.orchestra.run.smithi144.stderr:config diff 2023-09-23T20:02:26.806 INFO:teuthology.orchestra.run.smithi144.stderr:config diff get 2023-09-23T20:02:26.806 INFO:teuthology.orchestra.run.smithi144.stderr:config get 2023-09-23T20:02:26.806 INFO:teuthology.orchestra.run.smithi144.stderr:config help [] 2023-09-23T20:02:26.806 INFO:teuthology.orchestra.run.smithi144.stderr:config set ... 2023-09-23T20:02:26.806 INFO:teuthology.orchestra.run.smithi144.stderr:admin_socket: invalid command 2023-09-23T20:02:26.821 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:26 smithi144 ceph-mon[194833]: pgmap v666: 99 pgs: 99 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:02:26.851 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:02:26.890 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 39 16 57123 57107 91.4134 70.875 0.015208 0.0109322 2023-09-23T20:02:27.048 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:27.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:27.155 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:27.203 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:27.646 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:27.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:27.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:02:27.659 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-09-23T20:02:27.754 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:27.795 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:27.860 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:27.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:27.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:27.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:02:27.894151+0000 min lat: 0.0011489 max lat: 0.453318 avg lat: 0.0110534 2023-09-23T20:02:27.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:02:27.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 40 16 57946 57930 90.4085 51.4375 0.00777395 0.0110534 2023-09-23T20:02:27.895 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:27.909 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:27.967 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:28.009 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:28.074 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:28.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:28.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:28.181 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:28.263 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:28.578 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:02:28.596 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:28.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:28.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:28.619 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:02:28.672 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:02:28.708 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:28.731 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:02:28.765 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:02:28.791 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:28 smithi135 ceph-mon[213871]: pgmap v667: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:02:28.799 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:28 smithi135 ceph-mon[221113]: pgmap v667: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:02:28.799 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:28.807 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:02:28.866 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:02:28.867 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:02:28.867 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:02:28.867 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:02:28.867 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:02:28.867 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:02:28.867 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:02:28.867 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:02:28.867 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:02:28.881 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:28 smithi144 conmon[262273]: 2023-09-23T20:02:28.688+0000 7f3c59ed4540 -1 osd.6 143 log_to_monitors true 2023-09-23T20:02:28.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:28 smithi144 ceph-mon[194833]: pgmap v667: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:02:28.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 41 16 58961 58945 89.7512 63.4375 0.0100976 0.0111357 2023-09-23T20:02:29.881 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:02:29 smithi144 conmon[262273]: 2023-09-23T20:02:29.636+0000 7f3c4bcb6700 -1 osd.6 250 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:02:29.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3111787396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:29.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:29 smithi144 ceph-mon[194833]: from='osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:02:29.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:29 smithi144 ceph-mon[194833]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:02:29.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 42 16 60063 60047 89.2545 68.875 0.0163236 0.011199 2023-09-23T20:02:29.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3111787396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:29.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:29 smithi135 ceph-mon[221113]: from='osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:02:29.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:29 smithi135 ceph-mon[221113]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:02:29.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3111787396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:29.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:29 smithi135 ceph-mon[213871]: from='osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:02:29.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:29 smithi135 ceph-mon[213871]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:02:30.124 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:30.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:30.232 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:30.258 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:30.301 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:30.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:30.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:30.406 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:30.471 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:30.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[213871]: pgmap v668: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:02:30.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[213871]: failed to encode map e250 with expected crc 2023-09-23T20:02:30.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[213871]: failed to encode map e250 with expected crc 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[213871]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[213871]: osdmap e250: 8 total, 7 up, 7 in 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[213871]: from='osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[213871]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[221113]: pgmap v668: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[221113]: failed to encode map e250 with expected crc 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[221113]: failed to encode map e250 with expected crc 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[221113]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:02:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[221113]: osdmap e250: 8 total, 7 up, 7 in 2023-09-23T20:02:30.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[221113]: from='osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:02:30.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:30 smithi135 ceph-mon[221113]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:02:30.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:30 smithi144 ceph-mon[194833]: pgmap v668: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:02:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:30 smithi144 ceph-mon[194833]: failed to encode map e250 with expected crc 2023-09-23T20:02:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:30 smithi144 ceph-mon[194833]: failed to encode map e250 with expected crc 2023-09-23T20:02:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:30 smithi144 ceph-mon[194833]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:02:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:30 smithi144 ceph-mon[194833]: osdmap e250: 8 total, 7 up, 7 in 2023-09-23T20:02:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:30 smithi144 ceph-mon[194833]: from='osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:02:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:30 smithi144 ceph-mon[194833]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:02:30.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 43 16 60819 60803 88.2786 47.25 0.0590746 0.0113168 2023-09-23T20:02:31.329 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:02:31.330 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:02:31.361 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:31.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:31.472 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:31.502 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:31.573 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:31.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:31.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:31.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[213871]: osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283] boot 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[213871]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[213871]: failed to encode map e251 with expected crc 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[213871]: failed to encode map e251 with expected crc 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[221113]: osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283] boot 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[221113]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[221113]: failed to encode map e251 with expected crc 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:02:31.604 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:31 smithi135 ceph-mon[221113]: failed to encode map e251 with expected crc 2023-09-23T20:02:31.682 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:31.775 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:31.864 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:31.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:31 smithi144 ceph-mon[194833]: osd.6 [v2:172.21.15.144:6816/584583283,v1:172.21.15.144:6817/584583283] boot 2023-09-23T20:02:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:31 smithi144 ceph-mon[194833]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T20:02:31.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:31 smithi144 ceph-mon[194833]: failed to encode map e251 with expected crc 2023-09-23T20:02:31.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:31 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:02:31.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:31 smithi144 ceph-mon[194833]: failed to encode map e251 with expected crc 2023-09-23T20:02:31.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 44 16 61725 61709 87.5598 56.625 0.0121581 0.0114154 2023-09-23T20:02:31.974 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:32.034 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:32.249 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:32.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:32.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:32.353 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:32.450 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:32.544 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[213871]: failed to encode map e144 with expected crc 2023-09-23T20:02:32.544 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[213871]: failed to encode map e184 with expected crc 2023-09-23T20:02:32.544 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[213871]: failed to encode map e224 with expected crc 2023-09-23T20:02:32.544 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[213871]: failed to encode map e251 with expected crc 2023-09-23T20:02:32.545 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[213871]: failed to encode map e251 with expected crc 2023-09-23T20:02:32.545 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[213871]: pgmap v671: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s 2023-09-23T20:02:32.545 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[221113]: failed to encode map e144 with expected crc 2023-09-23T20:02:32.545 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[221113]: failed to encode map e184 with expected crc 2023-09-23T20:02:32.545 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[221113]: failed to encode map e224 with expected crc 2023-09-23T20:02:32.546 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[221113]: failed to encode map e251 with expected crc 2023-09-23T20:02:32.546 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[221113]: failed to encode map e251 with expected crc 2023-09-23T20:02:32.595 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:32.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:32.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:32.614 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:02:32.614 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:02:32.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.614 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:62379 7.9 7:9e522c3b:::benchmark_data_smithi135_404353_object62378:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2023-09-23T20:02:32.614 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:02:32.604559+0000", 2023-09-23T20:02:32.614 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0082447830000000003, 2023-09-23T20:02:32.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0082816020000000008, 2023-09-23T20:02:32.614 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 62379 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604559+0000", 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604559+0000", 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604558+0000", 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604584+0000", 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6451000000000001e-05 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604595+0000", 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0227999999999999e-05 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:02:32.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604602+0000", 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.233e-06 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.608210+0000", 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036078350000000002 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.608344+0000", 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000134392 2023-09-23T20:02:32.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.608484+0000", 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000140257 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610744+0000", 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022598890000000002 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:62380 7.b 7:dfaac586:::benchmark_data_smithi135_404353_object62379:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:02:32.604608+0000", 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.008196264, 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083447480000000008, 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:02:32.620 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 62380 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604608+0000", 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604608+0000", 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604608+0000", 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604639+0000", 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1296e-05 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604646+0000", 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0949999999999998e-06 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604651+0000", 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1780000000000002e-06 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.604938+0000", 2023-09-23T20:02:32.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000287053 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.605083+0000", 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000144826 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.605225+0000", 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000141657 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.608193+0000", 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029684680000000001 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.35358.0:62380 7.b e251/240)", 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:02:32.610628+0000", 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0021763030000000001, 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002400952, 2023-09-23T20:02:32.625 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610628+0000", 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610628+0000", 2023-09-23T20:02:32.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610627+0000", 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610633+0000", 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8270000000000003e-06 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610643+0000", 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8090000000000004e-06 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610650+0000", 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6919999999999996e-06 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:62386 7.7 7:ecb659d7:::benchmark_data_smithi135_404353_object62385:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:02:32.610677+0000", 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0021274660000000002, 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002405346, 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 62386 2023-09-23T20:02:32.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610677+0000", 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610677+0000", 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610675+0000", 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610698+0000", 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2399999999999999e-05 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:02:32.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610706+0000", 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4810000000000006e-06 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610712+0000", 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7409999999999998e-06 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610759+0000", 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6628000000000001e-05 2023-09-23T20:02:32.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.610901+0000", 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000142781 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.611058+0000", 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015650500000000001 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:02:32.633 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.35358.0:62380 7.b e251/240)", 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:02:32.611027+0000", 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0017776770000000001, 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002124959, 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:02:32.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.611027+0000", 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.611027+0000", 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.611026+0000", 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:02:32.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.611029+0000", 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8090000000000001e-06 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.611038+0000", 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6510000000000007e-06 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:32.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:02:32.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:32.611043+0000", 2023-09-23T20:02:32.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.879e-06 2023-09-23T20:02:32.637 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.637 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:02:32.637 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.638 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:32.638 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:02:32.638 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 5 2023-09-23T20:02:32.638 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:02:32.660 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:02:32.710 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:32.794 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:32 smithi135 ceph-mon[221113]: pgmap v671: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s 2023-09-23T20:02:32.795 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:32 smithi144 ceph-mon[194833]: failed to encode map e144 with expected crc 2023-09-23T20:02:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:32 smithi144 ceph-mon[194833]: failed to encode map e184 with expected crc 2023-09-23T20:02:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:32 smithi144 ceph-mon[194833]: failed to encode map e224 with expected crc 2023-09-23T20:02:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:32 smithi144 ceph-mon[194833]: failed to encode map e251 with expected crc 2023-09-23T20:02:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:32 smithi144 ceph-mon[194833]: failed to encode map e251 with expected crc 2023-09-23T20:02:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:32 smithi144 ceph-mon[194833]: pgmap v671: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s 2023-09-23T20:02:32.893 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 45 16 62694 62678 86.9603 60.5625 0.00806354 0.0114958 2023-09-23T20:02:32.960 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:32.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:33.064 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:33.110 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:33.120 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:02:33.175 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:33.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:33.282 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:33.331 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:02:33.336 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:33.417 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:33.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:33.520 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:33.550 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:33.560 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:02:33.584 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:33 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/644043269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:02:33.584 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:33 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:02:33.584 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:33 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/644043269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:02:33.584 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:33 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:02:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:33 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/644043269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:02:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:33 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:02:33.753 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:33.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:33.858 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:33.892 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:33.895 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 46 16 63588 63572 86.2828 55.875 0.0112013 0.0115851 2023-09-23T20:02:33.956 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:33.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:02:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:02:33] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2023-09-23T20:02:34.060 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:34.126 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:34.808 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: pgmap v672: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T20:02:34.808 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:34.808 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.808 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.808 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:02:34.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T20:02:34.809 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[221113]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.810 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:02:34.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: pgmap v672: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T20:02:34.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:34.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.810 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:02:34.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T20:02:34.811 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:34 smithi135 ceph-mon[213871]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: pgmap v672: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T20:02:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:02:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T20:02:34.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:34 smithi144 ceph-mon[194833]: failed to encode map e252 with expected crc 2023-09-23T20:02:34.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 47 16 64387 64371 85.5048 49.9375 0.0124396 0.0116893 2023-09-23T20:02:35.014 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:35.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:35.125 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:35.214 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:35.268 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:35.307 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:35.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:35.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:35.423 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:35.464 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:35.588 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:35.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:35.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:35.668 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:02:35.668 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:02:35.668 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:02:35.668 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:02:35.695 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:35.780 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:35.871 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:35.899 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 48 16 65361 65345 84.9905 60.875 0.00625589 0.0117621 2023-09-23T20:02:35.907 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:35.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:35.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:36.016 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:36.114 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:36.226 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:36.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:36.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:36.335 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:36.386 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:36.441 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:36.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:36.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:36.520 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:02:36.550 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:36.555 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:36 smithi135 ceph-mon[213871]: pgmap v674: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T20:02:36.585 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:36.640 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:36.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:36.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:36.746 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:36.831 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:36 smithi135 ceph-mon[221113]: pgmap v674: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T20:02:36.835 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:36.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:36 smithi144 ceph-mon[194833]: pgmap v674: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T20:02:36.899 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 49 16 66326 66310 84.4872 60.3125 0.00294364 0.0118125 2023-09-23T20:02:37.130 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:02:37.162 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:37.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:37.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:37.273 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:37.392 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:37.772 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:37.773 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]: dispatch 2023-09-23T20:02:37.773 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:37.774 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]: dispatch 2023-09-23T20:02:37.774 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:37.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:37.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:37.876 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]: dispatch 2023-09-23T20:02:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:37.899 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 50 16 67222 67206 83.9179 56 0.0107718 0.0119116 2023-09-23T20:02:37.909 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:37.957 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:02:37.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:37.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:38.064 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:38.096 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:38.156 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:38.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:38.262 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:38.356 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:38.596 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:38 smithi144 ceph-mon[194833]: pgmap v675: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s 2023-09-23T20:02:38.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:38 smithi144 ceph-mon[194833]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:38 smithi144 ceph-mon[194833]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]': finished 2023-09-23T20:02:38.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:38 smithi144 ceph-mon[194833]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T20:02:38.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:38 smithi144 ceph-mon[194833]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.597 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:38 smithi144 ceph-mon[194833]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.597 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[213871]: pgmap v675: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s 2023-09-23T20:02:38.598 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[213871]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.598 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[213871]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.598 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]': finished 2023-09-23T20:02:38.598 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[213871]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T20:02:38.598 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[213871]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.598 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[213871]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.837 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:38.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 51 16 68094 68078 83.3398 54.5 0.0193767 0.0119923 2023-09-23T20:02:38.901 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:38.905 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[221113]: pgmap v675: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s 2023-09-23T20:02:38.905 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[221113]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.905 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[221113]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.905 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]': finished 2023-09-23T20:02:38.906 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[221113]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T20:02:38.906 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[221113]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.906 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:38 smithi135 ceph-mon[221113]: failed to encode map e253 with expected crc 2023-09-23T20:02:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:38.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:39.007 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:39.038 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:39.089 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:39.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:39.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:39.159 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:39.195 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:39.212 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:02:39.256 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:39.267 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:02:39.306 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:02:39.367 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:02:39.411 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:02:39.455 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:02:39.508 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:39.518 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:02:39.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:39.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:39.563 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:02:39.585 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:02:39.586 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:02:39.586 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:02:39.586 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:02:39.586 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:02:39.586 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:02:39.586 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:02:39.586 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:02:39.621 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:39.638 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:02:39.638 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:02:39.660 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:39.723 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:02:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:39.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:39.795 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[213871]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.795 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[213871]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T20:02:39.795 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[213871]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.795 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[213871]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.796 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[213871]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.796 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3502659105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:39.796 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:02:39.798 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[221113]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.798 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[221113]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T20:02:39.798 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[221113]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.798 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[221113]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.798 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[221113]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.798 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3502659105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:39.798 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:02:39.833 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:39.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:39 smithi144 ceph-mon[194833]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:39 smithi144 ceph-mon[194833]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T20:02:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:39 smithi144 ceph-mon[194833]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:39 smithi144 ceph-mon[194833]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:39 smithi144 ceph-mon[194833]: failed to encode map e254 with expected crc 2023-09-23T20:02:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3502659105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:02:39.900 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 52 16 68911 68895 82.7196 51.0625 0.00964792 0.0120841 2023-09-23T20:02:39.929 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:40.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:40 smithi144 ceph-mon[194833]: pgmap v678: 99 pgs: 1 peering, 98 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:40.900 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 53 16 69938 69922 82.3701 64.1875 0.00302063 0.012126 2023-09-23T20:02:40.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:40 smithi135 ceph-mon[221113]: pgmap v678: 99 pgs: 1 peering, 98 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:40.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:40 smithi135 ceph-mon[213871]: pgmap v678: 99 pgs: 1 peering, 98 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:02:41.900 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 54 16 70854 70838 81.9053 57.25 0.0104302 0.0122043 2023-09-23T20:02:41.919 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:41.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:42.024 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:42.047 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:02:42.093 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:42.694 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:42.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:42.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:42.804 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:42.876 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:42 smithi135 ceph-mon[213871]: pgmap v679: 99 pgs: 1 peering, 98 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-09-23T20:02:42.877 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:42 smithi135 ceph-mon[221113]: pgmap v679: 99 pgs: 1 peering, 98 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-09-23T20:02:42.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:42 smithi144 ceph-mon[194833]: pgmap v679: 99 pgs: 1 peering, 98 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-09-23T20:02:42.885 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:42.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 55 16 71612 71596 81.278 47.375 0.0109842 0.0122977 2023-09-23T20:02:43.003 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:43.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:43.120 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:43.154 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:43.223 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:02:43.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:43.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:43.328 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:02:43.407 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:43.604 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:02:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:43.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:02:43.710 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:43.796 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:43.810 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:43.900 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 56 16 72405 72389 80.712 49.5625 0.0209781 0.0123834 2023-09-23T20:02:43.951 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:43.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:43.963 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:02:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:02:43] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2023-09-23T20:02:43.999 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:02:44.061 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:44.165 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:44.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:44 smithi135 ceph-mon[221113]: pgmap v680: 99 pgs: 1 peering, 98 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s 2023-09-23T20:02:44.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:44.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:44.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:44 smithi135 ceph-mon[213871]: pgmap v680: 99 pgs: 1 peering, 98 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s 2023-09-23T20:02:44.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:44.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:44.375 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:44 smithi144 ceph-mon[194833]: pgmap v680: 99 pgs: 1 peering, 98 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s 2023-09-23T20:02:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:44.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:44.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:44.498 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:44.531 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:44.586 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:02:44.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:44.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:44.693 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:44.724 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:44.771 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:02:44.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:44.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:44.878 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:44.900 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 57 16 73210 73194 80.1791 50.3125 0.0100068 0.0124664 2023-09-23T20:02:44.950 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:45.095 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/677698924' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:45.095 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3330915576' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:02:45.119 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:02:45.128 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:02:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/677698924' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3330915576' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:02:45.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/677698924' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:02:45.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3330915576' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:02:45.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 58 16 73875 73859 79.5138 41.5625 0.0138269 0.012571 2023-09-23T20:02:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: pgmap v681: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:02:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/677698924' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3330915576' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:46 smithi144 ceph-mon[194833]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: pgmap v681: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:02:46.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:02:46.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/677698924' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3330915576' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[221113]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: pgmap v681: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/677698924' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3330915576' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:46 smithi135 ceph-mon[213871]: failed to encode map e255 with expected crc 2023-09-23T20:02:46.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 59 16 74669 74653 79.0076 49.625 0.0101807 0.0126516 2023-09-23T20:02:47.655 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:47.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:47.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:47.765 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:02:47.766 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:47.887 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:47.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:02:47.902114+0000 min lat: 0.0011489 max lat: 0.453318 avg lat: 0.0127132 2023-09-23T20:02:47.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:02:47.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 60 16 75557 75541 78.616 55.5 0.0188073 0.0127132 2023-09-23T20:02:47.903 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:48.158 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:48.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:48.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:48.264 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:48.293 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:48.355 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:48.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:48.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:48.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:48 smithi135 ceph-mon[213871]: pgmap v683: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s 2023-09-23T20:02:48.458 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:02:48.489 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:48.530 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:02:48.530 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:02:48.545 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:02:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:48.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:02:48.552 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:48 smithi144 ceph-mon[194833]: pgmap v683: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s 2023-09-23T20:02:48.653 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:48.683 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:48.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:48 smithi135 ceph-mon[221113]: pgmap v683: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s 2023-09-23T20:02:48.737 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:48.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:48.841 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:48.897 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:48.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 61 16 76391 76375 78.182 52.125 0.00378769 0.0127792 2023-09-23T20:02:49.681 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:02:49.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 62 16 77146 77130 77.6824 47.1875 0.0126413 0.0128663 2023-09-23T20:02:50.082 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:50.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:50.209 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:50.235 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:50.440 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:50 smithi135 ceph-mon[213871]: pgmap v684: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s 2023-09-23T20:02:50.719 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:50 smithi135 ceph-mon[221113]: pgmap v684: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s 2023-09-23T20:02:50.720 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:02:50.776 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:50.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:50.795 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:50.881 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:50.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:50 smithi144 ceph-mon[194833]: pgmap v684: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s 2023-09-23T20:02:50.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 63 16 77943 77927 77.2403 49.8125 0.0322115 0.0129391 2023-09-23T20:02:51.002 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:51.071 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:51.224 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:51.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:51.332 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:51.362 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:51.418 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:02:51.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:51.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:51.526 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:51.615 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:51.901 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 64 16 78726 78710 76.7984 48.9375 0.012978 0.0130155 2023-09-23T20:02:52.268 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:02:52.268 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:02:52.275 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:02:52.314 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:52.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:52.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:52.339 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:02:52.393 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:02:52.426 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:52.428 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:02:52.456 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:52.461 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:02:52.500 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:02:52.510 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:02:52.510 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:02:52.510 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:02:52.510 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:02:52.510 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:02:52.510 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:02:52.511 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:02:52.511 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:02:52.519 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:02:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:52.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:52.624 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:52.699 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[213871]: pgmap v685: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T20:02:52.700 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1606581734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:52.700 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:52.700 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:52.700 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:52.701 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[221113]: pgmap v685: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T20:02:52.701 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1606581734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:52.701 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:52.701 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:52.701 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:52.703 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:52 smithi144 ceph-mon[194833]: pgmap v685: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T20:02:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1606581734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:02:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:02:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:02:52.902 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 65 15 79560 79545 76.4198 52.1875 0.0117089 0.0130687 2023-09-23T20:02:53.278 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:53.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:53.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:53.383 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:53.415 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:53.469 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:53.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:53.574 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:02:53.629 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:53.902 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 66 16 80363 80347 76.0216 50.125 0.018466 0.0131497 2023-09-23T20:02:53.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:02:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:02:53] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.43.0" 2023-09-23T20:02:54.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:54 smithi135 ceph-mon[213871]: pgmap v686: 99 pgs: 99 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T20:02:54.597 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:02:54.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:54.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:02:54.704 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:02:54.711 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:54 smithi135 ceph-mon[221113]: pgmap v686: 99 pgs: 99 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T20:02:54.736 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:54 smithi144 ceph-mon[194833]: pgmap v686: 99 pgs: 99 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T20:02:54.741 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:54.800 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:02:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:54.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:02:54.904 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 67 16 81094 81078 75.5666 45.6875 0.0197385 0.013228 2023-09-23T20:02:54.906 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:54.981 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:55.502 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:55.504 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:55.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:55.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:55.608 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:55.736 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:55.908 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 68 16 81830 81814 75.1321 46 0.0139522 0.0133054 2023-09-23T20:02:55.908 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:02:55.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:55.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:02:56.018 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:56.120 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:56.502 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:02:56.507 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:02:56.712 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:02:56.760 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:56 smithi135 ceph-mon[221113]: pgmap v687: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-09-23T20:02:56.760 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/432476427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:02:56.760 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1684524069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:56.761 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:56 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:56.761 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:56 smithi135 ceph-mon[213871]: pgmap v687: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-09-23T20:02:56.761 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/432476427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:02:56.761 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1684524069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:56.761 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:56 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:56.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:56 smithi144 ceph-mon[194833]: pgmap v687: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-09-23T20:02:56.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/432476427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:02:56.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1684524069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:56.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:56 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:02:56.904 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 69 16 82621 82605 74.7599 49.4375 0.0151417 0.0133712 2023-09-23T20:02:57.749 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:57.754 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:57.754 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[221113]: failed to encode map e256 with expected crc 2023-09-23T20:02:57.754 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[221113]: failed to encode map e256 with expected crc 2023-09-23T20:02:57.754 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/432476427' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:02:57.754 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:57.755 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[221113]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T20:02:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:57.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:57.756 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:57.756 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[213871]: failed to encode map e256 with expected crc 2023-09-23T20:02:57.756 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[213871]: failed to encode map e256 with expected crc 2023-09-23T20:02:57.756 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/432476427' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:02:57.757 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:57.757 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:57 smithi135 ceph-mon[213871]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T20:02:57.856 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:02:57.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:57 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:02:57.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:57 smithi144 ceph-mon[194833]: failed to encode map e256 with expected crc 2023-09-23T20:02:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:57 smithi144 ceph-mon[194833]: failed to encode map e256 with expected crc 2023-09-23T20:02:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/432476427' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:02:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:57 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:02:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:57 smithi144 ceph-mon[194833]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T20:02:57.892 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:57.904 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 70 16 83312 83296 74.3092 43.1875 0.0193726 0.0134462 2023-09-23T20:02:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:02:58 smithi135 ceph-mon[213871]: pgmap v689: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T20:02:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:02:58 smithi135 ceph-mon[221113]: pgmap v689: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T20:02:58.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:02:58 smithi144 ceph-mon[194833]: pgmap v689: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T20:02:58.904 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 71 16 84133 84117 73.9854 51.3125 0.0132206 0.013511 2023-09-23T20:02:58.924 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:02:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:58.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:02:59.035 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:59.068 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:59.127 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:59.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:59.232 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:02:59.243 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:02:59.261 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:59.317 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:02:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:59.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:02:59.422 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:02:59.450 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:59.510 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:02:59.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:02:59.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:02:59.615 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:02:59.739 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:02:59.904 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 72 16 84874 84858 73.6013 46.3125 0.0336072 0.0135809 2023-09-23T20:02:59.953 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:02:59.975 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:02:59.975 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:02:59.975 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.975 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:84912 7.2 e256/240 7:44e6580d:::benchmark_data_smithi135_404353_object84911:head v 256'5316, mlcod=256'5313)", 2023-09-23T20:02:59.975 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:02:59.958953+0000", 2023-09-23T20:02:59.975 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.010690641000000001, 2023-09-23T20:02:59.975 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010721948, 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.958953+0000", 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.976 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.958953+0000", 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.958952+0000", 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.977 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.958976+0000", 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3861000000000001e-05 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.958989+0000", 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3419e-05 2023-09-23T20:02:59.978 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.958997+0000", 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2479999999999999e-06 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.959704+0000", 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00070714599999999997 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.959731+0000", 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7382999999999999e-05 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:84917 7.e 7:767338a6:::benchmark_data_smithi135_404353_object84916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:02:59.960338+0000", 2023-09-23T20:02:59.980 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0093051660000000001, 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094833439999999995, 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 84917 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:02:59.981 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960338+0000", 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960338+0000", 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:59.982 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960337+0000", 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960358+0000", 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1132e-05 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960370+0000", 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2058e-05 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960378+0000", 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6830000000000008e-06 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.984 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:02:59.985 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960433+0000", 2023-09-23T20:02:59.985 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5339e-05 2023-09-23T20:02:59.985 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.985 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:02:59.985 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960590+0000", 2023-09-23T20:02:59.985 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015710299999999999 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.960745+0000", 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000154463 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.964635+0000", 2023-09-23T20:02:59.986 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003890241 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.967075+0000", 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024395839999999998 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:59.987 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:84918 7.a e256/240)", 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:02:59.965203+0000", 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0044408240000000003, 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046973789999999998, 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:02:59.988 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.965203+0000", 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.965203+0000", 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:02:59.989 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.965202+0000", 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.965234+0000", 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.205e-05 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.965247+0000", 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3684e-05 2023-09-23T20:02:59.990 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:02:59.965255+0000", 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7400000000000004e-06 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 3 2023-09-23T20:02:59.991 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:03:00.371 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:00.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:00.479 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:00.508 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:00.565 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:00.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:00.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:00.670 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:00.702 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:00.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:00 smithi135 ceph-mon[213871]: pgmap v690: 99 pgs: 99 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s 2023-09-23T20:03:00.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:00 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:00.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:00 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:00.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:00 smithi135 ceph-mon[221113]: pgmap v690: 99 pgs: 99 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s 2023-09-23T20:03:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:00 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:00 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:00.759 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:00.762 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:00 smithi144 ceph-mon[194833]: pgmap v690: 99 pgs: 99 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s 2023-09-23T20:03:00.762 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:00 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:00.763 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:00 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:00.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:00.864 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:00.903 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:00.904 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 73 16 85719 85703 73.3167 52.8125 0.0114821 0.013634 2023-09-23T20:03:00.961 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:00.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:00.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:01.069 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:01.102 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:01.161 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:01.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:01.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:01.268 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:01.297 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:01.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:03:01.351 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:01.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:01.456 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:01.485 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:01.539 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:01.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:01 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:03:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:01 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:03:01.643 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:03:01.645 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:01.647 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:01 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:03:01.648 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:01 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:03:01.648 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:01 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:03:01.648 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:01 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:03:01.791 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:01.910 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 74 16 86519 86503 72.9967 50 0.0287904 0.0136932 2023-09-23T20:03:01.981 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:01.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:01.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:02.086 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:02.114 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:02.176 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:02.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:02.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:02.281 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:02.308 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:02.373 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:02.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:02.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:02.478 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:02.517 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:02.576 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:02.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:02.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:02 smithi144 ceph-mon[194833]: pgmap v691: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s 2023-09-23T20:03:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:02.682 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:02.714 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:02.785 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:02.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:02.854 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:02.893 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:02.911 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 75 16 87287 87271 72.6623 48 0.00712054 0.013756 2023-09-23T20:03:02.926 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:02 smithi135 ceph-mon[213871]: pgmap v691: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s 2023-09-23T20:03:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:02.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:02 smithi135 ceph-mon[221113]: pgmap v691: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s 2023-09-23T20:03:02.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:02.986 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:02.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:02.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:03.096 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:03.127 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:03.179 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:03.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:03.285 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:03.313 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:03.369 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:03.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:03.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:03.475 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:03.576 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:03.723 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:03:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:03:03] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.43.0" 2023-09-23T20:03:03.759 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:03.849 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:03.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:03.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:03.856 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:03.918 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 76 16 88086 88070 72.3573 49.9375 0.0204309 0.013814 2023-09-23T20:03:03.954 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:04.034 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:04.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:04 smithi135 ceph-mon[213871]: pgmap v692: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T20:03:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:04 smithi135 ceph-mon[221113]: pgmap v692: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T20:03:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:04 smithi144 ceph-mon[194833]: pgmap v692: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T20:03:04.920 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 77 16 88924 88908 72.0961 52.375 0.0182678 0.0138648 2023-09-23T20:03:04.983 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:03:04.992 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:04.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:04.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:05.027 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:03:05.069 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:03:05.096 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:05.111 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:03:05.148 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:03:05.170 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:05.198 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:03:05.240 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:03:05.286 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:03:05.293 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:03:05.293 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:03:05.293 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:03:05.293 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:03:05.293 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:03:05.293 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:03:05.293 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:03:05.294 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:03:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2116025635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:05.920 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 78 16 89731 89715 71.8186 50.4375 0.0116324 0.0139175 2023-09-23T20:03:05.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2116025635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:05.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2116025635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:06.471 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:06.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:06.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:06.582 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:06.617 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:06.677 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:06.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:06.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:06.782 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:06.851 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:06 smithi135 ceph-mon[213871]: pgmap v693: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s 2023-09-23T20:03:06.852 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:06 smithi135 ceph-mon[221113]: pgmap v693: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s 2023-09-23T20:03:06.863 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:06 smithi144 ceph-mon[194833]: pgmap v693: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s 2023-09-23T20:03:06.921 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 79 16 90511 90495 71.5268 48.75 0.0350332 0.0139733 2023-09-23T20:03:07.908 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:07.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:07.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:07.923 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:03:07.924442+0000 min lat: 0.0011489 max lat: 0.453318 avg lat: 0.0140357 2023-09-23T20:03:07.923 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:03:07.923 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 80 16 91274 91258 71.2262 47.6875 0.00793032 0.0140357 2023-09-23T20:03:08.021 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:08.109 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:08.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:08 smithi135 ceph-mon[221113]: pgmap v694: 99 pgs: 99 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 759 op/s 2023-09-23T20:03:08.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:08.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:08 smithi135 ceph-mon[213871]: pgmap v694: 99 pgs: 99 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 759 op/s 2023-09-23T20:03:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:08 smithi144 ceph-mon[194833]: pgmap v694: 99 pgs: 99 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 759 op/s 2023-09-23T20:03:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:08.907 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:08.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:08.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:08.923 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 81 16 92021 92005 70.9234 46.6875 0.0178366 0.0140936 2023-09-23T20:03:09.016 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:09.109 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:09.147 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/779274518' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:03:09.147 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/779274518' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:03:09.173 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:03:09.427 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:09.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:09.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:09.534 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:09.560 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/779274518' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:03:09.562 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:09.614 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:09.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:09.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:09.665 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:03:09.720 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:09.749 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:09.924 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 82 16 92525 92509 70.4429 31.5 0.00451028 0.0141187 2023-09-23T20:03:10.171 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:03:10.240 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:03:10.240 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:03:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[221113]: pgmap v695: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T20:03:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[221113]: failed to encode map e257 with expected crc 2023-09-23T20:03:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/779274518' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[221113]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[221113]: failed to encode map e257 with expected crc 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3593102028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[213871]: pgmap v695: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[213871]: failed to encode map e257 with expected crc 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/779274518' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[213871]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T20:03:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[213871]: failed to encode map e257 with expected crc 2023-09-23T20:03:10.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3593102028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:10.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:10 smithi144 ceph-mon[194833]: pgmap v695: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T20:03:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:10 smithi144 ceph-mon[194833]: failed to encode map e257 with expected crc 2023-09-23T20:03:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/779274518' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:03:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:10 smithi144 ceph-mon[194833]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T20:03:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:10 smithi144 ceph-mon[194833]: failed to encode map e257 with expected crc 2023-09-23T20:03:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3593102028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:10.924 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 83 16 92525 92509 69.5949 0 - 0.0141187 2023-09-23T20:03:11.190 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:11.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:11.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:11.298 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:11.385 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:11.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:11.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[213871]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[213871]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[213871]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:11.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[213871]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T20:03:11.413 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[213871]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.414 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:11.414 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[221113]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.414 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[221113]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.414 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[221113]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.415 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:11.415 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[221113]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T20:03:11.415 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:11 smithi135 ceph-mon[221113]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.416 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:03:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:11 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:11 smithi144 ceph-mon[194833]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:11 smithi144 ceph-mon[194833]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:11 smithi144 ceph-mon[194833]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:11 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:11 smithi144 ceph-mon[194833]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T20:03:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:11 smithi144 ceph-mon[194833]: failed to encode map e258 with expected crc 2023-09-23T20:03:11.689 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:11.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:11.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:11.794 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:03:11.807 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:11.840 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:11.888 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:11.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:11.924 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 84 16 92988 92972 69.1112 14.4688 0.0288165 0.0144616 2023-09-23T20:03:11.996 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:12.028 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:12.087 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:12.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:12.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:12.120 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:12.192 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:12.294 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:12 smithi144 ceph-mon[194833]: pgmap v698: 99 pgs: 1 remapped+peering, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s 2023-09-23T20:03:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:12 smithi144 ceph-mon[194833]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:12 smithi144 ceph-mon[194833]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:12 smithi144 ceph-mon[194833]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T20:03:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:12 smithi144 ceph-mon[194833]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:12 smithi144 ceph-mon[194833]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:12 smithi144 ceph-mon[194833]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 256'5730 2023-09-23T20:03:12.443 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:12.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:12.453 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[221113]: pgmap v698: 99 pgs: 1 remapped+peering, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s 2023-09-23T20:03:12.453 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[221113]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.453 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[221113]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[221113]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T20:03:12.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[221113]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[221113]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[221113]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 256'5730 2023-09-23T20:03:12.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:12.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[213871]: pgmap v698: 99 pgs: 1 remapped+peering, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s 2023-09-23T20:03:12.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[213871]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[213871]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[213871]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T20:03:12.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[213871]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[213871]: failed to encode map e259 with expected crc 2023-09-23T20:03:12.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:12 smithi135 ceph-mon[213871]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 256'5730 2023-09-23T20:03:12.497 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:12.556 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:12.676 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:12.924 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 85 16 93683 93667 68.8094 43.4375 0.0207766 0.0145274 2023-09-23T20:03:13.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:13 smithi135 ceph-mon[213871]: failed to encode map e260 with expected crc 2023-09-23T20:03:13.198 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:13 smithi135 ceph-mon[213871]: failed to encode map e260 with expected crc 2023-09-23T20:03:13.198 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:13 smithi135 ceph-mon[213871]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T20:03:13.207 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:13.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:13.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:13.316 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:13.347 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:13.414 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:13.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:13.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:13.478 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:13 smithi135 ceph-mon[221113]: failed to encode map e260 with expected crc 2023-09-23T20:03:13.478 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:13 smithi135 ceph-mon[221113]: failed to encode map e260 with expected crc 2023-09-23T20:03:13.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:13 smithi135 ceph-mon[221113]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T20:03:13.521 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:13.595 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:13 smithi144 ceph-mon[194833]: failed to encode map e260 with expected crc 2023-09-23T20:03:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:13 smithi144 ceph-mon[194833]: failed to encode map e260 with expected crc 2023-09-23T20:03:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:13 smithi144 ceph-mon[194833]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T20:03:13.920 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:03:13.924 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 86 16 94490 94474 68.5959 50.4375 0.00754039 0.0145727 2023-09-23T20:03:13.946 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:03:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:03:13] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2023-09-23T20:03:14.479 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:14 smithi144 ceph-mon[194833]: pgmap v701: 99 pgs: 1 remapped+peering, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 358 op/s 2023-09-23T20:03:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:14 smithi144 ceph-mon[194833]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:14 smithi144 ceph-mon[194833]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:14 smithi144 ceph-mon[194833]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T20:03:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:14 smithi144 ceph-mon[194833]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:14 smithi144 ceph-mon[194833]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:14 smithi144 ceph-mon[194833]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[213871]: pgmap v701: 99 pgs: 1 remapped+peering, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 358 op/s 2023-09-23T20:03:14.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[213871]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[213871]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[213871]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T20:03:14.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[213871]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[213871]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[213871]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[221113]: pgmap v701: 99 pgs: 1 remapped+peering, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 358 op/s 2023-09-23T20:03:14.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[221113]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[221113]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[221113]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T20:03:14.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[221113]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[221113]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.691 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:14 smithi135 ceph-mon[221113]: failed to encode map e261 with expected crc 2023-09-23T20:03:14.924 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 87 16 95079 95063 68.2308 36.8125 0.0166219 0.0146512 2023-09-23T20:03:15.310 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:15.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:15.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:15.426 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:15.454 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:15.508 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:15.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:15.615 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:15.643 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:15.693 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:15.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:15.798 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:15.880 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:15.925 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 88 16 95710 95694 67.9034 39.4375 0.015445 0.0147209 2023-09-23T20:03:16.155 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:16.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:16 smithi135 ceph-mon[221113]: pgmap v703: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 587 op/s; 122 B/s, 0 objects/s recovering 2023-09-23T20:03:16.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:16 smithi135 ceph-mon[213871]: pgmap v703: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 587 op/s; 122 B/s, 0 objects/s recovering 2023-09-23T20:03:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:16 smithi144 ceph-mon[194833]: pgmap v703: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 587 op/s; 122 B/s, 0 objects/s recovering 2023-09-23T20:03:16.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:03:16.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:03:16.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:03:16.925 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 89 16 96370 96354 67.6041 41.25 0.00721188 0.0147864 2023-09-23T20:03:16.957 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:03:17.007 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:03:17.052 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:03:17.079 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:03:17.115 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:03:17.130 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:03:17.130 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:03:17.130 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:03:17.130 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:03:17.130 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:03:17.130 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:03:17.130 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:03:17.130 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:03:17.414 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:17.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:17.426 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2263781209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:17.426 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2263781209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:17.527 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:17.629 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2263781209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:17.925 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total time run: 90.0116 2023-09-23T20:03:17.925 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total writes made: 97059 2023-09-23T20:03:17.925 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Write size: 65536 2023-09-23T20:03:17.925 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object size: 65536 2023-09-23T20:03:17.925 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Bandwidth (MB/sec): 67.3934 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Bandwidth: 31.4105 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max bandwidth (MB/sec): 196.375 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average IOPS: 1078 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev IOPS: 502.577 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max IOPS: 3142 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min IOPS: 0 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average Latency(s): 0.0148344 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Latency(s): 0.0290645 2023-09-23T20:03:17.926 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max latency(s): 2.05023 2023-09-23T20:03:17.927 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min latency(s): 0.0011489 2023-09-23T20:03:17.930 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:03:18.581 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:18.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:18.687 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:18 smithi135 ceph-mon[221113]: pgmap v704: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 27 MiB/s wr, 433 op/s; 5512/282426 objects misplaced (1.952%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:03:18.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:18 smithi135 ceph-mon[213871]: pgmap v704: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 27 MiB/s wr, 433 op/s; 5512/282426 objects misplaced (1.952%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:03:18.691 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:18.716 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:18 smithi144 ceph-mon[194833]: pgmap v704: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 27 MiB/s wr, 433 op/s; 5512/282426 objects misplaced (1.952%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:03:18.719 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:18.776 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:18.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:18.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:18.882 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:18.914 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:18.968 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:18.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:18.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:19.073 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:19.232 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:19.404 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:19.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:19.534 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:19.613 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:19.855 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:03:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195610 5.8 5:13ab778b:::benchmark_data_smithi135_284570_object79826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086316+0000", 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.52616693700003, 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32473328200000001, 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195610 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086316+0000", 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086316+0000", 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086315+0000", 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086318+0000", 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.616e-06 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086324+0000", 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0000000000000002e-06 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086329+0000", 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0629999999999996e-06 2023-09-23T20:03:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099057+0000", 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012727894999999999 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099206+0000", 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014897700000000001 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099330+0000", 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012414300000000001 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.105020+0000", 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056901069999999998 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410872+0000", 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30585215500000001 2023-09-23T20:03:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410959+0000", 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7231999999999994e-05 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411010+0000", 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.024e-05 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411049+0000", 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9240999999999999e-05 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195612 5.a 5:51964534:::benchmark_data_smithi135_284570_object79828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086334+0000", 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.52614822800001, 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31885195999999999, 2023-09-23T20:03:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195612 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T20:03:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086334+0000", 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086336+0000", 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2479999999999999e-06 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086341+0000", 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0610000000000003e-06 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086346+0000", 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5310000000000003e-06 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086419+0000", 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3072999999999996e-05 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086523+0000", 2023-09-23T20:03:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000103767 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086633+0000", 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110221 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.095308+0000", 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0086747639999999997 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099671+0000", 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043637499999999996 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405096+0000", 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30542450199999999 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405140+0000", 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3736000000000003e-05 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405186+0000", 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6621e-05 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195613 5.a 5:51e9d81c:::benchmark_data_smithi135_284570_object79829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.086357+0000", 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.52612507700002, 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.321475978, 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195613 2023-09-23T20:03:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T20:03:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086357+0000", 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086360+0000", 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6060000000000001e-06 2023-09-23T20:03:19.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086365+0000", 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8440000000000003e-06 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.086371+0000", 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.096e-06 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.090840+0000", 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004469794 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.091046+0000", 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000205508 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.091154+0000", 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107816 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099701+0000", 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085474149999999992 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.099715+0000", 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4365999999999999e-05 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405243+0000", 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30552804300000003 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405285+0000", 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1223999999999997e-05 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.407833+0000", 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025487410000000002 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.878 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195623 5.8 5:1605c7cf:::benchmark_data_smithi135_284570_object79839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106079+0000", 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.50640337300001, 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30509230399999998, 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195623 2023-09-23T20:03:19.879 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106079+0000", 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106079+0000", 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106078+0000", 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106087+0000", 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8689999999999993e-06 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106101+0000", 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3971000000000001e-05 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106110+0000", 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9800000000000004e-06 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106177+0000", 2023-09-23T20:03:19.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7464000000000003e-05 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106310+0000", 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000132712 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106435+0000", 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000125499 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111791+0000", 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053554220000000003 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410920+0000", 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29912940799999999 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411095+0000", 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017488599999999999 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411136+0000", 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1517999999999998e-05 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411171+0000", 2023-09-23T20:03:19.885 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5009999999999999e-05 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195626 5.6 5:6edb6a83:::benchmark_data_smithi135_284570_object79842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106123+0000", 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.50636001300001, 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30425704100000001, 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.886 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195626 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106123+0000", 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106123+0000", 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106122+0000", 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106126+0000", 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1300000000000003e-06 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106133+0000", 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8109999999999997e-06 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106138+0000", 2023-09-23T20:03:19.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2569999999999998e-06 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106419+0000", 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028042200000000001 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106516+0000", 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7001000000000002e-05 2023-09-23T20:03:19.890 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106616+0000", 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100272 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405132+0000", 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.298516432 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405142+0000", 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2189999999999993e-06 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410291+0000", 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051497000000000001 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410341+0000", 2023-09-23T20:03:19.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0207000000000002e-05 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410380+0000", 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8109999999999999e-05 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.893 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:195612 5.a e151/112 ondisk, result = 0)", 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.106148+0000", 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.50633425500001, 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29904286699999999, 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.894 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106148+0000", 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.895 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106152+0000", 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1230000000000002e-06 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106163+0000", 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0951000000000001e-05 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.106169+0000", 2023-09-23T20:03:19.896 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8220000000000001e-06 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405078+0000", 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.298908224 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405088+0000", 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0686999999999999e-05 2023-09-23T20:03:19.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405191+0000", 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010278 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195629 5.8 5:14b98469:::benchmark_data_smithi135_284570_object79845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.110999+0000", 2023-09-23T20:03:19.899 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.50148372799998, 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31090222000000001, 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195629 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110999+0000", 2023-09-23T20:03:19.900 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110999+0000", 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.110998+0000", 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:19.901 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111004+0000", 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8809999999999998e-06 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111022+0000", 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7354999999999999e-05 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111032+0000", 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0251000000000001e-05 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111900+0000", 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00086782700000000005 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112014+0000", 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000113835 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112123+0000", 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010916000000000001 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411233+0000", 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.299110243 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.904 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.411284+0000", 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0732999999999997e-05 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421815+0000", 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010530813999999999 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.905 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421859+0000", 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4240000000000003e-05 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.421901+0000", 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2107000000000002e-05 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.906 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195630 5.6 5:65c744e5:::benchmark_data_smithi135_284570_object79846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.111039+0000", 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.50144322599999, 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.304778728, 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195630 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.907 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.908 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111039+0000", 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111042+0000", 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.507e-06 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111049+0000", 2023-09-23T20:03:19.909 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9399999999999996e-06 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111055+0000", 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4439999999999999e-06 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111663+0000", 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060795100000000002 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.910 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111801+0000", 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013784600000000001 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.111940+0000", 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000139574 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.911 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405147+0000", 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29320712999999998 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410463+0000", 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053154830000000002 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415730+0000", 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052668560000000003 2023-09-23T20:03:19.912 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415777+0000", 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7123000000000003e-05 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415818+0000", 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1347999999999998e-05 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.913 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:195610 5.8 e151/112 ondisk, result = 0)", 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.112245+0000", 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.50023707100002, 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29862952999999998, 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.914 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112245+0000", 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:03:19.915 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.112250+0000", 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0350000000000001e-06 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405043+0000", 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29279347100000003 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.916 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405055+0000", 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1627000000000001e-05 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410857+0000", 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058025580000000002 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410866+0000", 2023-09-23T20:03:19.917 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3450000000000006e-06 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.410875+0000", 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.217e-06 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:195633 5.b 5:d886e22d:::benchmark_data_smithi135_284570_object79849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:03:19.918 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:48.113988+0000", 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 451.49849500699997, 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32055229600000001, 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 195633 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.919 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113988+0000", 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113988+0000", 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113986+0000", 2023-09-23T20:03:19.920 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.113993+0000", 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5889999999999999e-06 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405037+0000", 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29104433699999999 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.921 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405056+0000", 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8836999999999999e-05 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405147+0000", 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1032000000000005e-05 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.922 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405287+0000", 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014026500000000001 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.405410+0000", 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000122636 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.415431+0000", 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010021077 2023-09-23T20:03:19.923 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.423733+0000", 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083021039999999994 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434462+0000", 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010729140999999999 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.924 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434502+0000", 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9712000000000002e-05 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:48.434540+0000", 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7759999999999998e-05 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.925 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:203734 5.b 5:dd140fcc:::benchmark_data_smithi135_284570_object87950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:55:59.309502+0000", 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 440.302980945, 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.29768655599999999, 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 203734 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.926 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309502+0000", 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309502+0000", 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.927 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309501+0000", 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309504+0000", 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.225e-06 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.928 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309513+0000", 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4309999999999994e-06 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309517+0000", 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0199999999999996e-06 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309557+0000", 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9762999999999999e-05 2023-09-23T20:03:19.929 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309632+0000", 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5908999999999994e-05 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.309704+0000", 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1668000000000006e-05 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.930 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.313325+0000", 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036211889999999999 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.313868+0000", 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00054303499999999998 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607103+0000", 2023-09-23T20:03:19.931 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.293234781 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607154+0000", 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0914000000000001e-05 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:55:59.607188+0000", 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4112999999999997e-05 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.932 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220400 5.b 5:d413819b:::benchmark_data_smithi135_284570_object104616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.640447+0000", 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.97203525399999, 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32328775900000001, 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.933 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220400 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640447+0000", 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640447+0000", 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.934 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640446+0000", 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640451+0000", 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4940000000000003e-06 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.935 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640463+0000", 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1299e-05 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640471+0000", 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8220000000000007e-06 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640519+0000", 2023-09-23T20:03:19.936 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8375999999999999e-05 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640629+0000", 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011054 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640741+0000", 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111857 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.937 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650179+0000", 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094374469999999998 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963581+0000", 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31340195599999998 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.938 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963649+0000", 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8036999999999999e-05 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963693+0000", 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4162000000000002e-05 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963735+0000", 2023-09-23T20:03:19.939 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2098999999999998e-05 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220403 5.b 5:d5744800:::benchmark_data_smithi135_284570_object104619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.640988+0000", 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.97149408000001, 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32284452200000002, 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.940 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220403 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.941 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640993+0000", 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.604e-06 2023-09-23T20:03:19.942 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.641005+0000", 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1365e-05 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.641012+0000", 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1470000000000004e-06 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.943 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645957+0000", 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049455829999999999 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.646045+0000", 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7979999999999995e-05 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.646112+0000", 2023-09-23T20:03:19.944 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6251e-05 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650257+0000", 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041449950000000003 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963620+0000", 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31336345500000001 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.945 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963767+0000", 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000147169 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963799+0000", 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2060000000000001e-05 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.946 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963833+0000", 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3733999999999999e-05 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220406 5.d 5:b474da8d:::benchmark_data_smithi135_284570_object104622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.645154+0000", 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.96732882499998, 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32365115, 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.947 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220406 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645154+0000", 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.948 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645154+0000", 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645152+0000", 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.949 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645158+0000", 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9819999999999996e-06 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645168+0000", 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7580000000000004e-06 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645175+0000", 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9009999999999997e-06 2023-09-23T20:03:19.950 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645213+0000", 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8535999999999998e-05 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645311+0000", 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7647999999999997e-05 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.951 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645397+0000", 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5791999999999994e-05 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650541+0000", 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051441569999999999 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968678+0000", 2023-09-23T20:03:19.952 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.318137369 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968716+0000", 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7765999999999997e-05 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968765+0000", 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8520000000000003e-05 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.953 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968805+0000", 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0170000000000003e-05 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220411 5.6 5:65202dbd:::benchmark_data_smithi135_284570_object104627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.650884+0000", 2023-09-23T20:03:19.954 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.96159878899999, 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31285145399999997, 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220411 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.955 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650884+0000", 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650884+0000", 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650883+0000", 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:03:19.956 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650889+0000", 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7460000000000003e-06 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650902+0000", 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2266000000000001e-05 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.957 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650936+0000", 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4878e-05 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650982+0000", 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5259000000000002e-05 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651089+0000", 2023-09-23T20:03:19.958 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107784 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651201+0000", 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111297 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963506+0000", 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.312305098 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.959 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963534+0000", 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8495000000000001e-05 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963649+0000", 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011424 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.960 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963692+0000", 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2907000000000001e-05 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963735+0000", 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.367e-05 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220414 5.d 5:b1d7f748:::benchmark_data_smithi135_284570_object104630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:03:19.961 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.650962+0000", 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.96152093400002, 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.317977958, 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220414 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.962 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650962+0000", 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650962+0000", 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.963 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650961+0000", 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650965+0000", 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.997e-06 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650972+0000", 2023-09-23T20:03:19.964 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0190000000000001e-06 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650977+0000", 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1340000000000003e-06 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651027+0000", 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9509000000000003e-05 2023-09-23T20:03:19.965 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651141+0000", 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000114848 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651250+0000", 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000108455 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.966 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968686+0000", 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31743582999999997 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968839+0000", 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015357700000000001 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.967 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968869+0000", 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9839e-05 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968902+0000", 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2449000000000001e-05 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968940+0000", 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7911999999999999e-05 2023-09-23T20:03:19.968 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220415 5.d 5:bcbf5941:::benchmark_data_smithi135_284570_object104631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.654307+0000", 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.95817563100002, 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33038375800000003, 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:19.969 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220415 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654307+0000", 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.970 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654307+0000", 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654305+0000", 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654313+0000", 2023-09-23T20:03:19.971 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0530000000000007e-06 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654330+0000", 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6290000000000002e-05 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654339+0000", 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.499e-06 2023-09-23T20:03:19.972 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655392+0000", 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010529370000000001 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655521+0000", 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012935900000000001 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.973 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655599+0000", 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7442999999999997e-05 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.979556+0000", 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32395747899999999 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:19.974 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.979597+0000", 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1085999999999999e-05 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984634+0000", 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005036769 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984665+0000", 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0960000000000002e-05 2023-09-23T20:03:19.975 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984691+0000", 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5607999999999999e-05 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:220413 5.4 e154/112 5:23b6015e:::benchmark_data_smithi135_284570_object104629:head v 154'13837, mlcod=154'13835)", 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.654925+0000", 2023-09-23T20:03:19.976 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.957557931, 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.308583475, 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654925+0000", 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.977 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654925+0000", 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654924+0000", 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654932+0000", 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9510000000000006e-06 2023-09-23T20:03:19.978 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654942+0000", 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0047999999999999e-05 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654947+0000", 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2089999999999996e-06 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.979 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654985+0000", 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8016999999999999e-05 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654999+0000", 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.454e-05 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963414+0000", 2023-09-23T20:03:19.980 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30841476099999998 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963508+0000", 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3911000000000003e-05 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.981 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:220406 5.d e154/112 ondisk, result = 0)", 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.657056+0000", 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.95542671599998, 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31163112799999998, 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657056+0000", 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.982 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657056+0000", 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657055+0000", 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.983 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657061+0000", 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4999999999999996e-06 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963465+0000", 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30640367400000001 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963476+0000", 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0861e-05 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.984 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968664+0000", 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051881879999999998 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968672+0000", 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0690000000000002e-06 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.985 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968687+0000", 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4715000000000001e-05 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:220411 5.6 e154/112 ondisk, result = 0)", 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.658010+0000", 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 417.95447202100002, 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30549902800000001, 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:19.986 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.987 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658015+0000", 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6190000000000002e-06 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:19.988 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963397+0000", 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30538196699999998 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963412+0000", 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4925e-05 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963488+0000", 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6477000000000007e-05 2023-09-23T20:03:19.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963498+0000", 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0478e-05 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963510+0000", 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1058e-05 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.990 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:19.991 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:19.991 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:03:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:19.992 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:19.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:20.093 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:20.118 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:20.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:20 smithi135 ceph-mon[213871]: pgmap v705: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 668 op/s; 5512/289131 objects misplaced (1.906%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T20:03:20.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1686826056' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:03:20.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:20 smithi135 ceph-mon[221113]: pgmap v705: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 668 op/s; 5512/289131 objects misplaced (1.906%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T20:03:20.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1686826056' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:03:20.495 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:03:20.749 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:03:20.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:20 smithi144 ceph-mon[194833]: pgmap v705: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 668 op/s; 5512/289131 objects misplaced (1.906%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T20:03:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1686826056' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:03:21.459 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:21.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:21.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:21.572 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:21.598 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:21.639 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:21.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:21.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:21.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:21 smithi135 ceph-mon[213871]: failed to encode map e262 with expected crc 2023-09-23T20:03:21.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:21 smithi135 ceph-mon[213871]: failed to encode map e262 with expected crc 2023-09-23T20:03:21.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1686826056' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:03:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:21 smithi135 ceph-mon[213871]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T20:03:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:21 smithi135 ceph-mon[221113]: failed to encode map e262 with expected crc 2023-09-23T20:03:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:21 smithi135 ceph-mon[221113]: failed to encode map e262 with expected crc 2023-09-23T20:03:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1686826056' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:03:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:21 smithi135 ceph-mon[221113]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T20:03:21.745 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:21.774 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:21 smithi144 ceph-mon[194833]: failed to encode map e262 with expected crc 2023-09-23T20:03:21.774 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:21 smithi144 ceph-mon[194833]: failed to encode map e262 with expected crc 2023-09-23T20:03:21.774 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1686826056' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:03:21.774 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:21 smithi144 ceph-mon[194833]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T20:03:21.776 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:21.818 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:21.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:21.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:21.922 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:21.954 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:22.002 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:22.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:22.107 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:22.518 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:22.649 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:03:22.708 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:22.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:22 smithi135 ceph-mon[221113]: pgmap v707: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 37 MiB/s wr, 661 op/s; 5031/289143 objects misplaced (1.740%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T20:03:22.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:22.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:22 smithi135 ceph-mon[213871]: pgmap v707: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 37 MiB/s wr, 661 op/s; 5031/289143 objects misplaced (1.740%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T20:03:22.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:22.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:22.751 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:03:22.832 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:22.857 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:22 smithi144 ceph-mon[194833]: pgmap v707: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 37 MiB/s wr, 661 op/s; 5031/289143 objects misplaced (1.740%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T20:03:22.857 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:22.860 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:22.910 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:22.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:22.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:23.015 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:23.042 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:23.094 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:23.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:23.200 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:23.299 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:23.482 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:23.509 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:23.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:23.614 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:23.714 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:23.827 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:23.829 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:23.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:03:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:03:23] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2023-09-23T20:03:23.977 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:23.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:23.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:24.082 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:24.166 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:24.525 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:03:24.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:24 smithi144 ceph-mon[194833]: pgmap v708: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 108 B/s rd, 30 MiB/s wr, 599 op/s; 5031/287709 objects misplaced (1.749%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:03:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3686253760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:24 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:24.887 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:24.908 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:24 smithi135 ceph-mon[221113]: pgmap v708: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 108 B/s rd, 30 MiB/s wr, 599 op/s; 5031/287709 objects misplaced (1.749%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:03:24.908 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3686253760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:24.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:24 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:24.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:24.910 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:24 smithi135 ceph-mon[213871]: pgmap v708: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 108 B/s rd, 30 MiB/s wr, 599 op/s; 5031/287709 objects misplaced (1.749%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:03:24.910 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3686253760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:24.910 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:24 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:03:25.011 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:25.117 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:25 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:25 smithi144 ceph-mon[194833]: failed to encode map e263 with expected crc 2023-09-23T20:03:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:25 smithi144 ceph-mon[194833]: failed to encode map e263 with expected crc 2023-09-23T20:03:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:25 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:25 smithi144 ceph-mon[194833]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T20:03:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:25 smithi144 ceph-mon[194833]: failed to encode map e263 with expected crc 2023-09-23T20:03:26.040 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:26.041 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[213871]: failed to encode map e263 with expected crc 2023-09-23T20:03:26.041 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[213871]: failed to encode map e263 with expected crc 2023-09-23T20:03:26.041 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:26.041 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[213871]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T20:03:26.041 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[213871]: failed to encode map e263 with expected crc 2023-09-23T20:03:26.041 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:26.042 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[221113]: failed to encode map e263 with expected crc 2023-09-23T20:03:26.042 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[221113]: failed to encode map e263 with expected crc 2023-09-23T20:03:26.042 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:03:26.042 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[221113]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T20:03:26.042 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:25 smithi135 ceph-mon[221113]: failed to encode map e263 with expected crc 2023-09-23T20:03:26.140 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:26.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:26.247 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:26.278 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:26.331 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:26.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:26.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:26.436 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:26.464 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:26.520 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:26.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:26.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:26 smithi144 ceph-mon[194833]: pgmap v710: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 23 MiB/s wr, 734 op/s; 5031/282471 objects misplaced (1.781%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:03:26.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:26.632 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:26.657 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:26.707 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:26.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:26.812 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:26.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:26 smithi135 ceph-mon[221113]: pgmap v710: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 23 MiB/s wr, 734 op/s; 5031/282471 objects misplaced (1.781%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:03:26.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:26 smithi135 ceph-mon[213871]: pgmap v710: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 23 MiB/s wr, 734 op/s; 5031/282471 objects misplaced (1.781%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:03:27.277 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:27.581 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:27.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:27.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:27.684 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:27.711 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:28.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:28 smithi135 ceph-mon[221113]: pgmap v711: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 4.8 MiB/s wr, 545 op/s; 4488/279777 objects misplaced (1.604%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:03:28.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:28 smithi135 ceph-mon[213871]: pgmap v711: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 4.8 MiB/s wr, 545 op/s; 4488/279777 objects misplaced (1.604%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:03:28.739 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:28.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:28.863 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:28.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:28 smithi144 ceph-mon[194833]: pgmap v711: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 4.8 MiB/s wr, 545 op/s; 4488/279777 objects misplaced (1.604%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:03:28.978 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:29.718 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:29.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:29.831 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:29.857 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:29.906 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:29.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:29.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:30.012 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:30.040 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:30.093 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:30.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:30.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:30.201 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:30.228 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:30.277 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:30.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:30.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:30.385 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:30.456 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:30.526 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:30 smithi144 ceph-mon[194833]: pgmap v712: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 124 B/s rd, 4.7 MiB/s wr, 847 op/s; 4488/271914 objects misplaced (1.651%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:03:30.582 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:03:30.612 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:30.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:30.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:30.638 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:03:30.682 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:03:30.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:30 smithi135 ceph-mon[213871]: pgmap v712: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 124 B/s rd, 4.7 MiB/s wr, 847 op/s; 4488/271914 objects misplaced (1.651%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:03:30.702 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:30 smithi135 ceph-mon[221113]: pgmap v712: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 124 B/s rd, 4.7 MiB/s wr, 847 op/s; 4488/271914 objects misplaced (1.651%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:03:30.726 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:03:30.735 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:30.766 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:30.770 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:03:30.814 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:03:30.823 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:30.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:30.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:30.852 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:03:30.924 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:03:30.930 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:30.954 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:03:30.954 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:03:30.958 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:30.966 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:03:30.978 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:03:30.978 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:03:30.978 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:03:30.978 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:03:30.978 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:03:30.978 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:03:30.979 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:03:30.979 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:03:31.016 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:31.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:31.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:31.122 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:31.150 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:31.206 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:31.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:31.315 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:31.345 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:31.403 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:31.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:31.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:31.508 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:31.539 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:31.548 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:03:31.585 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:31 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2829770123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:03:31.586 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:31 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4092280707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:31.590 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:31 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2829770123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:03:31.591 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:31 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4092280707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:31.599 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:31.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:31.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:31.705 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:31.801 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:31 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2829770123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:03:31.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:31 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4092280707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:32.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:32 smithi144 ceph-mon[194833]: pgmap v713: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.1 MiB/s wr, 693 op/s; 3984/269382 objects misplaced (1.479%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:03:32.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2829770123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:03:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:32 smithi144 ceph-mon[194833]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T20:03:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:32 smithi144 ceph-mon[194833]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:32 smithi144 ceph-mon[194833]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:32 smithi144 ceph-mon[194833]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:32 smithi144 ceph-mon[194833]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[213871]: pgmap v713: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.1 MiB/s wr, 693 op/s; 3984/269382 objects misplaced (1.479%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:03:32.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2829770123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:03:32.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[213871]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T20:03:32.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[213871]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[213871]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[213871]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[213871]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[221113]: pgmap v713: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.1 MiB/s wr, 693 op/s; 3984/269382 objects misplaced (1.479%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2829770123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[221113]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[221113]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[221113]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[221113]: failed to encode map e264 with expected crc 2023-09-23T20:03:32.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:32 smithi135 ceph-mon[221113]: failed to encode map e264 with expected crc 2023-09-23T20:03:33.031 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:33.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:33.141 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:33.176 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:33.220 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:33.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:33.326 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:33.354 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:33.399 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:33.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:33.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:33.505 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:33.521 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:03:33.536 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:33.586 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:33.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:33.691 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:33.771 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:03:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:03:33] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2023-09-23T20:03:33.778 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:34.158 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:03:34.194 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:34.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:34.308 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:34.404 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:34.422 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:34.501 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:34.553 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:34.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:34.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:34.669 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:34.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:34 smithi135 ceph-mon[221113]: pgmap v715: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 3984/266763 objects misplaced (1.493%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:03:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:34 smithi135 ceph-mon[213871]: pgmap v715: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 3984/266763 objects misplaced (1.493%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:03:34.742 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:34 smithi144 ceph-mon[194833]: pgmap v715: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 3984/266763 objects misplaced (1.493%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:03:34.883 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:34.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:34.988 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:35.019 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:35.082 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:35.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:35.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:35.191 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:35.222 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:35.268 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:35.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:35.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:35.375 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:35.450 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:35.565 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:35.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:35.677 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:35.710 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:35.770 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:35.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:35.876 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:35.949 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:36.099 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:03:36.730 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:36.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:36.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:36.835 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:36.861 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:36 smithi144 ceph-mon[194833]: pgmap v716: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 3984/262224 objects misplaced (1.519%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:03:36.865 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:36.927 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:36.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:36 smithi135 ceph-mon[213871]: pgmap v716: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 3984/262224 objects misplaced (1.519%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:03:36.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:36 smithi135 ceph-mon[221113]: pgmap v716: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 3984/262224 objects misplaced (1.519%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:03:37.033 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:37.063 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:37.663 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:37.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:37.777 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:37.856 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:37.882 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:37.882 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:37.902 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:38.015 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:38.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:38.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:38.120 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:38.134 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:38.212 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:38.585 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:38.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:38.700 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:38.743 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:38.763 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:38 smithi135 ceph-mon[221113]: pgmap v717: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 3541/259968 objects misplaced (1.362%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:03:38.763 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-09-23T20:03:38.763 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [1, 7]}]: dispatch 2023-09-23T20:03:38.763 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:38 smithi135 ceph-mon[213871]: pgmap v717: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 3541/259968 objects misplaced (1.362%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:03:38.763 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-09-23T20:03:38.764 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [1, 7]}]: dispatch 2023-09-23T20:03:38.795 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:38.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:38.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:38 smithi144 ceph-mon[194833]: pgmap v717: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 3541/259968 objects misplaced (1.362%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:03:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-09-23T20:03:38.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [1, 7]}]: dispatch 2023-09-23T20:03:38.901 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:38.934 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:38.981 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:38.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:38.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:39.089 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:39.200 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:03:39.201 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:03:39.262 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:39.672 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:03:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-09-23T20:03:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:39 smithi144 ceph-mon[194833]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [1, 7]}]': finished 2023-09-23T20:03:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:39 smithi144 ceph-mon[194833]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T20:03:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:39 smithi144 ceph-mon[194833]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:39 smithi144 ceph-mon[194833]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2106907802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:39.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:39 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-09-23T20:03:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[213871]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [1, 7]}]': finished 2023-09-23T20:03:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[213871]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T20:03:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[213871]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[213871]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2106907802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-09-23T20:03:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[221113]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [1, 7]}]': finished 2023-09-23T20:03:39.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[221113]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T20:03:39.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[221113]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[221113]: failed to encode map e265 with expected crc 2023-09-23T20:03:39.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2106907802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:39.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:39 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:03:40.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:40 smithi144 ceph-mon[194833]: pgmap v719: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 3541/252549 objects misplaced (1.402%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T20:03:40.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:40 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:03:40.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:40 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:03:40.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:40 smithi144 ceph-mon[194833]: failed to encode map e266 with expected crc 2023-09-23T20:03:40.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:40 smithi144 ceph-mon[194833]: failed to encode map e266 with expected crc 2023-09-23T20:03:40.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:40 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:40.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:40 smithi144 ceph-mon[194833]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T20:03:40.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:40 smithi144 ceph-mon[194833]: failed to encode map e266 with expected crc 2023-09-23T20:03:40.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[221113]: pgmap v719: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 3541/252549 objects misplaced (1.402%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T20:03:40.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:03:40.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:03:40.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[221113]: failed to encode map e266 with expected crc 2023-09-23T20:03:40.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[221113]: failed to encode map e266 with expected crc 2023-09-23T20:03:40.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:40.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[221113]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T20:03:40.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[221113]: failed to encode map e266 with expected crc 2023-09-23T20:03:40.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[213871]: pgmap v719: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 3541/252549 objects misplaced (1.402%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T20:03:40.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:03:40.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:03:40.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[213871]: failed to encode map e266 with expected crc 2023-09-23T20:03:40.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[213871]: failed to encode map e266 with expected crc 2023-09-23T20:03:40.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:03:40.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[213871]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T20:03:40.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:40 smithi135 ceph-mon[213871]: failed to encode map e266 with expected crc 2023-09-23T20:03:42.037 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:03:42.196 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:42.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:42.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:42.300 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:42.336 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:42.392 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:42.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:42.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:42.502 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:42.594 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:42.873 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:42 smithi135 ceph-mon[213871]: pgmap v721: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 2958/250158 objects misplaced (1.182%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:03:42.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:42 smithi135 ceph-mon[221113]: pgmap v721: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 2958/250158 objects misplaced (1.182%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:03:42.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:42 smithi144 ceph-mon[194833]: pgmap v721: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 2958/250158 objects misplaced (1.182%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:03:43.253 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:43.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:43.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:43.363 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:43.434 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:43.882 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:43.884 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:03:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:03:43] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2023-09-23T20:03:43.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:43.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:43.989 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:44.019 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:44.078 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:44.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:44.183 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:44.212 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:44.261 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:44.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:44.373 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:44.381 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:44.481 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:44.611 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:44.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:44.723 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:44.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:44 smithi135 ceph-mon[221113]: pgmap v722: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 2958/247821 objects misplaced (1.194%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:03:44.730 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:44 smithi135 ceph-mon[213871]: pgmap v722: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 2958/247821 objects misplaced (1.194%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:03:44.828 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:44.911 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:03:44.952 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:44.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:44.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:44.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:03:45.014 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:03:45.061 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:03:45.064 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:45.110 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:03:45.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:44 smithi144 ceph-mon[194833]: pgmap v722: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 2958/247821 objects misplaced (1.194%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:03:45.143 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:03:45.165 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:45.185 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:03:45.221 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:03:45.273 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:03:45.307 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:03:45.307 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:03:45.307 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:03:45.307 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:03:45.307 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:03:45.308 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:03:45.308 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:03:45.308 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:03:45.378 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:45.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:45.495 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:45.588 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:45.693 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:03:45.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2740942386' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:03:45.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:45 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:03:45.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3728063665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:45.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2740942386' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:03:45.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:45 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:03:45.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3728063665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2740942386' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:03:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:45 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:03:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3728063665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[221113]: pgmap v723: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 2958/242958 objects misplaced (1.217%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:03:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[221113]: failed to encode map e267 with expected crc 2023-09-23T20:03:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[221113]: failed to encode map e267 with expected crc 2023-09-23T20:03:46.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:03:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[221113]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T20:03:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[221113]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T20:03:46.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[213871]: pgmap v723: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 2958/242958 objects misplaced (1.217%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:03:46.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[213871]: failed to encode map e267 with expected crc 2023-09-23T20:03:46.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[213871]: failed to encode map e267 with expected crc 2023-09-23T20:03:46.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:03:46.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[213871]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T20:03:46.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:46 smithi135 ceph-mon[213871]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T20:03:47.029 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:47.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:46 smithi144 ceph-mon[194833]: pgmap v723: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 2958/242958 objects misplaced (1.217%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:03:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:46 smithi144 ceph-mon[194833]: failed to encode map e267 with expected crc 2023-09-23T20:03:47.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:46 smithi144 ceph-mon[194833]: failed to encode map e267 with expected crc 2023-09-23T20:03:47.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:03:47.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:46 smithi144 ceph-mon[194833]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T20:03:47.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:46 smithi144 ceph-mon[194833]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T20:03:47.135 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:47.208 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:47.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[213871]: failed to encode map e268 with expected crc 2023-09-23T20:03:47.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[213871]: failed to encode map e268 with expected crc 2023-09-23T20:03:47.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[213871]: failed to encode map e268 with expected crc 2023-09-23T20:03:47.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[213871]: failed to encode map e268 with expected crc 2023-09-23T20:03:47.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[213871]: pgmap v726: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 2483/240711 objects misplaced (1.032%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T20:03:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[221113]: failed to encode map e268 with expected crc 2023-09-23T20:03:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[221113]: failed to encode map e268 with expected crc 2023-09-23T20:03:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[221113]: failed to encode map e268 with expected crc 2023-09-23T20:03:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[221113]: failed to encode map e268 with expected crc 2023-09-23T20:03:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:47 smithi135 ceph-mon[221113]: pgmap v726: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 2483/240711 objects misplaced (1.032%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T20:03:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:47 smithi144 ceph-mon[194833]: failed to encode map e268 with expected crc 2023-09-23T20:03:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:47 smithi144 ceph-mon[194833]: failed to encode map e268 with expected crc 2023-09-23T20:03:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:47 smithi144 ceph-mon[194833]: failed to encode map e268 with expected crc 2023-09-23T20:03:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:47 smithi144 ceph-mon[194833]: failed to encode map e268 with expected crc 2023-09-23T20:03:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:47 smithi144 ceph-mon[194833]: pgmap v726: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 2483/240711 objects misplaced (1.032%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T20:03:48.342 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:48.363 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:03:48.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:48.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:48.467 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:48.609 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:48.686 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:48.745 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:48.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:48.852 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:48.867 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:48 smithi135 ceph-mon[221113]: failed to encode map e269 with expected crc 2023-09-23T20:03:48.867 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:48 smithi135 ceph-mon[221113]: failed to encode map e269 with expected crc 2023-09-23T20:03:48.867 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:48 smithi135 ceph-mon[221113]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T20:03:48.868 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:48 smithi135 ceph-mon[213871]: failed to encode map e269 with expected crc 2023-09-23T20:03:48.868 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:48 smithi135 ceph-mon[213871]: failed to encode map e269 with expected crc 2023-09-23T20:03:48.868 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:48 smithi135 ceph-mon[213871]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T20:03:48.880 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:48.935 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:48.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:49.043 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:49.073 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:48 smithi144 ceph-mon[194833]: failed to encode map e269 with expected crc 2023-09-23T20:03:49.073 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:48 smithi144 ceph-mon[194833]: failed to encode map e269 with expected crc 2023-09-23T20:03:49.073 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:48 smithi144 ceph-mon[194833]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T20:03:49.076 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:49.132 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:49.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:49.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:49.240 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:49.255 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:03:49.255 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:03:49.255 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:03:49.255 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:03:49.255 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.255 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148788 6.f 6:f326eaaa:::benchmark_data_smithi135_344736_object46132:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 246.31881767300001, 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5928274390000001, 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148788 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928731+0000", 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928740+0000", 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4009999999999997e-06 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928755+0000", 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5481999999999998e-05 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928765+0000", 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0239e-05 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928822+0000", 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6518000000000003e-05 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928829+0000", 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3819999999999998e-06 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944149+0000", 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015319442000000001 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944158+0000", 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3000000000000007e-06 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049126+0000", 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.104968422 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049138+0000", 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1644999999999999e-05 2023-09-23T20:03:49.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.265679+0000", 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2165408280000001 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.265834+0000", 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015490799999999999 2023-09-23T20:03:49.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.521437+0000", 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25560343699999999 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.521561+0000", 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123342 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.263 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148791 6.f 6:fce301f6:::benchmark_data_smithi135_344736_object46135:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 246.31875980800001, 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5984101370000001, 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.264 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148791 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928790+0000", 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928794+0000", 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8800000000000001e-06 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928802+0000", 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0499999999999992e-06 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928809+0000", 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1800000000000001e-06 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928882+0000", 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3524999999999998e-05 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928887+0000", 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0010000000000001e-06 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944176+0000", 2023-09-23T20:03:49.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015289166 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944182+0000", 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.105e-06 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049162+0000", 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1049793050000001 2023-09-23T20:03:49.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049170+0000", 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3869999999999995e-06 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268117+0000", 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2189466419999999 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268204+0000", 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7319999999999997e-05 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527103+0000", 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25889852899999999 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527201+0000", 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8562999999999998e-05 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148804 6.b 6:d3899bd3:::benchmark_data_smithi135_344736_object46148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 246.31193397300001, 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1034934710000002, 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.272 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148804 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935615+0000", 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935620+0000", 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.775e-06 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935630+0000", 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9310000000000003e-06 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935637+0000", 2023-09-23T20:03:49.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3440000000000003e-06 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.936071+0000", 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043330099999999998 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.936077+0000", 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6900000000000003e-06 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.941346+0000", 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052686859999999999 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.941360+0000", 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3464e-05 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050328+0000", 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.108968468 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050338+0000", 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0088999999999999e-05 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.003742+0000", 2023-09-23T20:03:49.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.95340369300000005 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.003918+0000", 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017656799999999999 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.004058+0000", 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000139682 2023-09-23T20:03:49.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.007338+0000", 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003280303 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.021476+0000", 2023-09-23T20:03:49.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014137442 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.038947+0000", 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017471477999999999 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.039000+0000", 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3022000000000002e-05 2023-09-23T20:03:49.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.039110+0000", 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000109966 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148813 6.f 6:f77e0ce5:::benchmark_data_smithi135_344736_object46157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:03:49.282 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 246.28660784100001, 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.545090514, 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148813 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960941+0000", 2023-09-23T20:03:49.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960950+0000", 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6949999999999995e-06 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960969+0000", 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8785999999999999e-05 2023-09-23T20:03:49.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960980+0000", 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0703000000000001e-05 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.961043+0000", 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2600000000000004e-05 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.961057+0000", 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4535999999999999e-05 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049192+0000", 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0881350000000001 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049200+0000", 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9049999999999997e-06 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268242+0000", 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.219042129 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268348+0000", 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010592 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268839+0000", 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00049129899999999995 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279896+0000", 2023-09-23T20:03:49.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011056725 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279913+0000", 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6673000000000001e-05 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.502819+0000", 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22290622900000001 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.502884+0000", 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4616000000000006e-05 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506033+0000", 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031498979999999999 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148815 6.f 6:f7552838:::benchmark_data_smithi135_344736_object46159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 245.12455672900001, 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.385714227, 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.292 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148815 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:03:49.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122993+0000", 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123000+0000", 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9290000000000001e-06 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123009+0000", 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5689999999999994e-06 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123015+0000", 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8189999999999997e-06 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.152030+0000", 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029014671999999998 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.152042+0000", 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1756000000000001e-05 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276241+0000", 2023-09-23T20:03:49.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.124199189 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276452+0000", 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000210934 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276679+0000", 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00022696700000000001 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279921+0000", 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032419129999999999 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279990+0000", 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9807000000000004e-05 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506095+0000", 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22610454399999999 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506139+0000", 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3659000000000003e-05 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.508708+0000", 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002569661 2023-09-23T20:03:49.299 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148816 6.b 6:dde61f8d:::benchmark_data_smithi135_344736_object46160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 245.124357003, 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.001808705, 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.300 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148816 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123193+0000", 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123196+0000", 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9679999999999998e-06 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123202+0000", 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9349999999999999e-06 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123206+0000", 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9960000000000004e-06 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123247+0000", 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1238000000000001e-05 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123256+0000", 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1999999999999994e-06 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.304 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018207+0000", 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.89495181199999996 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018364+0000", 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000156269 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:03:49.305 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018501+0000", 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013728199999999999 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.033263+0000", 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014761793 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.045761+0000", 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012498489 2023-09-23T20:03:49.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.124820+0000", 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.079059044999999994 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.124879+0000", 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9138999999999997e-05 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.125003+0000", 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012307900000000001 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148820 6.f 6:f8320d95:::benchmark_data_smithi135_344736_object46164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 244.19597319100001, 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.478686642, 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148820 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051576+0000", 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051584+0000", 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6049999999999998e-06 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051600+0000", 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5387999999999999e-05 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051616+0000", 2023-09-23T20:03:49.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6228000000000001e-05 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.100356+0000", 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.048739598000000002 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.100371+0000", 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5068999999999999e-05 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280046+0000", 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17967550800000001 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280245+0000", 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019827899999999999 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280450+0000", 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000205689 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.287010+0000", 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065595749999999998 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.287104+0000", 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4639e-05 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527260+0000", 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24015514399999999 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527323+0000", 2023-09-23T20:03:49.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3256000000000006e-05 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.530264+0000", 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029414630000000001 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.316 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182372 6.b 6:d373db44:::benchmark_data_smithi135_344736_object79716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 195.79776373600001, 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1349242150000001, 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.317 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182372 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.318 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449786+0000", 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449790+0000", 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4469999999999999e-06 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.319 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449800+0000", 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6909999999999992e-06 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449809+0000", 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3009999999999995e-06 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.320 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449948+0000", 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013898400000000001 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.450078+0000", 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012999600000000001 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.450185+0000", 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010655299999999999 2023-09-23T20:03:49.321 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.322 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.322 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.322 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.457033+0000", 2023-09-23T20:03:49.322 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0068485000000000004 2023-09-23T20:03:49.322 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.322 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.323 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.323 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.457047+0000", 2023-09-23T20:03:49.323 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.376e-05 2023-09-23T20:03:49.323 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.323 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.323 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.323 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.567645+0000", 2023-09-23T20:03:49.323 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1105975640000001 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.567815+0000", 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000170104 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584606+0000", 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016791006000000001 2023-09-23T20:03:49.324 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584711+0000", 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000105554 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182400 6.b 6:ded8ad1e:::benchmark_data_smithi135_344736_object79744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:03:49.325 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 195.74366155300001, 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0838019940000001, 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182400 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.326 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503888+0000", 2023-09-23T20:03:49.327 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503895+0000", 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3239999999999997e-06 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503912+0000", 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7125999999999998e-05 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.328 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503921+0000", 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8990000000000007e-06 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503982+0000", 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0566999999999998e-05 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.329 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503995+0000", 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3181e-05 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568139+0000", 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0641444200000001 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568266+0000", 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012720799999999999 2023-09-23T20:03:49.330 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568776+0000", 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050917599999999996 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580439+0000", 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011663902 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.331 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580591+0000", 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000151371 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584754+0000", 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041630570000000004 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.332 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587511+0000", 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027568660000000002 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587567+0000", 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6041000000000001e-05 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.333 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587691+0000", 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012448400000000001 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182410 6.b 6:d95be848:::benchmark_data_smithi135_344736_object79754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:03:49.334 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 195.72584717199999, 2023-09-23T20:03:49.335 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0661552510000001, 2023-09-23T20:03:49.335 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.335 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.335 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.335 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.335 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.335 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182410 2023-09-23T20:03:49.335 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.336 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521702+0000", 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521709+0000", 2023-09-23T20:03:49.337 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.64e-06 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521725+0000", 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6078000000000001e-05 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521733+0000", 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6809999999999998e-06 2023-09-23T20:03:49.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521799+0000", 2023-09-23T20:03:49.339 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5988999999999994e-05 2023-09-23T20:03:49.339 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521812+0000", 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3307e-05 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573040+0000", 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.051228013 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573162+0000", 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012158 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573254+0000", 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2051000000000003e-05 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.341 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580447+0000", 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0071931850000000004 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580639+0000", 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000191631 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.343 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.343 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584823+0000", 2023-09-23T20:03:49.343 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041845129999999999 2023-09-23T20:03:49.343 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.343 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.343 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.343 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587735+0000", 2023-09-23T20:03:49.343 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029121139999999999 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587767+0000", 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1823999999999999e-05 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.344 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587859+0000", 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1924999999999998e-05 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182448 6.b 6:d9f3c59c:::benchmark_data_smithi135_344736_object79792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 195.64050673700001, 2023-09-23T20:03:49.345 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98102556500000004, 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182448 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.346 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.347 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607042+0000", 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607050+0000", 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6240000000000002e-06 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607066+0000", 2023-09-23T20:03:49.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5667000000000001e-05 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607074+0000", 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4460000000000001e-06 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607133+0000", 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8736999999999997e-05 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607146+0000", 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2845e-05 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.576979+0000", 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96983302800000004 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.350 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.577095+0000", 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011613899999999999 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.577196+0000", 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101648 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.351 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580685+0000", 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00348869 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580699+0000", 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3509999999999999e-05 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584873+0000", 2023-09-23T20:03:49.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041742089999999999 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587899+0000", 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003026543 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587941+0000", 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1307000000000003e-05 2023-09-23T20:03:49.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588070+0000", 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012892900000000001 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182450 6.b 6:d37fd40c:::benchmark_data_smithi135_344736_object79794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:03:49.354 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 195.63429444799999, 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97506342899999998, 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182450 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.355 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613255+0000", 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613262+0000", 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8210000000000002e-06 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613278+0000", 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5359999999999999e-05 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613286+0000", 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613340+0000", 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3922999999999999e-05 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613349+0000", 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0340000000000002e-06 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580746+0000", 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96739630700000001 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580902+0000", 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015640900000000001 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.581089+0000", 2023-09-23T20:03:49.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018731100000000001 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584767+0000", 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036773520000000001 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.585362+0000", 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00059505599999999995 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587366+0000", 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020038109999999999 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588137+0000", 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00077110600000000003 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.362 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588191+0000", 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4521999999999997e-05 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588320+0000", 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012857600000000001 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201805 6.0 6:04364655:::benchmark_data_smithi135_344736_object99149:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 164.78107576900001, 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.08500007, 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201805 2023-09-23T20:03:49.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466479+0000", 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3380000000000004e-06 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466490+0000", 2023-09-23T20:03:49.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.129e-05 2023-09-23T20:03:49.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466500+0000", 2023-09-23T20:03:49.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0305999999999999e-05 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468241+0000", 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001740143 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468253+0000", 2023-09-23T20:03:49.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2268e-05 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542090+0000", 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.073837172 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542227+0000", 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013645199999999999 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542739+0000", 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00051271599999999997 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551138+0000", 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083988829999999993 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551197+0000", 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9363000000000001e-05 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551211+0000", 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3148e-05 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551266+0000", 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5229000000000003e-05 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551475+0000", 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020925699999999999 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201830 6.10 6:08e5b611:::benchmark_data_smithi135_344736_object99174:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:03:49.373 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 164.78104012, 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1094386300000001, 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201830 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.374 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:03:49.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466510+0000", 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466513+0000", 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4109999999999999e-06 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466519+0000", 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2500000000000003e-06 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466524+0000", 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5350000000000004e-06 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468278+0000", 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00175377 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468286+0000", 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9920000000000007e-06 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551513+0000", 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0832270980000001 2023-09-23T20:03:49.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.552197+0000", 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00068353800000000003 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556569+0000", 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043722819999999999 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556637+0000", 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8348000000000004e-05 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.557117+0000", 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00047992400000000001 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560978+0000", 2023-09-23T20:03:49.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038601740000000001 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561510+0000", 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00053195499999999997 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575543+0000", 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014033406 2023-09-23T20:03:49.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575578+0000", 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5404999999999997e-05 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575949+0000", 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037097900000000001 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.383 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201836 6.10 6:0c24ea53:::benchmark_data_smithi135_344736_object99180:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 164.78102126300001, 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1159160830000001, 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.384 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201836 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466529+0000", 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466532+0000", 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.43e-06 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466549+0000", 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7680999999999999e-05 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466556+0000", 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0380000000000002e-06 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468309+0000", 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001753075 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468317+0000", 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8699999999999992e-06 2023-09-23T20:03:49.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554470+0000", 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.086153084 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555087+0000", 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000616151 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.569798+0000", 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014711695 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.569892+0000", 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3364999999999998e-05 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:49.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.570445+0000", 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00055320300000000005 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576273+0000", 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058281779999999998 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582033+0000", 2023-09-23T20:03:49.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0057600059999999998 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582060+0000", 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7141999999999999e-05 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582101+0000", 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1272000000000001e-05 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582446+0000", 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034424599999999998 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201842 6.0 6:03218c77:::benchmark_data_smithi135_344736_object99186:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:03:49.393 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 164.780988753, 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.088181898, 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201842 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466565+0000", 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2610000000000002e-06 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466571+0000", 2023-09-23T20:03:49.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6059999999999999e-06 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466576+0000", 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9280000000000001e-06 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468350+0000", 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017738249999999999 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468359+0000", 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4190000000000002e-06 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548143+0000", 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.079783959 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548255+0000", 2023-09-23T20:03:49.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011229400000000001 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548588+0000", 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00033316999999999998 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554427+0000", 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058389430000000001 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554444+0000", 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7081000000000001e-05 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554491+0000", 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6495999999999997e-05 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554542+0000", 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1561999999999999e-05 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554744+0000", 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000201883 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201845 6.10 6:0a5d128c:::benchmark_data_smithi135_344736_object99189:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 164.780970164, 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1274511190000001, 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201845 2023-09-23T20:03:49.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466580+0000", 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466583+0000", 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4509999999999999e-06 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466588+0000", 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2159999999999997e-06 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466592+0000", 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4220000000000002e-06 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468381+0000", 2023-09-23T20:03:49.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001788778 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468389+0000", 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9689999999999994e-06 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555129+0000", 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.086740209 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555800+0000", 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00067015300000000005 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.572929+0000", 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017129485999999999 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.572997+0000", 2023-09-23T20:03:49.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7753e-05 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.573486+0000", 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00048920300000000001 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582000+0000", 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085139759999999995 2023-09-23T20:03:49.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585622+0000", 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003622489 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.593594+0000", 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0079714260000000002 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.410 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.593638+0000", 2023-09-23T20:03:49.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4351000000000003e-05 2023-09-23T20:03:49.410 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.410 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594032+0000", 2023-09-23T20:03:49.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00039355800000000002 2023-09-23T20:03:49.411 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.411 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.411 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.411 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201858 6.10 6:0e829f63:::benchmark_data_smithi135_344736_object99202:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 164.78095481700001, 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.12779443, 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201858 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466598+0000", 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8920000000000001e-06 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466601+0000", 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0869999999999998e-06 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466604+0000", 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.596e-06 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468607+0000", 2023-09-23T20:03:49.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020026620000000001 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468616+0000", 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7739999999999993e-06 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560293+0000", 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0916775329999999 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560929+0000", 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00063605599999999997 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576296+0000", 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01536671 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576359+0000", 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3100000000000002e-05 2023-09-23T20:03:49.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576772+0000", 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00041272599999999998 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582458+0000", 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056866119999999997 2023-09-23T20:03:49.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585662+0000", 2023-09-23T20:03:49.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032034730000000001 2023-09-23T20:03:49.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594087+0000", 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084249849999999994 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594137+0000", 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9614000000000003e-05 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594390+0000", 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00025390100000000001 2023-09-23T20:03:49.422 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201894 6.0 6:00f5be23:::benchmark_data_smithi135_344736_object99238:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 164.78094256200001, 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0976265000000001, 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.423 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201894 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466610+0000", 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3309999999999998e-06 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466616+0000", 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3630000000000002e-06 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466621+0000", 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1719999999999998e-06 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468641+0000", 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020203489999999998 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468650+0000", 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3739999999999998e-06 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551524+0000", 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0828746149999999 2023-09-23T20:03:49.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551624+0000", 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100259 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551973+0000", 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034831199999999998 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556557+0000", 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045842579999999999 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556597+0000", 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0182000000000003e-05 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.563996+0000", 2023-09-23T20:03:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073992540000000001 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.564035+0000", 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8602000000000001e-05 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.564235+0000", 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019976 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.430 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201912 6.10 6:0a9a0b94:::benchmark_data_smithi135_344736_object99256:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 164.78092564299999, 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1498471859999999, 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:03:49.431 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201912 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:03:49.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466628+0000", 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6249999999999999e-06 2023-09-23T20:03:49.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466633+0000", 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6210000000000001e-06 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466638+0000", 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6809999999999997e-06 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468671+0000", 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020332229999999998 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468677+0000", 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8409999999999996e-06 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561014+0000", 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0923374370000001 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561497+0000", 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00048270000000000002 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579372+0000", 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017874660000000001 2023-09-23T20:03:49.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579464+0000", 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2050000000000001e-05 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579912+0000", 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00044868700000000002 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582554+0000", 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00264197 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585730+0000", 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031761210000000001 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:03:49.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.615988+0000", 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030257373000000001 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.616043+0000", 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4648000000000003e-05 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.616472+0000", 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00042985000000000001 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.439 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.440 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.440 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:03:49.440 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:03:49.440 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:03:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:49.441 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:49.441 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:49.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:49.542 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:03:49.604 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:50.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:49 smithi144 ceph-mon[194833]: pgmap v728: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 781 op/s; 2483/233952 objects misplaced (1.061%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:03:50.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:49 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:03:50.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:49 smithi144 ceph-mon[194833]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:49 smithi144 ceph-mon[194833]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:49 smithi144 ceph-mon[194833]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:49 smithi144 ceph-mon[194833]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T20:03:50.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:49 smithi144 ceph-mon[194833]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[213871]: pgmap v728: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 781 op/s; 2483/233952 objects misplaced (1.061%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:03:50.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:03:50.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[213871]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[213871]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[213871]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[213871]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T20:03:50.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[213871]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[221113]: pgmap v728: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 781 op/s; 2483/233952 objects misplaced (1.061%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:03:50.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:03:50.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[221113]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[221113]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[221113]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[221113]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T20:03:50.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:49 smithi135 ceph-mon[221113]: failed to encode map e270 with expected crc 2023-09-23T20:03:50.224 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:03:50.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:50.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:03:50.244 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:03:50.331 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:50.497 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:50.649 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:50.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:50.765 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:50.771 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[221113]: failed to encode map e271 with expected crc 2023-09-23T20:03:50.771 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[221113]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T20:03:50.771 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[221113]: failed to encode map e271 with expected crc 2023-09-23T20:03:50.772 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[221113]: failed to encode map e271 with expected crc 2023-09-23T20:03:50.772 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[221113]: failed to encode map e271 with expected crc 2023-09-23T20:03:50.772 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[213871]: failed to encode map e271 with expected crc 2023-09-23T20:03:50.772 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[213871]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T20:03:50.772 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[213871]: failed to encode map e271 with expected crc 2023-09-23T20:03:50.772 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[213871]: failed to encode map e271 with expected crc 2023-09-23T20:03:50.772 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:50 smithi135 ceph-mon[213871]: failed to encode map e271 with expected crc 2023-09-23T20:03:50.792 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:50.854 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:50.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:50.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:50.959 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:50.984 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:51.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:50 smithi144 ceph-mon[194833]: failed to encode map e271 with expected crc 2023-09-23T20:03:51.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:50 smithi144 ceph-mon[194833]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T20:03:51.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:50 smithi144 ceph-mon[194833]: failed to encode map e271 with expected crc 2023-09-23T20:03:51.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:50 smithi144 ceph-mon[194833]: failed to encode map e271 with expected crc 2023-09-23T20:03:51.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:50 smithi144 ceph-mon[194833]: failed to encode map e271 with expected crc 2023-09-23T20:03:51.658 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:03:51.687 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:51.724 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:51.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:51.831 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:51.863 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:51.887 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:03:51.925 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:51.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:51.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:51 smithi135 ceph-mon[213871]: pgmap v731: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 755 op/s; 2042/231813 objects misplaced (0.881%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:03:51.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:51 smithi135 ceph-mon[213871]: 1.2 deep-scrub starts 2023-09-23T20:03:51.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:51 smithi135 ceph-mon[213871]: 1.2 deep-scrub ok 2023-09-23T20:03:51.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:51 smithi135 ceph-mon[221113]: pgmap v731: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 755 op/s; 2042/231813 objects misplaced (0.881%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:03:51.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:51 smithi135 ceph-mon[221113]: 1.2 deep-scrub starts 2023-09-23T20:03:51.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:51 smithi135 ceph-mon[221113]: 1.2 deep-scrub ok 2023-09-23T20:03:52.029 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:52.077 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:51 smithi144 ceph-mon[194833]: pgmap v731: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 755 op/s; 2042/231813 objects misplaced (0.881%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:03:52.077 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:51 smithi144 ceph-mon[194833]: 1.2 deep-scrub starts 2023-09-23T20:03:52.077 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:51 smithi144 ceph-mon[194833]: 1.2 deep-scrub ok 2023-09-23T20:03:52.080 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:52.113 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:03:52.142 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:52.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:52.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:52.246 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:52.276 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:52.391 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:52.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:52.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:52.497 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:52.577 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:52.749 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:52.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:52.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:52.853 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:52.890 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:52.948 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:52.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:52.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:53.052 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:53.081 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:53.133 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:53.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:53.160 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:53.160 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:53.160 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/745814468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:03:53.160 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:03:53.163 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:53.163 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:53.163 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/745814468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:03:53.163 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:03:53.239 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:53.347 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:03:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:03:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/745814468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:03:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:03:53.454 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:03:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:03:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:03:53] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-09-23T20:03:54.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[213871]: pgmap v732: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 649 op/s; 2042/229212 objects misplaced (0.891%); 3.8 MiB/s, 59 objects/s recovering 2023-09-23T20:03:54.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:54.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[213871]: failed to encode map e272 with expected crc 2023-09-23T20:03:54.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:03:54.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[213871]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T20:03:54.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[213871]: failed to encode map e272 with expected crc 2023-09-23T20:03:54.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[221113]: pgmap v732: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 649 op/s; 2042/229212 objects misplaced (0.891%); 3.8 MiB/s, 59 objects/s recovering 2023-09-23T20:03:54.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:54.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[221113]: failed to encode map e272 with expected crc 2023-09-23T20:03:54.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:03:54.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[221113]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T20:03:54.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:54 smithi135 ceph-mon[221113]: failed to encode map e272 with expected crc 2023-09-23T20:03:54.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:54 smithi144 ceph-mon[194833]: pgmap v732: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 649 op/s; 2042/229212 objects misplaced (0.891%); 3.8 MiB/s, 59 objects/s recovering 2023-09-23T20:03:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:54 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:03:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:54 smithi144 ceph-mon[194833]: failed to encode map e272 with expected crc 2023-09-23T20:03:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:54 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:03:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:54 smithi144 ceph-mon[194833]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T20:03:54.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:54 smithi144 ceph-mon[194833]: failed to encode map e272 with expected crc 2023-09-23T20:03:54.951 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:54.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:54.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:55.060 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:55.166 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:55.183 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:121077 7.1 7:8e0eda1d:::benchmark_data_smithi135_404353_object24015:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:03:55.179633+0000", 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0036173139999999999, 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0036483399999999999, 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:03:55.184 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 121077 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179633+0000", 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:03:55.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179633+0000", 2023-09-23T20:03:55.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:03:55.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:03:55.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179632+0000", 2023-09-23T20:03:55.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:03:55.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179638+0000", 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8200000000000002e-06 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179653+0000", 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5140000000000001e-05 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179660+0000", 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.109e-06 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179705+0000", 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4505000000000002e-05 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179835+0000", 2023-09-23T20:03:55.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013029200000000001 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179964+0000", 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012921800000000001 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:121085 7.1 7:831822bf:::benchmark_data_smithi135_404353_object24023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T20:03:55.189 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:03:55.179723+0000", 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0035275150000000002, 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0036546840000000001, 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 121085 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179723+0000", 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179723+0000", 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179722+0000", 2023-09-23T20:03:55.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179728+0000", 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.891e-06 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179741+0000", 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2799000000000001e-05 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.179747+0000", 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.844e-06 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.182514+0000", 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027669410000000002 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:03:55.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.182642+0000", 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012746599999999999 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.182761+0000", 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000119258 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.194 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35358.0:121075 7.3 e272/240 7:c3a0e658:::benchmark_data_smithi135_404353_object24013:head v 272'7660, mlcod=272'7658)", 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:03:55.180491+0000", 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0027592739999999999, 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029585929999999998, 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.180491+0000", 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.180491+0000", 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.180491+0000", 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:03:55.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.180497+0000", 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3219999999999998e-06 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.180510+0000", 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2632e-05 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.180515+0000", 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2170000000000002e-06 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.180558+0000", 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3436999999999998e-05 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:03:55.180574+0000", 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6079999999999999e-05 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:03:55.198 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:03:55.199 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2023-09-23T20:03:55.199 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 3 2023-09-23T20:03:55.199 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:03:55.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:55 smithi144 ceph-mon[194833]: failed to encode map e272 with expected crc 2023-09-23T20:03:55.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:55 smithi144 ceph-mon[194833]: failed to encode map e272 with expected crc 2023-09-23T20:03:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:55 smithi135 ceph-mon[221113]: failed to encode map e272 with expected crc 2023-09-23T20:03:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:55 smithi135 ceph-mon[221113]: failed to encode map e272 with expected crc 2023-09-23T20:03:55.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:55 smithi135 ceph-mon[213871]: failed to encode map e272 with expected crc 2023-09-23T20:03:55.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:55 smithi135 ceph-mon[213871]: failed to encode map e272 with expected crc 2023-09-23T20:03:55.362 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:55.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:55.473 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:55.506 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:55.552 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:55.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:55.656 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:03:55.688 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:55.949 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:03:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:55.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:03:56.011 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:03:56.052 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:56.133 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:56.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:56 smithi135 ceph-mon[221113]: pgmap v734: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 2042/224244 objects misplaced (0.911%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:03:56.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:56 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:03:56.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:56 smithi135 ceph-mon[213871]: pgmap v734: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 2042/224244 objects misplaced (0.911%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:03:56.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:56 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:03:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:56 smithi144 ceph-mon[194833]: pgmap v734: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 2042/224244 objects misplaced (0.911%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:03:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:56 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:03:56.437 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:03:56.467 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:56.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:56.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:56.582 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:56.683 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:56.818 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:56.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:56.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:56.836 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:03:56.885 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:03:56.924 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:03:56.941 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:03:56.990 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:03:57.038 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:57.054 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:03:57.106 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:03:57.143 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:03:57.167 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:03:57.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:57.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:03:57.190 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:03:57.194 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:03:57.194 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:03:57.195 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:03:57.195 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:03:57.195 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:03:57.195 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:03:57.195 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:03:57.195 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:03:57.278 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:03:57.305 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/293914936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:57.306 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/293914936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:57.309 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:57.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/293914936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:03:57.544 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:03:57.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:57.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:03:57.652 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:03:57.693 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:57.777 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:03:57.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:57.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:03:57.883 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:57.969 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:58.101 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:58.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:58.138 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:03:58.210 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:58.217 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:03:58.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:58 smithi135 ceph-mon[221113]: pgmap v735: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 1462/221967 objects misplaced (0.659%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:03:58.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/183368623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:03:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:58 smithi135 ceph-mon[213871]: pgmap v735: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 1462/221967 objects misplaced (0.659%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:03:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/183368623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:03:58.269 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:58.271 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:03:58.271 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:03:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:58 smithi144 ceph-mon[194833]: pgmap v735: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 1462/221967 objects misplaced (0.659%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:03:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/183368623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:03:58.483 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:03:58.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:58.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:03:58.586 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:03:58.615 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:58.658 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:03:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:58.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:03:58.761 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:03:58.788 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:59.028 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:03:59.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:03:59.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:03:59.076 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:03:59.132 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:03:59.159 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:03:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:59 smithi144 ceph-mon[194833]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:59 smithi144 ceph-mon[194833]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:59 smithi144 ceph-mon[194833]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/183368623' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:03:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:03:59 smithi144 ceph-mon[194833]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T20:03:59.431 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:03:59.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[221113]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[221113]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[221113]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/183368623' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:03:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[221113]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T20:03:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[213871]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[213871]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[213871]: failed to encode map e273 with expected crc 2023-09-23T20:03:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/183368623' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:03:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:03:59 smithi135 ceph-mon[213871]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T20:04:00.001 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:00.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:00.084 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:04:00.084 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:04:00.104 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:00.160 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:00.197 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:00.218 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:00.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:00.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:00.326 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:00.351 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:00 smithi144 ceph-mon[194833]: pgmap v737: 99 pgs: 7 peering, 1 active+remapped+backfilling, 91 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 1462/215877 objects misplaced (0.677%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:04:00.351 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:00 smithi144 ceph-mon[194833]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.351 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:00 smithi144 ceph-mon[194833]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.351 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:00 smithi144 ceph-mon[194833]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.351 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:00 smithi144 ceph-mon[194833]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T20:04:00.351 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:00 smithi144 ceph-mon[194833]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.354 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:00.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[221113]: pgmap v737: 99 pgs: 7 peering, 1 active+remapped+backfilling, 91 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 1462/215877 objects misplaced (0.677%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:04:00.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[221113]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[221113]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[221113]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[221113]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T20:04:00.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[221113]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.459 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[213871]: pgmap v737: 99 pgs: 7 peering, 1 active+remapped+backfilling, 91 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 1462/215877 objects misplaced (0.677%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:04:00.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[213871]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[213871]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[213871]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[213871]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T20:04:00.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:00 smithi135 ceph-mon[213871]: failed to encode map e274 with expected crc 2023-09-23T20:04:00.475 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:00.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:00.578 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:00.613 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:00.694 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:00.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:00.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:00.798 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:00.926 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:01.016 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:04:01.016 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:04:01.147 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:01.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:01.251 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:01.332 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T20:04:01.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: 7.6 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7797 2023-09-23T20:04:01.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: 7.c starting backfill to osd.3 from (0'0,0'0] MAX to 272'7593 2023-09-23T20:04:01.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: 7.7 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7712 2023-09-23T20:04:01.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:01 smithi144 ceph-mon[194833]: 7.1 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7635 2023-09-23T20:04:01.403 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: 7.6 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7797 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: 7.c starting backfill to osd.3 from (0'0,0'0] MAX to 272'7593 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: 7.7 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7712 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[221113]: 7.1 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7635 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: 7.6 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7797 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: 7.c starting backfill to osd.3 from (0'0,0'0] MAX to 272'7593 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: 7.7 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7712 2023-09-23T20:04:01.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: failed to encode map e275 with expected crc 2023-09-23T20:04:01.407 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:01 smithi135 ceph-mon[213871]: 7.1 starting backfill to osd.3 from (0'0,0'0] MAX to 272'7635 2023-09-23T20:04:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:01.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:01.509 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:01.546 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:01.630 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:01.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:01.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:01.734 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:01.764 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:01.768 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:04:01.890 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:01.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:01.945 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:04:01.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:02.026 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:02.090 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:02.137 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:02.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:02.244 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:02.263 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:02.282 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:02.346 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:02.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:02.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[221113]: pgmap v740: 99 pgs: 1 active+clean+remapped, 7 peering, 91 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 20 MiB/s, 323 objects/s recovering 2023-09-23T20:04:02.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[221113]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[221113]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[221113]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[221113]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T20:04:02.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[221113]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:04:02.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[213871]: pgmap v740: 99 pgs: 1 active+clean+remapped, 7 peering, 91 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 20 MiB/s, 323 objects/s recovering 2023-09-23T20:04:02.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[213871]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[213871]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[213871]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[213871]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T20:04:02.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[213871]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:04:02.384 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:02.456 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:02.491 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:02.620 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:02.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:02.624 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:02 smithi144 ceph-mon[194833]: pgmap v740: 99 pgs: 1 active+clean+remapped, 7 peering, 91 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 20 MiB/s, 323 objects/s recovering 2023-09-23T20:04:02.624 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:02 smithi144 ceph-mon[194833]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.624 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:02 smithi144 ceph-mon[194833]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.625 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:02 smithi144 ceph-mon[194833]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.625 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:02 smithi144 ceph-mon[194833]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T20:04:02.625 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:02 smithi144 ceph-mon[194833]: failed to encode map e276 with expected crc 2023-09-23T20:04:02.625 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:04:02.723 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:02.768 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:02.826 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:02.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:02.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:02.934 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:03.045 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:03.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[221113]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T20:04:03.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[221113]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[221113]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[221113]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.544 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[221113]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.544 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[221113]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.545 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[213871]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T20:04:03.545 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[213871]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.545 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[213871]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.545 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[213871]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.545 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[213871]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.545 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:03 smithi135 ceph-mon[213871]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:03 smithi144 ceph-mon[194833]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T20:04:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:03 smithi144 ceph-mon[194833]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:03 smithi144 ceph-mon[194833]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:03 smithi144 ceph-mon[194833]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:03 smithi144 ceph-mon[194833]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:03 smithi144 ceph-mon[194833]: failed to encode map e277 with expected crc 2023-09-23T20:04:03.978 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:04:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:04:03] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-09-23T20:04:04.347 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:04:04.596 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:04:04.597 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:04:04.622 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:04 smithi144 ceph-mon[194833]: pgmap v743: 99 pgs: 2 active+remapped+backfill_wait, 1 active+clean+remapped, 9 peering, 87 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 8837/212736 objects misplaced (4.154%); 22 MiB/s, 359 objects/s recovering 2023-09-23T20:04:04.622 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1748120397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:04.622 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:04 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:04.716 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:04.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:04.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:04.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:04 smithi135 ceph-mon[221113]: pgmap v743: 99 pgs: 2 active+remapped+backfill_wait, 1 active+clean+remapped, 9 peering, 87 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 8837/212736 objects misplaced (4.154%); 22 MiB/s, 359 objects/s recovering 2023-09-23T20:04:04.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1748120397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:04.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:04 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:04.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:04 smithi135 ceph-mon[213871]: pgmap v743: 99 pgs: 2 active+remapped+backfill_wait, 1 active+clean+remapped, 9 peering, 87 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 8837/212736 objects misplaced (4.154%); 22 MiB/s, 359 objects/s recovering 2023-09-23T20:04:04.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1748120397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:04.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:04 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:04.825 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:04.951 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:05.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:05.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:05.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[213871]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T20:04:05.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[213871]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[213871]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[213871]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[213871]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[213871]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.290 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:05.290 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:05.290 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[221113]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T20:04:05.350 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:04:05.382 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:05.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:05.488 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:05.519 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:05.585 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:05.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:05 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:05.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:05 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:05.588 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:05 smithi144 ceph-mon[194833]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T20:04:05.588 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:05 smithi144 ceph-mon[194833]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.588 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:05 smithi144 ceph-mon[194833]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.588 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:05 smithi144 ceph-mon[194833]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.588 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:05 smithi144 ceph-mon[194833]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.588 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:05 smithi144 ceph-mon[194833]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:05.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:05.689 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:05.715 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[221113]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[221113]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[221113]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[221113]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:05 smithi135 ceph-mon[221113]: failed to encode map e278 with expected crc 2023-09-23T20:04:05.825 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:05.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:05.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:05.934 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:05.970 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:05.980 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:06.122 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:06.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:06.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:06.226 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:06.297 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:06.317 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:06 smithi144 ceph-mon[194833]: pgmap v745: 99 pgs: 4 active+remapped+backfill_wait, 1 active+clean+remapped, 2 peering, 92 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 17309/211137 objects misplaced (8.198%); 26 MiB/s, 420 objects/s recovering 2023-09-23T20:04:06.561 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:06 smithi135 ceph-mon[221113]: pgmap v745: 99 pgs: 4 active+remapped+backfill_wait, 1 active+clean+remapped, 2 peering, 92 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 17309/211137 objects misplaced (8.198%); 26 MiB/s, 420 objects/s recovering 2023-09-23T20:04:06.562 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:06 smithi135 ceph-mon[213871]: pgmap v745: 99 pgs: 4 active+remapped+backfill_wait, 1 active+clean+remapped, 2 peering, 92 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 17309/211137 objects misplaced (8.198%); 26 MiB/s, 420 objects/s recovering 2023-09-23T20:04:06.683 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:06.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:06.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:06.792 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:06.883 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:07.040 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:07.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:07.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:07.131 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:04:07.148 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:07.176 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:04:07.181 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:07.205 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:04:07.206 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:04:07.258 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:04:07.319 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:04:07.360 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:07.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:07.369 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:04:07.415 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:04:07.450 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:04:07.468 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:07.497 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:04:07.517 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:04:07.517 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:04:07.517 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:04:07.517 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:04:07.517 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:04:07.517 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:04:07.517 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:04:07.517 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:04:07.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:07 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3539384002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:07.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:07.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:07.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:07 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3539384002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:07.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:07.544 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:07.550 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:07 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3539384002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:07.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:08.178 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:04:08.179 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:04:08.179 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:04:08.180 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:04:08.180 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:04:08.297 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:04:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:08 smithi135 ceph-mon[221113]: pgmap v746: 99 pgs: 4 active+remapped+backfill_wait, 2 peering, 93 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 17309/209622 objects misplaced (8.257%); 4.5 MiB/s, 74 objects/s recovering 2023-09-23T20:04:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:08 smithi135 ceph-mon[221113]: from='client.55216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:08.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:08.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:08 smithi135 ceph-mon[213871]: pgmap v746: 99 pgs: 4 active+remapped+backfill_wait, 2 peering, 93 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 17309/209622 objects misplaced (8.257%); 4.5 MiB/s, 74 objects/s recovering 2023-09-23T20:04:08.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:08 smithi135 ceph-mon[213871]: from='client.55216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:08.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:08.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:08 smithi144 ceph-mon[194833]: pgmap v746: 99 pgs: 4 active+remapped+backfill_wait, 2 peering, 93 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 17309/209622 objects misplaced (8.257%); 4.5 MiB/s, 74 objects/s recovering 2023-09-23T20:04:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:08 smithi144 ceph-mon[194833]: from='client.55216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:08.992 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:04:09.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:09.087 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:09.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:09.188 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:09.221 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:09.290 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:09.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:09.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:09.398 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:09.424 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:09.491 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:09.663 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":746,"stamp":"2023-09-23T20:04:06.702481+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4582152139,"num_objects":69874,"num_object_clones":0,"num_object_copies":209622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":17309,"num_objects_unfound":0,"num_objects_dirty":69874,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":125181,"num_write_kb":6224502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5648,"num_bytes_recovered":363010549,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106945,"ondisk_log_size":106945,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16288404,"kb_used_data":13789212,"kb_used_omap":32,"kb_used_meta":2499039,"kb_avail":639927660,"statfs":{"total":671965249536,"available":655285923840,"internally_reserved":0,"allocated":14120153088,"data_stored":14112705501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":2559016454},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-116850688,"num_objects":-1783,"num_object_clones":0,"num_object_copies":-5349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":17309,"num_objects_unfound":0,"num_objects_dirty":-1783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1783,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":445,"num_bytes_recovered":28246016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.007731"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":255,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267819+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T20:04:01.267819+0000","last_peered":"2023-09-23T20:04:01.267819+0000","last_clean":"2023-09-23T20:04:01.267819+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T20:04:01.267819+0000","last_undegraded":"2023-09-23T20:04:01.267819+0000","last_fullsized":"2023-09-23T20:04:01.267819+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":97,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.475711+0000","last_change":"2023-09-23T19:59:09.151117+0000","last_active":"2023-09-23T20:04:00.475711+0000","last_peered":"2023-09-23T20:04:00.475711+0000","last_clean":"2023-09-23T20:04:00.475711+0000","last_became_active":"2023-09-23T19:59:09.150795+0000","last_became_peered":"2023-09-23T19:59:09.150795+0000","last_unstale":"2023-09-23T20:04:00.475711+0000","last_undegraded":"2023-09-23T20:04:00.475711+0000","last_fullsized":"2023-09-23T20:04:00.475711+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:34.994574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":238,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.261112+0000","last_change":"2023-09-23T19:59:10.156070+0000","last_active":"2023-09-23T20:04:02.261112+0000","last_peered":"2023-09-23T20:04:02.261112+0000","last_clean":"2023-09-23T20:04:02.261112+0000","last_became_active":"2023-09-23T19:59:10.155813+0000","last_became_peered":"2023-09-23T19:59:10.155813+0000","last_unstale":"2023-09-23T20:04:02.261112+0000","last_undegraded":"2023-09-23T20:04:02.261112+0000","last_fullsized":"2023-09-23T20:04:02.261112+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:42.582380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":29,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261271+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:04:01.261271+0000","last_peered":"2023-09-23T20:04:01.261271+0000","last_clean":"2023-09-23T20:04:01.261271+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:04:01.261271+0000","last_undegraded":"2023-09-23T20:04:01.261271+0000","last_fullsized":"2023-09-23T20:04:01.261271+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":98,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.266914+0000","last_change":"2023-09-23T19:59:09.163182+0000","last_active":"2023-09-23T20:04:01.266914+0000","last_peered":"2023-09-23T20:04:01.266914+0000","last_clean":"2023-09-23T20:04:01.266914+0000","last_became_active":"2023-09-23T19:59:09.161612+0000","last_became_peered":"2023-09-23T19:59:09.161612+0000","last_unstale":"2023-09-23T20:04:01.266914+0000","last_undegraded":"2023-09-23T20:04:01.266914+0000","last_fullsized":"2023-09-23T20:04:01.266914+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:59.524152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":202,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258379+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T20:04:01.258379+0000","last_peered":"2023-09-23T20:04:01.258379+0000","last_clean":"2023-09-23T20:04:01.258379+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T20:04:01.258379+0000","last_undegraded":"2023-09-23T20:04:01.258379+0000","last_fullsized":"2023-09-23T20:04:01.258379+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:59.043547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":203,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258257+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:04:01.258257+0000","last_peered":"2023-09-23T20:04:01.258257+0000","last_clean":"2023-09-23T20:04:01.258257+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:04:01.258257+0000","last_undegraded":"2023-09-23T20:04:01.258257+0000","last_fullsized":"2023-09-23T20:04:01.258257+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":98,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.475449+0000","last_change":"2023-09-23T19:59:09.146009+0000","last_active":"2023-09-23T20:04:00.475449+0000","last_peered":"2023-09-23T20:04:00.475449+0000","last_clean":"2023-09-23T20:04:00.475449+0000","last_became_active":"2023-09-23T19:59:09.145687+0000","last_became_peered":"2023-09-23T19:59:09.145687+0000","last_unstale":"2023-09-23T20:04:00.475449+0000","last_undegraded":"2023-09-23T20:04:00.475449+0000","last_fullsized":"2023-09-23T20:04:00.475449+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.189450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":236,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258099+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T20:04:01.258099+0000","last_peered":"2023-09-23T20:04:01.258099+0000","last_clean":"2023-09-23T20:04:01.258099+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T20:04:01.258099+0000","last_undegraded":"2023-09-23T20:04:01.258099+0000","last_fullsized":"2023-09-23T20:04:01.258099+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":117,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.581749+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:04:00.581749+0000","last_peered":"2023-09-23T20:04:00.581749+0000","last_clean":"2023-09-23T20:04:00.581749+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:04:00.581749+0000","last_undegraded":"2023-09-23T20:04:00.581749+0000","last_fullsized":"2023-09-23T20:04:00.581749+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":272,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261475+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:04:01.261475+0000","last_peered":"2023-09-23T20:04:01.261475+0000","last_clean":"2023-09-23T20:04:01.261475+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:04:01.261475+0000","last_undegraded":"2023-09-23T20:04:01.261475+0000","last_fullsized":"2023-09-23T20:04:01.261475+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":235,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.260794+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T20:04:02.260794+0000","last_peered":"2023-09-23T20:04:02.260794+0000","last_clean":"2023-09-23T20:04:02.260794+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T20:04:02.260794+0000","last_undegraded":"2023-09-23T20:04:02.260794+0000","last_fullsized":"2023-09-23T20:04:02.260794+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:51.825640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":291,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268156+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:04:01.268156+0000","last_peered":"2023-09-23T20:04:01.268156+0000","last_clean":"2023-09-23T20:04:01.268156+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:04:01.268156+0000","last_undegraded":"2023-09-23T20:04:01.268156+0000","last_fullsized":"2023-09-23T20:04:01.268156+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":253,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268336+0000","last_change":"2023-09-23T20:01:52.439375+0000","last_active":"2023-09-23T20:04:01.268336+0000","last_peered":"2023-09-23T20:04:01.268336+0000","last_clean":"2023-09-23T20:04:01.268336+0000","last_became_active":"2023-09-23T20:01:52.439074+0000","last_became_peered":"2023-09-23T20:01:52.439074+0000","last_unstale":"2023-09-23T20:04:01.268336+0000","last_undegraded":"2023-09-23T20:04:01.268336+0000","last_fullsized":"2023-09-23T20:04:01.268336+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:31.103633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":239,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261398+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:04:01.261398+0000","last_peered":"2023-09-23T20:04:01.261398+0000","last_clean":"2023-09-23T20:04:01.261398+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:04:01.261398+0000","last_undegraded":"2023-09-23T20:04:01.261398+0000","last_fullsized":"2023-09-23T20:04:01.261398+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":98,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.589338+0000","last_change":"2023-09-23T19:59:09.195560+0000","last_active":"2023-09-23T20:04:00.589338+0000","last_peered":"2023-09-23T20:04:00.589338+0000","last_clean":"2023-09-23T20:04:00.589338+0000","last_became_active":"2023-09-23T19:59:09.195145+0000","last_became_peered":"2023-09-23T19:59:09.195145+0000","last_unstale":"2023-09-23T20:04:00.589338+0000","last_undegraded":"2023-09-23T20:04:00.589338+0000","last_fullsized":"2023-09-23T20:04:00.589338+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.190798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":242,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261112+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:04:01.261112+0000","last_peered":"2023-09-23T20:04:01.261112+0000","last_clean":"2023-09-23T20:04:01.261112+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:04:01.261112+0000","last_undegraded":"2023-09-23T20:04:01.261112+0000","last_fullsized":"2023-09-23T20:04:01.261112+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":255,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258828+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:04:01.258828+0000","last_peered":"2023-09-23T20:04:01.258828+0000","last_clean":"2023-09-23T20:04:01.258828+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:04:01.258828+0000","last_undegraded":"2023-09-23T20:04:01.258828+0000","last_fullsized":"2023-09-23T20:04:01.258828+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":237,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261124+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:04:01.261124+0000","last_peered":"2023-09-23T20:04:01.261124+0000","last_clean":"2023-09-23T20:04:01.261124+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:04:01.261124+0000","last_undegraded":"2023-09-23T20:04:01.261124+0000","last_fullsized":"2023-09-23T20:04:01.261124+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":241,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.582954+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:04:00.582954+0000","last_peered":"2023-09-23T20:04:00.582954+0000","last_clean":"2023-09-23T20:04:00.582954+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:04:00.582954+0000","last_undegraded":"2023-09-23T20:04:00.582954+0000","last_fullsized":"2023-09-23T20:04:00.582954+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":70,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.257897+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:04:01.257897+0000","last_peered":"2023-09-23T20:04:01.257897+0000","last_clean":"2023-09-23T20:04:01.257897+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:04:01.257897+0000","last_undegraded":"2023-09-23T20:04:01.257897+0000","last_fullsized":"2023-09-23T20:04:01.257897+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.279534+0000","last_change":"2023-09-23T20:03:59.232022+0000","last_active":"2023-09-23T20:04:01.279534+0000","last_peered":"2023-09-23T20:04:01.279534+0000","last_clean":"2023-09-23T20:04:01.279534+0000","last_became_active":"2023-09-23T20:03:59.231689+0000","last_became_peered":"2023-09-23T20:03:59.231689+0000","last_unstale":"2023-09-23T20:04:01.279534+0000","last_undegraded":"2023-09-23T20:04:01.279534+0000","last_fullsized":"2023-09-23T20:04:01.279534+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:29.669102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"278'7757","reported_seq":18039,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:05.487623+0000","last_change":"2023-09-23T20:04:02.290062+0000","last_active":"2023-09-23T20:04:05.487623+0000","last_peered":"2023-09-23T20:04:05.487623+0000","last_clean":"2023-09-23T20:04:05.487623+0000","last_became_active":"2023-09-23T20:04:02.271287+0000","last_became_peered":"2023-09-23T20:04:02.271287+0000","last_unstale":"2023-09-23T20:04:05.487623+0000","last_undegraded":"2023-09-23T20:04:05.487623+0000","last_fullsized":"2023-09-23T20:04:05.487623+0000","mapping_epoch":276,"log_start":"247'1700","ondisk_log_start":"247'1700","created":240,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6057,"log_dups_size":0,"ondisk_log_size":6057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:47.344179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283574272,"num_objects":4327,"num_object_clones":0,"num_object_copies":12981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7757,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5100,"num_bytes_recovered":334233600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":238,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261346+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:04:01.261346+0000","last_peered":"2023-09-23T20:04:01.261346+0000","last_clean":"2023-09-23T20:04:01.261346+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:04:01.261346+0000","last_undegraded":"2023-09-23T20:04:01.261346+0000","last_fullsized":"2023-09-23T20:04:01.261346+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":265,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.257806+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:04:01.257806+0000","last_peered":"2023-09-23T20:04:01.257806+0000","last_clean":"2023-09-23T20:04:01.257806+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:04:01.257806+0000","last_undegraded":"2023-09-23T20:04:01.257806+0000","last_fullsized":"2023-09-23T20:04:01.257806+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"276'7692","reported_seq":7738,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.686224+0000","last_change":"2023-09-23T20:01:39.911588+0000","last_active":"2023-09-23T20:04:01.686224+0000","last_peered":"2023-09-23T20:04:01.686224+0000","last_clean":"2023-09-23T20:04:01.686224+0000","last_became_active":"2023-09-23T20:01:39.911364+0000","last_became_peered":"2023-09-23T20:01:39.911364+0000","last_unstale":"2023-09-23T20:04:01.686224+0000","last_undegraded":"2023-09-23T20:04:01.686224+0000","last_fullsized":"2023-09-23T20:04:01.686224+0000","mapping_epoch":240,"log_start":"247'1995","ondisk_log_start":"247'1995","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:35.715001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285605888,"num_objects":4358,"num_object_clones":0,"num_object_copies":13074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7692,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":239,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261798+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T20:04:01.261798+0000","last_peered":"2023-09-23T20:04:01.261798+0000","last_clean":"2023-09-23T20:04:01.261798+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T20:04:01.261798+0000","last_undegraded":"2023-09-23T20:04:01.261798+0000","last_fullsized":"2023-09-23T20:04:01.261798+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":293,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267774+0000","last_change":"2023-09-23T20:03:59.253784+0000","last_active":"2023-09-23T20:04:01.267774+0000","last_peered":"2023-09-23T20:04:01.267774+0000","last_clean":"2023-09-23T20:04:01.267774+0000","last_became_active":"2023-09-23T20:03:59.253552+0000","last_became_peered":"2023-09-23T20:03:59.253552+0000","last_unstale":"2023-09-23T20:04:01.267774+0000","last_undegraded":"2023-09-23T20:04:01.267774+0000","last_fullsized":"2023-09-23T20:04:01.267774+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:56.110746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"278'7817","reported_seq":7865,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:05.394243+0000","last_change":"2023-09-23T20:01:39.976582+0000","last_active":"2023-09-23T20:04:05.394243+0000","last_peered":"2023-09-23T20:04:05.394243+0000","last_clean":"2023-09-23T20:04:05.394243+0000","last_became_active":"2023-09-23T20:01:39.976244+0000","last_became_peered":"2023-09-23T20:01:39.976244+0000","last_unstale":"2023-09-23T20:04:05.394243+0000","last_undegraded":"2023-09-23T20:04:05.394243+0000","last_fullsized":"2023-09-23T20:04:05.394243+0000","mapping_epoch":240,"log_start":"247'1700","ondisk_log_start":"247'1700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6117,"log_dups_size":0,"ondisk_log_size":6117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.952295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277151744,"num_objects":4229,"num_object_clones":0,"num_object_copies":12687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7817,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":242,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261351+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:04:01.261351+0000","last_peered":"2023-09-23T20:04:01.261351+0000","last_clean":"2023-09-23T20:04:01.261351+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:04:01.261351+0000","last_undegraded":"2023-09-23T20:04:01.261351+0000","last_fullsized":"2023-09-23T20:04:01.261351+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":205,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.261063+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T20:04:02.261063+0000","last_peered":"2023-09-23T20:04:02.261063+0000","last_clean":"2023-09-23T20:04:02.261063+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T20:04:02.261063+0000","last_undegraded":"2023-09-23T20:04:02.261063+0000","last_fullsized":"2023-09-23T20:04:02.261063+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.284038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"277'8008","reported_seq":8055,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.968431+0000","last_change":"2023-09-23T20:01:39.912566+0000","last_active":"2023-09-23T20:04:02.968431+0000","last_peered":"2023-09-23T20:04:02.968431+0000","last_clean":"2023-09-23T20:04:02.968431+0000","last_became_active":"2023-09-23T20:01:39.912259+0000","last_became_peered":"2023-09-23T20:01:39.912259+0000","last_unstale":"2023-09-23T20:04:02.968431+0000","last_undegraded":"2023-09-23T20:04:02.968431+0000","last_fullsized":"2023-09-23T20:04:02.968431+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8008,"log_dups_size":0,"ondisk_log_size":8008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:19.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290848768,"num_objects":4438,"num_object_clones":0,"num_object_copies":13314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8008,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":295,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258728+0000","last_change":"2023-09-23T20:00:34.468296+0000","last_active":"2023-09-23T20:04:01.258728+0000","last_peered":"2023-09-23T20:04:01.258728+0000","last_clean":"2023-09-23T20:04:01.258728+0000","last_became_active":"2023-09-23T20:00:34.467957+0000","last_became_peered":"2023-09-23T20:00:34.467957+0000","last_unstale":"2023-09-23T20:04:01.258728+0000","last_undegraded":"2023-09-23T20:04:01.258728+0000","last_fullsized":"2023-09-23T20:04:01.258728+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:50.449027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":93,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.262209+0000","last_change":"2023-09-23T20:01:04.470558+0000","last_active":"2023-09-23T20:04:02.262209+0000","last_peered":"2023-09-23T20:04:02.262209+0000","last_clean":"2023-09-23T20:04:02.262209+0000","last_became_active":"2023-09-23T20:01:04.470276+0000","last_became_peered":"2023-09-23T20:01:04.470276+0000","last_unstale":"2023-09-23T20:04:02.262209+0000","last_undegraded":"2023-09-23T20:04:02.262209+0000","last_fullsized":"2023-09-23T20:04:02.262209+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:17.788583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"276'7724","reported_seq":7770,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.664926+0000","last_change":"2023-09-23T20:01:39.974611+0000","last_active":"2023-09-23T20:04:01.664926+0000","last_peered":"2023-09-23T20:04:01.664926+0000","last_clean":"2023-09-23T20:04:01.664926+0000","last_became_active":"2023-09-23T20:01:39.974318+0000","last_became_peered":"2023-09-23T20:01:39.974318+0000","last_unstale":"2023-09-23T20:04:01.664926+0000","last_undegraded":"2023-09-23T20:04:01.664926+0000","last_fullsized":"2023-09-23T20:04:01.664926+0000","mapping_epoch":240,"log_start":"247'2027","ondisk_log_start":"247'2027","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:31.047765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283377664,"num_objects":4324,"num_object_clones":0,"num_object_copies":12972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7724,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":727,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.260968+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:04:01.260968+0000","last_peered":"2023-09-23T20:04:01.260968+0000","last_clean":"2023-09-23T20:04:01.260968+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:04:01.260968+0000","last_undegraded":"2023-09-23T20:04:01.260968+0000","last_fullsized":"2023-09-23T20:04:01.260968+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":102,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.475729+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:04:00.475729+0000","last_peered":"2023-09-23T20:04:00.475729+0000","last_clean":"2023-09-23T20:04:00.475729+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:04:00.475729+0000","last_undegraded":"2023-09-23T20:04:00.475729+0000","last_fullsized":"2023-09-23T20:04:00.475729+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":273,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268064+0000","last_change":"2023-09-23T19:59:09.206075+0000","last_active":"2023-09-23T20:04:01.268064+0000","last_peered":"2023-09-23T20:04:01.268064+0000","last_clean":"2023-09-23T20:04:01.268064+0000","last_became_active":"2023-09-23T19:59:09.205766+0000","last_became_peered":"2023-09-23T19:59:09.205766+0000","last_unstale":"2023-09-23T20:04:01.268064+0000","last_undegraded":"2023-09-23T20:04:01.268064+0000","last_fullsized":"2023-09-23T20:04:01.268064+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:14.627703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"277'7615","reported_seq":7662,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:03.079105+0000","last_change":"2023-09-23T20:01:39.976550+0000","last_active":"2023-09-23T20:04:03.079105+0000","last_peered":"2023-09-23T20:04:03.079105+0000","last_clean":"2023-09-23T20:04:03.079105+0000","last_became_active":"2023-09-23T20:01:39.976245+0000","last_became_peered":"2023-09-23T20:01:39.976245+0000","last_unstale":"2023-09-23T20:04:03.079105+0000","last_undegraded":"2023-09-23T20:04:03.079105+0000","last_fullsized":"2023-09-23T20:04:03.079105+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7615,"log_dups_size":0,"ondisk_log_size":7615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:45.979020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278462464,"num_objects":4249,"num_object_clones":0,"num_object_copies":12747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7615,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":636,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.257730+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:04:01.257730+0000","last_peered":"2023-09-23T20:04:01.257730+0000","last_clean":"2023-09-23T20:04:01.257730+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:04:01.257730+0000","last_undegraded":"2023-09-23T20:04:01.257730+0000","last_fullsized":"2023-09-23T20:04:01.257730+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":223,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267818+0000","last_change":"2023-09-23T19:59:09.172601+0000","last_active":"2023-09-23T20:04:01.267818+0000","last_peered":"2023-09-23T20:04:01.267818+0000","last_clean":"2023-09-23T20:04:01.267818+0000","last_became_active":"2023-09-23T19:59:09.172297+0000","last_became_peered":"2023-09-23T19:59:09.172297+0000","last_unstale":"2023-09-23T20:04:01.267818+0000","last_undegraded":"2023-09-23T20:04:01.267818+0000","last_fullsized":"2023-09-23T20:04:01.267818+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:09.528266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":253,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268079+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T20:04:01.268079+0000","last_peered":"2023-09-23T20:04:01.268079+0000","last_clean":"2023-09-23T20:04:01.268079+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T20:04:01.268079+0000","last_undegraded":"2023-09-23T20:04:01.268079+0000","last_fullsized":"2023-09-23T20:04:01.268079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"278'7834","reported_seq":7883,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:05.463054+0000","last_change":"2023-09-23T20:01:39.977019+0000","last_active":"2023-09-23T20:04:05.463054+0000","last_peered":"2023-09-23T20:04:05.463054+0000","last_clean":"2023-09-23T20:04:05.463054+0000","last_became_active":"2023-09-23T20:01:39.976270+0000","last_became_peered":"2023-09-23T20:01:39.976270+0000","last_unstale":"2023-09-23T20:04:05.463054+0000","last_undegraded":"2023-09-23T20:04:05.463054+0000","last_fullsized":"2023-09-23T20:04:05.463054+0000","mapping_epoch":240,"log_start":"247'1700","ondisk_log_start":"247'1700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.226759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278986776,"num_objects":4258,"num_object_clones":0,"num_object_copies":12774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4258,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7834,"num_write_kb":386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":237,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267615+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:04:01.267615+0000","last_peered":"2023-09-23T20:04:01.267615+0000","last_clean":"2023-09-23T20:04:01.267615+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:04:01.267615+0000","last_undegraded":"2023-09-23T20:04:01.267615+0000","last_fullsized":"2023-09-23T20:04:01.267615+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":283,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268008+0000","last_change":"2023-09-23T19:59:09.171926+0000","last_active":"2023-09-23T20:04:01.268008+0000","last_peered":"2023-09-23T20:04:01.268008+0000","last_clean":"2023-09-23T20:04:01.268008+0000","last_became_active":"2023-09-23T19:59:09.171609+0000","last_became_peered":"2023-09-23T19:59:09.171609+0000","last_unstale":"2023-09-23T20:04:01.268008+0000","last_undegraded":"2023-09-23T20:04:01.268008+0000","last_fullsized":"2023-09-23T20:04:01.268008+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:21.795329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"276'7637","reported_seq":7688,"reported_epoch":276,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:01.664197+0000","last_change":"2023-09-23T20:04:00.616600+0000","last_active":"2023-09-23T20:04:01.664197+0000","last_peered":"2023-09-23T20:04:01.664197+0000","last_clean":"2023-09-23T20:03:58.092468+0000","last_became_active":"2023-09-23T20:04:00.591086+0000","last_became_peered":"2023-09-23T20:04:00.591086+0000","last_unstale":"2023-09-23T20:04:01.664197+0000","last_undegraded":"2023-09-23T20:04:01.664197+0000","last_fullsized":"2023-09-23T20:04:01.664197+0000","mapping_epoch":274,"log_start":"246'1072","ondisk_log_start":"246'1072","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:19.737714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286064640,"num_objects":4365,"num_object_clones":0,"num_object_copies":13095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4365,"num_objects_unfound":0,"num_objects_dirty":4365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7637,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[0,7,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":4365}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":615,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258655+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:04:01.258655+0000","last_peered":"2023-09-23T20:04:01.258655+0000","last_clean":"2023-09-23T20:04:01.258655+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:04:01.258655+0000","last_undegraded":"2023-09-23T20:04:01.258655+0000","last_fullsized":"2023-09-23T20:04:01.258655+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.266022+0000","last_change":"2023-09-23T20:03:59.233742+0000","last_active":"2023-09-23T20:04:01.266022+0000","last_peered":"2023-09-23T20:04:01.266022+0000","last_clean":"2023-09-23T20:04:01.266022+0000","last_became_active":"2023-09-23T20:03:59.233209+0000","last_became_peered":"2023-09-23T20:03:59.233209+0000","last_unstale":"2023-09-23T20:04:01.266022+0000","last_undegraded":"2023-09-23T20:04:01.266022+0000","last_fullsized":"2023-09-23T20:04:01.266022+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:50.879925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":238,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.262939+0000","last_change":"2023-09-23T19:59:09.153852+0000","last_active":"2023-09-23T20:04:02.262939+0000","last_peered":"2023-09-23T20:04:02.262939+0000","last_clean":"2023-09-23T20:04:02.262939+0000","last_became_active":"2023-09-23T19:59:09.153514+0000","last_became_peered":"2023-09-23T19:59:09.153514+0000","last_unstale":"2023-09-23T20:04:02.262939+0000","last_undegraded":"2023-09-23T20:04:02.262939+0000","last_fullsized":"2023-09-23T20:04:02.262939+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:45.179600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"277'7719","reported_seq":7792,"reported_epoch":277,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:03.218219+0000","last_change":"2023-09-23T20:04:02.297387+0000","last_active":"2023-09-23T20:04:03.218219+0000","last_peered":"2023-09-23T20:04:03.218219+0000","last_clean":"2023-09-23T20:03:58.080766+0000","last_became_active":"2023-09-23T20:04:00.582511+0000","last_became_peered":"2023-09-23T20:04:00.582511+0000","last_unstale":"2023-09-23T20:04:03.218219+0000","last_undegraded":"2023-09-23T20:04:03.218219+0000","last_fullsized":"2023-09-23T20:04:03.218219+0000","mapping_epoch":274,"log_start":"246'1200","ondisk_log_start":"246'1200","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6519,"log_dups_size":0,"ondisk_log_size":6519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:56.089695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291962880,"num_objects":4455,"num_object_clones":0,"num_object_copies":13365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4452,"num_objects_unfound":0,"num_objects_dirty":4455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7719,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,1,0],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":4455}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":591,"reported_epoch":276,"state":"peering","last_fresh":"2023-09-23T20:04:00.474316+0000","last_change":"2023-09-23T20:04:02.045656+0000","last_active":"2023-09-23T20:04:02.045656+0000","last_peered":"2023-09-23T20:04:00.474316+0000","last_clean":"2023-09-23T20:02:56.478893+0000","last_became_active":"2023-09-23T20:04:00.474058+0000","last_became_peered":"2023-09-23T20:04:00.474058+0000","last_unstale":"2023-09-23T20:04:00.474316+0000","last_undegraded":"2023-09-23T20:04:00.474316+0000","last_fullsized":"2023-09-23T20:04:00.474316+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":616,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.263399+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T20:04:02.263399+0000","last_peered":"2023-09-23T20:04:02.263399+0000","last_clean":"2023-09-23T20:04:02.263399+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T20:04:02.263399+0000","last_undegraded":"2023-09-23T20:04:02.263399+0000","last_fullsized":"2023-09-23T20:04:02.263399+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":277,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258782+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:04:01.258782+0000","last_peered":"2023-09-23T20:04:01.258782+0000","last_clean":"2023-09-23T20:04:01.258782+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:04:01.258782+0000","last_undegraded":"2023-09-23T20:04:01.258782+0000","last_fullsized":"2023-09-23T20:04:01.258782+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":205,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.260866+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T20:04:02.260866+0000","last_peered":"2023-09-23T20:04:02.260866+0000","last_clean":"2023-09-23T20:04:02.260866+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T20:04:02.260866+0000","last_undegraded":"2023-09-23T20:04:02.260866+0000","last_fullsized":"2023-09-23T20:04:02.260866+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"276'7762","reported_seq":7822,"reported_epoch":276,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:01.704449+0000","last_change":"2023-09-23T20:04:00.730874+0000","last_active":"2023-09-23T20:04:01.704449+0000","last_peered":"2023-09-23T20:04:01.704449+0000","last_clean":"2023-09-23T20:03:58.087054+0000","last_became_active":"2023-09-23T20:04:00.560120+0000","last_became_peered":"2023-09-23T20:04:00.560120+0000","last_unstale":"2023-09-23T20:04:01.704449+0000","last_undegraded":"2023-09-23T20:04:01.704449+0000","last_fullsized":"2023-09-23T20:04:01.704449+0000","mapping_epoch":274,"log_start":"247'1800","ondisk_log_start":"247'1800","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:17.026687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293076992,"num_objects":4472,"num_object_clones":0,"num_object_copies":13416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4472,"num_objects_unfound":0,"num_objects_dirty":4472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7762,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":4472}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":588,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.257918+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:04:01.257918+0000","last_peered":"2023-09-23T20:04:01.257918+0000","last_clean":"2023-09-23T20:04:01.257918+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:04:01.257918+0000","last_undegraded":"2023-09-23T20:04:01.257918+0000","last_fullsized":"2023-09-23T20:04:01.257918+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":837,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.257753+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T20:04:01.257753+0000","last_peered":"2023-09-23T20:04:01.257753+0000","last_clean":"2023-09-23T20:04:01.257753+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T20:04:01.257753+0000","last_undegraded":"2023-09-23T20:04:01.257753+0000","last_fullsized":"2023-09-23T20:04:01.257753+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:58.623015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":276,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268415+0000","last_change":"2023-09-23T20:00:34.447064+0000","last_active":"2023-09-23T20:04:01.268415+0000","last_peered":"2023-09-23T20:04:01.268415+0000","last_clean":"2023-09-23T20:04:01.268415+0000","last_became_active":"2023-09-23T20:00:34.446754+0000","last_became_peered":"2023-09-23T20:00:34.446754+0000","last_unstale":"2023-09-23T20:04:01.268415+0000","last_undegraded":"2023-09-23T20:04:01.268415+0000","last_fullsized":"2023-09-23T20:04:01.268415+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:39.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":271,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.263319+0000","last_change":"2023-09-23T20:03:59.236594+0000","last_active":"2023-09-23T20:04:02.263319+0000","last_peered":"2023-09-23T20:04:02.263319+0000","last_clean":"2023-09-23T20:04:02.263319+0000","last_became_active":"2023-09-23T20:03:59.236305+0000","last_became_peered":"2023-09-23T20:03:59.236305+0000","last_unstale":"2023-09-23T20:04:02.263319+0000","last_undegraded":"2023-09-23T20:04:02.263319+0000","last_fullsized":"2023-09-23T20:04:02.263319+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:23.103726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"277'7695","reported_seq":7739,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:03.302955+0000","last_change":"2023-09-23T20:01:39.977018+0000","last_active":"2023-09-23T20:04:03.302955+0000","last_peered":"2023-09-23T20:04:03.302955+0000","last_clean":"2023-09-23T20:04:03.302955+0000","last_became_active":"2023-09-23T20:01:39.976359+0000","last_became_peered":"2023-09-23T20:01:39.976359+0000","last_unstale":"2023-09-23T20:04:03.302955+0000","last_undegraded":"2023-09-23T20:04:03.302955+0000","last_fullsized":"2023-09-23T20:04:03.302955+0000","mapping_epoch":240,"log_start":"246'1200","ondisk_log_start":"246'1200","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6495,"log_dups_size":0,"ondisk_log_size":6495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.221346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285802496,"num_objects":4361,"num_object_clones":0,"num_object_copies":13083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7695,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":512,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261012+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:04:01.261012+0000","last_peered":"2023-09-23T20:04:01.261012+0000","last_clean":"2023-09-23T20:04:01.261012+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:04:01.261012+0000","last_undegraded":"2023-09-23T20:04:01.261012+0000","last_fullsized":"2023-09-23T20:04:01.261012+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":909,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.271306+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:04:02.271306+0000","last_peered":"2023-09-23T20:04:02.271306+0000","last_clean":"2023-09-23T20:04:02.271306+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:04:02.271306+0000","last_undegraded":"2023-09-23T20:04:02.271306+0000","last_fullsized":"2023-09-23T20:04:02.271306+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":235,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.261427+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T20:04:02.261427+0000","last_peered":"2023-09-23T20:04:02.261427+0000","last_clean":"2023-09-23T20:04:02.261427+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T20:04:02.261427+0000","last_undegraded":"2023-09-23T20:04:02.261427+0000","last_fullsized":"2023-09-23T20:04:02.261427+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.564126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":249,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.263012+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T20:04:02.263012+0000","last_peered":"2023-09-23T20:04:02.263012+0000","last_clean":"2023-09-23T20:04:02.263012+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T20:04:02.263012+0000","last_undegraded":"2023-09-23T20:04:02.263012+0000","last_fullsized":"2023-09-23T20:04:02.263012+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"277'7876","reported_seq":8789,"reported_epoch":277,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:03.275272+0000","last_change":"2023-09-23T20:04:02.279075+0000","last_active":"2023-09-23T20:04:03.275272+0000","last_peered":"2023-09-23T20:04:03.275272+0000","last_clean":"2023-09-23T20:03:58.125973+0000","last_became_active":"2023-09-23T20:04:00.586359+0000","last_became_peered":"2023-09-23T20:04:00.586359+0000","last_unstale":"2023-09-23T20:04:03.275272+0000","last_undegraded":"2023-09-23T20:04:03.275272+0000","last_fullsized":"2023-09-23T20:04:03.275272+0000","mapping_epoch":274,"log_start":"245'756","ondisk_log_start":"245'756","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7120,"log_dups_size":0,"ondisk_log_size":7120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:05.232047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291110912,"num_objects":4442,"num_object_clones":0,"num_object_copies":13326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4020,"num_objects_unfound":0,"num_objects_dirty":4442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7876,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":426,"num_bytes_recovered":27918336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,5],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":4442}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1085,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.259260+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:04:01.259260+0000","last_peered":"2023-09-23T20:04:01.259260+0000","last_clean":"2023-09-23T20:04:01.259260+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:04:01.259260+0000","last_undegraded":"2023-09-23T20:04:01.259260+0000","last_fullsized":"2023-09-23T20:04:01.259260+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":687,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.260969+0000","last_change":"2023-09-23T20:00:34.445036+0000","last_active":"2023-09-23T20:04:01.260969+0000","last_peered":"2023-09-23T20:04:01.260969+0000","last_clean":"2023-09-23T20:04:01.260969+0000","last_became_active":"2023-09-23T20:00:34.444689+0000","last_became_peered":"2023-09-23T20:00:34.444689+0000","last_unstale":"2023-09-23T20:04:01.260969+0000","last_undegraded":"2023-09-23T20:04:01.260969+0000","last_fullsized":"2023-09-23T20:04:01.260969+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:44.485721+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267336+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:04:01.267336+0000","last_peered":"2023-09-23T20:04:01.267336+0000","last_clean":"2023-09-23T20:04:01.267336+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:04:01.267336+0000","last_undegraded":"2023-09-23T20:04:01.267336+0000","last_fullsized":"2023-09-23T20:04:01.267336+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268295+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:04:01.268295+0000","last_peered":"2023-09-23T20:04:01.268295+0000","last_clean":"2023-09-23T20:04:01.268295+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:04:01.268295+0000","last_undegraded":"2023-09-23T20:04:01.268295+0000","last_fullsized":"2023-09-23T20:04:01.268295+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"277'7852","reported_seq":7897,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:03.507196+0000","last_change":"2023-09-23T20:01:39.977010+0000","last_active":"2023-09-23T20:04:03.507196+0000","last_peered":"2023-09-23T20:04:03.507196+0000","last_clean":"2023-09-23T20:04:03.507196+0000","last_became_active":"2023-09-23T20:01:39.976722+0000","last_became_peered":"2023-09-23T20:01:39.976722+0000","last_unstale":"2023-09-23T20:04:03.507196+0000","last_undegraded":"2023-09-23T20:04:03.507196+0000","last_fullsized":"2023-09-23T20:04:03.507196+0000","mapping_epoch":240,"log_start":"245'731","ondisk_log_start":"245'731","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7121,"log_dups_size":0,"ondisk_log_size":7121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:13.592964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287178752,"num_objects":4382,"num_object_clones":0,"num_object_copies":13146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7852,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":618,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.262357+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T20:04:02.262357+0000","last_peered":"2023-09-23T20:04:02.262357+0000","last_clean":"2023-09-23T20:04:02.262357+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T20:04:02.262357+0000","last_undegraded":"2023-09-23T20:04:02.262357+0000","last_fullsized":"2023-09-23T20:04:02.262357+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.842550+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":995,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.260960+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T20:04:01.260960+0000","last_peered":"2023-09-23T20:04:01.260960+0000","last_clean":"2023-09-23T20:04:01.260960+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T20:04:01.260960+0000","last_undegraded":"2023-09-23T20:04:01.260960+0000","last_fullsized":"2023-09-23T20:04:01.260960+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":243,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267802+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:04:01.267802+0000","last_peered":"2023-09-23T20:04:01.267802+0000","last_clean":"2023-09-23T20:04:01.267802+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:04:01.267802+0000","last_undegraded":"2023-09-23T20:04:01.267802+0000","last_fullsized":"2023-09-23T20:04:01.267802+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":239,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258647+0000","last_change":"2023-09-23T20:03:10.175462+0000","last_active":"2023-09-23T20:04:01.258647+0000","last_peered":"2023-09-23T20:04:01.258647+0000","last_clean":"2023-09-23T20:04:01.258647+0000","last_became_active":"2023-09-23T20:03:10.175250+0000","last_became_peered":"2023-09-23T20:03:10.175250+0000","last_unstale":"2023-09-23T20:04:01.258647+0000","last_undegraded":"2023-09-23T20:04:01.258647+0000","last_fullsized":"2023-09-23T20:04:01.258647+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:42.961619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"277'7637","reported_seq":7681,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:03.034152+0000","last_change":"2023-09-23T20:01:39.913385+0000","last_active":"2023-09-23T20:04:03.034152+0000","last_peered":"2023-09-23T20:04:03.034152+0000","last_clean":"2023-09-23T20:04:03.034152+0000","last_became_active":"2023-09-23T20:01:39.912872+0000","last_became_peered":"2023-09-23T20:01:39.912872+0000","last_unstale":"2023-09-23T20:04:03.034152+0000","last_undegraded":"2023-09-23T20:04:03.034152+0000","last_fullsized":"2023-09-23T20:04:03.034152+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7637,"log_dups_size":0,"ondisk_log_size":7637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.619813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285671424,"num_objects":4359,"num_object_clones":0,"num_object_copies":13077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7637,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":583,"reported_epoch":276,"state":"peering","last_fresh":"2023-09-23T20:04:00.586515+0000","last_change":"2023-09-23T20:04:02.045558+0000","last_active":"2023-09-23T20:04:02.045558+0000","last_peered":"2023-09-23T20:04:00.586515+0000","last_clean":"2023-09-23T20:00:51.691805+0000","last_became_active":"2023-09-23T20:04:00.586298+0000","last_became_peered":"2023-09-23T20:04:00.586298+0000","last_unstale":"2023-09-23T20:04:00.586515+0000","last_undegraded":"2023-09-23T20:04:00.586515+0000","last_fullsized":"2023-09-23T20:04:00.227057+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":355,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261420+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:04:01.261420+0000","last_peered":"2023-09-23T20:04:01.261420+0000","last_clean":"2023-09-23T20:04:01.261420+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:04:01.261420+0000","last_undegraded":"2023-09-23T20:04:01.261420+0000","last_fullsized":"2023-09-23T20:04:01.261420+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":237,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.263473+0000","last_change":"2023-09-23T20:03:10.174842+0000","last_active":"2023-09-23T20:04:02.263473+0000","last_peered":"2023-09-23T20:04:02.263473+0000","last_clean":"2023-09-23T20:04:02.263473+0000","last_became_active":"2023-09-23T20:03:10.174511+0000","last_became_peered":"2023-09-23T20:03:10.174511+0000","last_unstale":"2023-09-23T20:04:02.263473+0000","last_undegraded":"2023-09-23T20:04:02.263473+0000","last_fullsized":"2023-09-23T20:04:02.263473+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:12.723811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":73,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.259334+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:04:01.259334+0000","last_peered":"2023-09-23T20:04:01.259334+0000","last_clean":"2023-09-23T20:04:01.259334+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:04:01.259334+0000","last_undegraded":"2023-09-23T20:04:01.259334+0000","last_fullsized":"2023-09-23T20:04:01.259334+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"276'7827","reported_seq":7873,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.603708+0000","last_change":"2023-09-23T20:01:39.912135+0000","last_active":"2023-09-23T20:04:01.603708+0000","last_peered":"2023-09-23T20:04:01.603708+0000","last_clean":"2023-09-23T20:04:01.603708+0000","last_became_active":"2023-09-23T20:01:39.911834+0000","last_became_peered":"2023-09-23T20:01:39.911834+0000","last_unstale":"2023-09-23T20:04:01.603708+0000","last_undegraded":"2023-09-23T20:04:01.603708+0000","last_fullsized":"2023-09-23T20:04:01.603708+0000","mapping_epoch":240,"log_start":"247'2126","ondisk_log_start":"247'2126","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:46.725578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289210368,"num_objects":4413,"num_object_clones":0,"num_object_copies":13239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7827,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":655,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267999+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:04:01.267999+0000","last_peered":"2023-09-23T20:04:01.267999+0000","last_clean":"2023-09-23T20:04:01.267999+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:04:01.267999+0000","last_undegraded":"2023-09-23T20:04:01.267999+0000","last_fullsized":"2023-09-23T20:04:01.267999+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":102,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.559406+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:04:00.559406+0000","last_peered":"2023-09-23T20:04:00.559406+0000","last_clean":"2023-09-23T20:04:00.559406+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:04:00.559406+0000","last_undegraded":"2023-09-23T20:04:00.559406+0000","last_fullsized":"2023-09-23T20:04:00.559406+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":683,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.262841+0000","last_change":"2023-09-23T19:59:40.899753+0000","last_active":"2023-09-23T20:04:02.262841+0000","last_peered":"2023-09-23T20:04:02.262841+0000","last_clean":"2023-09-23T20:04:02.262841+0000","last_became_active":"2023-09-23T19:59:40.899147+0000","last_became_peered":"2023-09-23T19:59:40.899147+0000","last_unstale":"2023-09-23T20:04:02.262841+0000","last_undegraded":"2023-09-23T20:04:02.262841+0000","last_fullsized":"2023-09-23T20:04:02.262841+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:08.938755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.266102+0000","last_change":"2023-09-23T20:03:59.233560+0000","last_active":"2023-09-23T20:04:01.266102+0000","last_peered":"2023-09-23T20:04:01.266102+0000","last_clean":"2023-09-23T20:04:01.266102+0000","last_became_active":"2023-09-23T20:03:59.232842+0000","last_became_peered":"2023-09-23T20:03:59.232842+0000","last_unstale":"2023-09-23T20:04:01.266102+0000","last_undegraded":"2023-09-23T20:04:01.266102+0000","last_fullsized":"2023-09-23T20:04:01.266102+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:16.909413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"276'7822","reported_seq":7868,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.679393+0000","last_change":"2023-09-23T20:01:39.913698+0000","last_active":"2023-09-23T20:04:01.679393+0000","last_peered":"2023-09-23T20:04:01.679393+0000","last_clean":"2023-09-23T20:04:01.679393+0000","last_became_active":"2023-09-23T20:01:39.913367+0000","last_became_peered":"2023-09-23T20:01:39.913367+0000","last_unstale":"2023-09-23T20:04:01.679393+0000","last_undegraded":"2023-09-23T20:04:01.679393+0000","last_fullsized":"2023-09-23T20:04:01.679393+0000","mapping_epoch":240,"log_start":"247'2125","ondisk_log_start":"247'2125","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289275904,"num_objects":4414,"num_object_clones":0,"num_object_copies":13242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7822,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":532,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.589431+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:04:00.589431+0000","last_peered":"2023-09-23T20:04:00.589431+0000","last_clean":"2023-09-23T20:04:00.589431+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:04:00.589431+0000","last_undegraded":"2023-09-23T20:04:00.589431+0000","last_fullsized":"2023-09-23T20:04:00.589431+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":33,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.261382+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:04:01.261382+0000","last_peered":"2023-09-23T20:04:01.261382+0000","last_clean":"2023-09-23T20:04:01.261382+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:04:01.261382+0000","last_undegraded":"2023-09-23T20:04:01.261382+0000","last_fullsized":"2023-09-23T20:04:01.261382+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":81,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.262903+0000","last_change":"2023-09-23T20:01:04.470744+0000","last_active":"2023-09-23T20:04:02.262903+0000","last_peered":"2023-09-23T20:04:02.262903+0000","last_clean":"2023-09-23T20:04:02.262903+0000","last_became_active":"2023-09-23T20:01:04.470302+0000","last_became_peered":"2023-09-23T20:01:04.470302+0000","last_unstale":"2023-09-23T20:04:02.262903+0000","last_undegraded":"2023-09-23T20:04:02.262903+0000","last_fullsized":"2023-09-23T20:04:02.262903+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:25.452596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":236,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267701+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:04:01.267701+0000","last_peered":"2023-09-23T20:04:01.267701+0000","last_clean":"2023-09-23T20:04:01.267701+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:04:01.267701+0000","last_undegraded":"2023-09-23T20:04:01.267701+0000","last_fullsized":"2023-09-23T20:04:01.267701+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":22,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.283054+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:04:01.283054+0000","last_peered":"2023-09-23T20:04:01.283054+0000","last_clean":"2023-09-23T20:04:01.283054+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:04:01.283054+0000","last_undegraded":"2023-09-23T20:04:01.283054+0000","last_fullsized":"2023-09-23T20:04:01.283054+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":30,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.284054+0000","last_change":"2023-09-23T20:03:10.178662+0000","last_active":"2023-09-23T20:04:01.284054+0000","last_peered":"2023-09-23T20:04:01.284054+0000","last_clean":"2023-09-23T20:04:01.284054+0000","last_became_active":"2023-09-23T20:03:10.178352+0000","last_became_peered":"2023-09-23T20:03:10.178352+0000","last_unstale":"2023-09-23T20:04:01.284054+0000","last_undegraded":"2023-09-23T20:04:01.284054+0000","last_fullsized":"2023-09-23T20:04:01.284054+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.469897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":238,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258644+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T20:04:01.258644+0000","last_peered":"2023-09-23T20:04:01.258644+0000","last_clean":"2023-09-23T20:04:01.258644+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T20:04:01.258644+0000","last_undegraded":"2023-09-23T20:04:01.258644+0000","last_fullsized":"2023-09-23T20:04:01.258644+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":273,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268458+0000","last_change":"2023-09-23T19:59:09.172452+0000","last_active":"2023-09-23T20:04:01.268458+0000","last_peered":"2023-09-23T20:04:01.268458+0000","last_clean":"2023-09-23T20:04:01.268458+0000","last_became_active":"2023-09-23T19:59:09.172080+0000","last_became_peered":"2023-09-23T19:59:09.172080+0000","last_unstale":"2023-09-23T20:04:01.268458+0000","last_undegraded":"2023-09-23T20:04:01.268458+0000","last_fullsized":"2023-09-23T20:04:01.268458+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:24.687991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":242,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258985+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T20:04:01.258985+0000","last_peered":"2023-09-23T20:04:01.258985+0000","last_clean":"2023-09-23T20:04:01.258985+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T20:04:01.258985+0000","last_undegraded":"2023-09-23T20:04:01.258985+0000","last_fullsized":"2023-09-23T20:04:01.258985+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:07.560646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":273,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267728+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:04:01.267728+0000","last_peered":"2023-09-23T20:04:01.267728+0000","last_clean":"2023-09-23T20:04:01.267728+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:04:01.267728+0000","last_undegraded":"2023-09-23T20:04:01.267728+0000","last_fullsized":"2023-09-23T20:04:01.267728+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":98,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T20:04:00.583164+0000","last_change":"2023-09-23T19:59:09.145224+0000","last_active":"2023-09-23T20:04:00.583164+0000","last_peered":"2023-09-23T20:04:00.583164+0000","last_clean":"2023-09-23T20:04:00.583164+0000","last_became_active":"2023-09-23T19:59:09.144819+0000","last_became_peered":"2023-09-23T19:59:09.144819+0000","last_unstale":"2023-09-23T20:04:00.583164+0000","last_undegraded":"2023-09-23T20:04:00.583164+0000","last_fullsized":"2023-09-23T20:04:00.583164+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:04.048181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.268227+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:04:01.268227+0000","last_peered":"2023-09-23T20:04:01.268227+0000","last_clean":"2023-09-23T20:04:01.268227+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:04:01.268227+0000","last_undegraded":"2023-09-23T20:04:01.268227+0000","last_fullsized":"2023-09-23T20:04:01.268227+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":96,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.258176+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:04:01.258176+0000","last_peered":"2023-09-23T20:04:01.258176+0000","last_clean":"2023-09-23T20:04:01.258176+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:04:01.258176+0000","last_undegraded":"2023-09-23T20:04:01.258176+0000","last_fullsized":"2023-09-23T20:04:01.258176+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":303,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T20:04:01.267517+0000","last_change":"2023-09-23T20:03:59.235232+0000","last_active":"2023-09-23T20:04:01.267517+0000","last_peered":"2023-09-23T20:04:01.267517+0000","last_clean":"2023-09-23T20:04:01.267517+0000","last_became_active":"2023-09-23T20:03:59.234941+0000","last_became_peered":"2023-09-23T20:03:59.234941+0000","last_unstale":"2023-09-23T20:04:01.267517+0000","last_undegraded":"2023-09-23T20:04:01.267517+0000","last_fullsized":"2023-09-23T20:04:01.267517+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:45.591449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4577361944,"num_objects":69846,"num_object_clones":0,"num_object_copies":209538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":17309,"num_objects_unfound":0,"num_objects_dirty":69846,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124274,"num_write_kb":6211777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5532,"num_bytes_recovered":362479616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14047195136,"data_stored":14047182920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104142,"ondisk_log_size":104142,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234275,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1962912,"kb_used_data":1698072,"kb_used_omap":0,"kb_used_meta":264832,"kb_avail":91782240,"statfs":{"total":95995035648,"available":93985013760,"internally_reserved":0,"allocated":1738825728,"data_stored":1737987332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":271187968},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:18 2023","interfaces":[{"interface":"back","average":{"1min":3.952,"5min":2.1850000000000001,"15min":1.8160000000000001},"min":{"1min":0.22700000000000001,"5min":0.22,"15min":0.19700000000000001},"max":{"1min":60.140000000000001,"5min":60.140000000000001,"15min":62.061999999999998},"last":0.254},{"interface":"front","average":{"1min":4.4889999999999999,"5min":2.0779999999999998,"15min":2.0110000000000001},"min":{"1min":0.20399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":62.389000000000003,"5min":62.389000000000003,"15min":62.389000000000003},"last":1.5429999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:03:18 2023","interfaces":[{"interface":"back","average":{"1min":14.236000000000001,"5min":5.7750000000000004,"15min":2.7639999999999998},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":104.16800000000001,"5min":104.16800000000001,"15min":104.16800000000001},"last":0.81799999999999995},{"interface":"front","average":{"1min":14.055,"5min":5.7489999999999997,"15min":2.831},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":104.003,"5min":104.003,"15min":104.003},"last":0.70099999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:03:36 2023","interfaces":[{"interface":"back","average":{"1min":7.5460000000000003,"5min":3.016,"15min":3.0070000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":105.842,"5min":105.842,"15min":130.673},"last":0.43099999999999999},{"interface":"front","average":{"1min":7.0190000000000001,"5min":3.4649999999999999,"15min":3.1930000000000001},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":105.66,"5min":105.66,"15min":130.71100000000001},"last":0.60699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":9.5630000000000006,"5min":3.9380000000000002,"15min":2.8290000000000002},"min":{"1min":0.316,"5min":0.247,"15min":0.247},"max":{"1min":146.38800000000001,"5min":146.38800000000001,"15min":146.38800000000001},"last":0.316},{"interface":"front","average":{"1min":9.1679999999999993,"5min":4.0839999999999996,"15min":2.8039999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":146.851,"5min":146.851,"15min":146.851},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.878,"15min":1.885},"min":{"1min":0.28299999999999997,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.3109999999999999,"5min":19.215,"15min":143.255},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.84499999999999997,"15min":1.7969999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.0419999999999998,"5min":19.364000000000001,"15min":143.41200000000001},"last":0.28899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.1819999999999999,"15min":1.444},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":2.9380000000000002,"5min":35.488,"15min":84.566999999999993},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.085,"15min":1.3089999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.343,"5min":34.896000000000001,"15min":81.244},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9820000000000002,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.38}]}]},{"osd":6,"up_from":0,"seq":1078036791316,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762477,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2603220,"kb_used_data":2226828,"kb_used_omap":7,"kb_used_meta":376376,"kb_avail":91141932,"statfs":{"total":95995035648,"available":93329338368,"internally_reserved":0,"allocated":2280271872,"data_stored":2279043226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":385409850},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.1360000000000001,"15min":3.6930000000000001},"min":{"1min":0.19500000000000001,"5min":0.16900000000000001,"15min":0.151},"max":{"1min":7.7409999999999997,"5min":25.238,"15min":202.917},"last":0.247},{"interface":"front","average":{"1min":2.948,"5min":2.1509999999999998,"15min":3.778},"min":{"1min":0.26000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":7.6189999999999998,"5min":22.337,"15min":203.34899999999999},"last":0.23200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":5.0609999999999999,"15min":5.2329999999999997},"min":{"1min":0.315,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":11.917999999999999,"5min":134.72499999999999,"15min":231.57499999999999},"last":9.3300000000000001},{"interface":"front","average":{"1min":3.8370000000000002,"5min":5.4279999999999999,"15min":5.2359999999999998},"min":{"1min":0.29399999999999998,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":20.677,"5min":135.74600000000001,"15min":236.077},"last":10.244999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":3.1179999999999999,"5min":3.0230000000000001,"15min":2.7970000000000002},"min":{"1min":0.28399999999999997,"5min":0.247,"15min":0.217},"max":{"1min":12.679,"5min":56.564,"15min":68.180999999999997},"last":0.27500000000000002},{"interface":"front","average":{"1min":3.0609999999999999,"5min":2.8900000000000001,"15min":2.5190000000000001},"min":{"1min":0.309,"5min":0.249,"15min":0.20100000000000001},"max":{"1min":12.286,"5min":56.939999999999998,"15min":66.509},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":3.9420000000000002,"5min":1.8700000000000001,"15min":2.3250000000000002},"min":{"1min":0.34599999999999997,"5min":0.254,"15min":0.25},"max":{"1min":35.497999999999998,"5min":35.497999999999998,"15min":161.68299999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":4.8200000000000003,"5min":1.968,"15min":2.282},"min":{"1min":0.42799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":35.521000000000001,"5min":35.521000000000001,"15min":161.286},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":4.9790000000000001,"5min":2.3769999999999998,"15min":3.1920000000000002},"min":{"1min":0.41199999999999998,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":51.698,"5min":51.698,"15min":219.048},"last":0.64000000000000001},{"interface":"front","average":{"1min":4.2729999999999997,"5min":2.3039999999999998,"15min":3.3490000000000002},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.29599999999999999},"max":{"1min":51.173000000000002,"5min":51.173000000000002,"15min":218.69499999999999},"last":5.2400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:03:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.2469999999999999,"15min":2.2469999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":9.8879999999999999,"5min":9.8879999999999999,"15min":9.8879999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":3.4460000000000002,"5min":3.4460000000000002,"15min":3.4460000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":20.734999999999999,"5min":20.734999999999999,"15min":20.734999999999999},"last":0.41299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1749999999999998,"5min":2.0249999999999999,"15min":2.0059999999999998},"min":{"1min":0.40000000000000002,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":14.007,"5min":14.23,"15min":60.213000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.1389999999999998,"5min":1.9330000000000001,"15min":1.6619999999999999},"min":{"1min":0.377,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":14.188000000000001,"5min":14.188000000000001,"15min":61.298999999999999},"last":0.53900000000000003}]}]},{"osd":0,"up_from":94,"seq":403726926011,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3251248,"kb_used_data":2823784,"kb_used_omap":8,"kb_used_meta":427447,"kb_avail":90493904,"statfs":{"total":95995035648,"available":92665757696,"internally_reserved":0,"allocated":2891554816,"data_stored":2890344942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":437706337},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6819999999999999,"5min":5.0289999999999999,"15min":2.6389999999999998},"min":{"1min":0.185,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":20.396000000000001,"5min":94.055999999999997,"15min":94.055999999999997},"last":0.19700000000000001},{"interface":"front","average":{"1min":3.964,"5min":5.1909999999999998,"15min":2.972},"min":{"1min":0.29099999999999998,"5min":0.183,"15min":0.183},"max":{"1min":22.056999999999999,"5min":91.632999999999996,"15min":91.632999999999996},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":5.9409999999999998,"15min":3.5910000000000002},"min":{"1min":0.23300000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.75,"5min":270.51100000000002,"15min":270.51100000000002},"last":0.248},{"interface":"front","average":{"1min":2.8109999999999999,"5min":5.5460000000000003,"15min":3.6419999999999999},"min":{"1min":0.25700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.038,"5min":269.69299999999998,"15min":269.69299999999998},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":6.2779999999999996,"5min":5.9699999999999998,"15min":4.4119999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":48.588000000000001,"5min":133.422,"15min":145.72800000000001},"last":0.21299999999999999},{"interface":"front","average":{"1min":6.4850000000000003,"5min":6.0259999999999998,"15min":4.5570000000000004},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":48.619999999999997,"5min":134.249,"15min":145.50700000000001},"last":0.25700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:03:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":2.1989999999999998,"15min":2.2639999999999998},"min":{"1min":0.35999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.4460000000000002,"5min":12.372,"15min":110.312},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.3959999999999999,"15min":1.859},"min":{"1min":0.29399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.2759999999999998,"5min":8.3499999999999996,"15min":110.256},"last":0.31}]},{"osd":5,"last update":"Sat Sep 23 20:03:04 2023","interfaces":[{"interface":"back","average":{"1min":4.8689999999999998,"5min":2.8170000000000002,"15min":2.4209999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28199999999999997},"max":{"1min":64.725999999999999,"5min":64.725999999999999,"15min":131.34399999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.4610000000000001,"5min":2.6059999999999999,"15min":2.5369999999999999},"min":{"1min":0.38500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":3.5150000000000001,"5min":27.609999999999999,"15min":131.386},"last":0.35399999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.607,"15min":1.607},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.4290000000000003,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.9059999999999999,"5min":1.9059999999999999,"15min":1.9059999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.0629999999999997,"5min":5.0629999999999997,"15min":5.0629999999999997},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:03:04 2023","interfaces":[{"interface":"back","average":{"1min":4.8979999999999997,"5min":2.5339999999999998,"15min":1.9330000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":65.081999999999994,"5min":65.081999999999994,"15min":77.132999999999996},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.601,"5min":2.2890000000000001,"15min":2.0259999999999998},"min":{"1min":0.47199999999999998,"5min":0.35199999999999998,"15min":0.34300000000000003},"max":{"1min":3.7429999999999999,"5min":12.452,"15min":75.465999999999994},"last":0.442}]}]},{"osd":2,"up_from":104,"seq":446676598950,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2727792,"kb_used_data":2243304,"kb_used_omap":12,"kb_used_meta":484467,"kb_avail":91017360,"statfs":{"total":95995035648,"available":93201776640,"internally_reserved":0,"allocated":2297143296,"data_stored":2295943814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":496095230},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":3.6309999999999998,"15min":2.5019999999999998},"min":{"1min":0.20899999999999999,"5min":0.192,"15min":0.154},"max":{"1min":10.073,"5min":48.966999999999999,"15min":50.506999999999998},"last":0.23400000000000001},{"interface":"front","average":{"1min":1.4690000000000001,"5min":3.2509999999999999,"15min":2.3969999999999998},"min":{"1min":0.20200000000000001,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":7.8840000000000003,"5min":48.783000000000001,"15min":50.802999999999997},"last":4.7060000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":6.4400000000000004,"5min":5.4279999999999999,"15min":2.9529999999999998},"min":{"1min":0.247,"5min":0.21299999999999999,"15min":0.184},"max":{"1min":50.192999999999998,"5min":94.768000000000001,"15min":94.768000000000001},"last":1.3260000000000001},{"interface":"front","average":{"1min":6.7729999999999997,"5min":6.0060000000000002,"15min":3.1890000000000001},"min":{"1min":0.218,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":47.987000000000002,"5min":96.808000000000007,"15min":96.808000000000007},"last":1.1910000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:03:20 2023","interfaces":[{"interface":"back","average":{"1min":6.4880000000000004,"5min":3.4740000000000002,"15min":2.5489999999999999},"min":{"1min":0.23899999999999999,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":44.408000000000001,"5min":44.408000000000001,"15min":50.186999999999998},"last":1.363},{"interface":"front","average":{"1min":4.4829999999999997,"5min":3.5529999999999999,"15min":2.5979999999999999},"min":{"1min":0.28100000000000003,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":43.981999999999999,"5min":43.981999999999999,"15min":50.585000000000001},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.893,"15min":1.6890000000000001},"min":{"1min":0.30499999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.9249999999999998,"5min":20.591999999999999,"15min":50.640000000000001},"last":0.314},{"interface":"front","average":{"1min":2.3039999999999998,"5min":1.819,"15min":1.6990000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7629999999999999,"5min":23.062000000000001,"15min":50.154000000000003},"last":1.3939999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":10.351000000000001,"5min":3.9199999999999999,"15min":2.5009999999999999},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":172.505,"5min":172.505,"15min":172.505},"last":0.34200000000000003},{"interface":"front","average":{"1min":10.007,"5min":3.5630000000000002,"15min":2.347},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":172.375,"5min":172.375,"15min":172.375},"last":1.054}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":1.6259999999999999,"15min":1.6259999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.144,"5min":10.144,"15min":10.144},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.9690000000000001,"5min":1.9690000000000001,"15min":1.9690000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":6.3520000000000003,"5min":6.3520000000000003,"15min":6.3520000000000003},"last":1.0880000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":1.9530000000000001,"15min":1.796},"min":{"1min":0.436,"5min":0.33700000000000002,"15min":0.27400000000000002},"max":{"1min":9.7330000000000005,"5min":23.079999999999998,"15min":50.118000000000002},"last":1.421},{"interface":"front","average":{"1min":1.734,"5min":2.395,"15min":2.0680000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":10.17,"5min":20.614999999999998,"15min":50.664999999999999},"last":0.45400000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435423,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2106524,"kb_used_data":1693908,"kb_used_omap":4,"kb_used_meta":412539,"kb_avail":91638628,"statfs":{"total":95995035648,"available":93837955072,"internally_reserved":0,"allocated":1734561792,"data_stored":1733250790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":422440045},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":3.4249999999999998,"5min":3.1949999999999998,"15min":3.7130000000000001},"min":{"1min":0.20399999999999999,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":21.100999999999999,"5min":57.149000000000001,"15min":240.40600000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":5.5540000000000003,"5min":3.9319999999999999,"15min":4.2450000000000001},"min":{"1min":0.19700000000000001,"5min":0.19600000000000001,"15min":0.17199999999999999},"max":{"1min":53.201000000000001,"5min":60.496000000000002,"15min":240.352},"last":0.26300000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":4.2699999999999996,"5min":3.1600000000000001,"15min":1.8080000000000001},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":18.395,"5min":22.041,"15min":22.041},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.9159999999999999,"5min":2.7189999999999999,"15min":1.6579999999999999},"min":{"1min":0.26900000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":18.524000000000001,"5min":47.951000000000001,"15min":47.951000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":3.9300000000000002,"5min":3.27,"15min":4.9089999999999998},"min":{"1min":0.39200000000000002,"5min":0.219,"15min":0.219},"max":{"1min":28.872,"5min":28.872,"15min":264.97800000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":4.4420000000000002,"5min":2.5630000000000002,"15min":4.7149999999999999},"min":{"1min":0.35399999999999998,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":30.768999999999998,"5min":30.768999999999998,"15min":257.40899999999999},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":2.8700000000000001,"15min":1.917},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.8340000000000001,"5min":74.866,"15min":74.866},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.121,"5min":3.4239999999999999,"15min":2.3570000000000002},"min":{"1min":0.311,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.6500000000000004,"5min":74.893000000000001,"15min":74.893000000000001},"last":0.38800000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":1.599,"5min":3.5150000000000001,"15min":2.1030000000000002},"min":{"1min":0.314,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.375,"5min":113.881,"15min":113.881},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.601,"5min":3.6459999999999999,"15min":2.036},"min":{"1min":0.34499999999999997,"5min":0.31,"15min":0.31},"max":{"1min":5.9820000000000002,"5min":114.598,"15min":114.598},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":5.7519999999999998,"5min":5.7519999999999998,"15min":5.7519999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":82.281999999999996,"5min":82.281999999999996,"15min":82.281999999999996},"last":0.496},{"interface":"front","average":{"1min":5.75,"5min":5.75,"15min":5.75},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":82.233000000000004,"5min":82.233000000000004,"15min":82.233000000000004},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":3.8420000000000001,"15min":2.1309999999999998},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.0090000000000003,"5min":138.035,"15min":138.035},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.1990000000000001,"5min":3.359,"15min":1.9119999999999999},"min":{"1min":0.33100000000000002,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":3.915,"5min":138.054,"15min":138.054},"last":0.56299999999999994}]}]},{"osd":4,"up_from":30,"seq":128849019185,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1944220,"kb_used_data":1693332,"kb_used_omap":0,"kb_used_meta":250880,"kb_avail":91800932,"statfs":{"total":95995035648,"available":94004154368,"internally_reserved":0,"allocated":1733971968,"data_stored":1733137656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":256901120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:06 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":3.036,"15min":2.9060000000000001},"min":{"1min":0.22,"5min":0.189,"15min":0.189},"max":{"1min":17.061,"5min":89.004999999999995,"15min":104.604},"last":4.226},{"interface":"front","average":{"1min":1.726,"5min":3.012,"15min":2.8679999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.23400000000000001},"max":{"1min":17.088999999999999,"5min":89.024000000000001,"15min":104.687},"last":5.4509999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:03:16 2023","interfaces":[{"interface":"back","average":{"1min":10.568,"5min":4.6589999999999998,"15min":2.819},"min":{"1min":0.252,"5min":0.248,"15min":0.216},"max":{"1min":141.22900000000001,"5min":141.22900000000001,"15min":141.22900000000001},"last":4.2649999999999997},{"interface":"front","average":{"1min":10.423,"5min":4.9329999999999998,"15min":2.7120000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":141.21299999999999,"5min":141.21299999999999,"15min":141.21299999999999},"last":4.4450000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":14.250999999999999,"5min":9.3689999999999998,"15min":5.0190000000000001},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":109.38200000000001,"5min":144.20099999999999,"15min":144.20099999999999},"last":4.4029999999999996},{"interface":"front","average":{"1min":14.645,"5min":8.7840000000000007,"15min":4.7690000000000001},"min":{"1min":0.34200000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":107.74299999999999,"5min":144.184,"15min":144.184},"last":11.048}]},{"osd":3,"last update":"Sat Sep 23 20:03:16 2023","interfaces":[{"interface":"back","average":{"1min":4.4850000000000003,"5min":8.5050000000000008,"15min":4.3129999999999997},"min":{"1min":0.375,"5min":0.255,"15min":0.255},"max":{"1min":32.033000000000001,"5min":266.54700000000003,"15min":266.54700000000003},"last":1.2030000000000001},{"interface":"front","average":{"1min":4.069,"5min":5.0410000000000004,"15min":3.327},"min":{"1min":0.39300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":31.959,"5min":139.911,"15min":139.911},"last":0.78800000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:03:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":3.7069999999999999,"15min":1.855},"min":{"1min":0.23599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.2930000000000001,"5min":266.59899999999999,"15min":266.59899999999999},"last":4.077},{"interface":"front","average":{"1min":0.51300000000000001,"5min":3.7069999999999999,"15min":1.9590000000000001},"min":{"1min":0.28299999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.988,"5min":266.72899999999998,"15min":266.72899999999998},"last":1.24}]},{"osd":6,"last update":"Sat Sep 23 20:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.7,"5min":1.7,"15min":1.7},"last":4.0410000000000004},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":4.2439999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:03:47 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.81399999999999995,"15min":0.97499999999999998},"min":{"1min":0.318,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.276,"5min":7.6210000000000004,"15min":11.59},"last":1.2669999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.95099999999999996,"15min":0.999},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":1.3240000000000001,"5min":18.012,"15min":18.012},"last":4.3700000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757549,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1692488,"kb_used_data":1409984,"kb_used_omap":0,"kb_used_meta":282496,"kb_avail":92052664,"statfs":{"total":95995035648,"available":94261927936,"internally_reserved":0,"allocated":1443823616,"data_stored":1442997741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":289275904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.171,"15min":1.2709999999999999},"min":{"1min":0.22900000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":1.677,"5min":9.1699999999999999,"15min":24.852},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.113,"5min":1.3320000000000001,"15min":1.2709999999999999},"min":{"1min":0.22700000000000001,"5min":0.223,"15min":0.216},"max":{"1min":3.4980000000000002,"5min":9.0530000000000008,"15min":24.638999999999999},"last":0.45800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:03:16 2023","interfaces":[{"interface":"back","average":{"1min":7.9859999999999998,"5min":8.3239999999999998,"15min":4.1319999999999997},"min":{"1min":0.33200000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":84.602000000000004,"5min":266.197,"15min":266.197},"last":0.34399999999999997},{"interface":"front","average":{"1min":8.6099999999999994,"5min":8.5299999999999994,"15min":4.1340000000000003},"min":{"1min":0.28100000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":88.775999999999996,"5min":266.14999999999998,"15min":266.14999999999998},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:03:49 2023","interfaces":[{"interface":"back","average":{"1min":10.763,"5min":6.2130000000000001,"15min":5.5830000000000002},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":177.816,"5min":238.96199999999999,"15min":238.96199999999999},"last":0.439},{"interface":"front","average":{"1min":10.779999999999999,"5min":6.2439999999999998,"15min":5.6189999999999998},"min":{"1min":0.33800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":177.851,"5min":238.905,"15min":238.905},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:03:16 2023","interfaces":[{"interface":"back","average":{"1min":6.9279999999999999,"5min":3.7040000000000002,"15min":2.706},"min":{"1min":0.41499999999999998,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":85.099999999999994,"5min":85.099999999999994,"15min":85.099999999999994},"last":0.47899999999999998},{"interface":"front","average":{"1min":6.7629999999999999,"5min":4.0030000000000001,"15min":2.722},"min":{"1min":0.38200000000000001,"5min":0.249,"15min":0.249},"max":{"1min":86.233999999999995,"5min":86.233999999999995,"15min":86.233999999999995},"last":0.44700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:03:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":3.0529999999999999,"15min":1.605},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.8079999999999998,"5min":191.78899999999999,"15min":191.78899999999999},"last":0.315},{"interface":"front","average":{"1min":0.36799999999999999,"5min":2.879,"15min":1.589},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53500000000000003,"5min":191.821,"15min":191.821},"last":0.36799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4420000000000002,"5min":2.4420000000000002,"15min":2.4420000000000002},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":3.2530000000000001,"15min":1.7330000000000001},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.996,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.497},{"interface":"front","average":{"1min":0.58799999999999997,"5min":3.379,"15min":1.9239999999999999},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.8169999999999999,"5min":191.69,"15min":191.69},"last":0.48999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2881622016,"data_stored":2881617944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2253656064,"data_stored":2253651992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2289172480,"data_stored":2289172480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1744506880,"data_stored":1744502808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1725825024,"data_stored":1725825024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1429602304,"data_stored":1429602304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1722810368,"data_stored":1722810368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:04:09.671 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:04:09.890 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:09.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:09.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:10.010 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:10.132 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:10.163 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:10.260 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:10.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:10.370 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:10.427 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:10.561 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:10.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:10.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:10.671 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:10 smithi135 ceph-mon[213871]: pgmap v747: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 16238/202788 objects misplaced (8.007%); 6.6 MiB/s, 107 objects/s recovering 2023-09-23T20:04:10.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:10 smithi135 ceph-mon[221113]: pgmap v747: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 16238/202788 objects misplaced (8.007%); 6.6 MiB/s, 107 objects/s recovering 2023-09-23T20:04:10.782 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:10.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:10 smithi144 ceph-mon[194833]: pgmap v747: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 16238/202788 objects misplaced (8.007%); 6.6 MiB/s, 107 objects/s recovering 2023-09-23T20:04:11.259 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149325 5.f 5:fde8a226:::benchmark_data_smithi135_284570_object33541:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.280934+0000", 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 566.97626661300001, 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1011945750000001, 2023-09-23T20:04:11.260 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149325 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.261 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280934+0000", 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280934+0000", 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.262 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280933+0000", 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280936+0000", 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0869999999999998e-06 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.263 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280942+0000", 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2829999999999998e-06 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280947+0000", 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3780000000000003e-06 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.264 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.265 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.280992+0000", 2023-09-23T20:04:11.265 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5649000000000003e-05 2023-09-23T20:04:11.265 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.265 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.265 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.266 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.281004+0000", 2023-09-23T20:04:11.266 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1311999999999999e-05 2023-09-23T20:04:11.266 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.266 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.266 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.266 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.355856+0000", 2023-09-23T20:04:11.267 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0748525609999999 2023-09-23T20:04:11.267 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.267 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.267 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.267 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.355989+0000", 2023-09-23T20:04:11.267 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000133029 2023-09-23T20:04:11.268 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.268 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.268 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:04:11.268 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.356119+0000", 2023-09-23T20:04:11.268 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013006199999999999 2023-09-23T20:04:11.268 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.269 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.269 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.269 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.373040+0000", 2023-09-23T20:04:11.269 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016920418 2023-09-23T20:04:11.269 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.269 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.270 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.270 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.381970+0000", 2023-09-23T20:04:11.270 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0089305089999999997 2023-09-23T20:04:11.270 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.270 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.271 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.271 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382036+0000", 2023-09-23T20:04:11.271 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5178000000000005e-05 2023-09-23T20:04:11.271 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.271 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382128+0000", 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2517999999999997e-05 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.272 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149335 5.f 5:f870a179:::benchmark_data_smithi135_284570_object33551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.296109+0000", 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 566.96109091799997, 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0862026, 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149335 2023-09-23T20:04:11.273 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296109+0000", 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.274 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296109+0000", 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296108+0000", 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296115+0000", 2023-09-23T20:04:11.275 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6409999999999996e-06 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296127+0000", 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2825e-05 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296133+0000", 2023-09-23T20:04:11.276 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6659999999999996e-06 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296179+0000", 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5927999999999998e-05 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.296187+0000", 2023-09-23T20:04:11.277 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1319999999999994e-06 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360611+0000", 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0644241830000001 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360726+0000", 2023-09-23T20:04:11.278 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114379 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360849+0000", 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012293100000000001 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.376986+0000", 2023-09-23T20:04:11.279 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016137459999999999 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382195+0000", 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052092550000000003 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382238+0000", 2023-09-23T20:04:11.280 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2998999999999999e-05 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382312+0000", 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3454000000000005e-05 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.281 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149337 5.f 5:f4eccdf5:::benchmark_data_smithi135_284570_object33553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.300192+0000", 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 566.95700860399995, 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0823250200000001, 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:04:11.282 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149337 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300192+0000", 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.283 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300192+0000", 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300191+0000", 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.284 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300196+0000", 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9629999999999997e-06 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300205+0000", 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1470000000000007e-06 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.285 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300211+0000", 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9739999999999999e-06 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300251+0000", 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0595e-05 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.286 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.300259+0000", 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6510000000000001e-06 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.365192+0000", 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0649325730000001 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.287 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.365303+0000", 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111457 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.365381+0000", 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8209999999999998e-05 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.288 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.376996+0000", 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011615138000000001 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382367+0000", 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005370934 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.289 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382424+0000", 2023-09-23T20:04:11.290 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6459999999999998e-05 2023-09-23T20:04:11.290 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.290 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.290 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.290 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382517+0000", 2023-09-23T20:04:11.290 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2748e-05 2023-09-23T20:04:11.290 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.290 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149354 5.f 5:f2a5a837:::benchmark_data_smithi135_284570_object33570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.332074+0000", 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 566.92512603900002, 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.050649921, 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.291 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149354 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332074+0000", 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.292 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332074+0000", 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332073+0000", 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.293 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332079+0000", 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3609999999999997e-06 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332095+0000", 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5537e-05 2023-09-23T20:04:11.294 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332103+0000", 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2630000000000003e-06 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332158+0000", 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5228e-05 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.295 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.332167+0000", 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1810000000000007e-06 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.368671+0000", 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0365039060000001 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.296 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.368789+0000", 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000117879 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.368928+0000", 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013848199999999999 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.297 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.381710+0000", 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012782188999999999 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382576+0000", 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00086563600000000003 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.298 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382631+0000", 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5569999999999998e-05 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382724+0000", 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2875999999999996e-05 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.299 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149356 5.f 5:ff7f40ec:::benchmark_data_smithi135_284570_object33572:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.335103+0000", 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 566.92209750999996, 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0478277620000001, 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.300 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149356 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335103+0000", 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.301 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335103+0000", 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335101+0000", 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:11.302 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335111+0000", 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6360000000000006e-06 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335124+0000", 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3264e-05 2023-09-23T20:04:11.303 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335129+0000", 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0250000000000002e-06 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335165+0000", 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5111000000000003e-05 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.304 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.335170+0000", 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5910000000000003e-06 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.373070+0000", 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0378999600000001 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.305 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.373192+0000", 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000121709 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.373325+0000", 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013317599999999999 2023-09-23T20:04:11.306 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.381895+0000", 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0085699360000000002 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382785+0000", 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00089052200000000004 2023-09-23T20:04:11.307 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382834+0000", 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8702000000000002e-05 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382930+0000", 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6261000000000006e-05 2023-09-23T20:04:11.308 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149360 5.f 5:fbe4ed6a:::benchmark_data_smithi135_284570_object33576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.346857+0000", 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 566.910343227, 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0362635920000001, 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.309 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.310 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:04:11.310 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:04:11.310 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149360 2023-09-23T20:04:11.310 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.310 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.310 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.310 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.311 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346857+0000", 2023-09-23T20:04:11.311 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.311 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.311 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.311 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.311 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346857+0000", 2023-09-23T20:04:11.311 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.312 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.312 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.312 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.312 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346856+0000", 2023-09-23T20:04:11.312 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:11.312 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.313 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.313 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.313 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346871+0000", 2023-09-23T20:04:11.313 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4905e-05 2023-09-23T20:04:11.313 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.314 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.314 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.314 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346883+0000", 2023-09-23T20:04:11.314 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2498e-05 2023-09-23T20:04:11.314 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.315 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.315 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.315 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346890+0000", 2023-09-23T20:04:11.315 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.477e-06 2023-09-23T20:04:11.315 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.315 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.316 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.316 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346942+0000", 2023-09-23T20:04:11.316 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2201999999999999e-05 2023-09-23T20:04:11.316 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.316 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.316 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.317 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.346953+0000", 2023-09-23T20:04:11.317 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0879999999999999e-05 2023-09-23T20:04:11.317 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.317 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.317 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.318 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.377037+0000", 2023-09-23T20:04:11.318 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0300842379999999 2023-09-23T20:04:11.318 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.318 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.318 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.318 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.377177+0000", 2023-09-23T20:04:11.319 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013932300000000001 2023-09-23T20:04:11.319 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.319 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.319 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:04:11.319 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.377326+0000", 2023-09-23T20:04:11.319 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000149615 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382944+0000", 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0056173849999999999 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.382985+0000", 2023-09-23T20:04:11.320 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1128000000000003e-05 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.383034+0000", 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9357999999999997e-05 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.383120+0000", 2023-09-23T20:04:11.321 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6485999999999998e-05 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148787 6.a 6:54a43917:::benchmark_data_smithi135_344736_object46131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.901403+0000", 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 268.35579687699999, 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2834841199999998, 2023-09-23T20:04:11.322 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148787 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.323 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901403+0000", 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901403+0000", 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.324 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901402+0000", 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901407+0000", 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6229999999999996e-06 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.325 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901422+0000", 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5262999999999999e-05 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.901433+0000", 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0353000000000001e-05 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.326 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.905928+0000", 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0044957510000000001 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.906049+0000", 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012040900000000001 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:04:11.327 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.906125+0000", 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6507000000000005e-05 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.909603+0000", 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0034781069999999998 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.328 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.911131+0000", 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001527608 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924795+0000", 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.01366392 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.329 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924807+0000", 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1746000000000001e-05 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925187+0000", 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0003801859999999 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.330 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925201+0000", 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3718e-05 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.184795+0000", 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.25959485 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.331 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.184887+0000", 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1875999999999999e-05 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148806 6.c 6:36c6f17c:::benchmark_data_smithi135_344736_object46150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.937119+0000", 2023-09-23T20:04:11.332 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 268.32008118599998, 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2790684319999999, 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148806 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.333 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937119+0000", 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937119+0000", 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.334 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937119+0000", 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937121+0000", 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.773e-06 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.335 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937128+0000", 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3579999999999999e-06 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937132+0000", 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0319999999999997e-06 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.336 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937188+0000", 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5967000000000001e-05 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937195+0000", 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1380000000000001e-06 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.337 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925897+0000", 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98870237500000002 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925908+0000", 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.064e-05 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.338 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.190584+0000", 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2646758870000001 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.190670+0000", 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6321000000000002e-05 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.339 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.190748+0000", 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7558000000000005e-05 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.195309+0000", 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045612430000000004 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.340 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.195781+0000", 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00047211300000000002 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.199253+0000", 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0034722970000000001 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.341 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.215988+0000", 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016734480999999999 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.216074+0000", 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6388e-05 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.342 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.216187+0000", 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113292 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148809 6.a 6:512c2dc0:::benchmark_data_smithi135_344736_object46153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:04:11.343 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.937144+0000", 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 268.32005638499999, 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6061435479999999, 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148809 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.344 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937144+0000", 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937144+0000", 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.345 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937143+0000", 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937146+0000", 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0800000000000002e-06 2023-09-23T20:04:11.346 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937152+0000", 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6200000000000004e-06 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937156+0000", 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4999999999999999e-06 2023-09-23T20:04:11.347 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937211+0000", 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.533e-05 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937221+0000", 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.013e-05 2023-09-23T20:04:11.348 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925294+0000", 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98807346500000004 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925306+0000", 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1430000000000001e-05 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.349 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.184916+0000", 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2596101420000001 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.185002+0000", 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5881e-05 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.350 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.185443+0000", 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00044055800000000002 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.196356+0000", 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010913440999999999 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.351 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.209973+0000", 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013617402000000001 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.219759+0000", 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0097860410000000005 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.352 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543097+0000", 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.32333798800000002 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543161+0000", 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3901999999999999e-05 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.353 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543287+0000", 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012600900000000001 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148810 6.a 6:5d5fc8b8:::benchmark_data_smithi135_344736_object46154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.937169+0000", 2023-09-23T20:04:11.354 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 268.320031417, 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6063484140000002, 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148810 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.355 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937169+0000", 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937169+0000", 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.356 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937168+0000", 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937171+0000", 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9000000000000002e-06 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.357 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937178+0000", 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9020000000000002e-06 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937182+0000", 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.631e-06 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.358 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937246+0000", 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3986999999999996e-05 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.937252+0000", 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.993e-06 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.359 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925335+0000", 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98808339700000003 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.925342+0000", 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4300000000000003e-06 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.360 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.193209+0000", 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.267867882 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.193313+0000", 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000104054 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:04:11.361 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.193429+0000", 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115164 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.196466+0000", 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003037363 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.362 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.219303+0000", 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022836921 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.219784+0000", 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000481552 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.363 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543348+0000", 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.323563989 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543403+0000", 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4934e-05 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.364 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.543517+0000", 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113706 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182362 6.a 6:587b6d41:::benchmark_data_smithi135_344736_object79706:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.437025+0000", 2023-09-23T20:04:11.365 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 217.82017545400001, 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1483356659999999, 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182362 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.366 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437025+0000", 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437025+0000", 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.367 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437024+0000", 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437029+0000", 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8280000000000002e-06 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.368 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437040+0000", 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1742999999999999e-05 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437046+0000", 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5149999999999997e-06 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.369 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437099+0000", 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3625999999999998e-05 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437203+0000", 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103088 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.370 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.437285+0000", 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2295999999999999e-05 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.440032+0000", 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027470379999999998 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.371 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.445210+0000", 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005178177 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.445223+0000", 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2746e-05 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.372 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585252+0000", 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1400295599999999 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585360+0000", 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107982 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.373 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182380 6.a 6:5ee9a171:::benchmark_data_smithi135_344736_object79724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.471936+0000", 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 217.785264326, 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.141209084, 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.374 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182380 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471936+0000", 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.375 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471936+0000", 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471934+0000", 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.376 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471940+0000", 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5969999999999999e-06 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471957+0000", 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7572999999999999e-05 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.471963+0000", 2023-09-23T20:04:11.377 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0950000000000001e-06 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.472016+0000", 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2750999999999997e-05 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.472027+0000", 2023-09-23T20:04:11.378 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0507e-05 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585386+0000", 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.11335973 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585520+0000", 2023-09-23T20:04:11.379 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013361199999999999 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.585677+0000", 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015681199999999999 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.597609+0000", 2023-09-23T20:04:11.380 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011932096 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599956+0000", 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023472800000000002 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599990+0000", 2023-09-23T20:04:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3587000000000002e-05 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.612967+0000", 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012977572999999999 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613024+0000", 2023-09-23T20:04:11.382 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6620999999999999e-05 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613145+0000", 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000120937 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.383 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182383 6.a 6:52d5eae7:::benchmark_data_smithi135_344736_object79727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.477344+0000", 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 217.779855846, 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.136035479, 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182383 2023-09-23T20:04:11.384 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477344+0000", 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477344+0000", 2023-09-23T20:04:11.385 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477343+0000", 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477350+0000", 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:04:11.386 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477365+0000", 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5081e-05 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477371+0000", 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5199999999999997e-06 2023-09-23T20:04:11.387 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477423+0000", 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2051e-05 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.477432+0000", 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7399999999999999e-06 2023-09-23T20:04:11.388 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.588349+0000", 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1109166269999999 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.588439+0000", 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9716000000000006e-05 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.588516+0000", 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7050000000000003e-05 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599746+0000", 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011229784 2023-09-23T20:04:11.390 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600017+0000", 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00027105100000000002 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600043+0000", 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6009000000000001e-05 2023-09-23T20:04:11.391 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613199+0000", 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013156407 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613258+0000", 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9326000000000001e-05 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.392 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613380+0000", 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012138000000000001 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182389 6.a 6:57ff510e:::benchmark_data_smithi135_344736_object79733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:04:11.393 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.485261+0000", 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 217.771938887, 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1283127610000001, 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182389 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.394 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485261+0000", 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485261+0000", 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.395 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485261+0000", 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485263+0000", 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5150000000000001e-06 2023-09-23T20:04:11.396 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485271+0000", 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.96e-06 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485275+0000", 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9639999999999997e-06 2023-09-23T20:04:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485337+0000", 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2015999999999997e-05 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.485348+0000", 2023-09-23T20:04:11.398 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1174e-05 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.591401+0000", 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1060528839999999 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.591542+0000", 2023-09-23T20:04:11.399 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000140307 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.591625+0000", 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3741999999999999e-05 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599753+0000", 2023-09-23T20:04:11.400 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0081273339999999999 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600068+0000", 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00031554800000000002 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600143+0000", 2023-09-23T20:04:11.401 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4919999999999994e-05 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613424+0000", 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013280973 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613469+0000", 2023-09-23T20:04:11.402 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4427e-05 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613574+0000", 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105371 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.403 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182397 6.a 6:5da29aad:::benchmark_data_smithi135_344736_object79741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.500485+0000", 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 217.756715157, 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.113300274, 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.404 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182397 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500485+0000", 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500485+0000", 2023-09-23T20:04:11.405 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500484+0000", 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500489+0000", 2023-09-23T20:04:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0440000000000003e-06 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500502+0000", 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3233e-05 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.407 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500523+0000", 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0930000000000001e-05 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500591+0000", 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8384000000000001e-05 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.500604+0000", 2023-09-23T20:04:11.408 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.295e-05 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.594382+0000", 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0937782300000001 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.594572+0000", 2023-09-23T20:04:11.409 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000189526 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.594707+0000", 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013493399999999999 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599919+0000", 2023-09-23T20:04:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052121670000000002 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.600093+0000", 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017360800000000001 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.612698+0000", 2023-09-23T20:04:11.411 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012605016 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613628+0000", 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00093064799999999996 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613677+0000", 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8222e-05 2023-09-23T20:04:11.412 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613785+0000", 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000108701 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:182416 6.a 6:5629a20a:::benchmark_data_smithi135_344736_object79760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:33.530109+0000", 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 217.727091089, 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0838080189999999, 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182416 2023-09-23T20:04:11.414 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530109+0000", 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530109+0000", 2023-09-23T20:04:11.415 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530108+0000", 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530114+0000", 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5289999999999997e-06 2023-09-23T20:04:11.416 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530126+0000", 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1975e-05 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530131+0000", 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0470000000000001e-06 2023-09-23T20:04:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530180+0000", 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8770999999999998e-05 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:33.530187+0000", 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1779999999999997e-06 2023-09-23T20:04:11.418 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.597643+0000", 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0674553579999999 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.597758+0000", 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114963 2023-09-23T20:04:11.419 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.597842+0000", 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4295999999999994e-05 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.599996+0000", 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002154319 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.601872+0000", 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001875959 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.612762+0000", 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010889598 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613821+0000", 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001059459 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613850+0000", 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8738000000000002e-05 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.422 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:34.613917+0000", 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7012000000000001e-05 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:184941 6.17 6:eb7ca16f:::benchmark_data_smithi135_344736_object82285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T20:04:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:39.639104+0000", 2023-09-23T20:04:11.424 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 211.61809627599999, 2023-09-23T20:04:11.424 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.94413358700000005, 2023-09-23T20:04:11.424 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.424 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.424 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.424 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.425 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.425 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 184941 2023-09-23T20:04:11.425 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.425 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.425 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.425 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.425 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639104+0000", 2023-09-23T20:04:11.426 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.426 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.426 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.426 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.426 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639104+0000", 2023-09-23T20:04:11.426 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.426 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.427 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.427 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.427 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639104+0000", 2023-09-23T20:04:11.427 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:11.427 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.427 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.428 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.428 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639105+0000", 2023-09-23T20:04:11.428 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.751e-06 2023-09-23T20:04:11.428 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.428 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.428 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.428 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639109+0000", 2023-09-23T20:04:11.429 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5260000000000002e-06 2023-09-23T20:04:11.429 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.429 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.429 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.429 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639111+0000", 2023-09-23T20:04:11.429 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.26e-06 2023-09-23T20:04:11.429 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.430 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.430 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.430 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639176+0000", 2023-09-23T20:04:11.430 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4807000000000005e-05 2023-09-23T20:04:11.430 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.430 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.431 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.431 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.639187+0000", 2023-09-23T20:04:11.431 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0727000000000001e-05 2023-09-23T20:04:11.431 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.431 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.431 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.431 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.552727+0000", 2023-09-23T20:04:11.432 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.91354033999999995 2023-09-23T20:04:11.432 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.432 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.432 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.432 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.552834+0000", 2023-09-23T20:04:11.432 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000106863 2023-09-23T20:04:11.432 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.433 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.433 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:04:11.433 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.553245+0000", 2023-09-23T20:04:11.433 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000410744 2023-09-23T20:04:11.433 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.433 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.433 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.434 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.567487+0000", 2023-09-23T20:04:11.434 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.01424215 2023-09-23T20:04:11.434 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.434 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.434 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.434 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583067+0000", 2023-09-23T20:04:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015580112 2023-09-23T20:04:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583143+0000", 2023-09-23T20:04:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6168000000000007e-05 2023-09-23T20:04:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.436 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.436 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.436 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583192+0000", 2023-09-23T20:04:11.436 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9024999999999999e-05 2023-09-23T20:04:11.436 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.436 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.436 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.437 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583237+0000", 2023-09-23T20:04:11.437 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5379e-05 2023-09-23T20:04:11.437 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.437 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.437 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.437 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.437 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.438 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:184942 6.17 6:ef4b34a6:::benchmark_data_smithi135_344736_object82286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T20:04:11.438 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:39.644227+0000", 2023-09-23T20:04:11.438 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 211.61297297199999, 2023-09-23T20:04:11.438 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.93935562299999997, 2023-09-23T20:04:11.438 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.438 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.439 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.439 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.439 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.439 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 184942 2023-09-23T20:04:11.439 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.439 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.439 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644227+0000", 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644227+0000", 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.440 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644226+0000", 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644231+0000", 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1479999999999997e-06 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.441 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644244+0000", 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2921e-05 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644249+0000", 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8590000000000004e-06 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644298+0000", 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9360000000000002e-05 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.644307+0000", 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2479999999999996e-06 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.443 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.555315+0000", 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.91100864500000001 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.555438+0000", 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012302999999999999 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.444 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.555806+0000", 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000368024 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.567497+0000", 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011690196999999999 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.445 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583113+0000", 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015616849 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583505+0000", 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00039157500000000002 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.446 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583550+0000", 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5334999999999999e-05 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583583+0000", 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2434000000000002e-05 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.447 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:185076 6.17 6:e9aef155:::benchmark_data_smithi135_344736_object82420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:39.844967+0000", 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 211.41223350300001, 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.73883129000000003, 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.448 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 185076 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844967+0000", 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.449 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844967+0000", 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844966+0000", 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.450 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844969+0000", 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.847e-06 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844982+0000", 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2542e-05 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.451 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844986+0000", 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5589999999999999e-06 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845037+0000", 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0881000000000003e-05 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.452 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845041+0000", 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.754e-06 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.561449+0000", 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.71640810200000005 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.453 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.561559+0000", 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011042799999999999 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.561854+0000", 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00029497200000000001 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.454 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.571424+0000", 2023-09-23T20:04:11.455 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0095692190000000003 2023-09-23T20:04:11.455 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.457 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583338+0000", 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011914401 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583726+0000", 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000387969 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.458 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.459 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.459 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583766+0000", 2023-09-23T20:04:11.459 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0232999999999999e-05 2023-09-23T20:04:11.459 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.459 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.459 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583798+0000", 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1764000000000003e-05 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:185077 6.17 6:ea269bbc:::benchmark_data_smithi135_344736_object82421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T20:04:11.460 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:00:39.844996+0000", 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 211.41220460900001, 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.73895094699999997, 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 185077 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.461 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844996+0000", 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844996+0000", 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:11.462 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844995+0000", 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.844998+0000", 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6759999999999999e-06 2023-09-23T20:04:11.463 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845003+0000", 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1699999999999996e-06 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845006+0000", 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.484e-06 2023-09-23T20:04:11.464 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845056+0000", 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9446000000000001e-05 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:39.845059+0000", 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5530000000000001e-06 2023-09-23T20:04:11.465 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.563857+0000", 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.71879795999999996 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.563981+0000", 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012405599999999999 2023-09-23T20:04:11.466 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.564269+0000", 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00028795600000000002 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.571432+0000", 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0071621860000000001 2023-09-23T20:04:11.467 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583461+0000", 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012029257 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583851+0000", 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00038999 2023-09-23T20:04:11.468 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583907+0000", 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6047000000000001e-05 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:00:40.583946+0000", 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9582999999999997e-05 2023-09-23T20:04:11.469 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.470 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.470 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.470 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:11.470 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:11.470 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:04:11.905 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:11.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:11.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:11.977 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:12.011 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:12.035 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:04:12.096 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:04:12.125 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:12.293 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:12.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:12.398 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:12.494 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:12.502 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:12.602 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:12 smithi135 ceph-mon[213871]: pgmap v748: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 16238/201243 objects misplaced (8.069%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T20:04:12.602 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:12 smithi135 ceph-mon[221113]: pgmap v748: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 16238/201243 objects misplaced (8.069%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T20:04:12.776 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:12 smithi144 ceph-mon[194833]: pgmap v748: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 16238/201243 objects misplaced (8.069%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T20:04:13.464 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:13.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:13.577 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:13.609 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:13.784 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:13.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:13.891 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:13.925 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:13.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:04:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:04:13] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.43.0" 2023-09-23T20:04:14.286 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:04:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:04:14.287 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:04:14.287 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:04:14.413 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:14.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:14.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:14.525 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:14.561 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:14.568 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:04:14.605 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:14.607 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:14 smithi144 ceph-mon[194833]: pgmap v749: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 15471/198051 objects misplaced (7.812%); 7.7 MiB/s, 124 objects/s recovering 2023-09-23T20:04:14.607 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3123516746' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:04:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:14.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:14.709 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:14 smithi135 ceph-mon[221113]: pgmap v749: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 15471/198051 objects misplaced (7.812%); 7.7 MiB/s, 124 objects/s recovering 2023-09-23T20:04:14.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3123516746' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:04:14.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:14 smithi135 ceph-mon[213871]: pgmap v749: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 15471/198051 objects misplaced (7.812%); 7.7 MiB/s, 124 objects/s recovering 2023-09-23T20:04:14.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3123516746' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:04:14.740 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:15.130 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:04:15.488 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:15.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:15 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:15.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:15 smithi144 ceph-mon[194833]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3123516746' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:04:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:15 smithi144 ceph-mon[194833]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T20:04:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:15 smithi144 ceph-mon[194833]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:15 smithi144 ceph-mon[194833]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:15 smithi144 ceph-mon[194833]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:15.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[221113]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3123516746' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:04:15.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[221113]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T20:04:15.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[221113]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[221113]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[221113]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:15.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[213871]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3123516746' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:04:15.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[213871]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T20:04:15.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[213871]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[213871]: failed to encode map e279 with expected crc 2023-09-23T20:04:15.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:15 smithi135 ceph-mon[213871]: failed to encode map e279 with expected crc 2023-09-23T20:04:16.062 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:16.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:16.157 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:16.174 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:16.278 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:16.465 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:16.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:16.571 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:16.603 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:16.707 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:16.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:16.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:16 smithi135 ceph-mon[213871]: failed to encode map e279 with expected crc 2023-09-23T20:04:16.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:16 smithi135 ceph-mon[213871]: pgmap v751: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 15132/194445 objects misplaced (7.782%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:04:16.734 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:16 smithi135 ceph-mon[221113]: failed to encode map e279 with expected crc 2023-09-23T20:04:16.734 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:16 smithi135 ceph-mon[221113]: pgmap v751: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 15132/194445 objects misplaced (7.782%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:04:16.814 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:16.840 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:16 smithi144 ceph-mon[194833]: failed to encode map e279 with expected crc 2023-09-23T20:04:16.841 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:16 smithi144 ceph-mon[194833]: pgmap v751: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 15132/194445 objects misplaced (7.782%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:04:16.844 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:16.899 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:16.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:16.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:17.005 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:17.056 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:04:17.105 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:04:17.106 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:17.240 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:04:17.240 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:04:17.240 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:04:17.240 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:04:17.240 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:04:18.124 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:04:18.184 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:04:18.505 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:18.817 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:18 smithi144 ceph-mon[194833]: pgmap v752: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 15132/192990 objects misplaced (7.841%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:04:18.817 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:18 smithi144 ceph-mon[194833]: from='client.55225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:18.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:18 smithi135 ceph-mon[213871]: pgmap v752: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 15132/192990 objects misplaced (7.841%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:04:18.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:18 smithi135 ceph-mon[213871]: from='client.55225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:18.951 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:18 smithi135 ceph-mon[221113]: pgmap v752: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 15132/192990 objects misplaced (7.841%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:04:18.951 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:18 smithi135 ceph-mon[221113]: from='client.55225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:18.982 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:18.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:19.091 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:19.121 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:19.165 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:19.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:19.268 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:19.338 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:19.764 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:19.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:19.850 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":752,"stamp":"2023-09-23T20:04:16.707900+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4218820555,"num_objects":64330,"num_object_clones":0,"num_object_copies":192990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":15132,"num_objects_unfound":0,"num_objects_dirty":64330,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":130725,"num_write_kb":6224502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6452,"num_bytes_recovered":415701493,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110089,"ondisk_log_size":110089,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15316756,"kb_used_data":12788828,"kb_used_omap":32,"kb_used_meta":2527583,"kb_avail":640899308,"statfs":{"total":671965249536,"available":656280891392,"internally_reserved":0,"allocated":13095759872,"data_stored":13088257834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":2588245510},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,1,8,9,10,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-363331584,"num_objects":-5544,"num_object_clones":0,"num_object_copies":-16632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2177,"num_objects_unfound":0,"num_objects_dirty":-5544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5544,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":804,"num_bytes_recovered":52690944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005418"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":256,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273583+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T20:04:02.273583+0000","last_peered":"2023-09-23T20:04:02.273583+0000","last_clean":"2023-09-23T20:04:02.273583+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T20:04:02.273583+0000","last_undegraded":"2023-09-23T20:04:02.273583+0000","last_fullsized":"2023-09-23T20:04:02.273583+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.316823+0000","last_change":"2023-09-23T19:59:09.151117+0000","last_active":"2023-09-23T20:04:04.316823+0000","last_peered":"2023-09-23T20:04:04.316823+0000","last_clean":"2023-09-23T20:04:04.316823+0000","last_became_active":"2023-09-23T19:59:09.150795+0000","last_became_peered":"2023-09-23T19:59:09.150795+0000","last_unstale":"2023-09-23T20:04:04.316823+0000","last_undegraded":"2023-09-23T20:04:04.316823+0000","last_fullsized":"2023-09-23T20:04:04.316823+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:34.994574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":239,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.321392+0000","last_change":"2023-09-23T19:59:10.156070+0000","last_active":"2023-09-23T20:04:04.321392+0000","last_peered":"2023-09-23T20:04:04.321392+0000","last_clean":"2023-09-23T20:04:04.321392+0000","last_became_active":"2023-09-23T19:59:10.155813+0000","last_became_peered":"2023-09-23T19:59:10.155813+0000","last_unstale":"2023-09-23T20:04:04.321392+0000","last_undegraded":"2023-09-23T20:04:04.321392+0000","last_fullsized":"2023-09-23T20:04:04.321392+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:42.582380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":30,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274543+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:04:02.274543+0000","last_peered":"2023-09-23T20:04:02.274543+0000","last_clean":"2023-09-23T20:04:02.274543+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:04:02.274543+0000","last_undegraded":"2023-09-23T20:04:02.274543+0000","last_fullsized":"2023-09-23T20:04:02.274543+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":100,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256612+0000","last_change":"2023-09-23T19:59:09.163182+0000","last_active":"2023-09-23T20:04:02.256612+0000","last_peered":"2023-09-23T20:04:02.256612+0000","last_clean":"2023-09-23T20:04:02.256612+0000","last_became_active":"2023-09-23T19:59:09.161612+0000","last_became_peered":"2023-09-23T19:59:09.161612+0000","last_unstale":"2023-09-23T20:04:02.256612+0000","last_undegraded":"2023-09-23T20:04:02.256612+0000","last_fullsized":"2023-09-23T20:04:02.256612+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:59.524152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.269385+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T20:04:02.269385+0000","last_peered":"2023-09-23T20:04:02.269385+0000","last_clean":"2023-09-23T20:04:02.269385+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T20:04:02.269385+0000","last_undegraded":"2023-09-23T20:04:02.269385+0000","last_fullsized":"2023-09-23T20:04:02.269385+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:59.043547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":204,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.269304+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:04:02.269304+0000","last_peered":"2023-09-23T20:04:02.269304+0000","last_clean":"2023-09-23T20:04:02.269304+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:04:02.269304+0000","last_undegraded":"2023-09-23T20:04:02.269304+0000","last_fullsized":"2023-09-23T20:04:02.269304+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":100,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256688+0000","last_change":"2023-09-23T19:59:09.146009+0000","last_active":"2023-09-23T20:04:02.256688+0000","last_peered":"2023-09-23T20:04:02.256688+0000","last_clean":"2023-09-23T20:04:02.256688+0000","last_became_active":"2023-09-23T19:59:09.145687+0000","last_became_peered":"2023-09-23T19:59:09.145687+0000","last_unstale":"2023-09-23T20:04:02.256688+0000","last_undegraded":"2023-09-23T20:04:02.256688+0000","last_fullsized":"2023-09-23T20:04:02.256688+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.189450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":237,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.269003+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T20:04:02.269003+0000","last_peered":"2023-09-23T20:04:02.269003+0000","last_clean":"2023-09-23T20:04:02.269003+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T20:04:02.269003+0000","last_undegraded":"2023-09-23T20:04:02.269003+0000","last_fullsized":"2023-09-23T20:04:02.269003+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":119,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256376+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:04:02.256376+0000","last_peered":"2023-09-23T20:04:02.256376+0000","last_clean":"2023-09-23T20:04:02.256376+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:04:02.256376+0000","last_undegraded":"2023-09-23T20:04:02.256376+0000","last_fullsized":"2023-09-23T20:04:02.256376+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":273,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274302+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:04:02.274302+0000","last_peered":"2023-09-23T20:04:02.274302+0000","last_clean":"2023-09-23T20:04:02.274302+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:04:02.274302+0000","last_undegraded":"2023-09-23T20:04:02.274302+0000","last_fullsized":"2023-09-23T20:04:02.274302+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":236,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.319065+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T20:04:04.319065+0000","last_peered":"2023-09-23T20:04:04.319065+0000","last_clean":"2023-09-23T20:04:04.319065+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T20:04:04.319065+0000","last_undegraded":"2023-09-23T20:04:04.319065+0000","last_fullsized":"2023-09-23T20:04:04.319065+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:51.825640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":292,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273015+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:04:02.273015+0000","last_peered":"2023-09-23T20:04:02.273015+0000","last_clean":"2023-09-23T20:04:02.273015+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:04:02.273015+0000","last_undegraded":"2023-09-23T20:04:02.273015+0000","last_fullsized":"2023-09-23T20:04:02.273015+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":254,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273227+0000","last_change":"2023-09-23T20:01:52.439375+0000","last_active":"2023-09-23T20:04:02.273227+0000","last_peered":"2023-09-23T20:04:02.273227+0000","last_clean":"2023-09-23T20:04:02.273227+0000","last_became_active":"2023-09-23T20:01:52.439074+0000","last_became_peered":"2023-09-23T20:01:52.439074+0000","last_unstale":"2023-09-23T20:04:02.273227+0000","last_undegraded":"2023-09-23T20:04:02.273227+0000","last_fullsized":"2023-09-23T20:04:02.273227+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:31.103633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":240,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274159+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:04:02.274159+0000","last_peered":"2023-09-23T20:04:02.274159+0000","last_clean":"2023-09-23T20:04:02.274159+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:04:02.274159+0000","last_undegraded":"2023-09-23T20:04:02.274159+0000","last_fullsized":"2023-09-23T20:04:02.274159+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256451+0000","last_change":"2023-09-23T19:59:09.195560+0000","last_active":"2023-09-23T20:04:02.256451+0000","last_peered":"2023-09-23T20:04:02.256451+0000","last_clean":"2023-09-23T20:04:02.256451+0000","last_became_active":"2023-09-23T19:59:09.195145+0000","last_became_peered":"2023-09-23T19:59:09.195145+0000","last_unstale":"2023-09-23T20:04:02.256451+0000","last_undegraded":"2023-09-23T20:04:02.256451+0000","last_fullsized":"2023-09-23T20:04:02.256451+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.190798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":243,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274008+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:04:02.274008+0000","last_peered":"2023-09-23T20:04:02.274008+0000","last_clean":"2023-09-23T20:04:02.274008+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:04:02.274008+0000","last_undegraded":"2023-09-23T20:04:02.274008+0000","last_fullsized":"2023-09-23T20:04:02.274008+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":256,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.268755+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:04:02.268755+0000","last_peered":"2023-09-23T20:04:02.268755+0000","last_clean":"2023-09-23T20:04:02.268755+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:04:02.268755+0000","last_undegraded":"2023-09-23T20:04:02.268755+0000","last_fullsized":"2023-09-23T20:04:02.268755+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":238,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273918+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:04:02.273918+0000","last_peered":"2023-09-23T20:04:02.273918+0000","last_clean":"2023-09-23T20:04:02.273918+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:04:02.273918+0000","last_undegraded":"2023-09-23T20:04:02.273918+0000","last_fullsized":"2023-09-23T20:04:02.273918+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":243,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.258786+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:04:02.258786+0000","last_peered":"2023-09-23T20:04:02.258786+0000","last_clean":"2023-09-23T20:04:02.258786+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:04:02.258786+0000","last_undegraded":"2023-09-23T20:04:02.258786+0000","last_fullsized":"2023-09-23T20:04:02.258786+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":71,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.267711+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:04:02.267711+0000","last_peered":"2023-09-23T20:04:02.267711+0000","last_clean":"2023-09-23T20:04:02.267711+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:04:02.267711+0000","last_undegraded":"2023-09-23T20:04:02.267711+0000","last_fullsized":"2023-09-23T20:04:02.267711+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256554+0000","last_change":"2023-09-23T20:03:59.232022+0000","last_active":"2023-09-23T20:04:02.256554+0000","last_peered":"2023-09-23T20:04:02.256554+0000","last_clean":"2023-09-23T20:04:02.256554+0000","last_became_active":"2023-09-23T20:03:59.231689+0000","last_became_peered":"2023-09-23T20:03:59.231689+0000","last_unstale":"2023-09-23T20:04:02.256554+0000","last_undegraded":"2023-09-23T20:04:02.256554+0000","last_fullsized":"2023-09-23T20:04:02.256554+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:29.669102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"279'8093","reported_seq":18376,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:15.475246+0000","last_change":"2023-09-23T20:04:02.290062+0000","last_active":"2023-09-23T20:04:15.475246+0000","last_peered":"2023-09-23T20:04:15.475246+0000","last_clean":"2023-09-23T20:04:15.475246+0000","last_became_active":"2023-09-23T20:04:02.271287+0000","last_became_peered":"2023-09-23T20:04:02.271287+0000","last_unstale":"2023-09-23T20:04:15.475246+0000","last_undegraded":"2023-09-23T20:04:15.475246+0000","last_fullsized":"2023-09-23T20:04:15.475246+0000","mapping_epoch":276,"log_start":"247'1700","ondisk_log_start":"247'1700","created":240,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:47.344179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261554176,"num_objects":3991,"num_object_clones":0,"num_object_copies":11973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8093,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5100,"num_bytes_recovered":334233600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":239,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.276706+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:04:02.276706+0000","last_peered":"2023-09-23T20:04:02.276706+0000","last_clean":"2023-09-23T20:04:02.276706+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:04:02.276706+0000","last_undegraded":"2023-09-23T20:04:02.276706+0000","last_fullsized":"2023-09-23T20:04:02.276706+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":266,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.267614+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:04:02.267614+0000","last_peered":"2023-09-23T20:04:02.267614+0000","last_clean":"2023-09-23T20:04:02.267614+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:04:02.267614+0000","last_undegraded":"2023-09-23T20:04:02.267614+0000","last_fullsized":"2023-09-23T20:04:02.267614+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"278'8040","reported_seq":8088,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:11.995203+0000","last_change":"2023-09-23T20:01:39.911588+0000","last_active":"2023-09-23T20:04:11.995203+0000","last_peered":"2023-09-23T20:04:11.995203+0000","last_clean":"2023-09-23T20:04:11.995203+0000","last_became_active":"2023-09-23T20:01:39.911364+0000","last_became_peered":"2023-09-23T20:01:39.911364+0000","last_unstale":"2023-09-23T20:04:11.995203+0000","last_undegraded":"2023-09-23T20:04:11.995203+0000","last_fullsized":"2023-09-23T20:04:11.995203+0000","mapping_epoch":240,"log_start":"247'2295","ondisk_log_start":"247'2295","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:35.715001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262799360,"num_objects":4010,"num_object_clones":0,"num_object_copies":12030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8040,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":240,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274705+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T20:04:02.274705+0000","last_peered":"2023-09-23T20:04:02.274705+0000","last_clean":"2023-09-23T20:04:02.274705+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T20:04:02.274705+0000","last_undegraded":"2023-09-23T20:04:02.274705+0000","last_fullsized":"2023-09-23T20:04:02.274705+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":294,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.272747+0000","last_change":"2023-09-23T20:03:59.253784+0000","last_active":"2023-09-23T20:04:02.272747+0000","last_peered":"2023-09-23T20:04:02.272747+0000","last_clean":"2023-09-23T20:04:02.272747+0000","last_became_active":"2023-09-23T20:03:59.253552+0000","last_became_peered":"2023-09-23T20:03:59.253552+0000","last_unstale":"2023-09-23T20:04:02.272747+0000","last_undegraded":"2023-09-23T20:04:02.272747+0000","last_fullsized":"2023-09-23T20:04:02.272747+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:56.110746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"279'8141","reported_seq":8190,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:15.281216+0000","last_change":"2023-09-23T20:01:39.976582+0000","last_active":"2023-09-23T20:04:15.281216+0000","last_peered":"2023-09-23T20:04:15.281216+0000","last_clean":"2023-09-23T20:04:15.281216+0000","last_became_active":"2023-09-23T20:01:39.976244+0000","last_became_peered":"2023-09-23T20:01:39.976244+0000","last_unstale":"2023-09-23T20:04:15.281216+0000","last_undegraded":"2023-09-23T20:04:15.281216+0000","last_fullsized":"2023-09-23T20:04:15.281216+0000","mapping_epoch":240,"log_start":"247'1700","ondisk_log_start":"247'1700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6441,"log_dups_size":0,"ondisk_log_size":6441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.952295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255918080,"num_objects":3905,"num_object_clones":0,"num_object_copies":11715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8141,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":243,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274118+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:04:02.274118+0000","last_peered":"2023-09-23T20:04:02.274118+0000","last_clean":"2023-09-23T20:04:02.274118+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:04:02.274118+0000","last_undegraded":"2023-09-23T20:04:02.274118+0000","last_fullsized":"2023-09-23T20:04:02.274118+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":206,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.319295+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T20:04:04.319295+0000","last_peered":"2023-09-23T20:04:04.319295+0000","last_clean":"2023-09-23T20:04:04.319295+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T20:04:04.319295+0000","last_undegraded":"2023-09-23T20:04:04.319295+0000","last_fullsized":"2023-09-23T20:04:04.319295+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.284038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"278'8388","reported_seq":8436,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:13.111944+0000","last_change":"2023-09-23T20:01:39.912566+0000","last_active":"2023-09-23T20:04:13.111944+0000","last_peered":"2023-09-23T20:04:13.111944+0000","last_clean":"2023-09-23T20:04:13.111944+0000","last_became_active":"2023-09-23T20:01:39.912259+0000","last_became_peered":"2023-09-23T20:01:39.912259+0000","last_unstale":"2023-09-23T20:04:13.111944+0000","last_undegraded":"2023-09-23T20:04:13.111944+0000","last_fullsized":"2023-09-23T20:04:13.111944+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8388,"log_dups_size":0,"ondisk_log_size":8388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:19.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265945088,"num_objects":4058,"num_object_clones":0,"num_object_copies":12174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8388,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":296,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.268555+0000","last_change":"2023-09-23T20:00:34.468296+0000","last_active":"2023-09-23T20:04:02.268555+0000","last_peered":"2023-09-23T20:04:02.268555+0000","last_clean":"2023-09-23T20:04:02.268555+0000","last_became_active":"2023-09-23T20:00:34.467957+0000","last_became_peered":"2023-09-23T20:00:34.467957+0000","last_unstale":"2023-09-23T20:04:02.268555+0000","last_undegraded":"2023-09-23T20:04:02.268555+0000","last_fullsized":"2023-09-23T20:04:02.268555+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:50.449027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":94,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.319765+0000","last_change":"2023-09-23T20:01:04.470558+0000","last_active":"2023-09-23T20:04:04.319765+0000","last_peered":"2023-09-23T20:04:04.319765+0000","last_clean":"2023-09-23T20:04:04.319765+0000","last_became_active":"2023-09-23T20:01:04.470276+0000","last_became_peered":"2023-09-23T20:01:04.470276+0000","last_unstale":"2023-09-23T20:04:04.319765+0000","last_undegraded":"2023-09-23T20:04:04.319765+0000","last_fullsized":"2023-09-23T20:04:04.319765+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:17.788583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"278'8071","reported_seq":8119,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:11.949315+0000","last_change":"2023-09-23T20:01:39.974611+0000","last_active":"2023-09-23T20:04:11.949315+0000","last_peered":"2023-09-23T20:04:11.949315+0000","last_clean":"2023-09-23T20:04:11.949315+0000","last_became_active":"2023-09-23T20:01:39.974318+0000","last_became_peered":"2023-09-23T20:01:39.974318+0000","last_unstale":"2023-09-23T20:04:11.949315+0000","last_undegraded":"2023-09-23T20:04:11.949315+0000","last_fullsized":"2023-09-23T20:04:11.949315+0000","mapping_epoch":240,"log_start":"247'2327","ondisk_log_start":"247'2327","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:31.047765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260636672,"num_objects":3977,"num_object_clones":0,"num_object_copies":11931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8071,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":728,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273711+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:04:02.273711+0000","last_peered":"2023-09-23T20:04:02.273711+0000","last_clean":"2023-09-23T20:04:02.273711+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:04:02.273711+0000","last_undegraded":"2023-09-23T20:04:02.273711+0000","last_fullsized":"2023-09-23T20:04:02.273711+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":104,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.257355+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:04:02.257355+0000","last_peered":"2023-09-23T20:04:02.257355+0000","last_clean":"2023-09-23T20:04:02.257355+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:04:02.257355+0000","last_undegraded":"2023-09-23T20:04:02.257355+0000","last_fullsized":"2023-09-23T20:04:02.257355+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":274,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.272923+0000","last_change":"2023-09-23T19:59:09.206075+0000","last_active":"2023-09-23T20:04:02.272923+0000","last_peered":"2023-09-23T20:04:02.272923+0000","last_clean":"2023-09-23T20:04:02.272923+0000","last_became_active":"2023-09-23T19:59:09.205766+0000","last_became_peered":"2023-09-23T19:59:09.205766+0000","last_unstale":"2023-09-23T20:04:02.272923+0000","last_undegraded":"2023-09-23T20:04:02.272923+0000","last_fullsized":"2023-09-23T20:04:02.272923+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:14.627703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"278'7946","reported_seq":7994,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:13.200497+0000","last_change":"2023-09-23T20:01:39.976550+0000","last_active":"2023-09-23T20:04:13.200497+0000","last_peered":"2023-09-23T20:04:13.200497+0000","last_clean":"2023-09-23T20:04:13.200497+0000","last_became_active":"2023-09-23T20:01:39.976245+0000","last_became_peered":"2023-09-23T20:01:39.976245+0000","last_unstale":"2023-09-23T20:04:13.200497+0000","last_undegraded":"2023-09-23T20:04:13.200497+0000","last_fullsized":"2023-09-23T20:04:13.200497+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:45.979020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256770048,"num_objects":3918,"num_object_clones":0,"num_object_copies":11754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7946,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":637,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.267537+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:04:02.267537+0000","last_peered":"2023-09-23T20:04:02.267537+0000","last_clean":"2023-09-23T20:04:02.267537+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:04:02.267537+0000","last_undegraded":"2023-09-23T20:04:02.267537+0000","last_fullsized":"2023-09-23T20:04:02.267537+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":224,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273523+0000","last_change":"2023-09-23T19:59:09.172601+0000","last_active":"2023-09-23T20:04:02.273523+0000","last_peered":"2023-09-23T20:04:02.273523+0000","last_clean":"2023-09-23T20:04:02.273523+0000","last_became_active":"2023-09-23T19:59:09.172297+0000","last_became_peered":"2023-09-23T19:59:09.172297+0000","last_unstale":"2023-09-23T20:04:02.273523+0000","last_undegraded":"2023-09-23T20:04:02.273523+0000","last_fullsized":"2023-09-23T20:04:02.273523+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:09.528266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":254,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.272928+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T20:04:02.272928+0000","last_peered":"2023-09-23T20:04:02.272928+0000","last_clean":"2023-09-23T20:04:02.272928+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T20:04:02.272928+0000","last_undegraded":"2023-09-23T20:04:02.272928+0000","last_fullsized":"2023-09-23T20:04:02.272928+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"279'8174","reported_seq":8224,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:15.492042+0000","last_change":"2023-09-23T20:01:39.977019+0000","last_active":"2023-09-23T20:04:15.492042+0000","last_peered":"2023-09-23T20:04:15.492042+0000","last_clean":"2023-09-23T20:04:15.492042+0000","last_became_active":"2023-09-23T20:01:39.976270+0000","last_became_peered":"2023-09-23T20:01:39.976270+0000","last_unstale":"2023-09-23T20:04:15.492042+0000","last_undegraded":"2023-09-23T20:04:15.492042+0000","last_fullsized":"2023-09-23T20:04:15.492042+0000","mapping_epoch":240,"log_start":"247'1700","ondisk_log_start":"247'1700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6474,"log_dups_size":0,"ondisk_log_size":6474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.226759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256704536,"num_objects":3918,"num_object_clones":0,"num_object_copies":11754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3918,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8174,"num_write_kb":386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":238,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.272589+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:04:02.272589+0000","last_peered":"2023-09-23T20:04:02.272589+0000","last_clean":"2023-09-23T20:04:02.272589+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:04:02.272589+0000","last_undegraded":"2023-09-23T20:04:02.272589+0000","last_fullsized":"2023-09-23T20:04:02.272589+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":284,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273225+0000","last_change":"2023-09-23T19:59:09.171926+0000","last_active":"2023-09-23T20:04:02.273225+0000","last_peered":"2023-09-23T20:04:02.273225+0000","last_clean":"2023-09-23T20:04:02.273225+0000","last_became_active":"2023-09-23T19:59:09.171609+0000","last_became_peered":"2023-09-23T19:59:09.171609+0000","last_unstale":"2023-09-23T20:04:02.273225+0000","last_undegraded":"2023-09-23T20:04:02.273225+0000","last_fullsized":"2023-09-23T20:04:02.273225+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:21.795329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"278'7996","reported_seq":9657,"reported_epoch":278,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:04:12.215683+0000","last_change":"2023-09-23T20:04:02.298192+0000","last_active":"2023-09-23T20:04:12.215683+0000","last_peered":"2023-09-23T20:04:12.215683+0000","last_clean":"2023-09-23T20:03:58.092468+0000","last_became_active":"2023-09-23T20:04:00.591086+0000","last_became_peered":"2023-09-23T20:04:00.591086+0000","last_unstale":"2023-09-23T20:04:12.215683+0000","last_undegraded":"2023-09-23T20:04:12.215683+0000","last_fullsized":"2023-09-23T20:04:12.215683+0000","mapping_epoch":274,"log_start":"247'1472","ondisk_log_start":"247'1472","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6524,"log_dups_size":0,"ondisk_log_size":6524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:19.737714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262537216,"num_objects":4006,"num_object_clones":0,"num_object_copies":12018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3245,"num_objects_unfound":0,"num_objects_dirty":4006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7996,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":804,"num_bytes_recovered":52690944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[0,7,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":4006}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":616,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.268125+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:04:02.268125+0000","last_peered":"2023-09-23T20:04:02.268125+0000","last_clean":"2023-09-23T20:04:02.268125+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:04:02.268125+0000","last_undegraded":"2023-09-23T20:04:02.268125+0000","last_fullsized":"2023-09-23T20:04:02.268125+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256274+0000","last_change":"2023-09-23T20:03:59.233742+0000","last_active":"2023-09-23T20:04:02.256274+0000","last_peered":"2023-09-23T20:04:02.256274+0000","last_clean":"2023-09-23T20:04:02.256274+0000","last_became_active":"2023-09-23T20:03:59.233209+0000","last_became_peered":"2023-09-23T20:03:59.233209+0000","last_unstale":"2023-09-23T20:04:02.256274+0000","last_undegraded":"2023-09-23T20:04:02.256274+0000","last_fullsized":"2023-09-23T20:04:02.256274+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:50.879925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":239,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.320284+0000","last_change":"2023-09-23T19:59:09.153852+0000","last_active":"2023-09-23T20:04:04.320284+0000","last_peered":"2023-09-23T20:04:04.320284+0000","last_clean":"2023-09-23T20:04:04.320284+0000","last_became_active":"2023-09-23T19:59:09.153514+0000","last_became_peered":"2023-09-23T19:59:09.153514+0000","last_unstale":"2023-09-23T20:04:04.320284+0000","last_undegraded":"2023-09-23T20:04:04.320284+0000","last_fullsized":"2023-09-23T20:04:04.320284+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:45.179600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"278'8095","reported_seq":8169,"reported_epoch":278,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:13.430508+0000","last_change":"2023-09-23T20:04:02.297387+0000","last_active":"2023-09-23T20:04:13.430508+0000","last_peered":"2023-09-23T20:04:13.430508+0000","last_clean":"2023-09-23T20:03:58.080766+0000","last_became_active":"2023-09-23T20:04:00.582511+0000","last_became_peered":"2023-09-23T20:04:00.582511+0000","last_unstale":"2023-09-23T20:04:13.430508+0000","last_undegraded":"2023-09-23T20:04:13.430508+0000","last_fullsized":"2023-09-23T20:04:13.430508+0000","mapping_epoch":274,"log_start":"247'1500","ondisk_log_start":"247'1500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6595,"log_dups_size":0,"ondisk_log_size":6595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:56.089695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267321344,"num_objects":4079,"num_object_clones":0,"num_object_copies":12237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4077,"num_objects_unfound":0,"num_objects_dirty":4079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8095,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,1,0],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":4079}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":601,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.278395+0000","last_change":"2023-09-23T20:04:02.277887+0000","last_active":"2023-09-23T20:04:02.278395+0000","last_peered":"2023-09-23T20:04:02.278395+0000","last_clean":"2023-09-23T20:04:02.278395+0000","last_became_active":"2023-09-23T20:04:02.277519+0000","last_became_peered":"2023-09-23T20:04:02.277519+0000","last_unstale":"2023-09-23T20:04:02.278395+0000","last_undegraded":"2023-09-23T20:04:02.278395+0000","last_fullsized":"2023-09-23T20:04:02.278395+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":617,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.320723+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T20:04:04.320723+0000","last_peered":"2023-09-23T20:04:04.320723+0000","last_clean":"2023-09-23T20:04:04.320723+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T20:04:04.320723+0000","last_undegraded":"2023-09-23T20:04:04.320723+0000","last_fullsized":"2023-09-23T20:04:04.320723+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":278,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.268679+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:04:02.268679+0000","last_peered":"2023-09-23T20:04:02.268679+0000","last_clean":"2023-09-23T20:04:02.268679+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:04:02.268679+0000","last_undegraded":"2023-09-23T20:04:02.268679+0000","last_fullsized":"2023-09-23T20:04:02.268679+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":206,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.319128+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T20:04:04.319128+0000","last_peered":"2023-09-23T20:04:04.319128+0000","last_clean":"2023-09-23T20:04:04.319128+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T20:04:04.319128+0000","last_undegraded":"2023-09-23T20:04:04.319128+0000","last_fullsized":"2023-09-23T20:04:04.319128+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"278'8122","reported_seq":8184,"reported_epoch":278,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:12.161258+0000","last_change":"2023-09-23T20:04:00.730874+0000","last_active":"2023-09-23T20:04:12.161258+0000","last_peered":"2023-09-23T20:04:12.161258+0000","last_clean":"2023-09-23T20:03:58.087054+0000","last_became_active":"2023-09-23T20:04:00.560120+0000","last_became_peered":"2023-09-23T20:04:00.560120+0000","last_unstale":"2023-09-23T20:04:12.161258+0000","last_undegraded":"2023-09-23T20:04:12.161258+0000","last_fullsized":"2023-09-23T20:04:12.161258+0000","mapping_epoch":274,"log_start":"247'1800","ondisk_log_start":"247'1800","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6322,"log_dups_size":0,"ondisk_log_size":6322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:17.026687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269484032,"num_objects":4112,"num_object_clones":0,"num_object_copies":12336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4112,"num_objects_unfound":0,"num_objects_dirty":4112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8122,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":4112}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":589,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.267678+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:04:02.267678+0000","last_peered":"2023-09-23T20:04:02.267678+0000","last_clean":"2023-09-23T20:04:02.267678+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:04:02.267678+0000","last_undegraded":"2023-09-23T20:04:02.267678+0000","last_fullsized":"2023-09-23T20:04:02.267678+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":838,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.267882+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T20:04:02.267882+0000","last_peered":"2023-09-23T20:04:02.267882+0000","last_clean":"2023-09-23T20:04:02.267882+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T20:04:02.267882+0000","last_undegraded":"2023-09-23T20:04:02.267882+0000","last_fullsized":"2023-09-23T20:04:02.267882+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:58.623015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":277,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273302+0000","last_change":"2023-09-23T20:00:34.447064+0000","last_active":"2023-09-23T20:04:02.273302+0000","last_peered":"2023-09-23T20:04:02.273302+0000","last_clean":"2023-09-23T20:04:02.273302+0000","last_became_active":"2023-09-23T20:00:34.446754+0000","last_became_peered":"2023-09-23T20:00:34.446754+0000","last_unstale":"2023-09-23T20:04:02.273302+0000","last_undegraded":"2023-09-23T20:04:02.273302+0000","last_fullsized":"2023-09-23T20:04:02.273302+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:39.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":272,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.320653+0000","last_change":"2023-09-23T20:03:59.236594+0000","last_active":"2023-09-23T20:04:04.320653+0000","last_peered":"2023-09-23T20:04:04.320653+0000","last_clean":"2023-09-23T20:04:04.320653+0000","last_became_active":"2023-09-23T20:03:59.236305+0000","last_became_peered":"2023-09-23T20:03:59.236305+0000","last_unstale":"2023-09-23T20:04:04.320653+0000","last_undegraded":"2023-09-23T20:04:04.320653+0000","last_fullsized":"2023-09-23T20:04:04.320653+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:23.103726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"278'8044","reported_seq":8090,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:13.496097+0000","last_change":"2023-09-23T20:01:39.977018+0000","last_active":"2023-09-23T20:04:13.496097+0000","last_peered":"2023-09-23T20:04:13.496097+0000","last_clean":"2023-09-23T20:04:13.496097+0000","last_became_active":"2023-09-23T20:01:39.976359+0000","last_became_peered":"2023-09-23T20:01:39.976359+0000","last_unstale":"2023-09-23T20:04:13.496097+0000","last_undegraded":"2023-09-23T20:04:13.496097+0000","last_fullsized":"2023-09-23T20:04:13.496097+0000","mapping_epoch":240,"log_start":"247'1500","ondisk_log_start":"247'1500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6544,"log_dups_size":0,"ondisk_log_size":6544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.221346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262930432,"num_objects":4012,"num_object_clones":0,"num_object_copies":12036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8044,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":513,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273918+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:04:02.273918+0000","last_peered":"2023-09-23T20:04:02.273918+0000","last_clean":"2023-09-23T20:04:02.273918+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:04:02.273918+0000","last_undegraded":"2023-09-23T20:04:02.273918+0000","last_fullsized":"2023-09-23T20:04:02.273918+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":910,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.271911+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:04:02.271911+0000","last_peered":"2023-09-23T20:04:02.271911+0000","last_clean":"2023-09-23T20:04:02.271911+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:04:02.271911+0000","last_undegraded":"2023-09-23T20:04:02.271911+0000","last_fullsized":"2023-09-23T20:04:02.271911+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":236,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.318831+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T20:04:04.318831+0000","last_peered":"2023-09-23T20:04:04.318831+0000","last_clean":"2023-09-23T20:04:04.318831+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T20:04:04.318831+0000","last_undegraded":"2023-09-23T20:04:04.318831+0000","last_fullsized":"2023-09-23T20:04:04.318831+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.564126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":250,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.320346+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T20:04:04.320346+0000","last_peered":"2023-09-23T20:04:04.320346+0000","last_clean":"2023-09-23T20:04:04.320346+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T20:04:04.320346+0000","last_undegraded":"2023-09-23T20:04:04.320346+0000","last_fullsized":"2023-09-23T20:04:04.320346+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"278'8227","reported_seq":9141,"reported_epoch":278,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:13.456619+0000","last_change":"2023-09-23T20:04:02.279075+0000","last_active":"2023-09-23T20:04:13.456619+0000","last_peered":"2023-09-23T20:04:13.456619+0000","last_clean":"2023-09-23T20:03:58.125973+0000","last_became_active":"2023-09-23T20:04:00.586359+0000","last_became_peered":"2023-09-23T20:04:00.586359+0000","last_unstale":"2023-09-23T20:04:13.456619+0000","last_undegraded":"2023-09-23T20:04:13.456619+0000","last_fullsized":"2023-09-23T20:04:13.456619+0000","mapping_epoch":274,"log_start":"246'856","ondisk_log_start":"246'856","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7371,"log_dups_size":0,"ondisk_log_size":7371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:05.232047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268107776,"num_objects":4091,"num_object_clones":0,"num_object_copies":12273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3698,"num_objects_unfound":0,"num_objects_dirty":4091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8227,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":426,"num_bytes_recovered":27918336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,5],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":4091}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1086,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.268556+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:04:02.268556+0000","last_peered":"2023-09-23T20:04:02.268556+0000","last_clean":"2023-09-23T20:04:02.268556+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:04:02.268556+0000","last_undegraded":"2023-09-23T20:04:02.268556+0000","last_fullsized":"2023-09-23T20:04:02.268556+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":688,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274121+0000","last_change":"2023-09-23T20:00:34.445036+0000","last_active":"2023-09-23T20:04:02.274121+0000","last_peered":"2023-09-23T20:04:02.274121+0000","last_clean":"2023-09-23T20:04:02.274121+0000","last_became_active":"2023-09-23T20:00:34.444689+0000","last_became_peered":"2023-09-23T20:00:34.444689+0000","last_unstale":"2023-09-23T20:04:02.274121+0000","last_undegraded":"2023-09-23T20:04:02.274121+0000","last_fullsized":"2023-09-23T20:04:02.274121+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:44.485721+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.257794+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:04:02.257794+0000","last_peered":"2023-09-23T20:04:02.257794+0000","last_clean":"2023-09-23T20:04:02.257794+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:04:02.257794+0000","last_undegraded":"2023-09-23T20:04:02.257794+0000","last_fullsized":"2023-09-23T20:04:02.257794+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":153,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273135+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:04:02.273135+0000","last_peered":"2023-09-23T20:04:02.273135+0000","last_clean":"2023-09-23T20:04:02.273135+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:04:02.273135+0000","last_undegraded":"2023-09-23T20:04:02.273135+0000","last_fullsized":"2023-09-23T20:04:02.273135+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"278'8180","reported_seq":8226,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:13.237641+0000","last_change":"2023-09-23T20:01:39.977010+0000","last_active":"2023-09-23T20:04:13.237641+0000","last_peered":"2023-09-23T20:04:13.237641+0000","last_clean":"2023-09-23T20:04:13.237641+0000","last_became_active":"2023-09-23T20:01:39.976722+0000","last_became_peered":"2023-09-23T20:01:39.976722+0000","last_unstale":"2023-09-23T20:04:13.237641+0000","last_undegraded":"2023-09-23T20:04:13.237641+0000","last_fullsized":"2023-09-23T20:04:13.237641+0000","mapping_epoch":240,"log_start":"246'831","ondisk_log_start":"246'831","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7349,"log_dups_size":0,"ondisk_log_size":7349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:13.592964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265682944,"num_objects":4054,"num_object_clones":0,"num_object_copies":12162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8180,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":619,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.319891+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T20:04:04.319891+0000","last_peered":"2023-09-23T20:04:04.319891+0000","last_clean":"2023-09-23T20:04:04.319891+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T20:04:04.319891+0000","last_undegraded":"2023-09-23T20:04:04.319891+0000","last_fullsized":"2023-09-23T20:04:04.319891+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.842550+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":996,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274011+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T20:04:02.274011+0000","last_peered":"2023-09-23T20:04:02.274011+0000","last_clean":"2023-09-23T20:04:02.274011+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T20:04:02.274011+0000","last_undegraded":"2023-09-23T20:04:02.274011+0000","last_fullsized":"2023-09-23T20:04:02.274011+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":244,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.272768+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:04:02.272768+0000","last_peered":"2023-09-23T20:04:02.272768+0000","last_clean":"2023-09-23T20:04:02.272768+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:04:02.272768+0000","last_undegraded":"2023-09-23T20:04:02.272768+0000","last_fullsized":"2023-09-23T20:04:02.272768+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":240,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.269082+0000","last_change":"2023-09-23T20:03:10.175462+0000","last_active":"2023-09-23T20:04:02.269082+0000","last_peered":"2023-09-23T20:04:02.269082+0000","last_clean":"2023-09-23T20:04:02.269082+0000","last_became_active":"2023-09-23T20:03:10.175250+0000","last_became_peered":"2023-09-23T20:03:10.175250+0000","last_unstale":"2023-09-23T20:04:02.269082+0000","last_undegraded":"2023-09-23T20:04:02.269082+0000","last_fullsized":"2023-09-23T20:04:02.269082+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:42.961619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"278'7967","reported_seq":8013,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:13.031454+0000","last_change":"2023-09-23T20:01:39.913385+0000","last_active":"2023-09-23T20:04:13.031454+0000","last_peered":"2023-09-23T20:04:13.031454+0000","last_clean":"2023-09-23T20:04:13.031454+0000","last_became_active":"2023-09-23T20:01:39.912872+0000","last_became_peered":"2023-09-23T20:01:39.912872+0000","last_unstale":"2023-09-23T20:04:13.031454+0000","last_undegraded":"2023-09-23T20:04:13.031454+0000","last_fullsized":"2023-09-23T20:04:13.031454+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7967,"log_dups_size":0,"ondisk_log_size":7967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.619813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264044544,"num_objects":4029,"num_object_clones":0,"num_object_copies":12087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7967,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":593,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.278380+0000","last_change":"2023-09-23T20:04:02.277648+0000","last_active":"2023-09-23T20:04:02.278380+0000","last_peered":"2023-09-23T20:04:02.278380+0000","last_clean":"2023-09-23T20:04:02.278380+0000","last_became_active":"2023-09-23T20:04:02.277271+0000","last_became_peered":"2023-09-23T20:04:02.277271+0000","last_unstale":"2023-09-23T20:04:02.278380+0000","last_undegraded":"2023-09-23T20:04:02.278380+0000","last_fullsized":"2023-09-23T20:04:02.278380+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":356,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.274207+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:04:02.274207+0000","last_peered":"2023-09-23T20:04:02.274207+0000","last_clean":"2023-09-23T20:04:02.274207+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:04:02.274207+0000","last_undegraded":"2023-09-23T20:04:02.274207+0000","last_fullsized":"2023-09-23T20:04:02.274207+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":238,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.320791+0000","last_change":"2023-09-23T20:03:10.174842+0000","last_active":"2023-09-23T20:04:04.320791+0000","last_peered":"2023-09-23T20:04:04.320791+0000","last_clean":"2023-09-23T20:04:04.320791+0000","last_became_active":"2023-09-23T20:03:10.174511+0000","last_became_peered":"2023-09-23T20:03:10.174511+0000","last_unstale":"2023-09-23T20:04:04.320791+0000","last_undegraded":"2023-09-23T20:04:04.320791+0000","last_fullsized":"2023-09-23T20:04:04.320791+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:12.723811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":74,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.268626+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:04:02.268626+0000","last_peered":"2023-09-23T20:04:02.268626+0000","last_clean":"2023-09-23T20:04:02.268626+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:04:02.268626+0000","last_undegraded":"2023-09-23T20:04:02.268626+0000","last_fullsized":"2023-09-23T20:04:02.268626+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"278'8176","reported_seq":8224,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:11.991691+0000","last_change":"2023-09-23T20:01:39.912135+0000","last_active":"2023-09-23T20:04:11.991691+0000","last_peered":"2023-09-23T20:04:11.991691+0000","last_clean":"2023-09-23T20:04:11.991691+0000","last_became_active":"2023-09-23T20:01:39.911834+0000","last_became_peered":"2023-09-23T20:01:39.911834+0000","last_unstale":"2023-09-23T20:04:11.991691+0000","last_undegraded":"2023-09-23T20:04:11.991691+0000","last_fullsized":"2023-09-23T20:04:11.991691+0000","mapping_epoch":240,"log_start":"247'2426","ondisk_log_start":"247'2426","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:46.725578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266338304,"num_objects":4064,"num_object_clones":0,"num_object_copies":12192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8176,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":656,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.272763+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:04:02.272763+0000","last_peered":"2023-09-23T20:04:02.272763+0000","last_clean":"2023-09-23T20:04:02.272763+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:04:02.272763+0000","last_undegraded":"2023-09-23T20:04:02.272763+0000","last_fullsized":"2023-09-23T20:04:02.272763+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":104,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256918+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:04:02.256918+0000","last_peered":"2023-09-23T20:04:02.256918+0000","last_clean":"2023-09-23T20:04:02.256918+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:04:02.256918+0000","last_undegraded":"2023-09-23T20:04:02.256918+0000","last_fullsized":"2023-09-23T20:04:02.256918+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":684,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.319822+0000","last_change":"2023-09-23T19:59:40.899753+0000","last_active":"2023-09-23T20:04:04.319822+0000","last_peered":"2023-09-23T20:04:04.319822+0000","last_clean":"2023-09-23T20:04:04.319822+0000","last_became_active":"2023-09-23T19:59:40.899147+0000","last_became_peered":"2023-09-23T19:59:40.899147+0000","last_unstale":"2023-09-23T20:04:04.319822+0000","last_undegraded":"2023-09-23T20:04:04.319822+0000","last_fullsized":"2023-09-23T20:04:04.319822+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:08.938755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256345+0000","last_change":"2023-09-23T20:03:59.233560+0000","last_active":"2023-09-23T20:04:02.256345+0000","last_peered":"2023-09-23T20:04:02.256345+0000","last_clean":"2023-09-23T20:04:02.256345+0000","last_became_active":"2023-09-23T20:03:59.232842+0000","last_became_peered":"2023-09-23T20:03:59.232842+0000","last_unstale":"2023-09-23T20:04:02.256345+0000","last_undegraded":"2023-09-23T20:04:02.256345+0000","last_fullsized":"2023-09-23T20:04:02.256345+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:16.909413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"278'8158","reported_seq":8206,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:11.971480+0000","last_change":"2023-09-23T20:01:39.913698+0000","last_active":"2023-09-23T20:04:11.971480+0000","last_peered":"2023-09-23T20:04:11.971480+0000","last_clean":"2023-09-23T20:04:11.971480+0000","last_became_active":"2023-09-23T20:01:39.913367+0000","last_became_peered":"2023-09-23T20:01:39.913367+0000","last_unstale":"2023-09-23T20:04:11.971480+0000","last_undegraded":"2023-09-23T20:04:11.971480+0000","last_fullsized":"2023-09-23T20:04:11.971480+0000","mapping_epoch":240,"log_start":"247'2425","ondisk_log_start":"247'2425","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267255808,"num_objects":4078,"num_object_clones":0,"num_object_copies":12234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8158,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":534,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256558+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:04:02.256558+0000","last_peered":"2023-09-23T20:04:02.256558+0000","last_clean":"2023-09-23T20:04:02.256558+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:04:02.256558+0000","last_undegraded":"2023-09-23T20:04:02.256558+0000","last_fullsized":"2023-09-23T20:04:02.256558+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":34,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.276705+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:04:02.276705+0000","last_peered":"2023-09-23T20:04:02.276705+0000","last_clean":"2023-09-23T20:04:02.276705+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:04:02.276705+0000","last_undegraded":"2023-09-23T20:04:02.276705+0000","last_fullsized":"2023-09-23T20:04:02.276705+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":82,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:04:04.319912+0000","last_change":"2023-09-23T20:01:04.470744+0000","last_active":"2023-09-23T20:04:04.319912+0000","last_peered":"2023-09-23T20:04:04.319912+0000","last_clean":"2023-09-23T20:04:04.319912+0000","last_became_active":"2023-09-23T20:01:04.470302+0000","last_became_peered":"2023-09-23T20:01:04.470302+0000","last_unstale":"2023-09-23T20:04:04.319912+0000","last_undegraded":"2023-09-23T20:04:04.319912+0000","last_fullsized":"2023-09-23T20:04:04.319912+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:25.452596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":237,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.272671+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:04:02.272671+0000","last_peered":"2023-09-23T20:04:02.272671+0000","last_clean":"2023-09-23T20:04:02.272671+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:04:02.272671+0000","last_undegraded":"2023-09-23T20:04:02.272671+0000","last_fullsized":"2023-09-23T20:04:02.272671+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":23,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256796+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:04:02.256796+0000","last_peered":"2023-09-23T20:04:02.256796+0000","last_clean":"2023-09-23T20:04:02.256796+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:04:02.256796+0000","last_undegraded":"2023-09-23T20:04:02.256796+0000","last_fullsized":"2023-09-23T20:04:02.256796+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":31,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.257542+0000","last_change":"2023-09-23T20:03:10.178662+0000","last_active":"2023-09-23T20:04:02.257542+0000","last_peered":"2023-09-23T20:04:02.257542+0000","last_clean":"2023-09-23T20:04:02.257542+0000","last_became_active":"2023-09-23T20:03:10.178352+0000","last_became_peered":"2023-09-23T20:03:10.178352+0000","last_unstale":"2023-09-23T20:04:02.257542+0000","last_undegraded":"2023-09-23T20:04:02.257542+0000","last_fullsized":"2023-09-23T20:04:02.257542+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.469897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":239,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.268478+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T20:04:02.268478+0000","last_peered":"2023-09-23T20:04:02.268478+0000","last_clean":"2023-09-23T20:04:02.268478+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T20:04:02.268478+0000","last_undegraded":"2023-09-23T20:04:02.268478+0000","last_fullsized":"2023-09-23T20:04:02.268478+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":274,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273375+0000","last_change":"2023-09-23T19:59:09.172452+0000","last_active":"2023-09-23T20:04:02.273375+0000","last_peered":"2023-09-23T20:04:02.273375+0000","last_clean":"2023-09-23T20:04:02.273375+0000","last_became_active":"2023-09-23T19:59:09.172080+0000","last_became_peered":"2023-09-23T19:59:09.172080+0000","last_unstale":"2023-09-23T20:04:02.273375+0000","last_undegraded":"2023-09-23T20:04:02.273375+0000","last_fullsized":"2023-09-23T20:04:02.273375+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:24.687991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":243,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.268587+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T20:04:02.268587+0000","last_peered":"2023-09-23T20:04:02.268587+0000","last_clean":"2023-09-23T20:04:02.268587+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T20:04:02.268587+0000","last_undegraded":"2023-09-23T20:04:02.268587+0000","last_fullsized":"2023-09-23T20:04:02.268587+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:07.560646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":274,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273462+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:04:02.273462+0000","last_peered":"2023-09-23T20:04:02.273462+0000","last_clean":"2023-09-23T20:04:02.273462+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:04:02.273462+0000","last_undegraded":"2023-09-23T20:04:02.273462+0000","last_fullsized":"2023-09-23T20:04:02.273462+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":100,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.258993+0000","last_change":"2023-09-23T19:59:09.145224+0000","last_active":"2023-09-23T20:04:02.258993+0000","last_peered":"2023-09-23T20:04:02.258993+0000","last_clean":"2023-09-23T20:04:02.258993+0000","last_became_active":"2023-09-23T19:59:09.144819+0000","last_became_peered":"2023-09-23T19:59:09.144819+0000","last_unstale":"2023-09-23T20:04:02.258993+0000","last_undegraded":"2023-09-23T20:04:02.258993+0000","last_fullsized":"2023-09-23T20:04:02.258993+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:04.048181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.256215+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:04:02.256215+0000","last_peered":"2023-09-23T20:04:02.256215+0000","last_clean":"2023-09-23T20:04:02.256215+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:04:02.256215+0000","last_undegraded":"2023-09-23T20:04:02.256215+0000","last_fullsized":"2023-09-23T20:04:02.256215+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":97,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.269285+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:04:02.269285+0000","last_peered":"2023-09-23T20:04:02.269285+0000","last_clean":"2023-09-23T20:04:02.269285+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:04:02.269285+0000","last_undegraded":"2023-09-23T20:04:02.269285+0000","last_fullsized":"2023-09-23T20:04:02.269285+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":304,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T20:04:02.273331+0000","last_change":"2023-09-23T20:03:59.235232+0000","last_active":"2023-09-23T20:04:02.273331+0000","last_peered":"2023-09-23T20:04:02.273331+0000","last_clean":"2023-09-23T20:04:02.273331+0000","last_became_active":"2023-09-23T20:03:59.234941+0000","last_became_peered":"2023-09-23T20:03:59.234941+0000","last_unstale":"2023-09-23T20:04:02.273331+0000","last_undegraded":"2023-09-23T20:04:02.273331+0000","last_fullsized":"2023-09-23T20:04:02.273331+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:45.591449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4214030360,"num_objects":64302,"num_object_clones":0,"num_object_copies":192906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":15132,"num_objects_unfound":0,"num_objects_dirty":64302,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129818,"num_write_kb":6211777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6336,"num_bytes_recovered":415170560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12986232832,"data_stored":12986220616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107286,"ondisk_log_size":107286,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234277,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1824492,"kb_used_data":1566052,"kb_used_omap":0,"kb_used_meta":258432,"kb_avail":91920660,"statfs":{"total":95995035648,"available":94126755840,"internally_reserved":0,"allocated":1603637248,"data_stored":1602794856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":264634368},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:18 2023","interfaces":[{"interface":"back","average":{"1min":3.952,"5min":2.1850000000000001,"15min":1.8160000000000001},"min":{"1min":0.22700000000000001,"5min":0.22,"15min":0.19700000000000001},"max":{"1min":60.140000000000001,"5min":60.140000000000001,"15min":62.061999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":4.4889999999999999,"5min":2.0779999999999998,"15min":2.0110000000000001},"min":{"1min":0.20399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":62.389000000000003,"5min":62.389000000000003,"15min":62.389000000000003},"last":10.041}]},{"osd":1,"last update":"Sat Sep 23 20:03:18 2023","interfaces":[{"interface":"back","average":{"1min":14.236000000000001,"5min":5.7750000000000004,"15min":2.7639999999999998},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":104.16800000000001,"5min":104.16800000000001,"15min":104.16800000000001},"last":4.4000000000000004},{"interface":"front","average":{"1min":14.055,"5min":5.7489999999999997,"15min":2.831},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":104.003,"5min":104.003,"15min":104.003},"last":0.36599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:03:36 2023","interfaces":[{"interface":"back","average":{"1min":7.5460000000000003,"5min":3.016,"15min":3.0070000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":105.842,"5min":105.842,"15min":130.673},"last":3.4820000000000002},{"interface":"front","average":{"1min":7.0190000000000001,"5min":3.4649999999999999,"15min":3.1930000000000001},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":105.66,"5min":105.66,"15min":130.71100000000001},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":9.5630000000000006,"5min":3.9380000000000002,"15min":2.8290000000000002},"min":{"1min":0.316,"5min":0.247,"15min":0.247},"max":{"1min":146.38800000000001,"5min":146.38800000000001,"15min":146.38800000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":9.1679999999999993,"5min":4.0839999999999996,"15min":2.8039999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":146.851,"5min":146.851,"15min":146.851},"last":3.5099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.878,"15min":1.885},"min":{"1min":0.28299999999999997,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.3109999999999999,"5min":19.215,"15min":143.255},"last":3.714},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.84499999999999997,"15min":1.7969999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.0419999999999998,"5min":19.364000000000001,"15min":143.41200000000001},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 20:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.1819999999999999,"15min":1.444},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":2.9380000000000002,"5min":35.488,"15min":84.566999999999993},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.085,"15min":1.3089999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.343,"5min":34.896000000000001,"15min":81.244},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9820000000000002,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.47499999999999998}]}]},{"osd":6,"up_from":0,"seq":1078036791318,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762479,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2428820,"kb_used_data":2044172,"kb_used_omap":7,"kb_used_meta":384568,"kb_avail":91316332,"statfs":{"total":95995035648,"available":93507923968,"internally_reserved":0,"allocated":2093232128,"data_stored":2092003482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":393798458},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,2,5,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.1360000000000001,"15min":3.6930000000000001},"min":{"1min":0.19500000000000001,"5min":0.16900000000000001,"15min":0.151},"max":{"1min":7.7409999999999997,"5min":25.238,"15min":202.917},"last":0.29399999999999998},{"interface":"front","average":{"1min":2.948,"5min":2.1509999999999998,"15min":3.778},"min":{"1min":0.26000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":7.6189999999999998,"5min":22.337,"15min":203.34899999999999},"last":3.1059999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":5.0609999999999999,"15min":5.2329999999999997},"min":{"1min":0.315,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":11.917999999999999,"5min":134.72499999999999,"15min":231.57499999999999},"last":1.028},{"interface":"front","average":{"1min":3.8370000000000002,"5min":5.4279999999999999,"15min":5.2359999999999998},"min":{"1min":0.29399999999999998,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":20.677,"5min":135.74600000000001,"15min":236.077},"last":3.556}]},{"osd":3,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":3.1179999999999999,"5min":3.0230000000000001,"15min":2.7970000000000002},"min":{"1min":0.28399999999999997,"5min":0.247,"15min":0.217},"max":{"1min":12.679,"5min":56.564,"15min":68.180999999999997},"last":2.5190000000000001},{"interface":"front","average":{"1min":3.0609999999999999,"5min":2.8900000000000001,"15min":2.5190000000000001},"min":{"1min":0.309,"5min":0.249,"15min":0.20100000000000001},"max":{"1min":12.286,"5min":56.939999999999998,"15min":66.509},"last":1.6850000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":3.9420000000000002,"5min":1.8700000000000001,"15min":2.3250000000000002},"min":{"1min":0.34599999999999997,"5min":0.254,"15min":0.25},"max":{"1min":35.497999999999998,"5min":35.497999999999998,"15min":161.68299999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":4.8200000000000003,"5min":1.968,"15min":2.282},"min":{"1min":0.42799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":35.521000000000001,"5min":35.521000000000001,"15min":161.286},"last":1.117}]},{"osd":5,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":4.9790000000000001,"5min":2.3769999999999998,"15min":3.1920000000000002},"min":{"1min":0.41199999999999998,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":51.698,"5min":51.698,"15min":219.048},"last":1.151},{"interface":"front","average":{"1min":4.2729999999999997,"5min":2.3039999999999998,"15min":3.3490000000000002},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.29599999999999999},"max":{"1min":51.173000000000002,"5min":51.173000000000002,"15min":218.69499999999999},"last":1.1339999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:03:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.2469999999999999,"15min":2.2469999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":9.8879999999999999,"5min":9.8879999999999999,"15min":9.8879999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":3.4460000000000002,"5min":3.4460000000000002,"15min":3.4460000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":20.734999999999999,"5min":20.734999999999999,"15min":20.734999999999999},"last":1.1759999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1749999999999998,"5min":2.0249999999999999,"15min":2.0059999999999998},"min":{"1min":0.40000000000000002,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":14.007,"5min":14.23,"15min":60.213000000000001},"last":1.079},{"interface":"front","average":{"1min":2.1389999999999998,"5min":1.9330000000000001,"15min":1.6619999999999999},"min":{"1min":0.377,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":14.188000000000001,"5min":14.188000000000001,"15min":61.298999999999999},"last":0.50900000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926013,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3035984,"kb_used_data":2616712,"kb_used_omap":8,"kb_used_meta":419255,"kb_avail":90709168,"statfs":{"total":95995035648,"available":92886188032,"internally_reserved":0,"allocated":2679513088,"data_stored":2678287832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":429317729},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6819999999999999,"5min":5.0289999999999999,"15min":2.6389999999999998},"min":{"1min":0.185,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":20.396000000000001,"5min":94.055999999999997,"15min":94.055999999999997},"last":0.32400000000000001},{"interface":"front","average":{"1min":3.964,"5min":5.1909999999999998,"15min":2.972},"min":{"1min":0.29099999999999998,"5min":0.183,"15min":0.183},"max":{"1min":22.056999999999999,"5min":91.632999999999996,"15min":91.632999999999996},"last":0.36799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":5.9409999999999998,"15min":3.5910000000000002},"min":{"1min":0.23300000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.75,"5min":270.51100000000002,"15min":270.51100000000002},"last":0.22800000000000001},{"interface":"front","average":{"1min":2.8109999999999999,"5min":5.5460000000000003,"15min":3.6419999999999999},"min":{"1min":0.25700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.038,"5min":269.69299999999998,"15min":269.69299999999998},"last":0.29199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":6.2779999999999996,"5min":5.9699999999999998,"15min":4.4119999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":48.588000000000001,"5min":133.422,"15min":145.72800000000001},"last":5.46},{"interface":"front","average":{"1min":6.4850000000000003,"5min":6.0259999999999998,"15min":4.5570000000000004},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":48.619999999999997,"5min":134.249,"15min":145.50700000000001},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.7430000000000001,"15min":2.3159999999999998},"min":{"1min":0.27600000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.3829999999999991,"5min":12.372,"15min":110.312},"last":0.60099999999999998},{"interface":"front","average":{"1min":2.27,"5min":1.659,"15min":1.964},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27600000000000002},"max":{"1min":8.7330000000000005,"5min":8.7330000000000005,"15min":110.256},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":2.181,"15min":2.4929999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.5419999999999998,"5min":64.725999999999999,"15min":131.34399999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.8380000000000001,"15min":2.569},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.4139999999999997,"5min":12.48,"15min":131.386},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.607,"15min":1.607},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.4290000000000003,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.9059999999999999,"5min":1.9059999999999999,"15min":1.9059999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.0629999999999997,"5min":5.0629999999999997,"15min":5.0629999999999997},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":2.2189999999999999,"15min":2.004},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":8.6039999999999992,"5min":65.081999999999994,"15min":77.132999999999996},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.8400000000000001,"5min":1.9419999999999999,"15min":2.0990000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.3900000000000006,"5min":12.452,"15min":75.465999999999994},"last":0.66300000000000003}]}]},{"osd":2,"up_from":104,"seq":446676598952,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2541248,"kb_used_data":2064888,"kb_used_omap":12,"kb_used_meta":476275,"kb_avail":91203904,"statfs":{"total":95995035648,"available":93392797696,"internally_reserved":0,"allocated":2114445312,"data_stored":2113238127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":487706622},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,6],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":3.6309999999999998,"15min":2.5019999999999998},"min":{"1min":0.20899999999999999,"5min":0.192,"15min":0.154},"max":{"1min":10.073,"5min":48.966999999999999,"15min":50.506999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.4690000000000001,"5min":3.2509999999999999,"15min":2.3969999999999998},"min":{"1min":0.20200000000000001,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":7.8840000000000003,"5min":48.783000000000001,"15min":50.802999999999997},"last":0.215}]},{"osd":1,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":6.4400000000000004,"5min":5.4279999999999999,"15min":2.9529999999999998},"min":{"1min":0.247,"5min":0.21299999999999999,"15min":0.184},"max":{"1min":50.192999999999998,"5min":94.768000000000001,"15min":94.768000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":6.7729999999999997,"5min":6.0060000000000002,"15min":3.1890000000000001},"min":{"1min":0.218,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":47.987000000000002,"5min":96.808000000000007,"15min":96.808000000000007},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 20:03:20 2023","interfaces":[{"interface":"back","average":{"1min":6.4880000000000004,"5min":3.4740000000000002,"15min":2.5489999999999999},"min":{"1min":0.23899999999999999,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":44.408000000000001,"5min":44.408000000000001,"15min":50.186999999999998},"last":2.4860000000000002},{"interface":"front","average":{"1min":4.4829999999999997,"5min":3.5529999999999999,"15min":2.5979999999999999},"min":{"1min":0.28100000000000003,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":43.981999999999999,"5min":43.981999999999999,"15min":50.585000000000001},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.893,"15min":1.6890000000000001},"min":{"1min":0.30499999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.9249999999999998,"5min":20.591999999999999,"15min":50.640000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.3039999999999998,"5min":1.819,"15min":1.6990000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7629999999999999,"5min":23.062000000000001,"15min":50.154000000000003},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":10.351000000000001,"5min":3.9199999999999999,"15min":2.5009999999999999},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":172.505,"5min":172.505,"15min":172.505},"last":0.65600000000000003},{"interface":"front","average":{"1min":10.007,"5min":3.5630000000000002,"15min":2.347},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":172.375,"5min":172.375,"15min":172.375},"last":1.647}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":1.6259999999999999,"15min":1.6259999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.144,"5min":10.144,"15min":10.144},"last":1.8140000000000001},{"interface":"front","average":{"1min":1.9690000000000001,"5min":1.9690000000000001,"15min":1.9690000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":6.3520000000000003,"5min":6.3520000000000003,"15min":6.3520000000000003},"last":1.6140000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":1.9530000000000001,"15min":1.796},"min":{"1min":0.436,"5min":0.33700000000000002,"15min":0.27400000000000002},"max":{"1min":9.7330000000000005,"5min":23.079999999999998,"15min":50.118000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.734,"5min":2.395,"15min":2.0680000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":10.17,"5min":20.614999999999998,"15min":50.664999999999999},"last":0.40100000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435425,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2087740,"kb_used_data":1634548,"kb_used_omap":4,"kb_used_meta":453115,"kb_avail":91657412,"statfs":{"total":95995035648,"available":93857189888,"internally_reserved":0,"allocated":1673777152,"data_stored":1672450768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":463989869},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,6,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":3.4249999999999998,"5min":3.1949999999999998,"15min":3.7130000000000001},"min":{"1min":0.20399999999999999,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":21.100999999999999,"5min":57.149000000000001,"15min":240.40600000000001},"last":2.9820000000000002},{"interface":"front","average":{"1min":5.5540000000000003,"5min":3.9319999999999999,"15min":4.2450000000000001},"min":{"1min":0.19700000000000001,"5min":0.19600000000000001,"15min":0.17199999999999999},"max":{"1min":53.201000000000001,"5min":60.496000000000002,"15min":240.352},"last":3.5819999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":4.2699999999999996,"5min":3.1600000000000001,"15min":1.8080000000000001},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":18.395,"5min":22.041,"15min":22.041},"last":5.3289999999999997},{"interface":"front","average":{"1min":2.9159999999999999,"5min":2.7189999999999999,"15min":1.6579999999999999},"min":{"1min":0.26900000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":18.524000000000001,"5min":47.951000000000001,"15min":47.951000000000001},"last":0.249}]},{"osd":2,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":3.9300000000000002,"5min":3.27,"15min":4.9089999999999998},"min":{"1min":0.39200000000000002,"5min":0.219,"15min":0.219},"max":{"1min":28.872,"5min":28.872,"15min":264.97800000000001},"last":5.3650000000000002},{"interface":"front","average":{"1min":4.4420000000000002,"5min":2.5630000000000002,"15min":4.7149999999999999},"min":{"1min":0.35399999999999998,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":30.768999999999998,"5min":30.768999999999998,"15min":257.40899999999999},"last":1.1100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":2.8700000000000001,"15min":1.917},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.8340000000000001,"5min":74.866,"15min":74.866},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.121,"5min":3.4239999999999999,"15min":2.3570000000000002},"min":{"1min":0.311,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.6500000000000004,"5min":74.893000000000001,"15min":74.893000000000001},"last":3.6400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":1.599,"5min":3.5150000000000001,"15min":2.1030000000000002},"min":{"1min":0.314,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.375,"5min":113.881,"15min":113.881},"last":3.6179999999999999},{"interface":"front","average":{"1min":1.601,"5min":3.6459999999999999,"15min":2.036},"min":{"1min":0.34499999999999997,"5min":0.31,"15min":0.31},"max":{"1min":5.9820000000000002,"5min":114.598,"15min":114.598},"last":5.3929999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":5.7519999999999998,"5min":5.7519999999999998,"15min":5.7519999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":82.281999999999996,"5min":82.281999999999996,"15min":82.281999999999996},"last":5.4530000000000003},{"interface":"front","average":{"1min":5.75,"5min":5.75,"15min":5.75},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":82.233000000000004,"5min":82.233000000000004,"15min":82.233000000000004},"last":0.33400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":3.8420000000000001,"15min":2.1309999999999998},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.0090000000000003,"5min":138.035,"15min":138.035},"last":5.4219999999999997},{"interface":"front","average":{"1min":1.1990000000000001,"5min":3.359,"15min":1.9119999999999999},"min":{"1min":0.33100000000000002,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":3.915,"5min":138.054,"15min":138.054},"last":0.35799999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019187,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1821992,"kb_used_data":1561504,"kb_used_omap":0,"kb_used_meta":260416,"kb_avail":91923160,"statfs":{"total":95995035648,"available":94129315840,"internally_reserved":0,"allocated":1598980096,"data_stored":1598141788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":266665984},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":2.9470000000000001,"15min":2.9249999999999998},"min":{"1min":0.251,"5min":0.189,"15min":0.189},"max":{"1min":5.508,"5min":89.004999999999995,"15min":104.604},"last":3.2589999999999999},{"interface":"front","average":{"1min":0.86899999999999999,"5min":2.8570000000000002,"15min":2.883},"min":{"1min":0.29499999999999998,"5min":0.249,"15min":0.23400000000000001},"max":{"1min":5.4509999999999996,"5min":89.024000000000001,"15min":104.687},"last":3.2650000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:03:16 2023","interfaces":[{"interface":"back","average":{"1min":10.568,"5min":4.6589999999999998,"15min":2.819},"min":{"1min":0.252,"5min":0.248,"15min":0.216},"max":{"1min":141.22900000000001,"5min":141.22900000000001,"15min":141.22900000000001},"last":3.0609999999999999},{"interface":"front","average":{"1min":10.423,"5min":4.9329999999999998,"15min":2.7120000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":141.21299999999999,"5min":141.21299999999999,"15min":141.21299999999999},"last":2.0219999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":14.250999999999999,"5min":9.3689999999999998,"15min":5.0190000000000001},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":109.38200000000001,"5min":144.20099999999999,"15min":144.20099999999999},"last":0.88200000000000001},{"interface":"front","average":{"1min":14.645,"5min":8.7840000000000007,"15min":4.7690000000000001},"min":{"1min":0.34200000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":107.74299999999999,"5min":144.184,"15min":144.184},"last":2.948}]},{"osd":3,"last update":"Sat Sep 23 20:03:16 2023","interfaces":[{"interface":"back","average":{"1min":4.4850000000000003,"5min":8.5050000000000008,"15min":4.3129999999999997},"min":{"1min":0.375,"5min":0.255,"15min":0.255},"max":{"1min":32.033000000000001,"5min":266.54700000000003,"15min":266.54700000000003},"last":3.9590000000000001},{"interface":"front","average":{"1min":4.069,"5min":5.0410000000000004,"15min":3.327},"min":{"1min":0.39300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":31.959,"5min":139.911,"15min":139.911},"last":1.847}]},{"osd":5,"last update":"Sat Sep 23 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.8640000000000001},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":4.077,"5min":5.0650000000000004,"15min":266.59899999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.60799999999999998,"15min":1.9570000000000001},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.395,"5min":4.8739999999999997,"15min":266.72899999999998},"last":0.35499999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.7,"5min":1.7,"15min":1.7},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.39300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:03:47 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.81399999999999995,"15min":0.97499999999999998},"min":{"1min":0.318,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.276,"5min":7.6210000000000004,"15min":11.59},"last":1.9510000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.95099999999999996,"15min":0.999},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":1.3240000000000001,"5min":18.012,"15min":18.012},"last":1.9630000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757551,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1576480,"kb_used_data":1300952,"kb_used_omap":0,"kb_used_meta":275520,"kb_avail":92168672,"statfs":{"total":95995035648,"available":94380720128,"internally_reserved":0,"allocated":1332174848,"data_stored":1331340981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":282132480},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.94499999999999995,"15min":1.27},"min":{"1min":0.22900000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":2.8490000000000002,"5min":9.1699999999999999,"15min":24.852},"last":3.3450000000000002},{"interface":"front","average":{"1min":1.7829999999999999,"5min":1.3149999999999999,"15min":1.353},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.216},"max":{"1min":7.4100000000000001,"5min":9.0530000000000008,"15min":24.638999999999999},"last":13.032}]},{"osd":1,"last update":"Sat Sep 23 20:03:16 2023","interfaces":[{"interface":"back","average":{"1min":7.9859999999999998,"5min":8.3239999999999998,"15min":4.1319999999999997},"min":{"1min":0.33200000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":84.602000000000004,"5min":266.197,"15min":266.197},"last":8.8079999999999998},{"interface":"front","average":{"1min":8.6099999999999994,"5min":8.5299999999999994,"15min":4.1340000000000003},"min":{"1min":0.28100000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":88.775999999999996,"5min":266.14999999999998,"15min":266.14999999999998},"last":3.7509999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:03:49 2023","interfaces":[{"interface":"back","average":{"1min":10.763,"5min":6.2130000000000001,"15min":5.5830000000000002},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":177.816,"5min":238.96199999999999,"15min":238.96199999999999},"last":0.317},{"interface":"front","average":{"1min":10.779999999999999,"5min":6.2439999999999998,"15min":5.6189999999999998},"min":{"1min":0.33800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":177.851,"5min":238.905,"15min":238.905},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:03:16 2023","interfaces":[{"interface":"back","average":{"1min":6.9279999999999999,"5min":3.7040000000000002,"15min":2.706},"min":{"1min":0.41499999999999998,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":85.099999999999994,"5min":85.099999999999994,"15min":85.099999999999994},"last":3.359},{"interface":"front","average":{"1min":6.7629999999999999,"5min":4.0030000000000001,"15min":2.722},"min":{"1min":0.38200000000000001,"5min":0.249,"15min":0.249},"max":{"1min":86.233999999999995,"5min":86.233999999999995,"15min":86.233999999999995},"last":3.218}]},{"osd":4,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.476,"15min":1.609},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.9129999999999998,"5min":191.78899999999999,"15min":191.78899999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":2.4849999999999999,"15min":1.5960000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.177,"5min":191.821,"15min":191.821},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4420000000000002,"5min":2.4420000000000002,"15min":2.4420000000000002},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":3.2530000000000001,"15min":1.7330000000000001},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.996,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":3.379,"15min":1.9239999999999999},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.8169999999999999,"5min":191.69,"15min":191.69},"last":0.47099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2655064064,"data_stored":2655059992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2075922432,"data_stored":2075918360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2103050240,"data_stored":2103050240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1663897600,"data_stored":1663893528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1588723712,"data_stored":1588723712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1317011456,"data_stored":1317011456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1582563328,"data_stored":1582563328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:04:19.858 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:04:19.899 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:19.930 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:20.042 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:20.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:19 smithi144 ceph-mon[194833]: pgmap v753: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 14019/185997 objects misplaced (7.537%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:04:20.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:20.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:20.146 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:20.180 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:19 smithi135 ceph-mon[221113]: pgmap v753: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 14019/185997 objects misplaced (7.537%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:04:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:19 smithi135 ceph-mon[213871]: pgmap v753: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 14019/185997 objects misplaced (7.537%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:04:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:20.341 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:20.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:20.405 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:04:20.405 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:04:20.405 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:04:20.405 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149309 5.9 5:90e65886:::benchmark_data_smithi135_284570_object33525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.246164+0000", 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 576.15575065799999, 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.106800241, 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:04:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149309 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246164+0000", 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246164+0000", 2023-09-23T20:04:20.407 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246162+0000", 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246172+0000", 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5470000000000002e-06 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.408 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246189+0000", 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6769000000000002e-05 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246197+0000", 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7260000000000008e-06 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.409 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246251+0000", 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.325e-05 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246392+0000", 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014115199999999999 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,6", 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.246525+0000", 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013331399999999999 2023-09-23T20:04:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.256018+0000", 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.009493352 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.265047+0000", 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090289589999999996 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.411 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.265056+0000", 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6139999999999992e-06 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352889+0000", 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0878331240000001 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352964+0000", 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5154000000000006e-05 2023-09-23T20:04:20.412 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44591.0:149362 5.9 5:96e7d67a:::benchmark_data_smithi135_284570_object33578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:54:44.354858+0000", 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 576.04705709500001, 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.005834238, 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44591", 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149362 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354858+0000", 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.414 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354858+0000", 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354857+0000", 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354866+0000", 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8339999999999994e-06 2023-09-23T20:04:20.415 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354879+0000", 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2714000000000001e-05 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354887+0000", 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0430000000000008e-06 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.416 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354939+0000", 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2045000000000001e-05 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:44.354947+0000", 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3210000000000005e-06 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.352991+0000", 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.998044025 2023-09-23T20:04:20.417 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.353109+0000", 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011731999999999999 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4", 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.353230+0000", 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000121514 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.358433+0000", 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052028939999999996 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360549+0000", 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021155750000000002 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360599+0000", 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9743000000000001e-05 2023-09-23T20:04:20.419 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:54:45.360692+0000", 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3247999999999999e-05 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:149989 6.15 6:afeafb5c:::benchmark_data_smithi135_344736_object47333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:04:20.420 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.37099116600001, 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0666702159999999, 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149989 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030922+0000", 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:04:20.422 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030930+0000", 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9789999999999993e-06 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030945+0000", 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5231999999999999e-05 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.423 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030952+0000", 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3880000000000002e-06 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.031049+0000", 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6500000000000001e-05 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.031062+0000", 2023-09-23T20:04:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2901000000000001e-05 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090321+0000", 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.059259199 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090433+0000", 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000112318 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.425 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090708+0000", 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00027474699999999999 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094761+0000", 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040525600000000002 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094797+0000", 2023-09-23T20:04:20.426 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6569000000000002e-05 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097524+0000", 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002726657 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097554+0000", 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0074e-05 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.427 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097594+0000", 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9827999999999999e-05 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:149996 6.15 6:ac0d5d46:::benchmark_data_smithi135_344736_object47340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.36149714300001, 2023-09-23T20:04:20.428 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.057323408, 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149996 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:04:20.429 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040417+0000", 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.430 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.431 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.431 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040422+0000", 2023-09-23T20:04:20.431 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5979999999999996e-06 2023-09-23T20:04:20.431 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.431 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.431 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.431 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040433+0000", 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0444e-05 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040445+0000", 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2872e-05 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040503+0000", 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7289e-05 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040512+0000", 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6269999999999995e-06 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092269+0000", 2023-09-23T20:04:20.434 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.05175639 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092337+0000", 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8631000000000001e-05 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092523+0000", 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018535200000000001 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.435 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.095904+0000", 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033811940000000001 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097643+0000", 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001738697 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097688+0000", 2023-09-23T20:04:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5427999999999999e-05 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097718+0000", 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0437999999999999e-05 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097741+0000", 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2449000000000001e-05 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.437 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150009 6.15 6:abeabe84:::benchmark_data_smithi135_344736_object47353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.050577+0000", 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.35133780000001, 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0527466489999999, 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.438 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150009 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050577+0000", 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.439 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050577+0000", 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050575+0000", 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050582+0000", 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2179999999999996e-06 2023-09-23T20:04:20.440 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050597+0000", 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5071e-05 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050603+0000", 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3289999999999999e-06 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.441 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050662+0000", 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9382000000000002e-05 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050672+0000", 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1940000000000004e-06 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.093723+0000", 2023-09-23T20:04:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.043051862 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.093790+0000", 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6422999999999997e-05 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.093945+0000", 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000155167 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.443 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097604+0000", 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0036585900000000002 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097794+0000", 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019022700000000001 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.444 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103257+0000", 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054631289999999997 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103299+0000", 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2123000000000003e-05 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103323+0000", 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4358999999999998e-05 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.445 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150016 6.15 6:a9052062:::benchmark_data_smithi135_344736_object47360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.062231+0000", 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.33968352400001, 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0411754929999999, 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.446 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150016 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062231+0000", 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.447 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062231+0000", 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062230+0000", 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062235+0000", 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9439999999999996e-06 2023-09-23T20:04:20.448 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062247+0000", 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.182e-05 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062252+0000", 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6800000000000001e-06 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.449 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062306+0000", 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4709000000000001e-05 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062319+0000", 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2625e-05 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094832+0000", 2023-09-23T20:04:20.450 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0325127649999999 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094901+0000", 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9375999999999994e-05 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.095056+0000", 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015530000000000001 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.451 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097752+0000", 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0026950899999999998 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097829+0000", 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7132999999999995e-05 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103361+0000", 2023-09-23T20:04:20.452 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055324709999999997 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103386+0000", 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5309000000000001e-05 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103407+0000", 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0146e-05 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.453 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150053 6.15 6:aea43896:::benchmark_data_smithi135_344736_object47397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.110539+0000", 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.29137588200001, 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99774726800000002, 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.454 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150053 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110539+0000", 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110539+0000", 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.455 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110537+0000", 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110544+0000", 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2309999999999999e-06 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.456 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110561+0000", 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6568000000000001e-05 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110568+0000", 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8569999999999997e-06 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110622+0000", 2023-09-23T20:04:20.457 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4069e-05 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110631+0000", 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1749999999999994e-06 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.095941+0000", 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98531004099999997 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.458 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.096043+0000", 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101562 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.096241+0000", 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019844400000000001 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.459 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097835+0000", 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0015935840000000001 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100545+0000", 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027106220000000002 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108195+0000", 2023-09-23T20:04:20.460 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0076494980000000002 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108241+0000", 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6554000000000002e-05 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108286+0000", 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4629999999999998e-05 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.461 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150055 6.15 6:a8b57976:::benchmark_data_smithi135_344736_object47399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.113842+0000", 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.28807302000001, 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.994575243, 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.462 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150055 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113842+0000", 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113842+0000", 2023-09-23T20:04:20.463 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113840+0000", 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113846+0000", 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9869999999999996e-06 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.464 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113859+0000", 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3286e-05 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113871+0000", 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1279e-05 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113914+0000", 2023-09-23T20:04:20.465 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3346000000000001e-05 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113920+0000", 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0190000000000003e-06 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097874+0000", 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98395352999999997 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.466 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097987+0000", 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011300500000000001 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.098190+0000", 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020352200000000001 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.467 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100550+0000", 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023602110000000001 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.105582+0000", 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0050321139999999999 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108329+0000", 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027466719999999999 2023-09-23T20:04:20.468 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108378+0000", 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9357000000000002e-05 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108417+0000", 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8427000000000003e-05 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.469 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150062 6.15 6:ab03f741:::benchmark_data_smithi135_344736_object47406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.119730+0000", 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.28218484899998, 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98880436100000002, 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.470 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150062 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119730+0000", 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119730+0000", 2023-09-23T20:04:20.471 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119729+0000", 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119733+0000", 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4789999999999998e-06 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.472 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119742+0000", 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4530000000000002e-06 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119747+0000", 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9110000000000001e-06 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.473 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119927+0000", 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018037199999999999 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119934+0000", 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4359999999999999e-06 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.099463+0000", 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97952930299999996 2023-09-23T20:04:20.474 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.099526+0000", 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3118000000000001e-05 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.099680+0000", 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015395099999999999 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.475 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103415+0000", 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037350009999999999 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108153+0000", 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0047377890000000001 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108456+0000", 2023-09-23T20:04:20.476 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00030299400000000001 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108498+0000", 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2416000000000002e-05 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108534+0000", 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5941e-05 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.477 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150100 6.15 6:adb76daf:::benchmark_data_smithi135_344736_object47444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.166966+0000", 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.23494864499997, 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.94168644899999998, 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.478 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150100 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166966+0000", 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166966+0000", 2023-09-23T20:04:20.479 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166965+0000", 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166971+0000", 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0070000000000003e-06 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.480 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166983+0000", 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2524000000000001e-05 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166988+0000", 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8590000000000004e-06 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.167042+0000", 2023-09-23T20:04:20.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3888000000000003e-05 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.167049+0000", 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4839999999999998e-06 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100572+0000", 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.933522352 2023-09-23T20:04:20.482 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100640+0000", 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8392999999999994e-05 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100820+0000", 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017973500000000001 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.483 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103421+0000", 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0026012230000000002 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108579+0000", 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0051578789999999998 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108610+0000", 2023-09-23T20:04:20.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1353999999999998e-05 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108634+0000", 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3447000000000001e-05 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108652+0000", 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8783e-05 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.485 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150109 6.15 6:aac4d8bd:::benchmark_data_smithi135_344736_object47453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.177679+0000", 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 273.22423609600003, 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.93341589000000003, 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.486 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150109 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177679+0000", 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.487 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177679+0000", 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177678+0000", 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177681+0000", 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4489999999999999e-06 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.488 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177689+0000", 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5599999999999996e-06 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177693+0000", 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7009999999999998e-06 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.489 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177725+0000", 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2536000000000002e-05 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177731+0000", 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6370000000000004e-06 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.101724+0000", 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.92399303899999996 2023-09-23T20:04:20.490 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.101791+0000", 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7607999999999994e-05 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.101971+0000", 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000179233 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.491 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.105545+0000", 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003574331 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.111014+0000", 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054689830000000002 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.111041+0000", 2023-09-23T20:04:20.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7325e-05 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.111072+0000", 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0772999999999998e-05 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.111094+0000", 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2456999999999999e-05 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.493 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201802 6.9 6:91b29cdb:::benchmark_data_smithi135_344736_object99146:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 195.935577112, 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1705403599999999, 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.494 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201802 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.495 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466336+0000", 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466346+0000", 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0098000000000001e-05 2023-09-23T20:04:20.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466364+0000", 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7572999999999999e-05 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466371+0000", 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.238e-06 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466440+0000", 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8442000000000006e-05 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466449+0000", 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9490000000000008e-06 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469797+0000", 2023-09-23T20:04:20.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033470090000000002 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469805+0000", 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2930000000000001e-06 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593229+0000", 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.123424108 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.499 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593367+0000", 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000138338 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593937+0000", 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00056937199999999996 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608226+0000", 2023-09-23T20:04:20.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014289792000000001 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614886+0000", 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0066598359999999997 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614925+0000", 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8294999999999998e-05 2023-09-23T20:04:20.501 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636724+0000", 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021799457000000001 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636765+0000", 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1247e-05 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.502 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636878+0000", 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000112652 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201804 6.19 6:9cfa48b5:::benchmark_data_smithi135_344736_object99148:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 195.93552717399999, 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1704891850000001, 2023-09-23T20:04:20.503 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201804 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.504 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466390+0000", 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.599e-06 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466405+0000", 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5133000000000001e-05 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466410+0000", 2023-09-23T20:04:20.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8169999999999998e-06 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5672000000000002e-05 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466510+0000", 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6859999999999997e-06 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.471432+0000", 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0049220280000000002 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.471445+0000", 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2568000000000001e-05 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.508 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594208+0000", 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1227630580000001 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594319+0000", 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111501 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594813+0000", 2023-09-23T20:04:20.509 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000494181 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612638+0000", 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.017824584000000001 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614937+0000", 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022991069999999999 2023-09-23T20:04:20.510 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614972+0000", 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4678999999999998e-05 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636699+0000", 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021726942999999999 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636748+0000", 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9106e-05 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636877+0000", 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012898200000000001 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.512 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201813 6.9 6:96974822:::benchmark_data_smithi135_344736_object99157:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 195.93549143999999, 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1705933580000001, 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201813 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.513 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466425+0000", 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5450000000000002e-06 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466432+0000", 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6059999999999999e-06 2023-09-23T20:04:20.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466436+0000", 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9979999999999997e-06 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466527+0000", 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0829999999999996e-05 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.516 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466531+0000", 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.765e-06 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469820+0000", 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032892910000000002 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.517 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469824+0000", 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5639999999999998e-06 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.598827+0000", 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1290025159999999 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.598953+0000", 2023-09-23T20:04:20.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012642799999999999 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.599456+0000", 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000502974 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614525+0000", 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015068381 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614956+0000", 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043108699999999997 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614987+0000", 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1103999999999999e-05 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636915+0000", 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021927720000000001 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636954+0000", 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8930000000000002e-05 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637017+0000", 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2964000000000002e-05 2023-09-23T20:04:20.521 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201824 6.9 6:912a71a1:::benchmark_data_smithi135_344736_object99168:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 195.93547298499999, 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1708038080000001, 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.522 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201824 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.523 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466441+0000", 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466444+0000", 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2010000000000002e-06 2023-09-23T20:04:20.524 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466450+0000", 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9680000000000003e-06 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466454+0000", 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4009999999999999e-06 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.525 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466551+0000", 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6948999999999996e-05 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466557+0000", 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4999999999999996e-06 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469839+0000", 2023-09-23T20:04:20.526 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032815219999999998 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469843+0000", 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.527e-06 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602356+0000", 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1325121519999999 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.527 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602467+0000", 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111412 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602926+0000", 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000458691 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.528 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614536+0000", 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011610646000000001 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615029+0000", 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00049241599999999997 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615073+0000", 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4329999999999997e-05 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.529 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637077+0000", 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022004391000000002 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637136+0000", 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8501000000000001e-05 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.530 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637245+0000", 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010948800000000001 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201827 6.9 6:937e540a:::benchmark_data_smithi135_344736_object99171:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 195.93545659, 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1709668390000001, 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.531 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201827 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.532 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.533 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466460+0000", 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1710000000000001e-06 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466466+0000", 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8119999999999996e-06 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466470+0000", 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8290000000000001e-06 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466732+0000", 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000262759 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466747+0000", 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4204e-05 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469858+0000", 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003111426 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469875+0000", 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6639000000000001e-05 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.536 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605181+0000", 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.135306167 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605324+0000", 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014373500000000001 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605687+0000", 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036256599999999998 2023-09-23T20:04:20.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614615+0000", 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0089275680000000003 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615114+0000", 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00049912899999999996 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.538 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615172+0000", 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8260000000000001e-05 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637300+0000", 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022127563999999999 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637335+0000", 2023-09-23T20:04:20.539 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5825999999999999e-05 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637425+0000", 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9488999999999996e-05 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201882 6.9 6:972c2747:::benchmark_data_smithi135_344736_object99226:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:04:20.540 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 195.935441121, 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.173680238, 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201882 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.541 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466473+0000", 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:20.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8220000000000001e-06 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466481+0000", 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7080000000000002e-06 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.543 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466485+0000", 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.749e-06 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466901+0000", 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000416542 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466909+0000", 2023-09-23T20:04:20.544 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469891+0000", 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029817060000000002 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469895+0000", 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4510000000000002e-06 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.545 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608273+0000", 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.138377135 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608376+0000", 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010315 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:04:20.546 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608741+0000", 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036480300000000002 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614621+0000", 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0058808610000000002 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615135+0000", 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00051361899999999999 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615214+0000", 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8611999999999999e-05 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640006+0000", 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.024792756999999999 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640051+0000", 2023-09-23T20:04:20.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4193999999999997e-05 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640154+0000", 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010318 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.549 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201893 6.9 6:9279cfff:::benchmark_data_smithi135_344736_object99237:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 195.935409566, 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.173837531, 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201893 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.550 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466500+0000", 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999952 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466507+0000", 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.905e-06 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466514+0000", 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6449999999999999e-06 2023-09-23T20:04:20.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466518+0000", 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.985e-06 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466950+0000", 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043239999999999999 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.553 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466957+0000", 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1590000000000004e-06 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469913+0000", 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029559510000000001 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469918+0000", 2023-09-23T20:04:20.554 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4739999999999999e-06 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.610696+0000", 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1407785420000001 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.610812+0000", 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115654 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.555 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.611100+0000", 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00028765000000000003 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614629+0000", 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035288590000000001 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.556 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615255+0000", 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00062691999999999999 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615334+0000", 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8597000000000007e-05 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640192+0000", 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.024858410000000001 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.557 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640237+0000", 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4200999999999999e-05 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640343+0000", 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010594000000000001 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.558 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201908 6.9 6:9284cb28:::benchmark_data_smithi135_344736_object99252:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 195.93538646100001, 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.174007815, 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.559 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201908 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0990000000000001e-06 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.561 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466541+0000", 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0922e-05 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466545+0000", 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.788e-06 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466984+0000", 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000439586 2023-09-23T20:04:20.562 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466991+0000", 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9190000000000002e-06 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469933+0000", 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002941534 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.563 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469937+0000", 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2810000000000002e-06 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612655+0000", 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1427177049999999 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612740+0000", 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5024000000000005e-05 2023-09-23T20:04:20.564 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612994+0000", 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00025448499999999999 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614834+0000", 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018393560000000001 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.565 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615298+0000", 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000464303 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615356+0000", 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7827e-05 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640379+0000", 2023-09-23T20:04:20.566 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.025023627999999999 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640429+0000", 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.922e-05 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640536+0000", 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107451 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.567 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201927 6.8 6:11b41467:::benchmark_data_smithi135_344736_object99271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:05.597022+0000", 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 194.80489280099999, 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.932576234, 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:04:20.568 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201927 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.597022+0000", 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.597022+0000", 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.569 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.597020+0000", 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.597027+0000", 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5100000000000004e-06 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.570 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.597038+0000", 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1234e-05 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.597044+0000", 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7729999999999996e-06 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.597110+0000", 2023-09-23T20:04:20.571 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6196000000000001e-05 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.597120+0000", 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7680000000000003e-06 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:06.494113+0000", 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.89699346000000002 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.572 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:06.494256+0000", 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000142948 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:06.494730+0000", 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00047379099999999999 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:06.504538+0000", 2023-09-23T20:04:20.573 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0098080379999999998 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:06.529456+0000", 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.024917642 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:06.529491+0000", 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5224e-05 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.574 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:06.529552+0000", 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1332000000000002e-05 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:06.529598+0000", 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5719000000000001e-05 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.575 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.576 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:04:20.576 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:04:20.576 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:04:20.576 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:20.577 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:20.607 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:20.684 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:04:20.757 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:20.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:20.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:20.762 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:04:20.795 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:20.819 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:04:20.861 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:20.879 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:04:20.888 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:20.939 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:20.941 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:04:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:20.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:20.992 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:04:21.029 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:04:21.043 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:21.063 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:04:21.069 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:04:21.069 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:04:21.069 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:21.069 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3799847420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:21.069 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:20 smithi144 ceph-mon[194833]: pgmap v754: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 14019/184455 objects misplaced (7.600%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:04:21.071 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:21.089 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:04:21.089 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:04:21.089 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:04:21.090 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:04:21.090 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:04:21.090 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:04:21.090 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:04:21.090 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:04:21.125 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:04:21.125 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:04:21.125 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:21.125 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3799847420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:21.125 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[221113]: pgmap v754: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 14019/184455 objects misplaced (7.600%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:04:21.126 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:04:21.126 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:04:21.126 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:21.126 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3799847420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:21.126 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:20 smithi135 ceph-mon[213871]: pgmap v754: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 14019/184455 objects misplaced (7.600%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:04:21.288 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:21.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:21.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:21.354 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:04:21.392 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:21.429 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:21.501 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:21.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:21.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:21.609 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:21.718 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:21.776 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:22.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:22.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:23.202 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:23.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:23.309 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:23.351 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:23.410 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:23.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:23.515 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:23.550 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:23.598 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:23.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:23 smithi144 ceph-mon[194833]: pgmap v755: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 13288/181185 objects misplaced (7.334%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:04:23.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:23.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:23.612 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:04:23.613 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:04:23.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:23 smithi135 ceph-mon[221113]: pgmap v755: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 13288/181185 objects misplaced (7.334%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:04:23.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:23 smithi135 ceph-mon[213871]: pgmap v755: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 13288/181185 objects misplaced (7.334%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:04:23.702 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:23.787 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:04:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:04:23] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-09-23T20:04:24.268 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:24.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:24.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:24.381 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:24.453 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:04:24.497 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:24.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:24 smithi144 ceph-mon[194833]: pgmap v756: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 12931/177456 objects misplaced (7.287%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:04:24.782 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:24 smithi135 ceph-mon[213871]: pgmap v756: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 12931/177456 objects misplaced (7.287%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:04:24.924 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:25.018 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:25.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:25.032 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:24 smithi135 ceph-mon[221113]: pgmap v756: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 12931/177456 objects misplaced (7.287%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:04:25.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:25.133 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:25.165 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:25.223 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:25.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:25.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:25.327 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:25.357 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:25.526 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:25.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:25.629 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:25.656 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:25.695 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:04:25.695 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1035,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":279,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":7,"osd_in_since":1695499311,"num_remapped_pgs":4},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":95},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":99,"num_pools":5,"num_objects":59152,"data_bytes":3879475147,"bytes_used":14712561664,"bytes_avail":657252687872,"bytes_total":671965249536,"misplaced_objects":12931,"misplaced_total":177456,"misplaced_ratio":0.072868767469119106,"recovering_objects_per_sec":90,"recovering_bytes_per_sec":5956049,"recovering_keys_per_sec":0,"num_objects_recovered":924,"num_bytes_recovered":60555264,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":556},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-23T20:03:30.680470+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"8f9ef775-0a40-45cd-8917-5824ea2cda14":{"message":"Global Recovery Event (70s)\n [==========================..] (remaining: 2s)","progress":0.95959597826004028,"add_to_ceph_s":true}}} 2023-09-23T20:04:25.700 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:25.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:25.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:25.788 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2391058571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:25.788 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:25 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:25.788 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1105824683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:04:25.803 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:25.832 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:25.836 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:04:26.065 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:26.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:26.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:26.169 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:26.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2391058571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:26.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:25 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:26.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1105824683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:04:26.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2391058571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:26.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:25 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:26.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1105824683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:04:26.253 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:04:26.267 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:26.679 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:04:26.679 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:04:27.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:04:27.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:04:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T20:04:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:26 smithi144 ceph-mon[194833]: pgmap v758: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 12931/175716 objects misplaced (7.359%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:04:27.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:04:27.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:04:27.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[213871]: pgmap v758: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 12931/175716 objects misplaced (7.359%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:27.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T20:04:27.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: failed to encode map e280 with expected crc 2023-09-23T20:04:27.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:26 smithi135 ceph-mon[221113]: pgmap v758: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 12931/175716 objects misplaced (7.359%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:04:27.487 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:04:27.910 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:04:27.924 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:27.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:27.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:28.033 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:28.065 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:04:28.065 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:04:28.113 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:28.445 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:28.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:28.564 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:28.569 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:28.596 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:28.651 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:28.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:28.754 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:28.839 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:28.953 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:28.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:28.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:29.057 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:29.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:28 smithi144 ceph-mon[194833]: pgmap v759: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 11882/168645 objects misplaced (7.046%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:04:29.140 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:28 smithi135 ceph-mon[213871]: pgmap v759: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 11882/168645 objects misplaced (7.046%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:04:29.140 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:28 smithi135 ceph-mon[221113]: pgmap v759: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 11882/168645 objects misplaced (7.046%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:04:29.151 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:29.217 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:29.350 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:29.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:29.457 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:29.537 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:29.673 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:29.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:29.738 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:04:29.740 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:04:29.778 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:29.805 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:29.846 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:29.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:29.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:29.950 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:29.981 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:30.041 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:30.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:30.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:30.146 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:30.175 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:30.221 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:30.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:30.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:30.324 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:30.406 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:30.449 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:04:30.634 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:04:30.697 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:30.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:30.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:30.810 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:30.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:30 smithi144 ceph-mon[194833]: pgmap v760: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 11882/167028 objects misplaced (7.114%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:04:30.932 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:30 smithi135 ceph-mon[221113]: pgmap v760: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 11882/167028 objects misplaced (7.114%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:04:30.941 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:30 smithi135 ceph-mon[213871]: pgmap v760: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 11882/167028 objects misplaced (7.114%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:04:30.947 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:31.349 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:31.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:31.458 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:31.539 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:32.318 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:32.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:32.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:32.419 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:32.449 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:32.482 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:04:32.498 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:32.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:32.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:32.510 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:32.520 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:04:32.565 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:04:32.605 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:32.630 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:04:32.632 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:32.633 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:32.675 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:04:32.682 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:32.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:32.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/402806929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:32.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/402806929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:32.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:04:32.770 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:04:32.787 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:32.814 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:04:32.851 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:04:32.851 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:04:32.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:04:32.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:04:32.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:04:32.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:04:32.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:04:32.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:04:32.859 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:32.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/402806929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:33.081 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:33.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:33.195 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:33.311 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:33.524 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:04:33.541 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:04:33.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:33 smithi135 ceph-mon[221113]: pgmap v761: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11328/163680 objects misplaced (6.921%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:04:33.670 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:33 smithi135 ceph-mon[213871]: pgmap v761: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11328/163680 objects misplaced (6.921%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:04:33.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:33 smithi144 ceph-mon[194833]: pgmap v761: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11328/163680 objects misplaced (6.921%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:04:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:04:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:04:33] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-09-23T20:04:34.805 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:34 smithi135 ceph-mon[213871]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:34.806 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:34 smithi135 ceph-mon[221113]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:34.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:34 smithi144 ceph-mon[194833]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:35.338 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:35.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:35.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:35.447 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:35.529 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:35.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:35 smithi135 ceph-mon[213871]: pgmap v762: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 10951/159945 objects misplaced (6.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:04:35.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:35 smithi135 ceph-mon[221113]: pgmap v762: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 10951/159945 objects misplaced (6.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:04:35.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:35 smithi144 ceph-mon[194833]: pgmap v762: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 10951/159945 objects misplaced (6.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:04:35.886 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":761,"stamp":"2023-09-23T20:04:32.714640+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3578533835,"num_objects":54560,"num_object_clones":0,"num_object_copies":163680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11328,"num_objects_unfound":0,"num_objects_dirty":54560,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":140495,"num_write_kb":6224502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8107,"num_bytes_recovered":524163573,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112159,"ondisk_log_size":112159,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13527000,"kb_used_data":10971552,"kb_used_omap":32,"kb_used_meta":2555103,"kb_avail":642689064,"statfs":{"total":671965249536,"available":658113601536,"internally_reserved":0,"allocated":11234869248,"data_stored":11227285317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":2616425990},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-382402560,"num_objects":-5835,"num_object_clones":0,"num_object_copies":-17505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1960,"num_objects_unfound":0,"num_objects_dirty":-5835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5835,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":731,"num_bytes_recovered":47906816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004841"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":258,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756087+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T20:04:14.756087+0000","last_peered":"2023-09-23T20:04:14.756087+0000","last_clean":"2023-09-23T20:04:14.756087+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T20:04:14.756087+0000","last_undegraded":"2023-09-23T20:04:14.756087+0000","last_fullsized":"2023-09-23T20:04:14.756087+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":102,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554265+0000","last_change":"2023-09-23T19:59:09.151117+0000","last_active":"2023-09-23T20:04:14.554265+0000","last_peered":"2023-09-23T20:04:14.554265+0000","last_clean":"2023-09-23T20:04:14.554265+0000","last_became_active":"2023-09-23T19:59:09.150795+0000","last_became_peered":"2023-09-23T19:59:09.150795+0000","last_unstale":"2023-09-23T20:04:14.554265+0000","last_undegraded":"2023-09-23T20:04:14.554265+0000","last_fullsized":"2023-09-23T20:04:14.554265+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:34.994574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":241,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:20.523193+0000","last_change":"2023-09-23T19:59:10.156070+0000","last_active":"2023-09-23T20:04:20.523193+0000","last_peered":"2023-09-23T20:04:20.523193+0000","last_clean":"2023-09-23T20:04:20.523193+0000","last_became_active":"2023-09-23T19:59:10.155813+0000","last_became_peered":"2023-09-23T19:59:10.155813+0000","last_unstale":"2023-09-23T20:04:20.523193+0000","last_undegraded":"2023-09-23T20:04:20.523193+0000","last_fullsized":"2023-09-23T20:04:20.523193+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:42.582380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":32,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.557396+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:04:14.557396+0000","last_peered":"2023-09-23T20:04:14.557396+0000","last_clean":"2023-09-23T20:04:14.557396+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:04:14.557396+0000","last_undegraded":"2023-09-23T20:04:14.557396+0000","last_fullsized":"2023-09-23T20:04:14.557396+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.553121+0000","last_change":"2023-09-23T19:59:09.163182+0000","last_active":"2023-09-23T20:04:14.553121+0000","last_peered":"2023-09-23T20:04:14.553121+0000","last_clean":"2023-09-23T20:04:14.553121+0000","last_became_active":"2023-09-23T19:59:09.161612+0000","last_became_peered":"2023-09-23T19:59:09.161612+0000","last_unstale":"2023-09-23T20:04:14.553121+0000","last_undegraded":"2023-09-23T20:04:14.553121+0000","last_fullsized":"2023-09-23T20:04:14.553121+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:59.524152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":205,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559826+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T20:04:14.559826+0000","last_peered":"2023-09-23T20:04:14.559826+0000","last_clean":"2023-09-23T20:04:14.559826+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T20:04:14.559826+0000","last_undegraded":"2023-09-23T20:04:14.559826+0000","last_fullsized":"2023-09-23T20:04:14.559826+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:59.043547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":206,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559702+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:04:14.559702+0000","last_peered":"2023-09-23T20:04:14.559702+0000","last_clean":"2023-09-23T20:04:14.559702+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:04:14.559702+0000","last_undegraded":"2023-09-23T20:04:14.559702+0000","last_fullsized":"2023-09-23T20:04:14.559702+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":102,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.553192+0000","last_change":"2023-09-23T19:59:09.146009+0000","last_active":"2023-09-23T20:04:14.553192+0000","last_peered":"2023-09-23T20:04:14.553192+0000","last_clean":"2023-09-23T20:04:14.553192+0000","last_became_active":"2023-09-23T19:59:09.145687+0000","last_became_peered":"2023-09-23T19:59:09.145687+0000","last_unstale":"2023-09-23T20:04:14.553192+0000","last_undegraded":"2023-09-23T20:04:14.553192+0000","last_fullsized":"2023-09-23T20:04:14.553192+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.189450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":239,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559217+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T20:04:14.559217+0000","last_peered":"2023-09-23T20:04:14.559217+0000","last_clean":"2023-09-23T20:04:14.559217+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T20:04:14.559217+0000","last_undegraded":"2023-09-23T20:04:14.559217+0000","last_fullsized":"2023-09-23T20:04:14.559217+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":121,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.551829+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:04:14.551829+0000","last_peered":"2023-09-23T20:04:14.551829+0000","last_clean":"2023-09-23T20:04:14.551829+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:04:14.551829+0000","last_undegraded":"2023-09-23T20:04:14.551829+0000","last_fullsized":"2023-09-23T20:04:14.551829+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":275,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.557138+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:04:14.557138+0000","last_peered":"2023-09-23T20:04:14.557138+0000","last_clean":"2023-09-23T20:04:14.557138+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:04:14.557138+0000","last_undegraded":"2023-09-23T20:04:14.557138+0000","last_fullsized":"2023-09-23T20:04:14.557138+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":237,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.578784+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T20:04:14.578784+0000","last_peered":"2023-09-23T20:04:14.578784+0000","last_clean":"2023-09-23T20:04:14.578784+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T20:04:14.578784+0000","last_undegraded":"2023-09-23T20:04:14.578784+0000","last_fullsized":"2023-09-23T20:04:14.578784+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:51.825640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":294,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756257+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:04:14.756257+0000","last_peered":"2023-09-23T20:04:14.756257+0000","last_clean":"2023-09-23T20:04:14.756257+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:04:14.756257+0000","last_undegraded":"2023-09-23T20:04:14.756257+0000","last_fullsized":"2023-09-23T20:04:14.756257+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":256,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.759542+0000","last_change":"2023-09-23T20:01:52.439375+0000","last_active":"2023-09-23T20:04:14.759542+0000","last_peered":"2023-09-23T20:04:14.759542+0000","last_clean":"2023-09-23T20:04:14.759542+0000","last_became_active":"2023-09-23T20:01:52.439074+0000","last_became_peered":"2023-09-23T20:01:52.439074+0000","last_unstale":"2023-09-23T20:04:14.759542+0000","last_undegraded":"2023-09-23T20:04:14.759542+0000","last_fullsized":"2023-09-23T20:04:14.759542+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:31.103633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556959+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:04:14.556959+0000","last_peered":"2023-09-23T20:04:14.556959+0000","last_clean":"2023-09-23T20:04:14.556959+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:04:14.556959+0000","last_undegraded":"2023-09-23T20:04:14.556959+0000","last_fullsized":"2023-09-23T20:04:14.556959+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":102,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.553803+0000","last_change":"2023-09-23T19:59:09.195560+0000","last_active":"2023-09-23T20:04:14.553803+0000","last_peered":"2023-09-23T20:04:14.553803+0000","last_clean":"2023-09-23T20:04:14.553803+0000","last_became_active":"2023-09-23T19:59:09.195145+0000","last_became_peered":"2023-09-23T19:59:09.195145+0000","last_unstale":"2023-09-23T20:04:14.553803+0000","last_undegraded":"2023-09-23T20:04:14.553803+0000","last_fullsized":"2023-09-23T20:04:14.553803+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.190798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":245,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556892+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:04:14.556892+0000","last_peered":"2023-09-23T20:04:14.556892+0000","last_clean":"2023-09-23T20:04:14.556892+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:04:14.556892+0000","last_undegraded":"2023-09-23T20:04:14.556892+0000","last_fullsized":"2023-09-23T20:04:14.556892+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":258,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559133+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:04:14.559133+0000","last_peered":"2023-09-23T20:04:14.559133+0000","last_clean":"2023-09-23T20:04:14.559133+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:04:14.559133+0000","last_undegraded":"2023-09-23T20:04:14.559133+0000","last_fullsized":"2023-09-23T20:04:14.559133+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556996+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:04:14.556996+0000","last_peered":"2023-09-23T20:04:14.556996+0000","last_clean":"2023-09-23T20:04:14.556996+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:04:14.556996+0000","last_undegraded":"2023-09-23T20:04:14.556996+0000","last_fullsized":"2023-09-23T20:04:14.556996+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":245,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.553813+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:04:14.553813+0000","last_peered":"2023-09-23T20:04:14.553813+0000","last_clean":"2023-09-23T20:04:14.553813+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:04:14.553813+0000","last_undegraded":"2023-09-23T20:04:14.553813+0000","last_fullsized":"2023-09-23T20:04:14.553813+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":73,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558816+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:04:14.558816+0000","last_peered":"2023-09-23T20:04:14.558816+0000","last_clean":"2023-09-23T20:04:14.558816+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:04:14.558816+0000","last_undegraded":"2023-09-23T20:04:14.558816+0000","last_fullsized":"2023-09-23T20:04:14.558816+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554417+0000","last_change":"2023-09-23T20:03:59.232022+0000","last_active":"2023-09-23T20:04:14.554417+0000","last_peered":"2023-09-23T20:04:14.554417+0000","last_clean":"2023-09-23T20:04:14.554417+0000","last_became_active":"2023-09-23T20:03:59.231689+0000","last_became_peered":"2023-09-23T20:03:59.231689+0000","last_unstale":"2023-09-23T20:04:14.554417+0000","last_undegraded":"2023-09-23T20:04:14.554417+0000","last_fullsized":"2023-09-23T20:04:14.554417+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:29.669102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"280'8648","reported_seq":18932,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:30.481989+0000","last_change":"2023-09-23T20:04:02.290062+0000","last_active":"2023-09-23T20:04:30.481989+0000","last_peered":"2023-09-23T20:04:30.481989+0000","last_clean":"2023-09-23T20:04:30.481989+0000","last_became_active":"2023-09-23T20:04:02.271287+0000","last_became_peered":"2023-09-23T20:04:02.271287+0000","last_unstale":"2023-09-23T20:04:30.481989+0000","last_undegraded":"2023-09-23T20:04:30.481989+0000","last_fullsized":"2023-09-23T20:04:30.481989+0000","mapping_epoch":276,"log_start":"247'2100","ondisk_log_start":"247'2100","created":240,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:47.344179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225181696,"num_objects":3436,"num_object_clones":0,"num_object_copies":10308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8648,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5100,"num_bytes_recovered":334233600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":241,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.757255+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:04:14.757255+0000","last_peered":"2023-09-23T20:04:14.757255+0000","last_clean":"2023-09-23T20:04:14.757255+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:04:14.757255+0000","last_undegraded":"2023-09-23T20:04:14.757255+0000","last_fullsized":"2023-09-23T20:04:14.757255+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":268,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558779+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:04:14.558779+0000","last_peered":"2023-09-23T20:04:14.558779+0000","last_clean":"2023-09-23T20:04:14.558779+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:04:14.558779+0000","last_undegraded":"2023-09-23T20:04:14.558779+0000","last_fullsized":"2023-09-23T20:04:14.558779+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"280'8781","reported_seq":8831,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:32.043415+0000","last_change":"2023-09-23T20:01:39.911588+0000","last_active":"2023-09-23T20:04:32.043415+0000","last_peered":"2023-09-23T20:04:32.043415+0000","last_clean":"2023-09-23T20:04:32.043415+0000","last_became_active":"2023-09-23T20:01:39.911364+0000","last_became_peered":"2023-09-23T20:01:39.911364+0000","last_unstale":"2023-09-23T20:04:32.043415+0000","last_undegraded":"2023-09-23T20:04:32.043415+0000","last_fullsized":"2023-09-23T20:04:32.043415+0000","mapping_epoch":240,"log_start":"248'3095","ondisk_log_start":"248'3095","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:35.715001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214237184,"num_objects":3269,"num_object_clones":0,"num_object_copies":9807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8781,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.757162+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T20:04:14.757162+0000","last_peered":"2023-09-23T20:04:14.757162+0000","last_clean":"2023-09-23T20:04:14.757162+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T20:04:14.757162+0000","last_undegraded":"2023-09-23T20:04:14.757162+0000","last_fullsized":"2023-09-23T20:04:14.757162+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":296,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756455+0000","last_change":"2023-09-23T20:03:59.253784+0000","last_active":"2023-09-23T20:04:14.756455+0000","last_peered":"2023-09-23T20:04:14.756455+0000","last_clean":"2023-09-23T20:04:14.756455+0000","last_became_active":"2023-09-23T20:03:59.253552+0000","last_became_peered":"2023-09-23T20:03:59.253552+0000","last_unstale":"2023-09-23T20:04:14.756455+0000","last_undegraded":"2023-09-23T20:04:14.756455+0000","last_fullsized":"2023-09-23T20:04:14.756455+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:56.110746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"280'8684","reported_seq":8733,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:30.496048+0000","last_change":"2023-09-23T20:01:39.976582+0000","last_active":"2023-09-23T20:04:30.496048+0000","last_peered":"2023-09-23T20:04:30.496048+0000","last_clean":"2023-09-23T20:04:30.496048+0000","last_became_active":"2023-09-23T20:01:39.976244+0000","last_became_peered":"2023-09-23T20:01:39.976244+0000","last_unstale":"2023-09-23T20:04:30.496048+0000","last_undegraded":"2023-09-23T20:04:30.496048+0000","last_fullsized":"2023-09-23T20:04:30.496048+0000","mapping_epoch":240,"log_start":"247'2100","ondisk_log_start":"247'2100","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6584,"log_dups_size":0,"ondisk_log_size":6584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.952295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220332032,"num_objects":3362,"num_object_clones":0,"num_object_copies":10086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8684,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":245,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556986+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:04:14.556986+0000","last_peered":"2023-09-23T20:04:14.556986+0000","last_clean":"2023-09-23T20:04:14.556986+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:04:14.556986+0000","last_undegraded":"2023-09-23T20:04:14.556986+0000","last_fullsized":"2023-09-23T20:04:14.556986+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":207,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.578984+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T20:04:14.578984+0000","last_peered":"2023-09-23T20:04:14.578984+0000","last_clean":"2023-09-23T20:04:14.578984+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T20:04:14.578984+0000","last_undegraded":"2023-09-23T20:04:14.578984+0000","last_fullsized":"2023-09-23T20:04:14.578984+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.284038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"280'8934","reported_seq":8984,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:28.166199+0000","last_change":"2023-09-23T20:01:39.912566+0000","last_active":"2023-09-23T20:04:28.166199+0000","last_peered":"2023-09-23T20:04:28.166199+0000","last_clean":"2023-09-23T20:04:28.166199+0000","last_became_active":"2023-09-23T20:01:39.912259+0000","last_became_peered":"2023-09-23T20:01:39.912259+0000","last_unstale":"2023-09-23T20:04:28.166199+0000","last_undegraded":"2023-09-23T20:04:28.166199+0000","last_fullsized":"2023-09-23T20:04:28.166199+0000","mapping_epoch":240,"log_start":"243'100","ondisk_log_start":"243'100","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8834,"log_dups_size":0,"ondisk_log_size":8834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:19.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230162432,"num_objects":3512,"num_object_clones":0,"num_object_copies":10536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8934,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":298,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558964+0000","last_change":"2023-09-23T20:00:34.468296+0000","last_active":"2023-09-23T20:04:14.558964+0000","last_peered":"2023-09-23T20:04:14.558964+0000","last_clean":"2023-09-23T20:04:14.558964+0000","last_became_active":"2023-09-23T20:00:34.467957+0000","last_became_peered":"2023-09-23T20:00:34.467957+0000","last_unstale":"2023-09-23T20:04:14.558964+0000","last_undegraded":"2023-09-23T20:04:14.558964+0000","last_fullsized":"2023-09-23T20:04:14.558964+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:50.449027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":95,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.579056+0000","last_change":"2023-09-23T20:01:04.470558+0000","last_active":"2023-09-23T20:04:14.579056+0000","last_peered":"2023-09-23T20:04:14.579056+0000","last_clean":"2023-09-23T20:04:14.579056+0000","last_became_active":"2023-09-23T20:01:04.470276+0000","last_became_peered":"2023-09-23T20:01:04.470276+0000","last_unstale":"2023-09-23T20:04:14.579056+0000","last_undegraded":"2023-09-23T20:04:14.579056+0000","last_fullsized":"2023-09-23T20:04:14.579056+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:17.788583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"280'8846","reported_seq":8896,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:32.062173+0000","last_change":"2023-09-23T20:01:39.974611+0000","last_active":"2023-09-23T20:04:32.062173+0000","last_peered":"2023-09-23T20:04:32.062173+0000","last_clean":"2023-09-23T20:04:32.062173+0000","last_became_active":"2023-09-23T20:01:39.974318+0000","last_became_peered":"2023-09-23T20:01:39.974318+0000","last_unstale":"2023-09-23T20:04:32.062173+0000","last_undegraded":"2023-09-23T20:04:32.062173+0000","last_fullsized":"2023-09-23T20:04:32.062173+0000","mapping_epoch":240,"log_start":"248'3127","ondisk_log_start":"248'3127","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:31.047765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209846272,"num_objects":3202,"num_object_clones":0,"num_object_copies":9606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8846,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":730,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556764+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:04:14.556764+0000","last_peered":"2023-09-23T20:04:14.556764+0000","last_clean":"2023-09-23T20:04:14.556764+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:04:14.556764+0000","last_undegraded":"2023-09-23T20:04:14.556764+0000","last_fullsized":"2023-09-23T20:04:14.556764+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":106,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554597+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:04:14.554597+0000","last_peered":"2023-09-23T20:04:14.554597+0000","last_clean":"2023-09-23T20:04:14.554597+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:04:14.554597+0000","last_undegraded":"2023-09-23T20:04:14.554597+0000","last_fullsized":"2023-09-23T20:04:14.554597+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":276,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756129+0000","last_change":"2023-09-23T19:59:09.206075+0000","last_active":"2023-09-23T20:04:14.756129+0000","last_peered":"2023-09-23T20:04:14.756129+0000","last_clean":"2023-09-23T20:04:14.756129+0000","last_became_active":"2023-09-23T19:59:09.205766+0000","last_became_peered":"2023-09-23T19:59:09.205766+0000","last_unstale":"2023-09-23T20:04:14.756129+0000","last_undegraded":"2023-09-23T20:04:14.756129+0000","last_fullsized":"2023-09-23T20:04:14.756129+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:14.627703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"280'8483","reported_seq":8533,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:28.101772+0000","last_change":"2023-09-23T20:01:39.976550+0000","last_active":"2023-09-23T20:04:28.101772+0000","last_peered":"2023-09-23T20:04:28.101772+0000","last_clean":"2023-09-23T20:04:28.101772+0000","last_became_active":"2023-09-23T20:01:39.976245+0000","last_became_peered":"2023-09-23T20:01:39.976245+0000","last_unstale":"2023-09-23T20:04:28.101772+0000","last_undegraded":"2023-09-23T20:04:28.101772+0000","last_fullsized":"2023-09-23T20:04:28.101772+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8483,"log_dups_size":0,"ondisk_log_size":8483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:45.979020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221577216,"num_objects":3381,"num_object_clones":0,"num_object_copies":10143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8483,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":639,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558712+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:04:14.558712+0000","last_peered":"2023-09-23T20:04:14.558712+0000","last_clean":"2023-09-23T20:04:14.558712+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:04:14.558712+0000","last_undegraded":"2023-09-23T20:04:14.558712+0000","last_fullsized":"2023-09-23T20:04:14.558712+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":226,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756115+0000","last_change":"2023-09-23T19:59:09.172601+0000","last_active":"2023-09-23T20:04:14.756115+0000","last_peered":"2023-09-23T20:04:14.756115+0000","last_clean":"2023-09-23T20:04:14.756115+0000","last_became_active":"2023-09-23T19:59:09.172297+0000","last_became_peered":"2023-09-23T19:59:09.172297+0000","last_unstale":"2023-09-23T20:04:14.756115+0000","last_undegraded":"2023-09-23T20:04:14.756115+0000","last_fullsized":"2023-09-23T20:04:14.756115+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:09.528266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":256,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756079+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T20:04:14.756079+0000","last_peered":"2023-09-23T20:04:14.756079+0000","last_clean":"2023-09-23T20:04:14.756079+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T20:04:14.756079+0000","last_undegraded":"2023-09-23T20:04:14.756079+0000","last_fullsized":"2023-09-23T20:04:14.756079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"280'8709","reported_seq":8760,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:30.481932+0000","last_change":"2023-09-23T20:01:39.977019+0000","last_active":"2023-09-23T20:04:30.481932+0000","last_peered":"2023-09-23T20:04:30.481932+0000","last_clean":"2023-09-23T20:04:30.481932+0000","last_became_active":"2023-09-23T20:01:39.976270+0000","last_became_peered":"2023-09-23T20:01:39.976270+0000","last_unstale":"2023-09-23T20:04:30.481932+0000","last_undegraded":"2023-09-23T20:04:30.481932+0000","last_fullsized":"2023-09-23T20:04:30.481932+0000","mapping_epoch":240,"log_start":"247'2100","ondisk_log_start":"247'2100","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.226759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221642776,"num_objects":3383,"num_object_clones":0,"num_object_copies":10149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8709,"num_write_kb":386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756306+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:04:14.756306+0000","last_peered":"2023-09-23T20:04:14.756306+0000","last_clean":"2023-09-23T20:04:14.756306+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:04:14.756306+0000","last_undegraded":"2023-09-23T20:04:14.756306+0000","last_fullsized":"2023-09-23T20:04:14.756306+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":286,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756241+0000","last_change":"2023-09-23T19:59:09.171926+0000","last_active":"2023-09-23T20:04:14.756241+0000","last_peered":"2023-09-23T20:04:14.756241+0000","last_clean":"2023-09-23T20:04:14.756241+0000","last_became_active":"2023-09-23T19:59:09.171609+0000","last_became_peered":"2023-09-23T19:59:09.171609+0000","last_unstale":"2023-09-23T20:04:14.756241+0000","last_undegraded":"2023-09-23T20:04:14.756241+0000","last_fullsized":"2023-09-23T20:04:14.756241+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:21.795329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"280'8689","reported_seq":13662,"reported_epoch":280,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:04:32.242761+0000","last_change":"2023-09-23T20:04:02.298192+0000","last_active":"2023-09-23T20:04:32.242761+0000","last_peered":"2023-09-23T20:04:32.242761+0000","last_clean":"2023-09-23T20:03:58.092468+0000","last_became_active":"2023-09-23T20:04:00.591086+0000","last_became_peered":"2023-09-23T20:04:00.591086+0000","last_unstale":"2023-09-23T20:04:32.242761+0000","last_undegraded":"2023-09-23T20:04:32.242761+0000","last_fullsized":"2023-09-23T20:04:32.242761+0000","mapping_epoch":274,"log_start":"247'2072","ondisk_log_start":"247'2072","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:19.737714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217120768,"num_objects":3313,"num_object_clones":0,"num_object_copies":9939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1184,"num_objects_unfound":0,"num_objects_dirty":3313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8689,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2459,"num_bytes_recovered":161153024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[0,7,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":3313}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":618,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.560245+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:04:14.560245+0000","last_peered":"2023-09-23T20:04:14.560245+0000","last_clean":"2023-09-23T20:04:14.560245+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:04:14.560245+0000","last_undegraded":"2023-09-23T20:04:14.560245+0000","last_fullsized":"2023-09-23T20:04:14.560245+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":17,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.551858+0000","last_change":"2023-09-23T20:03:59.233742+0000","last_active":"2023-09-23T20:04:14.551858+0000","last_peered":"2023-09-23T20:04:14.551858+0000","last_clean":"2023-09-23T20:04:14.551858+0000","last_became_active":"2023-09-23T20:03:59.233209+0000","last_became_peered":"2023-09-23T20:03:59.233209+0000","last_unstale":"2023-09-23T20:04:14.551858+0000","last_undegraded":"2023-09-23T20:04:14.551858+0000","last_fullsized":"2023-09-23T20:04:14.551858+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:50.879925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.579736+0000","last_change":"2023-09-23T19:59:09.153852+0000","last_active":"2023-09-23T20:04:14.579736+0000","last_peered":"2023-09-23T20:04:14.579736+0000","last_clean":"2023-09-23T20:04:14.579736+0000","last_became_active":"2023-09-23T19:59:09.153514+0000","last_became_peered":"2023-09-23T19:59:09.153514+0000","last_unstale":"2023-09-23T20:04:14.579736+0000","last_undegraded":"2023-09-23T20:04:14.579736+0000","last_fullsized":"2023-09-23T20:04:14.579736+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:45.179600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"280'8645","reported_seq":8721,"reported_epoch":280,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:28.356248+0000","last_change":"2023-09-23T20:04:02.297387+0000","last_active":"2023-09-23T20:04:28.356248+0000","last_peered":"2023-09-23T20:04:28.356248+0000","last_clean":"2023-09-23T20:03:58.080766+0000","last_became_active":"2023-09-23T20:04:00.582511+0000","last_became_peered":"2023-09-23T20:04:00.582511+0000","last_unstale":"2023-09-23T20:04:28.356248+0000","last_undegraded":"2023-09-23T20:04:28.356248+0000","last_fullsized":"2023-09-23T20:04:28.356248+0000","mapping_epoch":274,"log_start":"247'2100","ondisk_log_start":"247'2100","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6545,"log_dups_size":0,"ondisk_log_size":6545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:56.089695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231276544,"num_objects":3529,"num_object_clones":0,"num_object_copies":10587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3527,"num_objects_unfound":0,"num_objects_dirty":3529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8645,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,1,0],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":3529}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":603,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554510+0000","last_change":"2023-09-23T20:04:02.277887+0000","last_active":"2023-09-23T20:04:14.554510+0000","last_peered":"2023-09-23T20:04:14.554510+0000","last_clean":"2023-09-23T20:04:14.554510+0000","last_became_active":"2023-09-23T20:04:02.277519+0000","last_became_peered":"2023-09-23T20:04:02.277519+0000","last_unstale":"2023-09-23T20:04:14.554510+0000","last_undegraded":"2023-09-23T20:04:14.554510+0000","last_fullsized":"2023-09-23T20:04:14.554510+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":619,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.526631+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T20:04:25.526631+0000","last_peered":"2023-09-23T20:04:25.526631+0000","last_clean":"2023-09-23T20:04:25.526631+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T20:04:25.526631+0000","last_undegraded":"2023-09-23T20:04:25.526631+0000","last_fullsized":"2023-09-23T20:04:25.526631+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":280,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559130+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:04:14.559130+0000","last_peered":"2023-09-23T20:04:14.559130+0000","last_clean":"2023-09-23T20:04:14.559130+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:04:14.559130+0000","last_undegraded":"2023-09-23T20:04:14.559130+0000","last_fullsized":"2023-09-23T20:04:14.559130+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":208,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.526561+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T20:04:25.526561+0000","last_peered":"2023-09-23T20:04:25.526561+0000","last_clean":"2023-09-23T20:04:25.526561+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T20:04:25.526561+0000","last_undegraded":"2023-09-23T20:04:25.526561+0000","last_fullsized":"2023-09-23T20:04:25.526561+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"280'8822","reported_seq":8886,"reported_epoch":280,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:32.184514+0000","last_change":"2023-09-23T20:04:00.730874+0000","last_active":"2023-09-23T20:04:32.184514+0000","last_peered":"2023-09-23T20:04:32.184514+0000","last_clean":"2023-09-23T20:03:58.087054+0000","last_became_active":"2023-09-23T20:04:00.560120+0000","last_became_peered":"2023-09-23T20:04:00.560120+0000","last_unstale":"2023-09-23T20:04:32.184514+0000","last_undegraded":"2023-09-23T20:04:32.184514+0000","last_fullsized":"2023-09-23T20:04:32.184514+0000","mapping_epoch":274,"log_start":"247'2300","ondisk_log_start":"247'2300","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:17.026687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223608832,"num_objects":3412,"num_object_clones":0,"num_object_copies":10236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3412,"num_objects_unfound":0,"num_objects_dirty":3412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8822,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":3412}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":591,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558887+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:04:14.558887+0000","last_peered":"2023-09-23T20:04:14.558887+0000","last_clean":"2023-09-23T20:04:14.558887+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:04:14.558887+0000","last_undegraded":"2023-09-23T20:04:14.558887+0000","last_fullsized":"2023-09-23T20:04:14.558887+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":840,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.560003+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T20:04:14.560003+0000","last_peered":"2023-09-23T20:04:14.560003+0000","last_clean":"2023-09-23T20:04:14.560003+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T20:04:14.560003+0000","last_undegraded":"2023-09-23T20:04:14.560003+0000","last_fullsized":"2023-09-23T20:04:14.560003+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:58.623015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":279,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.759663+0000","last_change":"2023-09-23T20:00:34.447064+0000","last_active":"2023-09-23T20:04:14.759663+0000","last_peered":"2023-09-23T20:04:14.759663+0000","last_clean":"2023-09-23T20:04:14.759663+0000","last_became_active":"2023-09-23T20:00:34.446754+0000","last_became_peered":"2023-09-23T20:00:34.446754+0000","last_unstale":"2023-09-23T20:04:14.759663+0000","last_undegraded":"2023-09-23T20:04:14.759663+0000","last_fullsized":"2023-09-23T20:04:14.759663+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:39.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.526614+0000","last_change":"2023-09-23T20:03:59.236594+0000","last_active":"2023-09-23T20:04:25.526614+0000","last_peered":"2023-09-23T20:04:25.526614+0000","last_clean":"2023-09-23T20:04:25.526614+0000","last_became_active":"2023-09-23T20:03:59.236305+0000","last_became_peered":"2023-09-23T20:03:59.236305+0000","last_unstale":"2023-09-23T20:04:25.526614+0000","last_undegraded":"2023-09-23T20:04:25.526614+0000","last_fullsized":"2023-09-23T20:04:25.526614+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:23.103726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"280'8578","reported_seq":8623,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:28.512762+0000","last_change":"2023-09-23T20:01:39.977018+0000","last_active":"2023-09-23T20:04:28.512762+0000","last_peered":"2023-09-23T20:04:28.512762+0000","last_clean":"2023-09-23T20:04:28.512762+0000","last_became_active":"2023-09-23T20:01:39.976359+0000","last_became_peered":"2023-09-23T20:01:39.976359+0000","last_unstale":"2023-09-23T20:04:28.512762+0000","last_undegraded":"2023-09-23T20:04:28.512762+0000","last_fullsized":"2023-09-23T20:04:28.512762+0000","mapping_epoch":240,"log_start":"247'2000","ondisk_log_start":"247'2000","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.221346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227934208,"num_objects":3478,"num_object_clones":0,"num_object_copies":10434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8578,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":515,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556815+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:04:14.556815+0000","last_peered":"2023-09-23T20:04:14.556815+0000","last_clean":"2023-09-23T20:04:14.556815+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:04:14.556815+0000","last_undegraded":"2023-09-23T20:04:14.556815+0000","last_fullsized":"2023-09-23T20:04:14.556815+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":912,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554311+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:04:14.554311+0000","last_peered":"2023-09-23T20:04:14.554311+0000","last_clean":"2023-09-23T20:04:14.554311+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:04:14.554311+0000","last_undegraded":"2023-09-23T20:04:14.554311+0000","last_fullsized":"2023-09-23T20:04:14.554311+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":237,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.581639+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T20:04:14.581639+0000","last_peered":"2023-09-23T20:04:14.581639+0000","last_clean":"2023-09-23T20:04:14.581639+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T20:04:14.581639+0000","last_undegraded":"2023-09-23T20:04:14.581639+0000","last_fullsized":"2023-09-23T20:04:14.581639+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.564126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":251,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.579740+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T20:04:14.579740+0000","last_peered":"2023-09-23T20:04:14.579740+0000","last_clean":"2023-09-23T20:04:14.579740+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T20:04:14.579740+0000","last_undegraded":"2023-09-23T20:04:14.579740+0000","last_fullsized":"2023-09-23T20:04:14.579740+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"280'8769","reported_seq":9685,"reported_epoch":280,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:28.467611+0000","last_change":"2023-09-23T20:04:02.279075+0000","last_active":"2023-09-23T20:04:28.467611+0000","last_peered":"2023-09-23T20:04:28.467611+0000","last_clean":"2023-09-23T20:03:58.125973+0000","last_became_active":"2023-09-23T20:04:00.586359+0000","last_became_peered":"2023-09-23T20:04:00.586359+0000","last_unstale":"2023-09-23T20:04:28.467611+0000","last_undegraded":"2023-09-23T20:04:28.467611+0000","last_fullsized":"2023-09-23T20:04:28.467611+0000","mapping_epoch":274,"log_start":"246'1356","ondisk_log_start":"246'1356","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7413,"log_dups_size":0,"ondisk_log_size":7413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:05.232047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232587264,"num_objects":3549,"num_object_clones":0,"num_object_copies":10647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3205,"num_objects_unfound":0,"num_objects_dirty":3549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8769,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":426,"num_bytes_recovered":27918336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,5],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":3549}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1088,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.561595+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:04:14.561595+0000","last_peered":"2023-09-23T20:04:14.561595+0000","last_clean":"2023-09-23T20:04:14.561595+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:04:14.561595+0000","last_undegraded":"2023-09-23T20:04:14.561595+0000","last_fullsized":"2023-09-23T20:04:14.561595+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":690,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556894+0000","last_change":"2023-09-23T20:00:34.445036+0000","last_active":"2023-09-23T20:04:14.556894+0000","last_peered":"2023-09-23T20:04:14.556894+0000","last_clean":"2023-09-23T20:04:14.556894+0000","last_became_active":"2023-09-23T20:00:34.444689+0000","last_became_peered":"2023-09-23T20:00:34.444689+0000","last_unstale":"2023-09-23T20:04:14.556894+0000","last_undegraded":"2023-09-23T20:04:14.556894+0000","last_fullsized":"2023-09-23T20:04:14.556894+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:44.485721+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":17,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.553274+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:04:14.553274+0000","last_peered":"2023-09-23T20:04:14.553274+0000","last_clean":"2023-09-23T20:04:14.553274+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:04:14.553274+0000","last_undegraded":"2023-09-23T20:04:14.553274+0000","last_fullsized":"2023-09-23T20:04:14.553274+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":155,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756210+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:04:14.756210+0000","last_peered":"2023-09-23T20:04:14.756210+0000","last_clean":"2023-09-23T20:04:14.756210+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:04:14.756210+0000","last_undegraded":"2023-09-23T20:04:14.756210+0000","last_fullsized":"2023-09-23T20:04:14.756210+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"280'8739","reported_seq":8787,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:28.415080+0000","last_change":"2023-09-23T20:01:39.977010+0000","last_active":"2023-09-23T20:04:28.415080+0000","last_peered":"2023-09-23T20:04:28.415080+0000","last_clean":"2023-09-23T20:04:28.415080+0000","last_became_active":"2023-09-23T20:01:39.976722+0000","last_became_peered":"2023-09-23T20:01:39.976722+0000","last_unstale":"2023-09-23T20:04:28.415080+0000","last_undegraded":"2023-09-23T20:04:28.415080+0000","last_fullsized":"2023-09-23T20:04:28.415080+0000","mapping_epoch":240,"log_start":"246'1331","ondisk_log_start":"246'1331","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7408,"log_dups_size":0,"ondisk_log_size":7408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:13.592964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229048320,"num_objects":3495,"num_object_clones":0,"num_object_copies":10485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8739,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":620,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.579135+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T20:04:14.579135+0000","last_peered":"2023-09-23T20:04:14.579135+0000","last_clean":"2023-09-23T20:04:14.579135+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T20:04:14.579135+0000","last_undegraded":"2023-09-23T20:04:14.579135+0000","last_fullsized":"2023-09-23T20:04:14.579135+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.842550+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":998,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556819+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T20:04:14.556819+0000","last_peered":"2023-09-23T20:04:14.556819+0000","last_clean":"2023-09-23T20:04:14.556819+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T20:04:14.556819+0000","last_undegraded":"2023-09-23T20:04:14.556819+0000","last_fullsized":"2023-09-23T20:04:14.556819+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":246,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756506+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:04:14.756506+0000","last_peered":"2023-09-23T20:04:14.756506+0000","last_clean":"2023-09-23T20:04:14.756506+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:04:14.756506+0000","last_undegraded":"2023-09-23T20:04:14.756506+0000","last_fullsized":"2023-09-23T20:04:14.756506+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559272+0000","last_change":"2023-09-23T20:03:10.175462+0000","last_active":"2023-09-23T20:04:14.559272+0000","last_peered":"2023-09-23T20:04:14.559272+0000","last_clean":"2023-09-23T20:04:14.559272+0000","last_became_active":"2023-09-23T20:03:10.175250+0000","last_became_peered":"2023-09-23T20:03:10.175250+0000","last_unstale":"2023-09-23T20:04:14.559272+0000","last_undegraded":"2023-09-23T20:04:14.559272+0000","last_fullsized":"2023-09-23T20:04:14.559272+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:42.961619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"280'8485","reported_seq":8533,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:28.061330+0000","last_change":"2023-09-23T20:01:39.913385+0000","last_active":"2023-09-23T20:04:28.061330+0000","last_peered":"2023-09-23T20:04:28.061330+0000","last_clean":"2023-09-23T20:04:28.061330+0000","last_became_active":"2023-09-23T20:01:39.912872+0000","last_became_peered":"2023-09-23T20:01:39.912872+0000","last_unstale":"2023-09-23T20:04:28.061330+0000","last_undegraded":"2023-09-23T20:04:28.061330+0000","last_fullsized":"2023-09-23T20:04:28.061330+0000","mapping_epoch":240,"log_start":"243'100","ondisk_log_start":"243'100","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8385,"log_dups_size":0,"ondisk_log_size":8385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.619813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230096896,"num_objects":3511,"num_object_clones":0,"num_object_copies":10533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8485,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":595,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.553663+0000","last_change":"2023-09-23T20:04:02.277648+0000","last_active":"2023-09-23T20:04:14.553663+0000","last_peered":"2023-09-23T20:04:14.553663+0000","last_clean":"2023-09-23T20:04:14.553663+0000","last_became_active":"2023-09-23T20:04:02.277271+0000","last_became_peered":"2023-09-23T20:04:02.277271+0000","last_unstale":"2023-09-23T20:04:14.553663+0000","last_undegraded":"2023-09-23T20:04:14.553663+0000","last_fullsized":"2023-09-23T20:04:14.553663+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":358,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.557071+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:04:14.557071+0000","last_peered":"2023-09-23T20:04:14.557071+0000","last_clean":"2023-09-23T20:04:14.557071+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:04:14.557071+0000","last_undegraded":"2023-09-23T20:04:14.557071+0000","last_fullsized":"2023-09-23T20:04:14.557071+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.526648+0000","last_change":"2023-09-23T20:03:10.174842+0000","last_active":"2023-09-23T20:04:25.526648+0000","last_peered":"2023-09-23T20:04:25.526648+0000","last_clean":"2023-09-23T20:04:25.526648+0000","last_became_active":"2023-09-23T20:03:10.174511+0000","last_became_peered":"2023-09-23T20:03:10.174511+0000","last_unstale":"2023-09-23T20:04:25.526648+0000","last_undegraded":"2023-09-23T20:04:25.526648+0000","last_fullsized":"2023-09-23T20:04:25.526648+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:12.723811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":76,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.561652+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:04:14.561652+0000","last_peered":"2023-09-23T20:04:14.561652+0000","last_clean":"2023-09-23T20:04:14.561652+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:04:14.561652+0000","last_undegraded":"2023-09-23T20:04:14.561652+0000","last_fullsized":"2023-09-23T20:04:14.561652+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"280'8885","reported_seq":8935,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:32.067821+0000","last_change":"2023-09-23T20:01:39.912135+0000","last_active":"2023-09-23T20:04:32.067821+0000","last_peered":"2023-09-23T20:04:32.067821+0000","last_clean":"2023-09-23T20:04:32.067821+0000","last_became_active":"2023-09-23T20:01:39.911834+0000","last_became_peered":"2023-09-23T20:01:39.911834+0000","last_unstale":"2023-09-23T20:04:32.067821+0000","last_undegraded":"2023-09-23T20:04:32.067821+0000","last_fullsized":"2023-09-23T20:04:32.067821+0000","mapping_epoch":240,"log_start":"248'3126","ondisk_log_start":"248'3126","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:46.725578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219873280,"num_objects":3355,"num_object_clones":0,"num_object_copies":10065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8885,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":658,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756043+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:04:14.756043+0000","last_peered":"2023-09-23T20:04:14.756043+0000","last_clean":"2023-09-23T20:04:14.756043+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:04:14.756043+0000","last_undegraded":"2023-09-23T20:04:14.756043+0000","last_fullsized":"2023-09-23T20:04:14.756043+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":106,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554831+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:04:14.554831+0000","last_peered":"2023-09-23T20:04:14.554831+0000","last_clean":"2023-09-23T20:04:14.554831+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:04:14.554831+0000","last_undegraded":"2023-09-23T20:04:14.554831+0000","last_fullsized":"2023-09-23T20:04:14.554831+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":685,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.579516+0000","last_change":"2023-09-23T19:59:40.899753+0000","last_active":"2023-09-23T20:04:14.579516+0000","last_peered":"2023-09-23T20:04:14.579516+0000","last_clean":"2023-09-23T20:04:14.579516+0000","last_became_active":"2023-09-23T19:59:40.899147+0000","last_became_peered":"2023-09-23T19:59:40.899147+0000","last_unstale":"2023-09-23T20:04:14.579516+0000","last_undegraded":"2023-09-23T20:04:14.579516+0000","last_fullsized":"2023-09-23T20:04:14.579516+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:08.938755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554977+0000","last_change":"2023-09-23T20:03:59.233560+0000","last_active":"2023-09-23T20:04:14.554977+0000","last_peered":"2023-09-23T20:04:14.554977+0000","last_clean":"2023-09-23T20:04:14.554977+0000","last_became_active":"2023-09-23T20:03:59.232842+0000","last_became_peered":"2023-09-23T20:03:59.232842+0000","last_unstale":"2023-09-23T20:04:14.554977+0000","last_undegraded":"2023-09-23T20:04:14.554977+0000","last_fullsized":"2023-09-23T20:04:14.554977+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:16.909413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"280'8891","reported_seq":8941,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:31.947587+0000","last_change":"2023-09-23T20:01:39.913698+0000","last_active":"2023-09-23T20:04:31.947587+0000","last_peered":"2023-09-23T20:04:31.947587+0000","last_clean":"2023-09-23T20:04:31.947587+0000","last_became_active":"2023-09-23T20:01:39.913367+0000","last_became_peered":"2023-09-23T20:01:39.913367+0000","last_unstale":"2023-09-23T20:04:31.947587+0000","last_undegraded":"2023-09-23T20:04:31.947587+0000","last_fullsized":"2023-09-23T20:04:31.947587+0000","mapping_epoch":240,"log_start":"248'3225","ondisk_log_start":"248'3225","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219217920,"num_objects":3345,"num_object_clones":0,"num_object_copies":10035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8891,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":536,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554099+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:04:14.554099+0000","last_peered":"2023-09-23T20:04:14.554099+0000","last_clean":"2023-09-23T20:04:14.554099+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:04:14.554099+0000","last_undegraded":"2023-09-23T20:04:14.554099+0000","last_fullsized":"2023-09-23T20:04:14.554099+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":36,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756760+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:04:14.756760+0000","last_peered":"2023-09-23T20:04:14.756760+0000","last_clean":"2023-09-23T20:04:14.756760+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:04:14.756760+0000","last_undegraded":"2023-09-23T20:04:14.756760+0000","last_fullsized":"2023-09-23T20:04:14.756760+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":83,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.579561+0000","last_change":"2023-09-23T20:01:04.470744+0000","last_active":"2023-09-23T20:04:14.579561+0000","last_peered":"2023-09-23T20:04:14.579561+0000","last_clean":"2023-09-23T20:04:14.579561+0000","last_became_active":"2023-09-23T20:01:04.470302+0000","last_became_peered":"2023-09-23T20:01:04.470302+0000","last_unstale":"2023-09-23T20:04:14.579561+0000","last_undegraded":"2023-09-23T20:04:14.579561+0000","last_fullsized":"2023-09-23T20:04:14.579561+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:25.452596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":239,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756402+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:04:14.756402+0000","last_peered":"2023-09-23T20:04:14.756402+0000","last_clean":"2023-09-23T20:04:14.756402+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:04:14.756402+0000","last_undegraded":"2023-09-23T20:04:14.756402+0000","last_fullsized":"2023-09-23T20:04:14.756402+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":25,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554108+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:04:14.554108+0000","last_peered":"2023-09-23T20:04:14.554108+0000","last_clean":"2023-09-23T20:04:14.554108+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:04:14.554108+0000","last_undegraded":"2023-09-23T20:04:14.554108+0000","last_fullsized":"2023-09-23T20:04:14.554108+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":33,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554670+0000","last_change":"2023-09-23T20:03:10.178662+0000","last_active":"2023-09-23T20:04:14.554670+0000","last_peered":"2023-09-23T20:04:14.554670+0000","last_clean":"2023-09-23T20:04:14.554670+0000","last_became_active":"2023-09-23T20:03:10.178352+0000","last_became_peered":"2023-09-23T20:03:10.178352+0000","last_unstale":"2023-09-23T20:04:14.554670+0000","last_undegraded":"2023-09-23T20:04:14.554670+0000","last_fullsized":"2023-09-23T20:04:14.554670+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.469897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":241,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558812+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T20:04:14.558812+0000","last_peered":"2023-09-23T20:04:14.558812+0000","last_clean":"2023-09-23T20:04:14.558812+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T20:04:14.558812+0000","last_undegraded":"2023-09-23T20:04:14.558812+0000","last_fullsized":"2023-09-23T20:04:14.558812+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":276,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.759730+0000","last_change":"2023-09-23T19:59:09.172452+0000","last_active":"2023-09-23T20:04:14.759730+0000","last_peered":"2023-09-23T20:04:14.759730+0000","last_clean":"2023-09-23T20:04:14.759730+0000","last_became_active":"2023-09-23T19:59:09.172080+0000","last_became_peered":"2023-09-23T19:59:09.172080+0000","last_unstale":"2023-09-23T20:04:14.759730+0000","last_undegraded":"2023-09-23T20:04:14.759730+0000","last_fullsized":"2023-09-23T20:04:14.759730+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:24.687991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":245,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559021+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T20:04:14.559021+0000","last_peered":"2023-09-23T20:04:14.559021+0000","last_clean":"2023-09-23T20:04:14.559021+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T20:04:14.559021+0000","last_undegraded":"2023-09-23T20:04:14.559021+0000","last_fullsized":"2023-09-23T20:04:14.559021+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:07.560646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":276,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.755995+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:04:14.755995+0000","last_peered":"2023-09-23T20:04:14.755995+0000","last_clean":"2023-09-23T20:04:14.755995+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:04:14.755995+0000","last_undegraded":"2023-09-23T20:04:14.755995+0000","last_fullsized":"2023-09-23T20:04:14.755995+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":102,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554033+0000","last_change":"2023-09-23T19:59:09.145224+0000","last_active":"2023-09-23T20:04:14.554033+0000","last_peered":"2023-09-23T20:04:14.554033+0000","last_clean":"2023-09-23T20:04:14.554033+0000","last_became_active":"2023-09-23T19:59:09.144819+0000","last_became_peered":"2023-09-23T19:59:09.144819+0000","last_unstale":"2023-09-23T20:04:14.554033+0000","last_undegraded":"2023-09-23T20:04:14.554033+0000","last_fullsized":"2023-09-23T20:04:14.554033+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:04.048181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":17,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.551891+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:04:14.551891+0000","last_peered":"2023-09-23T20:04:14.551891+0000","last_clean":"2023-09-23T20:04:14.551891+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:04:14.551891+0000","last_undegraded":"2023-09-23T20:04:14.551891+0000","last_fullsized":"2023-09-23T20:04:14.551891+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":99,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559611+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:04:14.559611+0000","last_peered":"2023-09-23T20:04:14.559611+0000","last_clean":"2023-09-23T20:04:14.559611+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:04:14.559611+0000","last_undegraded":"2023-09-23T20:04:14.559611+0000","last_fullsized":"2023-09-23T20:04:14.559611+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":306,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.563830+0000","last_change":"2023-09-23T20:03:59.235232+0000","last_active":"2023-09-23T20:04:14.563830+0000","last_peered":"2023-09-23T20:04:14.563830+0000","last_clean":"2023-09-23T20:04:14.563830+0000","last_became_active":"2023-09-23T20:03:59.234941+0000","last_became_peered":"2023-09-23T20:03:59.234941+0000","last_unstale":"2023-09-23T20:04:14.563830+0000","last_undegraded":"2023-09-23T20:04:14.563830+0000","last_fullsized":"2023-09-23T20:04:14.563830+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:45.591449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3573743640,"num_objects":54532,"num_object_clones":0,"num_object_copies":163596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11328,"num_objects_unfound":0,"num_objects_dirty":54532,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139588,"num_write_kb":6211777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7991,"num_bytes_recovered":523632640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11138904064,"data_stored":11138891848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109356,"ondisk_log_size":109356,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234280,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1612292,"kb_used_data":1354940,"kb_used_omap":0,"kb_used_meta":257344,"kb_avail":92132860,"statfs":{"total":95995035648,"available":94344048640,"internally_reserved":0,"allocated":1387458560,"data_stored":1386608176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":263520256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:20 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5750000000000002,"15min":2.0430000000000001},"min":{"1min":0.245,"5min":0.22,"15min":0.19700000000000001},"max":{"1min":41.889000000000003,"5min":60.140000000000001,"15min":62.061999999999998},"last":0.23200000000000001},{"interface":"front","average":{"1min":4.5209999999999999,"5min":2.7120000000000002,"15min":2.27},"min":{"1min":0.25900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":41.936,"5min":62.389000000000003,"15min":62.389000000000003},"last":10.68}]},{"osd":1,"last update":"Sat Sep 23 20:04:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2240000000000002,"5min":5.7729999999999997,"15min":3.0739999999999998},"min":{"1min":0.27100000000000002,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":56.024999999999999,"5min":104.16800000000001,"15min":104.16800000000001},"last":3.601},{"interface":"front","average":{"1min":6.3220000000000001,"5min":6.0590000000000002,"15min":3.2149999999999999},"min":{"1min":0.34200000000000003,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":56.146000000000001,"5min":104.003,"15min":104.003},"last":1.8420000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:03:36 2023","interfaces":[{"interface":"back","average":{"1min":7.5460000000000003,"5min":3.016,"15min":3.0070000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":105.842,"5min":105.842,"15min":130.673},"last":8.0079999999999991},{"interface":"front","average":{"1min":7.0190000000000001,"5min":3.4649999999999999,"15min":3.1930000000000001},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":105.66,"5min":105.66,"15min":130.71100000000001},"last":1.365}]},{"osd":3,"last update":"Sat Sep 23 20:04:25 2023","interfaces":[{"interface":"back","average":{"1min":10.352,"5min":5.5999999999999996,"15min":3.4630000000000001},"min":{"1min":0.316,"5min":0.247,"15min":0.247},"max":{"1min":80.257999999999996,"5min":146.38800000000001,"15min":146.38800000000001},"last":1.236},{"interface":"front","average":{"1min":10.545999999999999,"5min":5.8099999999999996,"15min":3.464},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":82.281999999999996,"5min":146.851,"15min":146.851},"last":1.704}]},{"osd":4,"last update":"Sat Sep 23 20:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.878,"15min":1.885},"min":{"1min":0.28299999999999997,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.3109999999999999,"5min":19.215,"15min":143.255},"last":0.246},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.84499999999999997,"15min":1.7969999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.0419999999999998,"5min":19.364000000000001,"15min":143.41200000000001},"last":0.29599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.1819999999999999,"15min":1.444},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":2.9380000000000002,"5min":35.488,"15min":84.566999999999993},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.085,"15min":1.3089999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.343,"5min":34.896000000000001,"15min":81.244},"last":0.28599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9820000000000002,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.317}]}]},{"osd":6,"up_from":0,"seq":1078036791321,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762482,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2152564,"kb_used_data":1767980,"kb_used_omap":7,"kb_used_meta":384568,"kb_avail":91592588,"statfs":{"total":95995035648,"available":93790810112,"internally_reserved":0,"allocated":1810411520,"data_stored":1809167468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":393798458},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":2.2240000000000002,"15min":3.8799999999999999},"min":{"1min":0.247,"5min":0.16900000000000001,"15min":0.151},"max":{"1min":11.710000000000001,"5min":25.238,"15min":202.917},"last":16.843},{"interface":"front","average":{"1min":3.8780000000000001,"5min":2.4460000000000002,"15min":4.0119999999999996},"min":{"1min":0.23200000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":14.839,"5min":22.337,"15min":203.34899999999999},"last":13.4}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":5.0609999999999999,"15min":5.2329999999999997},"min":{"1min":0.315,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":11.917999999999999,"5min":134.72499999999999,"15min":231.57499999999999},"last":10.112},{"interface":"front","average":{"1min":3.8370000000000002,"5min":5.4279999999999999,"15min":5.2359999999999998},"min":{"1min":0.29399999999999998,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":20.677,"5min":135.74600000000001,"15min":236.077},"last":7.6200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.665,"5min":2.7349999999999999,"15min":3.016},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.217},"max":{"1min":11.98,"5min":15.571,"15min":68.180999999999997},"last":10.215},{"interface":"front","average":{"1min":5.2850000000000001,"5min":2.7749999999999999,"15min":2.8439999999999999},"min":{"1min":0.36799999999999999,"5min":0.249,"15min":0.20100000000000001},"max":{"1min":20.989999999999998,"5min":22.385999999999999,"15min":66.509},"last":13.329000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.9450000000000001,"15min":2.4159999999999999},"min":{"1min":0.33800000000000002,"5min":0.254,"15min":0.25},"max":{"1min":9.9190000000000005,"5min":35.497999999999998,"15min":161.68299999999999},"last":7.5259999999999998},{"interface":"front","average":{"1min":3.093,"5min":2.226,"15min":2.4470000000000001},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":20.847000000000001,"5min":35.521000000000001,"15min":161.286},"last":13.236000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":2.407,"15min":3.3610000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":20.960000000000001,"5min":51.698,"15min":219.048},"last":13.278},{"interface":"front","average":{"1min":3.6739999999999999,"5min":2.6459999999999999,"15min":3.5510000000000002},"min":{"1min":0.40400000000000003,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":15.507999999999999,"5min":51.173000000000002,"15min":218.69499999999999},"last":10.189}]},{"osd":6,"last update":"Sat Sep 23 20:03:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.2469999999999999,"15min":2.2469999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":9.8879999999999999,"5min":9.8879999999999999,"15min":9.8879999999999999},"last":7.5579999999999998},{"interface":"front","average":{"1min":3.4460000000000002,"5min":3.4460000000000002,"15min":3.4460000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":20.734999999999999,"5min":20.734999999999999,"15min":20.734999999999999},"last":13.304}]},{"osd":7,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":2.1869999999999998,"15min":2.1739999999999999},"min":{"1min":0.432,"5min":0.36199999999999999,"15min":0.26800000000000002},"max":{"1min":11.893000000000001,"5min":14.23,"15min":60.213000000000001},"last":10.162000000000001},{"interface":"front","average":{"1min":2.048,"5min":1.746,"15min":1.7529999999999999},"min":{"1min":0.42599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.8309999999999995,"5min":14.188000000000001,"15min":61.298999999999999},"last":7.5810000000000004}]}]},{"osd":0,"up_from":94,"seq":403726926017,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2574320,"kb_used_data":2146856,"kb_used_omap":8,"kb_used_meta":427447,"kb_avail":91170832,"statfs":{"total":95995035648,"available":93358931968,"internally_reserved":0,"allocated":2198380544,"data_stored":2197139882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":437706337},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6819999999999999,"5min":5.0289999999999999,"15min":2.6389999999999998},"min":{"1min":0.185,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":20.396000000000001,"5min":94.055999999999997,"15min":94.055999999999997},"last":4.0279999999999996},{"interface":"front","average":{"1min":3.964,"5min":5.1909999999999998,"15min":2.972},"min":{"1min":0.29099999999999998,"5min":0.183,"15min":0.183},"max":{"1min":22.056999999999999,"5min":91.632999999999996,"15min":91.632999999999996},"last":5.5990000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":5.9409999999999998,"15min":3.5910000000000002},"min":{"1min":0.23300000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.75,"5min":270.51100000000002,"15min":270.51100000000002},"last":0.68200000000000005},{"interface":"front","average":{"1min":2.8109999999999999,"5min":5.5460000000000003,"15min":3.6419999999999999},"min":{"1min":0.25700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.038,"5min":269.69299999999998,"15min":269.69299999999998},"last":0.71099999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":6.2779999999999996,"5min":5.9699999999999998,"15min":4.4119999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":48.588000000000001,"5min":133.422,"15min":145.72800000000001},"last":5.6989999999999998},{"interface":"front","average":{"1min":6.4850000000000003,"5min":6.0259999999999998,"15min":4.5570000000000004},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":48.619999999999997,"5min":134.249,"15min":145.50700000000001},"last":5.6779999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.7430000000000001,"15min":2.3159999999999998},"min":{"1min":0.27600000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.3829999999999991,"5min":12.372,"15min":110.312},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.27,"5min":1.659,"15min":1.964},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27600000000000002},"max":{"1min":8.7330000000000005,"5min":8.7330000000000005,"15min":110.256},"last":5.4690000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":2.181,"15min":2.4929999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.5419999999999998,"5min":64.725999999999999,"15min":131.34399999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.8380000000000001,"15min":2.569},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.4139999999999997,"5min":12.48,"15min":131.386},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.607,"15min":1.607},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.4290000000000003,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.9059999999999999,"5min":1.9059999999999999,"15min":1.9059999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.0629999999999997,"5min":5.0629999999999997,"15min":5.0629999999999997},"last":5.5110000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":2.2189999999999999,"15min":2.004},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":8.6039999999999992,"5min":65.081999999999994,"15min":77.132999999999996},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.8400000000000001,"5min":1.9419999999999999,"15min":2.0990000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.3900000000000006,"5min":12.452,"15min":75.465999999999994},"last":0.59999999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598955,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2269532,"kb_used_data":1780820,"kb_used_omap":12,"kb_used_meta":488563,"kb_avail":91475620,"statfs":{"total":95995035648,"available":93671034880,"internally_reserved":0,"allocated":1823559680,"data_stored":1822340796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":500289534},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":3.6309999999999998,"15min":2.5019999999999998},"min":{"1min":0.20899999999999999,"5min":0.192,"15min":0.154},"max":{"1min":10.073,"5min":48.966999999999999,"15min":50.506999999999998},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.4690000000000001,"5min":3.2509999999999999,"15min":2.3969999999999998},"min":{"1min":0.20200000000000001,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":7.8840000000000003,"5min":48.783000000000001,"15min":50.802999999999997},"last":3.8740000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":6.4400000000000004,"5min":5.4279999999999999,"15min":2.9529999999999998},"min":{"1min":0.247,"5min":0.21299999999999999,"15min":0.184},"max":{"1min":50.192999999999998,"5min":94.768000000000001,"15min":94.768000000000001},"last":3.1840000000000002},{"interface":"front","average":{"1min":6.7729999999999997,"5min":6.0060000000000002,"15min":3.1890000000000001},"min":{"1min":0.218,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":47.987000000000002,"5min":96.808000000000007,"15min":96.808000000000007},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:04:22 2023","interfaces":[{"interface":"back","average":{"1min":4.117,"5min":3.8380000000000001,"15min":2.7970000000000002},"min":{"1min":0.307,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":13.984,"5min":44.408000000000001,"15min":50.186999999999998},"last":3.6190000000000002},{"interface":"front","average":{"1min":2.2709999999999999,"5min":3.3290000000000002,"15min":2.7229999999999999},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.084,"5min":43.981999999999999,"15min":50.585000000000001},"last":0.64400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.893,"15min":1.6890000000000001},"min":{"1min":0.30499999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.9249999999999998,"5min":20.591999999999999,"15min":50.640000000000001},"last":8.2240000000000002},{"interface":"front","average":{"1min":2.3039999999999998,"5min":1.819,"15min":1.6990000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7629999999999999,"5min":23.062000000000001,"15min":50.154000000000003},"last":3.246}]},{"osd":5,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":10.351000000000001,"5min":3.9199999999999999,"15min":2.5009999999999999},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":172.505,"5min":172.505,"15min":172.505},"last":2.1440000000000001},{"interface":"front","average":{"1min":10.007,"5min":3.5630000000000002,"15min":2.347},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":172.375,"5min":172.375,"15min":172.375},"last":3.9009999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":1.6259999999999999,"15min":1.6259999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.144,"5min":10.144,"15min":10.144},"last":0.622},{"interface":"front","average":{"1min":1.9690000000000001,"5min":1.9690000000000001,"15min":1.9690000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":6.3520000000000003,"5min":6.3520000000000003,"15min":6.3520000000000003},"last":3.9180000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":1.9530000000000001,"15min":1.796},"min":{"1min":0.436,"5min":0.33700000000000002,"15min":0.27400000000000002},"max":{"1min":9.7330000000000005,"5min":23.079999999999998,"15min":50.118000000000002},"last":3.2200000000000002},{"interface":"front","average":{"1min":1.734,"5min":2.395,"15min":2.0680000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":10.17,"5min":20.614999999999998,"15min":50.664999999999999},"last":1.357}]}]},{"osd":3,"up_from":109,"seq":468151435429,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1882844,"kb_used_data":1433748,"kb_used_omap":4,"kb_used_meta":449019,"kb_avail":91862308,"statfs":{"total":95995035648,"available":94067003392,"internally_reserved":0,"allocated":1468157952,"data_stored":1466816162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":459795565},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0600000000000001,"5min":2.399,"15min":3.8929999999999998},"min":{"1min":0.193,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":10.401,"5min":21.100999999999999,"15min":240.40600000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":4.1799999999999997,"5min":3.327,"15min":4.4989999999999997},"min":{"1min":0.23100000000000001,"5min":0.19600000000000001,"15min":0.17199999999999999},"max":{"1min":16.978000000000002,"5min":53.201000000000001,"15min":240.352},"last":3.1429999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.4569999999999999,"5min":4.1920000000000002,"15min":2.2799999999999998},"min":{"1min":0.23000000000000001,"5min":0.217,"15min":0.20000000000000001},"max":{"1min":86.337999999999994,"5min":86.337999999999994,"15min":86.337999999999994},"last":1.579},{"interface":"front","average":{"1min":9.1300000000000008,"5min":3.7149999999999999,"15min":2.2410000000000001},"min":{"1min":0.249,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":84.299999999999997,"5min":84.299999999999997,"15min":84.299999999999997},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.2539999999999996,"5min":4.0030000000000001,"15min":5.3659999999999997},"min":{"1min":0.25800000000000001,"5min":0.255,"15min":0.219},"max":{"1min":82.111999999999995,"5min":82.111999999999995,"15min":264.97800000000001},"last":1.3280000000000001},{"interface":"front","average":{"1min":3.5790000000000002,"5min":2.7000000000000002,"15min":4.9269999999999996},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":17.657,"5min":30.768999999999998,"15min":257.40899999999999},"last":3.1760000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":13.324999999999999,"5min":4.5519999999999996,"15min":2.7669999999999999},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":109.66800000000001,"5min":109.66800000000001,"15min":109.66800000000001},"last":0.499},{"interface":"front","average":{"1min":13.428000000000001,"5min":5.0140000000000002,"15min":3.2130000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":110.295,"5min":110.295,"15min":110.295},"last":2.194}]},{"osd":5,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.0170000000000003,"5min":4.3049999999999997,"15min":2.5310000000000001},"min":{"1min":0.33400000000000002,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":82.138000000000005,"5min":113.881,"15min":113.881},"last":2.2309999999999999},{"interface":"front","average":{"1min":6.6020000000000003,"5min":4.3559999999999999,"15min":2.4340000000000002},"min":{"1min":0.43099999999999999,"5min":0.313,"15min":0.31},"max":{"1min":82.25,"5min":114.598,"15min":114.598},"last":1.0900000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":5.7519999999999998,"5min":5.7519999999999998,"15min":5.7519999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":82.281999999999996,"5min":82.281999999999996,"15min":82.281999999999996},"last":1.1240000000000001},{"interface":"front","average":{"1min":5.75,"5min":5.75,"15min":5.75},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":82.233000000000004,"5min":82.233000000000004,"15min":82.233000000000004},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":13.244,"5min":4.3380000000000001,"15min":2.9729999999999999},"min":{"1min":0.45200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":112.883,"5min":112.883,"15min":138.035},"last":1.1579999999999999},{"interface":"front","average":{"1min":13.119999999999999,"5min":3.8069999999999999,"15min":2.7440000000000002},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":111.85599999999999,"5min":111.85599999999999,"15min":138.054},"last":0.52700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019190,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1616064,"kb_used_data":1357624,"kb_used_omap":0,"kb_used_meta":258368,"kb_avail":92129088,"statfs":{"total":95995035648,"available":94340186112,"internally_reserved":0,"allocated":1390206976,"data_stored":1389360676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":264568832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":2.9470000000000001,"15min":2.9249999999999998},"min":{"1min":0.251,"5min":0.189,"15min":0.189},"max":{"1min":5.508,"5min":89.004999999999995,"15min":104.604},"last":10.063000000000001},{"interface":"front","average":{"1min":0.86899999999999999,"5min":2.8570000000000002,"15min":2.883},"min":{"1min":0.29499999999999998,"5min":0.249,"15min":0.23400000000000001},"max":{"1min":5.4509999999999996,"5min":89.024000000000001,"15min":104.687},"last":10.045999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":31.100000000000001,"5min":10.585000000000001,"15min":4.8559999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.216},"max":{"1min":161.613,"5min":161.613,"15min":161.613},"last":5.274},{"interface":"front","average":{"1min":31.327999999999999,"5min":10.795,"15min":4.7619999999999996},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":162.017,"5min":162.017,"15min":162.017},"last":5.9089999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":14.250999999999999,"5min":9.3689999999999998,"15min":5.0190000000000001},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":109.38200000000001,"5min":144.20099999999999,"15min":144.20099999999999},"last":4.6799999999999997},{"interface":"front","average":{"1min":14.645,"5min":8.7840000000000007,"15min":4.7690000000000001},"min":{"1min":0.34200000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":107.74299999999999,"5min":144.184,"15min":144.184},"last":2.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":14.276999999999999,"5min":7.5449999999999999,"15min":5.2300000000000004},"min":{"1min":0.376,"5min":0.255,"15min":0.255},"max":{"1min":99.930999999999997,"5min":139.976,"15min":266.54700000000003},"last":3.0459999999999998},{"interface":"front","average":{"1min":13.647,"5min":7.1440000000000001,"15min":4.1980000000000004},"min":{"1min":0.39300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":100.01600000000001,"5min":139.911,"15min":139.911},"last":6.7779999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.8640000000000001},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":4.077,"5min":5.0650000000000004,"15min":266.59899999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.60799999999999998,"15min":1.9570000000000001},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.395,"5min":4.8739999999999997,"15min":266.72899999999998},"last":3.004}]},{"osd":6,"last update":"Sat Sep 23 20:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.7,"5min":1.7,"15min":1.7},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":3.0779999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:03:47 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.81399999999999995,"15min":0.97499999999999998},"min":{"1min":0.318,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.276,"5min":7.6210000000000004,"15min":11.59},"last":3.109},{"interface":"front","average":{"1min":0.503,"5min":0.95099999999999996,"15min":0.999},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":1.3240000000000001,"5min":18.012,"15min":18.012},"last":4.7089999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757554,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1419384,"kb_used_data":1129584,"kb_used_omap":0,"kb_used_meta":289792,"kb_avail":92325768,"statfs":{"total":95995035648,"available":94541586432,"internally_reserved":0,"allocated":1156694016,"data_stored":1155852157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":296747008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.94499999999999995,"15min":1.27},"min":{"1min":0.22900000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":2.8490000000000002,"5min":9.1699999999999999,"15min":24.852},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.7829999999999999,"5min":1.3149999999999999,"15min":1.353},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.216},"max":{"1min":7.4100000000000001,"5min":9.0530000000000008,"15min":24.638999999999999},"last":2.8239999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":12.311,"5min":9.4450000000000003,"15min":4.9180000000000001},"min":{"1min":0.33100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":171.47300000000001,"5min":266.197,"15min":266.197},"last":4.2770000000000001},{"interface":"front","average":{"1min":12.787000000000001,"5min":9.6140000000000008,"15min":4.9509999999999996},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":171.399,"5min":266.14999999999998,"15min":266.14999999999998},"last":5.375}]},{"osd":2,"last update":"Sat Sep 23 20:03:49 2023","interfaces":[{"interface":"back","average":{"1min":10.763,"5min":6.2130000000000001,"15min":5.5830000000000002},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":177.816,"5min":238.96199999999999,"15min":238.96199999999999},"last":0.876},{"interface":"front","average":{"1min":10.779999999999999,"5min":6.2439999999999998,"15min":5.6189999999999998},"min":{"1min":0.33800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":177.851,"5min":238.905,"15min":238.905},"last":0.90500000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":12.673,"5min":4.883,"15min":3.5150000000000001},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":168.31399999999999,"5min":168.31399999999999,"15min":168.31399999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":10.301,"5min":4.806,"15min":3.3679999999999999},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":168.39500000000001,"5min":168.39500000000001,"15min":168.39500000000001},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.476,"15min":1.609},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.9129999999999998,"5min":191.78899999999999,"15min":191.78899999999999},"last":2.786},{"interface":"front","average":{"1min":0.59799999999999998,"5min":2.4849999999999999,"15min":1.5960000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.177,"5min":191.821,"15min":191.821},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4420000000000002,"5min":2.4420000000000002,"15min":2.4420000000000002},"last":2.8769999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.63300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":3.2530000000000001,"15min":1.7330000000000001},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.996,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":3.379,"15min":1.9239999999999999},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.8169999999999999,"5min":191.69,"15min":191.69},"last":2.851}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2182090752,"data_stored":2182086680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1789726720,"data_stored":1789722648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1816854528,"data_stored":1816854528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1463160832,"data_stored":1463156760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1379467264,"data_stored":1379467264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1139736576,"data_stored":1139736576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1367867392,"data_stored":1367867392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:04:35.900 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:35.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:35.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:36.010 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:36.039 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:36.095 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:36.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:36.202 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:36.231 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:36.283 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:36.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:36.393 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:36.475 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:36.782 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:36.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:36.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:36.895 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:36.934 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:37.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:36 smithi144 ceph-mon[194833]: pgmap v763: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10951/158238 objects misplaced (6.921%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:37.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:36 smithi135 ceph-mon[221113]: pgmap v763: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10951/158238 objects misplaced (6.921%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:37.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:36 smithi135 ceph-mon[213871]: pgmap v763: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 10951/158238 objects misplaced (6.921%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:37.620 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:37.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:37.727 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:37.756 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:37.808 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:37.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:37.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:37.915 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:37.938 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:04:37.938 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:04:37.938 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:144183 7.9 e280/240 7:931cecb3:::benchmark_data_smithi135_404353_object47121:head v 280'8978, mlcod=280'8976)", 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:04:37.933472+0000", 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0051648550000000003, 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051941879999999998, 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:37.933472+0000", 2023-09-23T20:04:37.939 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:37.933472+0000", 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:37.933472+0000", 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:04:37.940 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:37.933477+0000", 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9930000000000003e-06 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:37.933485+0000", 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1820000000000006e-06 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:04:37.941 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:37.933490+0000", 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4489999999999998e-06 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:37.937468+0000", 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0039778310000000002 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:37.937491+0000", 2023-09-23T20:04:37.942 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3249000000000001e-05 2023-09-23T20:04:37.943 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:04:37.943 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:04:37.943 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:04:37.943 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:04:37.943 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:04:37.943 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 1 2023-09-23T20:04:37.943 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:04:37.943 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:37.997 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:38.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:38.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:38.103 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:38.130 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:38.188 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:38.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:38.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:38.229 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:38.229 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:38.239 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:38.240 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:38.275 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:04:38.293 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:38.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:38.894 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:04:39.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:39 smithi135 ceph-mon[213871]: pgmap v764: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 9997/150882 objects misplaced (6.626%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:04:39.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:39 smithi135 ceph-mon[221113]: pgmap v764: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 9997/150882 objects misplaced (6.626%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:04:39.435 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:39.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:39.543 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:04:39.544 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:39 smithi144 ceph-mon[194833]: pgmap v764: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 9997/150882 objects misplaced (6.626%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:04:39.657 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:39.837 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:39.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:39.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:39.893 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:39.919 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:39.945 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:40.248 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:40 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2202500449' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:04:40.248 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:04:40.278 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:40.324 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:40.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:40.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:40.432 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:40.446 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:40 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2202500449' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:04:40.513 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:40 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2202500449' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:04:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[213871]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[213871]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[213871]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2202500449' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:04:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[213871]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T20:04:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[213871]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[213871]: pgmap v766: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9997/149232 objects misplaced (6.699%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:04:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[221113]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[221113]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[221113]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2202500449' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:04:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[221113]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T20:04:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[221113]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:41 smithi135 ceph-mon[221113]: pgmap v766: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9997/149232 objects misplaced (6.699%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:04:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:41 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:41 smithi144 ceph-mon[194833]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:41 smithi144 ceph-mon[194833]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:41 smithi144 ceph-mon[194833]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2202500449' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:04:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:41 smithi144 ceph-mon[194833]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T20:04:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:41 smithi144 ceph-mon[194833]: failed to encode map e281 with expected crc 2023-09-23T20:04:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:41 smithi144 ceph-mon[194833]: pgmap v766: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9997/149232 objects misplaced (6.699%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:04:42.676 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:42.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:42.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:42.793 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:42.820 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:42.869 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:42.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:42.965 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:42.977 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:43.065 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:43.419 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:43.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:43.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:43.534 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:43.561 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:43.602 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:43.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:43.709 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:43.734 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:43.776 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:43.780 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 20:04:43 smithi144 conmon[173765]: logger=cleanup t=2023-09-23T20:04:43.49217335Z level=info msg="Completed cleanup jobs" duration=64.714648ms 2023-09-23T20:04:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:43.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:43.881 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:43.910 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:43.967 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:04:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:04:43] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-09-23T20:04:43.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:43.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:44.080 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:44.152 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:44.436 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:04:44.463 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:44.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:44.568 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:44.593 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:44.644 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:44.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:44.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:44.671 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:44 smithi135 ceph-mon[213871]: pgmap v767: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 9447/145926 objects misplaced (6.474%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:44 smithi135 ceph-mon[221113]: pgmap v767: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 9447/145926 objects misplaced (6.474%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:44.742 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:04:44.749 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:44.773 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:04:44.833 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:44.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:44 smithi144 ceph-mon[194833]: pgmap v767: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 9447/145926 objects misplaced (6.474%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:45.720 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:45.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:45.778 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:45 smithi135 ceph-mon[213871]: from='client.55252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:45.778 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:45 smithi135 ceph-mon[221113]: from='client.55252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:45.829 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:45.857 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:45.857 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:45 smithi144 ceph-mon[194833]: from='client.55252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:04:45.907 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:45.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:45.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:46.012 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:46.043 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:46.091 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:46.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:46.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:46.199 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:46.257 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:46.548 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:46.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:46.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:46.596 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":767,"stamp":"2023-09-23T20:04:42.737012+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3190691787,"num_objects":48642,"num_object_clones":0,"num_object_copies":145926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9447,"num_objects_unfound":0,"num_objects_dirty":48642,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":146413,"num_write_kb":6224502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8775,"num_bytes_recovered":567941621,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112477,"ondisk_log_size":112477,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12412208,"kb_used_data":9831672,"kb_used_omap":32,"kb_used_meta":2579551,"kb_avail":643803856,"statfs":{"total":671965249536,"available":659255148544,"internally_reserved":0,"allocated":10067632128,"data_stored":10060040205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":2641460742},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,8,4,6,8,2,1,2,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-387842048,"num_objects":-5918,"num_object_clones":0,"num_object_copies":-17754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1881,"num_objects_unfound":0,"num_objects_dirty":-5918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5918,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":668,"num_bytes_recovered":43778048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.022371"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":258,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756087+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T20:04:14.756087+0000","last_peered":"2023-09-23T20:04:14.756087+0000","last_clean":"2023-09-23T20:04:14.756087+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T20:04:14.756087+0000","last_undegraded":"2023-09-23T20:04:14.756087+0000","last_fullsized":"2023-09-23T20:04:14.756087+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.820101+0000","last_change":"2023-09-23T19:59:09.151117+0000","last_active":"2023-09-23T20:04:25.820101+0000","last_peered":"2023-09-23T20:04:25.820101+0000","last_clean":"2023-09-23T20:04:25.820101+0000","last_became_active":"2023-09-23T19:59:09.150795+0000","last_became_peered":"2023-09-23T19:59:09.150795+0000","last_unstale":"2023-09-23T20:04:25.820101+0000","last_undegraded":"2023-09-23T20:04:25.820101+0000","last_fullsized":"2023-09-23T20:04:25.820101+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:34.994574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":242,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.841466+0000","last_change":"2023-09-23T19:59:10.156070+0000","last_active":"2023-09-23T20:04:25.841466+0000","last_peered":"2023-09-23T20:04:25.841466+0000","last_clean":"2023-09-23T20:04:25.841466+0000","last_became_active":"2023-09-23T19:59:10.155813+0000","last_became_peered":"2023-09-23T19:59:10.155813+0000","last_unstale":"2023-09-23T20:04:25.841466+0000","last_undegraded":"2023-09-23T20:04:25.841466+0000","last_fullsized":"2023-09-23T20:04:25.841466+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:42.582380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":32,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.557396+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:04:14.557396+0000","last_peered":"2023-09-23T20:04:14.557396+0000","last_clean":"2023-09-23T20:04:14.557396+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:04:14.557396+0000","last_undegraded":"2023-09-23T20:04:14.557396+0000","last_fullsized":"2023-09-23T20:04:14.557396+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.818949+0000","last_change":"2023-09-23T19:59:09.163182+0000","last_active":"2023-09-23T20:04:25.818949+0000","last_peered":"2023-09-23T20:04:25.818949+0000","last_clean":"2023-09-23T20:04:25.818949+0000","last_became_active":"2023-09-23T19:59:09.161612+0000","last_became_peered":"2023-09-23T19:59:09.161612+0000","last_unstale":"2023-09-23T20:04:25.818949+0000","last_undegraded":"2023-09-23T20:04:25.818949+0000","last_fullsized":"2023-09-23T20:04:25.818949+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:59.524152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":205,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559826+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T20:04:14.559826+0000","last_peered":"2023-09-23T20:04:14.559826+0000","last_clean":"2023-09-23T20:04:14.559826+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T20:04:14.559826+0000","last_undegraded":"2023-09-23T20:04:14.559826+0000","last_fullsized":"2023-09-23T20:04:14.559826+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:59.043547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":206,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559702+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:04:14.559702+0000","last_peered":"2023-09-23T20:04:14.559702+0000","last_clean":"2023-09-23T20:04:14.559702+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:04:14.559702+0000","last_undegraded":"2023-09-23T20:04:14.559702+0000","last_fullsized":"2023-09-23T20:04:14.559702+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.819023+0000","last_change":"2023-09-23T19:59:09.146009+0000","last_active":"2023-09-23T20:04:25.819023+0000","last_peered":"2023-09-23T20:04:25.819023+0000","last_clean":"2023-09-23T20:04:25.819023+0000","last_became_active":"2023-09-23T19:59:09.145687+0000","last_became_peered":"2023-09-23T19:59:09.145687+0000","last_unstale":"2023-09-23T20:04:25.819023+0000","last_undegraded":"2023-09-23T20:04:25.819023+0000","last_fullsized":"2023-09-23T20:04:25.819023+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.189450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":239,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559217+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T20:04:14.559217+0000","last_peered":"2023-09-23T20:04:14.559217+0000","last_clean":"2023-09-23T20:04:14.559217+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T20:04:14.559217+0000","last_undegraded":"2023-09-23T20:04:14.559217+0000","last_fullsized":"2023-09-23T20:04:14.559217+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":122,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.819030+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:04:25.819030+0000","last_peered":"2023-09-23T20:04:25.819030+0000","last_clean":"2023-09-23T20:04:25.819030+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:04:25.819030+0000","last_undegraded":"2023-09-23T20:04:25.819030+0000","last_fullsized":"2023-09-23T20:04:25.819030+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":275,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.557138+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:04:14.557138+0000","last_peered":"2023-09-23T20:04:14.557138+0000","last_clean":"2023-09-23T20:04:14.557138+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:04:14.557138+0000","last_undegraded":"2023-09-23T20:04:14.557138+0000","last_fullsized":"2023-09-23T20:04:14.557138+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":238,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839673+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T20:04:25.839673+0000","last_peered":"2023-09-23T20:04:25.839673+0000","last_clean":"2023-09-23T20:04:25.839673+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T20:04:25.839673+0000","last_undegraded":"2023-09-23T20:04:25.839673+0000","last_fullsized":"2023-09-23T20:04:25.839673+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:51.825640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":294,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756257+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:04:14.756257+0000","last_peered":"2023-09-23T20:04:14.756257+0000","last_clean":"2023-09-23T20:04:14.756257+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:04:14.756257+0000","last_undegraded":"2023-09-23T20:04:14.756257+0000","last_fullsized":"2023-09-23T20:04:14.756257+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":256,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.759542+0000","last_change":"2023-09-23T20:01:52.439375+0000","last_active":"2023-09-23T20:04:14.759542+0000","last_peered":"2023-09-23T20:04:14.759542+0000","last_clean":"2023-09-23T20:04:14.759542+0000","last_became_active":"2023-09-23T20:01:52.439074+0000","last_became_peered":"2023-09-23T20:01:52.439074+0000","last_unstale":"2023-09-23T20:04:14.759542+0000","last_undegraded":"2023-09-23T20:04:14.759542+0000","last_fullsized":"2023-09-23T20:04:14.759542+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:31.103633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556959+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:04:14.556959+0000","last_peered":"2023-09-23T20:04:14.556959+0000","last_clean":"2023-09-23T20:04:14.556959+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:04:14.556959+0000","last_undegraded":"2023-09-23T20:04:14.556959+0000","last_fullsized":"2023-09-23T20:04:14.556959+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.819686+0000","last_change":"2023-09-23T19:59:09.195560+0000","last_active":"2023-09-23T20:04:25.819686+0000","last_peered":"2023-09-23T20:04:25.819686+0000","last_clean":"2023-09-23T20:04:25.819686+0000","last_became_active":"2023-09-23T19:59:09.195145+0000","last_became_peered":"2023-09-23T19:59:09.195145+0000","last_unstale":"2023-09-23T20:04:25.819686+0000","last_undegraded":"2023-09-23T20:04:25.819686+0000","last_fullsized":"2023-09-23T20:04:25.819686+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.190798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":245,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556892+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:04:14.556892+0000","last_peered":"2023-09-23T20:04:14.556892+0000","last_clean":"2023-09-23T20:04:14.556892+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:04:14.556892+0000","last_undegraded":"2023-09-23T20:04:14.556892+0000","last_fullsized":"2023-09-23T20:04:14.556892+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":258,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559133+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:04:14.559133+0000","last_peered":"2023-09-23T20:04:14.559133+0000","last_clean":"2023-09-23T20:04:14.559133+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:04:14.559133+0000","last_undegraded":"2023-09-23T20:04:14.559133+0000","last_fullsized":"2023-09-23T20:04:14.559133+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556996+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:04:14.556996+0000","last_peered":"2023-09-23T20:04:14.556996+0000","last_clean":"2023-09-23T20:04:14.556996+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:04:14.556996+0000","last_undegraded":"2023-09-23T20:04:14.556996+0000","last_fullsized":"2023-09-23T20:04:14.556996+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":246,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.822136+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:04:25.822136+0000","last_peered":"2023-09-23T20:04:25.822136+0000","last_clean":"2023-09-23T20:04:25.822136+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:04:25.822136+0000","last_undegraded":"2023-09-23T20:04:25.822136+0000","last_fullsized":"2023-09-23T20:04:25.822136+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":73,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558816+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:04:14.558816+0000","last_peered":"2023-09-23T20:04:14.558816+0000","last_clean":"2023-09-23T20:04:14.558816+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:04:14.558816+0000","last_undegraded":"2023-09-23T20:04:14.558816+0000","last_fullsized":"2023-09-23T20:04:14.558816+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":18,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.821411+0000","last_change":"2023-09-23T20:03:59.232022+0000","last_active":"2023-09-23T20:04:25.821411+0000","last_peered":"2023-09-23T20:04:25.821411+0000","last_clean":"2023-09-23T20:04:25.821411+0000","last_became_active":"2023-09-23T20:03:59.231689+0000","last_became_peered":"2023-09-23T20:03:59.231689+0000","last_unstale":"2023-09-23T20:04:25.821411+0000","last_undegraded":"2023-09-23T20:04:25.821411+0000","last_fullsized":"2023-09-23T20:04:25.821411+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:29.669102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"281'9038","reported_seq":19323,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:04:40.514191+0000","last_change":"2023-09-23T20:04:02.290062+0000","last_active":"2023-09-23T20:04:40.514191+0000","last_peered":"2023-09-23T20:04:40.514191+0000","last_clean":"2023-09-23T20:04:40.514191+0000","last_became_active":"2023-09-23T20:04:02.271287+0000","last_became_peered":"2023-09-23T20:04:02.271287+0000","last_unstale":"2023-09-23T20:04:40.514191+0000","last_undegraded":"2023-09-23T20:04:40.514191+0000","last_fullsized":"2023-09-23T20:04:40.514191+0000","mapping_epoch":276,"log_start":"248'2500","ondisk_log_start":"248'2500","created":240,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6538,"log_dups_size":0,"ondisk_log_size":6538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:47.344179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199622656,"num_objects":3046,"num_object_clones":0,"num_object_copies":9138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9038,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5100,"num_bytes_recovered":334233600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":241,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.757255+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:04:14.757255+0000","last_peered":"2023-09-23T20:04:14.757255+0000","last_clean":"2023-09-23T20:04:14.757255+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:04:14.757255+0000","last_undegraded":"2023-09-23T20:04:14.757255+0000","last_fullsized":"2023-09-23T20:04:14.757255+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":268,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558779+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:04:14.558779+0000","last_peered":"2023-09-23T20:04:14.558779+0000","last_clean":"2023-09-23T20:04:14.558779+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:04:14.558779+0000","last_undegraded":"2023-09-23T20:04:14.558779+0000","last_fullsized":"2023-09-23T20:04:14.558779+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"281'9134","reported_seq":9185,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:04:41.946247+0000","last_change":"2023-09-23T20:01:39.911588+0000","last_active":"2023-09-23T20:04:41.946247+0000","last_peered":"2023-09-23T20:04:41.946247+0000","last_clean":"2023-09-23T20:04:41.946247+0000","last_became_active":"2023-09-23T20:01:39.911364+0000","last_became_peered":"2023-09-23T20:01:39.911364+0000","last_unstale":"2023-09-23T20:04:41.946247+0000","last_undegraded":"2023-09-23T20:04:41.946247+0000","last_fullsized":"2023-09-23T20:04:41.946247+0000","mapping_epoch":240,"log_start":"249'3395","ondisk_log_start":"249'3395","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:35.715001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191102976,"num_objects":2916,"num_object_clones":0,"num_object_copies":8748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9134,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.757162+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T20:04:14.757162+0000","last_peered":"2023-09-23T20:04:14.757162+0000","last_clean":"2023-09-23T20:04:14.757162+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T20:04:14.757162+0000","last_undegraded":"2023-09-23T20:04:14.757162+0000","last_fullsized":"2023-09-23T20:04:14.757162+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":296,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756455+0000","last_change":"2023-09-23T20:03:59.253784+0000","last_active":"2023-09-23T20:04:14.756455+0000","last_peered":"2023-09-23T20:04:14.756455+0000","last_clean":"2023-09-23T20:04:14.756455+0000","last_became_active":"2023-09-23T20:03:59.253552+0000","last_became_peered":"2023-09-23T20:03:59.253552+0000","last_unstale":"2023-09-23T20:04:14.756455+0000","last_undegraded":"2023-09-23T20:04:14.756455+0000","last_fullsized":"2023-09-23T20:04:14.756455+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:56.110746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"281'9073","reported_seq":9124,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:04:40.553070+0000","last_change":"2023-09-23T20:01:39.976582+0000","last_active":"2023-09-23T20:04:40.553070+0000","last_peered":"2023-09-23T20:04:40.553070+0000","last_clean":"2023-09-23T20:04:40.553070+0000","last_became_active":"2023-09-23T20:01:39.976244+0000","last_became_peered":"2023-09-23T20:01:39.976244+0000","last_unstale":"2023-09-23T20:04:40.553070+0000","last_undegraded":"2023-09-23T20:04:40.553070+0000","last_fullsized":"2023-09-23T20:04:40.553070+0000","mapping_epoch":240,"log_start":"247'2500","ondisk_log_start":"247'2500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.952295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194838528,"num_objects":2973,"num_object_clones":0,"num_object_copies":8919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9073,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":245,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556986+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:04:14.556986+0000","last_peered":"2023-09-23T20:04:14.556986+0000","last_clean":"2023-09-23T20:04:14.556986+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:04:14.556986+0000","last_undegraded":"2023-09-23T20:04:14.556986+0000","last_fullsized":"2023-09-23T20:04:14.556986+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":208,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839943+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T20:04:25.839943+0000","last_peered":"2023-09-23T20:04:25.839943+0000","last_clean":"2023-09-23T20:04:25.839943+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T20:04:25.839943+0000","last_undegraded":"2023-09-23T20:04:25.839943+0000","last_fullsized":"2023-09-23T20:04:25.839943+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.284038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"280'9285","reported_seq":9335,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:38.168259+0000","last_change":"2023-09-23T20:01:39.912566+0000","last_active":"2023-09-23T20:04:38.168259+0000","last_peered":"2023-09-23T20:04:38.168259+0000","last_clean":"2023-09-23T20:04:38.168259+0000","last_became_active":"2023-09-23T20:01:39.912259+0000","last_became_peered":"2023-09-23T20:01:39.912259+0000","last_unstale":"2023-09-23T20:04:38.168259+0000","last_undegraded":"2023-09-23T20:04:38.168259+0000","last_fullsized":"2023-09-23T20:04:38.168259+0000","mapping_epoch":240,"log_start":"243'400","ondisk_log_start":"243'400","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8885,"log_dups_size":0,"ondisk_log_size":8885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:19.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207159296,"num_objects":3161,"num_object_clones":0,"num_object_copies":9483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9285,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":298,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558964+0000","last_change":"2023-09-23T20:00:34.468296+0000","last_active":"2023-09-23T20:04:14.558964+0000","last_peered":"2023-09-23T20:04:14.558964+0000","last_clean":"2023-09-23T20:04:14.558964+0000","last_became_active":"2023-09-23T20:00:34.467957+0000","last_became_peered":"2023-09-23T20:00:34.467957+0000","last_unstale":"2023-09-23T20:04:14.558964+0000","last_undegraded":"2023-09-23T20:04:14.558964+0000","last_fullsized":"2023-09-23T20:04:14.558964+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:50.449027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":96,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.840608+0000","last_change":"2023-09-23T20:01:04.470558+0000","last_active":"2023-09-23T20:04:25.840608+0000","last_peered":"2023-09-23T20:04:25.840608+0000","last_clean":"2023-09-23T20:04:25.840608+0000","last_became_active":"2023-09-23T20:01:04.470276+0000","last_became_peered":"2023-09-23T20:01:04.470276+0000","last_unstale":"2023-09-23T20:04:25.840608+0000","last_undegraded":"2023-09-23T20:04:25.840608+0000","last_fullsized":"2023-09-23T20:04:25.840608+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:17.788583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"281'9228","reported_seq":9277,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:04:42.095282+0000","last_change":"2023-09-23T20:01:39.974611+0000","last_active":"2023-09-23T20:04:42.095282+0000","last_peered":"2023-09-23T20:04:42.095282+0000","last_clean":"2023-09-23T20:04:42.095282+0000","last_became_active":"2023-09-23T20:01:39.974318+0000","last_became_peered":"2023-09-23T20:01:39.974318+0000","last_unstale":"2023-09-23T20:04:42.095282+0000","last_undegraded":"2023-09-23T20:04:42.095282+0000","last_fullsized":"2023-09-23T20:04:42.095282+0000","mapping_epoch":240,"log_start":"249'3527","ondisk_log_start":"249'3527","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:31.047765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184811520,"num_objects":2820,"num_object_clones":0,"num_object_copies":8460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9228,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":730,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556764+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:04:14.556764+0000","last_peered":"2023-09-23T20:04:14.556764+0000","last_clean":"2023-09-23T20:04:14.556764+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:04:14.556764+0000","last_undegraded":"2023-09-23T20:04:14.556764+0000","last_fullsized":"2023-09-23T20:04:14.556764+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":107,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.819296+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:04:25.819296+0000","last_peered":"2023-09-23T20:04:25.819296+0000","last_clean":"2023-09-23T20:04:25.819296+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:04:25.819296+0000","last_undegraded":"2023-09-23T20:04:25.819296+0000","last_fullsized":"2023-09-23T20:04:25.819296+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":276,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756129+0000","last_change":"2023-09-23T19:59:09.206075+0000","last_active":"2023-09-23T20:04:14.756129+0000","last_peered":"2023-09-23T20:04:14.756129+0000","last_clean":"2023-09-23T20:04:14.756129+0000","last_became_active":"2023-09-23T19:59:09.205766+0000","last_became_peered":"2023-09-23T19:59:09.205766+0000","last_unstale":"2023-09-23T20:04:14.756129+0000","last_undegraded":"2023-09-23T20:04:14.756129+0000","last_fullsized":"2023-09-23T20:04:14.756129+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:14.627703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"280'8849","reported_seq":8897,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:38.241471+0000","last_change":"2023-09-23T20:01:39.976550+0000","last_active":"2023-09-23T20:04:38.241471+0000","last_peered":"2023-09-23T20:04:38.241471+0000","last_clean":"2023-09-23T20:04:38.241471+0000","last_became_active":"2023-09-23T20:01:39.976245+0000","last_became_peered":"2023-09-23T20:01:39.976245+0000","last_unstale":"2023-09-23T20:04:38.241471+0000","last_undegraded":"2023-09-23T20:04:38.241471+0000","last_fullsized":"2023-09-23T20:04:38.241471+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8849,"log_dups_size":0,"ondisk_log_size":8849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:45.979020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197591040,"num_objects":3015,"num_object_clones":0,"num_object_copies":9045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8849,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":639,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558712+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:04:14.558712+0000","last_peered":"2023-09-23T20:04:14.558712+0000","last_clean":"2023-09-23T20:04:14.558712+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:04:14.558712+0000","last_undegraded":"2023-09-23T20:04:14.558712+0000","last_fullsized":"2023-09-23T20:04:14.558712+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":226,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756115+0000","last_change":"2023-09-23T19:59:09.172601+0000","last_active":"2023-09-23T20:04:14.756115+0000","last_peered":"2023-09-23T20:04:14.756115+0000","last_clean":"2023-09-23T20:04:14.756115+0000","last_became_active":"2023-09-23T19:59:09.172297+0000","last_became_peered":"2023-09-23T19:59:09.172297+0000","last_unstale":"2023-09-23T20:04:14.756115+0000","last_undegraded":"2023-09-23T20:04:14.756115+0000","last_fullsized":"2023-09-23T20:04:14.756115+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:09.528266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":256,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756079+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T20:04:14.756079+0000","last_peered":"2023-09-23T20:04:14.756079+0000","last_clean":"2023-09-23T20:04:14.756079+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T20:04:14.756079+0000","last_undegraded":"2023-09-23T20:04:14.756079+0000","last_fullsized":"2023-09-23T20:04:14.756079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"281'9049","reported_seq":9101,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:04:40.544694+0000","last_change":"2023-09-23T20:01:39.977019+0000","last_active":"2023-09-23T20:04:40.544694+0000","last_peered":"2023-09-23T20:04:40.544694+0000","last_clean":"2023-09-23T20:04:40.544694+0000","last_became_active":"2023-09-23T20:01:39.976270+0000","last_became_peered":"2023-09-23T20:01:39.976270+0000","last_unstale":"2023-09-23T20:04:40.544694+0000","last_undegraded":"2023-09-23T20:04:40.544694+0000","last_fullsized":"2023-09-23T20:04:40.544694+0000","mapping_epoch":240,"log_start":"247'2500","ondisk_log_start":"247'2500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.226759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199360536,"num_objects":3043,"num_object_clones":0,"num_object_copies":9129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3043,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9049,"num_write_kb":386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756306+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:04:14.756306+0000","last_peered":"2023-09-23T20:04:14.756306+0000","last_clean":"2023-09-23T20:04:14.756306+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:04:14.756306+0000","last_undegraded":"2023-09-23T20:04:14.756306+0000","last_fullsized":"2023-09-23T20:04:14.756306+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":286,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756241+0000","last_change":"2023-09-23T19:59:09.171926+0000","last_active":"2023-09-23T20:04:14.756241+0000","last_peered":"2023-09-23T20:04:14.756241+0000","last_clean":"2023-09-23T20:04:14.756241+0000","last_became_active":"2023-09-23T19:59:09.171609+0000","last_became_peered":"2023-09-23T19:59:09.171609+0000","last_unstale":"2023-09-23T20:04:14.756241+0000","last_undegraded":"2023-09-23T20:04:14.756241+0000","last_fullsized":"2023-09-23T20:04:14.756241+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:21.795329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"281'9071","reported_seq":15382,"reported_epoch":281,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:04:42.275002+0000","last_change":"2023-09-23T20:04:02.298192+0000","last_active":"2023-09-23T20:04:42.275002+0000","last_peered":"2023-09-23T20:04:42.275002+0000","last_clean":"2023-09-23T20:03:58.092468+0000","last_became_active":"2023-09-23T20:04:00.591086+0000","last_became_peered":"2023-09-23T20:04:00.591086+0000","last_unstale":"2023-09-23T20:04:42.275002+0000","last_undegraded":"2023-09-23T20:04:42.275002+0000","last_fullsized":"2023-09-23T20:04:42.275002+0000","mapping_epoch":274,"log_start":"247'2472","ondisk_log_start":"247'2472","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6599,"log_dups_size":0,"ondisk_log_size":6599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:19.737714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192086016,"num_objects":2931,"num_object_clones":0,"num_object_copies":8793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":415,"num_objects_unfound":0,"num_objects_dirty":2931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9071,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3127,"num_bytes_recovered":204931072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[0,7,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":2931}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":618,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.560245+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:04:14.560245+0000","last_peered":"2023-09-23T20:04:14.560245+0000","last_clean":"2023-09-23T20:04:14.560245+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:04:14.560245+0000","last_undegraded":"2023-09-23T20:04:14.560245+0000","last_fullsized":"2023-09-23T20:04:14.560245+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":18,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.818589+0000","last_change":"2023-09-23T20:03:59.233742+0000","last_active":"2023-09-23T20:04:25.818589+0000","last_peered":"2023-09-23T20:04:25.818589+0000","last_clean":"2023-09-23T20:04:25.818589+0000","last_became_active":"2023-09-23T20:03:59.233209+0000","last_became_peered":"2023-09-23T20:03:59.233209+0000","last_unstale":"2023-09-23T20:04:25.818589+0000","last_undegraded":"2023-09-23T20:04:25.818589+0000","last_fullsized":"2023-09-23T20:04:25.818589+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:50.879925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":241,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839349+0000","last_change":"2023-09-23T19:59:09.153852+0000","last_active":"2023-09-23T20:04:25.839349+0000","last_peered":"2023-09-23T20:04:25.839349+0000","last_clean":"2023-09-23T20:04:25.839349+0000","last_became_active":"2023-09-23T19:59:09.153514+0000","last_became_peered":"2023-09-23T19:59:09.153514+0000","last_unstale":"2023-09-23T20:04:25.839349+0000","last_undegraded":"2023-09-23T20:04:25.839349+0000","last_fullsized":"2023-09-23T20:04:25.839349+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:45.179600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"280'9033","reported_seq":9109,"reported_epoch":280,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:38.281753+0000","last_change":"2023-09-23T20:04:02.297387+0000","last_active":"2023-09-23T20:04:38.281753+0000","last_peered":"2023-09-23T20:04:38.281753+0000","last_clean":"2023-09-23T20:03:58.080766+0000","last_became_active":"2023-09-23T20:04:00.582511+0000","last_became_peered":"2023-09-23T20:04:00.582511+0000","last_unstale":"2023-09-23T20:04:38.281753+0000","last_undegraded":"2023-09-23T20:04:38.281753+0000","last_fullsized":"2023-09-23T20:04:38.281753+0000","mapping_epoch":274,"log_start":"247'2500","ondisk_log_start":"247'2500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6533,"log_dups_size":0,"ondisk_log_size":6533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:56.089695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205848576,"num_objects":3141,"num_object_clones":0,"num_object_copies":9423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3139,"num_objects_unfound":0,"num_objects_dirty":3141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9033,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,1,0],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":3141}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":604,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.821500+0000","last_change":"2023-09-23T20:04:02.277887+0000","last_active":"2023-09-23T20:04:25.821500+0000","last_peered":"2023-09-23T20:04:25.821500+0000","last_clean":"2023-09-23T20:04:25.821500+0000","last_became_active":"2023-09-23T20:04:02.277519+0000","last_became_peered":"2023-09-23T20:04:02.277519+0000","last_unstale":"2023-09-23T20:04:25.821500+0000","last_undegraded":"2023-09-23T20:04:25.821500+0000","last_fullsized":"2023-09-23T20:04:25.821500+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":620,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839927+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T20:04:25.839927+0000","last_peered":"2023-09-23T20:04:25.839927+0000","last_clean":"2023-09-23T20:04:25.839927+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T20:04:25.839927+0000","last_undegraded":"2023-09-23T20:04:25.839927+0000","last_fullsized":"2023-09-23T20:04:25.839927+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":280,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559130+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:04:14.559130+0000","last_peered":"2023-09-23T20:04:14.559130+0000","last_clean":"2023-09-23T20:04:14.559130+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:04:14.559130+0000","last_undegraded":"2023-09-23T20:04:14.559130+0000","last_fullsized":"2023-09-23T20:04:14.559130+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":209,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839743+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T20:04:25.839743+0000","last_peered":"2023-09-23T20:04:25.839743+0000","last_clean":"2023-09-23T20:04:25.839743+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T20:04:25.839743+0000","last_undegraded":"2023-09-23T20:04:25.839743+0000","last_fullsized":"2023-09-23T20:04:25.839743+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"281'9192","reported_seq":9257,"reported_epoch":281,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:42.243162+0000","last_change":"2023-09-23T20:04:00.730874+0000","last_active":"2023-09-23T20:04:42.243162+0000","last_peered":"2023-09-23T20:04:42.243162+0000","last_clean":"2023-09-23T20:03:58.087054+0000","last_became_active":"2023-09-23T20:04:00.560120+0000","last_became_peered":"2023-09-23T20:04:00.560120+0000","last_unstale":"2023-09-23T20:04:42.243162+0000","last_undegraded":"2023-09-23T20:04:42.243162+0000","last_fullsized":"2023-09-23T20:04:42.243162+0000","mapping_epoch":274,"log_start":"248'2600","ondisk_log_start":"248'2600","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6592,"log_dups_size":0,"ondisk_log_size":6592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:17.026687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199360512,"num_objects":3042,"num_object_clones":0,"num_object_copies":9126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3042,"num_objects_unfound":0,"num_objects_dirty":3042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9192,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":3042}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":591,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558887+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:04:14.558887+0000","last_peered":"2023-09-23T20:04:14.558887+0000","last_clean":"2023-09-23T20:04:14.558887+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:04:14.558887+0000","last_undegraded":"2023-09-23T20:04:14.558887+0000","last_fullsized":"2023-09-23T20:04:14.558887+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":840,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.560003+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T20:04:14.560003+0000","last_peered":"2023-09-23T20:04:14.560003+0000","last_clean":"2023-09-23T20:04:14.560003+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T20:04:14.560003+0000","last_undegraded":"2023-09-23T20:04:14.560003+0000","last_fullsized":"2023-09-23T20:04:14.560003+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:58.623015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":279,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.759663+0000","last_change":"2023-09-23T20:00:34.447064+0000","last_active":"2023-09-23T20:04:14.759663+0000","last_peered":"2023-09-23T20:04:14.759663+0000","last_clean":"2023-09-23T20:04:14.759663+0000","last_became_active":"2023-09-23T20:00:34.446754+0000","last_became_peered":"2023-09-23T20:00:34.446754+0000","last_unstale":"2023-09-23T20:04:14.759663+0000","last_undegraded":"2023-09-23T20:04:14.759663+0000","last_fullsized":"2023-09-23T20:04:14.759663+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:39.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":275,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839837+0000","last_change":"2023-09-23T20:03:59.236594+0000","last_active":"2023-09-23T20:04:25.839837+0000","last_peered":"2023-09-23T20:04:25.839837+0000","last_clean":"2023-09-23T20:04:25.839837+0000","last_became_active":"2023-09-23T20:03:59.236305+0000","last_became_peered":"2023-09-23T20:03:59.236305+0000","last_unstale":"2023-09-23T20:04:25.839837+0000","last_undegraded":"2023-09-23T20:04:25.839837+0000","last_fullsized":"2023-09-23T20:04:25.839837+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:23.103726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"280'8940","reported_seq":8988,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:38.429105+0000","last_change":"2023-09-23T20:01:39.977018+0000","last_active":"2023-09-23T20:04:38.429105+0000","last_peered":"2023-09-23T20:04:38.429105+0000","last_clean":"2023-09-23T20:04:38.429105+0000","last_became_active":"2023-09-23T20:01:39.976359+0000","last_became_peered":"2023-09-23T20:01:39.976359+0000","last_unstale":"2023-09-23T20:04:38.429105+0000","last_undegraded":"2023-09-23T20:04:38.429105+0000","last_fullsized":"2023-09-23T20:04:38.429105+0000","mapping_epoch":240,"log_start":"247'2400","ondisk_log_start":"247'2400","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.221346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204210176,"num_objects":3116,"num_object_clones":0,"num_object_copies":9348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8940,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":515,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556815+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:04:14.556815+0000","last_peered":"2023-09-23T20:04:14.556815+0000","last_clean":"2023-09-23T20:04:14.556815+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:04:14.556815+0000","last_undegraded":"2023-09-23T20:04:14.556815+0000","last_fullsized":"2023-09-23T20:04:14.556815+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":912,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554311+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:04:14.554311+0000","last_peered":"2023-09-23T20:04:14.554311+0000","last_clean":"2023-09-23T20:04:14.554311+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:04:14.554311+0000","last_undegraded":"2023-09-23T20:04:14.554311+0000","last_fullsized":"2023-09-23T20:04:14.554311+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":238,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839468+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T20:04:25.839468+0000","last_peered":"2023-09-23T20:04:25.839468+0000","last_clean":"2023-09-23T20:04:25.839468+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T20:04:25.839468+0000","last_undegraded":"2023-09-23T20:04:25.839468+0000","last_fullsized":"2023-09-23T20:04:25.839468+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.564126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":252,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839424+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T20:04:25.839424+0000","last_peered":"2023-09-23T20:04:25.839424+0000","last_clean":"2023-09-23T20:04:25.839424+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T20:04:25.839424+0000","last_undegraded":"2023-09-23T20:04:25.839424+0000","last_fullsized":"2023-09-23T20:04:25.839424+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"280'9160","reported_seq":10072,"reported_epoch":280,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:38.500004+0000","last_change":"2023-09-23T20:04:02.279075+0000","last_active":"2023-09-23T20:04:38.500004+0000","last_peered":"2023-09-23T20:04:38.500004+0000","last_clean":"2023-09-23T20:03:58.125973+0000","last_became_active":"2023-09-23T20:04:00.586359+0000","last_became_peered":"2023-09-23T20:04:00.586359+0000","last_unstale":"2023-09-23T20:04:38.500004+0000","last_undegraded":"2023-09-23T20:04:38.500004+0000","last_fullsized":"2023-09-23T20:04:38.500004+0000","mapping_epoch":274,"log_start":"247'1756","ondisk_log_start":"247'1756","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:05.232047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206962688,"num_objects":3158,"num_object_clones":0,"num_object_copies":9474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2851,"num_objects_unfound":0,"num_objects_dirty":3158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9160,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":426,"num_bytes_recovered":27918336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,5],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":3158}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1088,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.561595+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:04:14.561595+0000","last_peered":"2023-09-23T20:04:14.561595+0000","last_clean":"2023-09-23T20:04:14.561595+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:04:14.561595+0000","last_undegraded":"2023-09-23T20:04:14.561595+0000","last_fullsized":"2023-09-23T20:04:14.561595+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":690,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556894+0000","last_change":"2023-09-23T20:00:34.445036+0000","last_active":"2023-09-23T20:04:14.556894+0000","last_peered":"2023-09-23T20:04:14.556894+0000","last_clean":"2023-09-23T20:04:14.556894+0000","last_became_active":"2023-09-23T20:00:34.444689+0000","last_became_peered":"2023-09-23T20:00:34.444689+0000","last_unstale":"2023-09-23T20:04:14.556894+0000","last_undegraded":"2023-09-23T20:04:14.556894+0000","last_fullsized":"2023-09-23T20:04:14.556894+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:44.485721+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":18,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.820491+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:04:25.820491+0000","last_peered":"2023-09-23T20:04:25.820491+0000","last_clean":"2023-09-23T20:04:25.820491+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:04:25.820491+0000","last_undegraded":"2023-09-23T20:04:25.820491+0000","last_fullsized":"2023-09-23T20:04:25.820491+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":155,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756210+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:04:14.756210+0000","last_peered":"2023-09-23T20:04:14.756210+0000","last_clean":"2023-09-23T20:04:14.756210+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:04:14.756210+0000","last_undegraded":"2023-09-23T20:04:14.756210+0000","last_fullsized":"2023-09-23T20:04:14.756210+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"280'9096","reported_seq":9144,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:38.454400+0000","last_change":"2023-09-23T20:01:39.977010+0000","last_active":"2023-09-23T20:04:38.454400+0000","last_peered":"2023-09-23T20:04:38.454400+0000","last_clean":"2023-09-23T20:04:38.454400+0000","last_became_active":"2023-09-23T20:01:39.976722+0000","last_became_peered":"2023-09-23T20:01:39.976722+0000","last_unstale":"2023-09-23T20:04:38.454400+0000","last_undegraded":"2023-09-23T20:04:38.454400+0000","last_fullsized":"2023-09-23T20:04:38.454400+0000","mapping_epoch":240,"log_start":"247'1731","ondisk_log_start":"247'1731","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7365,"log_dups_size":0,"ondisk_log_size":7365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:13.592964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205651968,"num_objects":3138,"num_object_clones":0,"num_object_copies":9414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9096,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":621,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.840731+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T20:04:25.840731+0000","last_peered":"2023-09-23T20:04:25.840731+0000","last_clean":"2023-09-23T20:04:25.840731+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T20:04:25.840731+0000","last_undegraded":"2023-09-23T20:04:25.840731+0000","last_fullsized":"2023-09-23T20:04:25.840731+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.842550+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":998,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.556819+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T20:04:14.556819+0000","last_peered":"2023-09-23T20:04:14.556819+0000","last_clean":"2023-09-23T20:04:14.556819+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T20:04:14.556819+0000","last_undegraded":"2023-09-23T20:04:14.556819+0000","last_fullsized":"2023-09-23T20:04:14.556819+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":246,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756506+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:04:14.756506+0000","last_peered":"2023-09-23T20:04:14.756506+0000","last_clean":"2023-09-23T20:04:14.756506+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:04:14.756506+0000","last_undegraded":"2023-09-23T20:04:14.756506+0000","last_fullsized":"2023-09-23T20:04:14.756506+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559272+0000","last_change":"2023-09-23T20:03:10.175462+0000","last_active":"2023-09-23T20:04:14.559272+0000","last_peered":"2023-09-23T20:04:14.559272+0000","last_clean":"2023-09-23T20:04:14.559272+0000","last_became_active":"2023-09-23T20:03:10.175250+0000","last_became_peered":"2023-09-23T20:03:10.175250+0000","last_unstale":"2023-09-23T20:04:14.559272+0000","last_undegraded":"2023-09-23T20:04:14.559272+0000","last_fullsized":"2023-09-23T20:04:14.559272+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:42.961619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"280'8862","reported_seq":8909,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:37.976727+0000","last_change":"2023-09-23T20:01:39.913385+0000","last_active":"2023-09-23T20:04:37.976727+0000","last_peered":"2023-09-23T20:04:37.976727+0000","last_clean":"2023-09-23T20:04:37.976727+0000","last_became_active":"2023-09-23T20:01:39.912872+0000","last_became_peered":"2023-09-23T20:01:39.912872+0000","last_unstale":"2023-09-23T20:04:37.976727+0000","last_undegraded":"2023-09-23T20:04:37.976727+0000","last_fullsized":"2023-09-23T20:04:37.976727+0000","mapping_epoch":240,"log_start":"244'500","ondisk_log_start":"244'500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8362,"log_dups_size":0,"ondisk_log_size":8362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.619813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205389824,"num_objects":3134,"num_object_clones":0,"num_object_copies":9402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8862,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":596,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.821998+0000","last_change":"2023-09-23T20:04:02.277648+0000","last_active":"2023-09-23T20:04:25.821998+0000","last_peered":"2023-09-23T20:04:25.821998+0000","last_clean":"2023-09-23T20:04:25.821998+0000","last_became_active":"2023-09-23T20:04:02.277271+0000","last_became_peered":"2023-09-23T20:04:02.277271+0000","last_unstale":"2023-09-23T20:04:25.821998+0000","last_undegraded":"2023-09-23T20:04:25.821998+0000","last_fullsized":"2023-09-23T20:04:25.821998+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":358,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.557071+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:04:14.557071+0000","last_peered":"2023-09-23T20:04:14.557071+0000","last_clean":"2023-09-23T20:04:14.557071+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:04:14.557071+0000","last_undegraded":"2023-09-23T20:04:14.557071+0000","last_fullsized":"2023-09-23T20:04:14.557071+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":241,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.840010+0000","last_change":"2023-09-23T20:03:10.174842+0000","last_active":"2023-09-23T20:04:25.840010+0000","last_peered":"2023-09-23T20:04:25.840010+0000","last_clean":"2023-09-23T20:04:25.840010+0000","last_became_active":"2023-09-23T20:03:10.174511+0000","last_became_peered":"2023-09-23T20:03:10.174511+0000","last_unstale":"2023-09-23T20:04:25.840010+0000","last_undegraded":"2023-09-23T20:04:25.840010+0000","last_fullsized":"2023-09-23T20:04:25.840010+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:12.723811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":76,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.561652+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:04:14.561652+0000","last_peered":"2023-09-23T20:04:14.561652+0000","last_clean":"2023-09-23T20:04:14.561652+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:04:14.561652+0000","last_undegraded":"2023-09-23T20:04:14.561652+0000","last_fullsized":"2023-09-23T20:04:14.561652+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"281'9246","reported_seq":9297,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:04:42.201685+0000","last_change":"2023-09-23T20:01:39.912135+0000","last_active":"2023-09-23T20:04:42.201685+0000","last_peered":"2023-09-23T20:04:42.201685+0000","last_clean":"2023-09-23T20:04:42.201685+0000","last_became_active":"2023-09-23T20:01:39.911834+0000","last_became_peered":"2023-09-23T20:01:39.911834+0000","last_unstale":"2023-09-23T20:04:42.201685+0000","last_undegraded":"2023-09-23T20:04:42.201685+0000","last_fullsized":"2023-09-23T20:04:42.201685+0000","mapping_epoch":240,"log_start":"249'3526","ondisk_log_start":"249'3526","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5720,"log_dups_size":0,"ondisk_log_size":5720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:46.725578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196214784,"num_objects":2994,"num_object_clones":0,"num_object_copies":8982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9246,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":658,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756043+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:04:14.756043+0000","last_peered":"2023-09-23T20:04:14.756043+0000","last_clean":"2023-09-23T20:04:14.756043+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:04:14.756043+0000","last_undegraded":"2023-09-23T20:04:14.756043+0000","last_fullsized":"2023-09-23T20:04:14.756043+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":107,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.821846+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:04:25.821846+0000","last_peered":"2023-09-23T20:04:25.821846+0000","last_clean":"2023-09-23T20:04:25.821846+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:04:25.821846+0000","last_undegraded":"2023-09-23T20:04:25.821846+0000","last_fullsized":"2023-09-23T20:04:25.821846+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":686,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839553+0000","last_change":"2023-09-23T19:59:40.899753+0000","last_active":"2023-09-23T20:04:25.839553+0000","last_peered":"2023-09-23T20:04:25.839553+0000","last_clean":"2023-09-23T20:04:25.839553+0000","last_became_active":"2023-09-23T19:59:40.899147+0000","last_became_peered":"2023-09-23T19:59:40.899147+0000","last_unstale":"2023-09-23T20:04:25.839553+0000","last_undegraded":"2023-09-23T20:04:25.839553+0000","last_fullsized":"2023-09-23T20:04:25.839553+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:08.938755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":18,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.818668+0000","last_change":"2023-09-23T20:03:59.233560+0000","last_active":"2023-09-23T20:04:25.818668+0000","last_peered":"2023-09-23T20:04:25.818668+0000","last_clean":"2023-09-23T20:04:25.818668+0000","last_became_active":"2023-09-23T20:03:59.232842+0000","last_became_peered":"2023-09-23T20:03:59.232842+0000","last_unstale":"2023-09-23T20:04:25.818668+0000","last_undegraded":"2023-09-23T20:04:25.818668+0000","last_fullsized":"2023-09-23T20:04:25.818668+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:16.909413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"281'9250","reported_seq":9298,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T20:04:41.944828+0000","last_change":"2023-09-23T20:01:39.913698+0000","last_active":"2023-09-23T20:04:41.944828+0000","last_peered":"2023-09-23T20:04:41.944828+0000","last_clean":"2023-09-23T20:04:41.944828+0000","last_became_active":"2023-09-23T20:01:39.913367+0000","last_became_peered":"2023-09-23T20:01:39.913367+0000","last_unstale":"2023-09-23T20:04:41.944828+0000","last_undegraded":"2023-09-23T20:04:41.944828+0000","last_fullsized":"2023-09-23T20:04:41.944828+0000","mapping_epoch":240,"log_start":"249'3525","ondisk_log_start":"249'3525","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195690496,"num_objects":2986,"num_object_clones":0,"num_object_copies":8958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9250,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":537,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.819760+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:04:25.819760+0000","last_peered":"2023-09-23T20:04:25.819760+0000","last_clean":"2023-09-23T20:04:25.819760+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:04:25.819760+0000","last_undegraded":"2023-09-23T20:04:25.819760+0000","last_fullsized":"2023-09-23T20:04:25.819760+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":36,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756760+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:04:14.756760+0000","last_peered":"2023-09-23T20:04:14.756760+0000","last_clean":"2023-09-23T20:04:14.756760+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:04:14.756760+0000","last_undegraded":"2023-09-23T20:04:14.756760+0000","last_fullsized":"2023-09-23T20:04:14.756760+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":84,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.839637+0000","last_change":"2023-09-23T20:01:04.470744+0000","last_active":"2023-09-23T20:04:25.839637+0000","last_peered":"2023-09-23T20:04:25.839637+0000","last_clean":"2023-09-23T20:04:25.839637+0000","last_became_active":"2023-09-23T20:01:04.470302+0000","last_became_peered":"2023-09-23T20:01:04.470302+0000","last_unstale":"2023-09-23T20:04:25.839637+0000","last_undegraded":"2023-09-23T20:04:25.839637+0000","last_fullsized":"2023-09-23T20:04:25.839637+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:25.452596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":239,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.756402+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:04:14.756402+0000","last_peered":"2023-09-23T20:04:14.756402+0000","last_clean":"2023-09-23T20:04:14.756402+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:04:14.756402+0000","last_undegraded":"2023-09-23T20:04:14.756402+0000","last_fullsized":"2023-09-23T20:04:14.756402+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":25,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554108+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:04:14.554108+0000","last_peered":"2023-09-23T20:04:14.554108+0000","last_clean":"2023-09-23T20:04:14.554108+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:04:14.554108+0000","last_undegraded":"2023-09-23T20:04:14.554108+0000","last_fullsized":"2023-09-23T20:04:14.554108+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":33,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.554670+0000","last_change":"2023-09-23T20:03:10.178662+0000","last_active":"2023-09-23T20:04:14.554670+0000","last_peered":"2023-09-23T20:04:14.554670+0000","last_clean":"2023-09-23T20:04:14.554670+0000","last_became_active":"2023-09-23T20:03:10.178352+0000","last_became_peered":"2023-09-23T20:03:10.178352+0000","last_unstale":"2023-09-23T20:04:14.554670+0000","last_undegraded":"2023-09-23T20:04:14.554670+0000","last_fullsized":"2023-09-23T20:04:14.554670+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.469897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":241,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.558812+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T20:04:14.558812+0000","last_peered":"2023-09-23T20:04:14.558812+0000","last_clean":"2023-09-23T20:04:14.558812+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T20:04:14.558812+0000","last_undegraded":"2023-09-23T20:04:14.558812+0000","last_fullsized":"2023-09-23T20:04:14.558812+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":276,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.759730+0000","last_change":"2023-09-23T19:59:09.172452+0000","last_active":"2023-09-23T20:04:14.759730+0000","last_peered":"2023-09-23T20:04:14.759730+0000","last_clean":"2023-09-23T20:04:14.759730+0000","last_became_active":"2023-09-23T19:59:09.172080+0000","last_became_peered":"2023-09-23T19:59:09.172080+0000","last_unstale":"2023-09-23T20:04:14.759730+0000","last_undegraded":"2023-09-23T20:04:14.759730+0000","last_fullsized":"2023-09-23T20:04:14.759730+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:24.687991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":245,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559021+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T20:04:14.559021+0000","last_peered":"2023-09-23T20:04:14.559021+0000","last_clean":"2023-09-23T20:04:14.559021+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T20:04:14.559021+0000","last_undegraded":"2023-09-23T20:04:14.559021+0000","last_fullsized":"2023-09-23T20:04:14.559021+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:07.560646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":276,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.755995+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:04:14.755995+0000","last_peered":"2023-09-23T20:04:14.755995+0000","last_clean":"2023-09-23T20:04:14.755995+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:04:14.755995+0000","last_undegraded":"2023-09-23T20:04:14.755995+0000","last_fullsized":"2023-09-23T20:04:14.755995+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.822310+0000","last_change":"2023-09-23T19:59:09.145224+0000","last_active":"2023-09-23T20:04:25.822310+0000","last_peered":"2023-09-23T20:04:25.822310+0000","last_clean":"2023-09-23T20:04:25.822310+0000","last_became_active":"2023-09-23T19:59:09.144819+0000","last_became_peered":"2023-09-23T19:59:09.144819+0000","last_unstale":"2023-09-23T20:04:25.822310+0000","last_undegraded":"2023-09-23T20:04:25.822310+0000","last_fullsized":"2023-09-23T20:04:25.822310+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:04.048181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":18,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T20:04:25.818891+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:04:25.818891+0000","last_peered":"2023-09-23T20:04:25.818891+0000","last_clean":"2023-09-23T20:04:25.818891+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:04:25.818891+0000","last_undegraded":"2023-09-23T20:04:25.818891+0000","last_fullsized":"2023-09-23T20:04:25.818891+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":99,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.559611+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:04:14.559611+0000","last_peered":"2023-09-23T20:04:14.559611+0000","last_clean":"2023-09-23T20:04:14.559611+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:04:14.559611+0000","last_undegraded":"2023-09-23T20:04:14.559611+0000","last_fullsized":"2023-09-23T20:04:14.559611+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":306,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:04:14.563830+0000","last_change":"2023-09-23T20:03:59.235232+0000","last_active":"2023-09-23T20:04:14.563830+0000","last_peered":"2023-09-23T20:04:14.563830+0000","last_clean":"2023-09-23T20:04:14.563830+0000","last_became_active":"2023-09-23T20:03:59.234941+0000","last_became_peered":"2023-09-23T20:03:59.234941+0000","last_unstale":"2023-09-23T20:04:14.563830+0000","last_undegraded":"2023-09-23T20:04:14.563830+0000","last_fullsized":"2023-09-23T20:04:14.563830+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:45.591449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3185901592,"num_objects":48614,"num_object_clones":0,"num_object_copies":145842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9447,"num_objects_unfound":0,"num_objects_dirty":48614,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":145506,"num_write_kb":6211777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8659,"num_bytes_recovered":567410688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9986453504,"data_stored":9986441288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109674,"ondisk_log_size":109674,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234282,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1479428,"kb_used_data":1211836,"kb_used_omap":0,"kb_used_meta":267584,"kb_avail":92265724,"statfs":{"total":95995035648,"available":94480101376,"internally_reserved":0,"allocated":1240920064,"data_stored":1240069680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":274006016},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:20 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5750000000000002,"15min":2.0430000000000001},"min":{"1min":0.245,"5min":0.22,"15min":0.19700000000000001},"max":{"1min":41.889000000000003,"5min":60.140000000000001,"15min":62.061999999999998},"last":11.256},{"interface":"front","average":{"1min":4.5209999999999999,"5min":2.7120000000000002,"15min":2.27},"min":{"1min":0.25900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":41.936,"5min":62.389000000000003,"15min":62.389000000000003},"last":2.7690000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:04:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2240000000000002,"5min":5.7729999999999997,"15min":3.0739999999999998},"min":{"1min":0.27100000000000002,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":56.024999999999999,"5min":104.16800000000001,"15min":104.16800000000001},"last":2.1389999999999998},{"interface":"front","average":{"1min":6.3220000000000001,"5min":6.0590000000000002,"15min":3.2149999999999999},"min":{"1min":0.34200000000000003,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":56.146000000000001,"5min":104.003,"15min":104.003},"last":3.2850000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:04:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.794,"15min":3.1379999999999999},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.6159999999999997,"5min":105.842,"15min":130.673},"last":3.5539999999999998},{"interface":"front","average":{"1min":4.0819999999999999,"5min":3.4249999999999998,"15min":3.4260000000000002},"min":{"1min":0.376,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":41.968000000000004,"5min":105.66,"15min":130.71100000000001},"last":3.4649999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:04:25 2023","interfaces":[{"interface":"back","average":{"1min":10.352,"5min":5.5999999999999996,"15min":3.4630000000000001},"min":{"1min":0.316,"5min":0.247,"15min":0.247},"max":{"1min":80.257999999999996,"5min":146.38800000000001,"15min":146.38800000000001},"last":3.3639999999999999},{"interface":"front","average":{"1min":10.545999999999999,"5min":5.8099999999999996,"15min":3.464},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":82.281999999999996,"5min":146.851,"15min":146.851},"last":2.1259999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.878,"15min":1.885},"min":{"1min":0.28299999999999997,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.3109999999999999,"5min":19.215,"15min":143.255},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.84499999999999997,"15min":1.7969999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.0419999999999998,"5min":19.364000000000001,"15min":143.41200000000001},"last":0.28399999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.1819999999999999,"15min":1.444},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":2.9380000000000002,"5min":35.488,"15min":84.566999999999993},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.085,"15min":1.3089999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.343,"5min":34.896000000000001,"15min":81.244},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:04:32 2023","interfaces":[{"interface":"back","average":{"1min":2.6709999999999998,"5min":1.1020000000000001,"15min":0.83999999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":41.902999999999999,"5min":41.902999999999999,"15min":41.902999999999999},"last":1.175},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.66900000000000004,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.3620000000000001,"5min":6.3620000000000001,"15min":6.3620000000000001},"last":0.496}]}]},{"osd":6,"up_from":0,"seq":1078036791323,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762484,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1952436,"kb_used_data":1575724,"kb_used_omap":7,"kb_used_meta":376376,"kb_avail":91792716,"statfs":{"total":95995035648,"available":93995741184,"internally_reserved":0,"allocated":1613541376,"data_stored":1612297324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":385409850},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":2.2240000000000002,"15min":3.8799999999999999},"min":{"1min":0.247,"5min":0.16900000000000001,"15min":0.151},"max":{"1min":11.710000000000001,"5min":25.238,"15min":202.917},"last":2.7490000000000001},{"interface":"front","average":{"1min":3.8780000000000001,"5min":2.4460000000000002,"15min":4.0119999999999996},"min":{"1min":0.23200000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":14.839,"5min":22.337,"15min":203.34899999999999},"last":6.5890000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":5.0609999999999999,"15min":5.2329999999999997},"min":{"1min":0.315,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":11.917999999999999,"5min":134.72499999999999,"15min":231.57499999999999},"last":1.9670000000000001},{"interface":"front","average":{"1min":3.8370000000000002,"5min":5.4279999999999999,"15min":5.2359999999999998},"min":{"1min":0.29399999999999998,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":20.677,"5min":135.74600000000001,"15min":236.077},"last":6.4109999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.665,"5min":2.7349999999999999,"15min":3.016},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.217},"max":{"1min":11.98,"5min":15.571,"15min":68.180999999999997},"last":5.9020000000000001},{"interface":"front","average":{"1min":5.2850000000000001,"5min":2.7749999999999999,"15min":2.8439999999999999},"min":{"1min":0.36799999999999999,"5min":0.249,"15min":0.20100000000000001},"max":{"1min":20.989999999999998,"5min":22.385999999999999,"15min":66.509},"last":6.5629999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.9450000000000001,"15min":2.4159999999999999},"min":{"1min":0.33800000000000002,"5min":0.254,"15min":0.25},"max":{"1min":9.9190000000000005,"5min":35.497999999999998,"15min":161.68299999999999},"last":2.7869999999999999},{"interface":"front","average":{"1min":3.093,"5min":2.226,"15min":2.4470000000000001},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":20.847000000000001,"5min":35.521000000000001,"15min":161.286},"last":1.98}]},{"osd":5,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":2.407,"15min":3.3610000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":20.960000000000001,"5min":51.698,"15min":219.048},"last":2.004},{"interface":"front","average":{"1min":3.6739999999999999,"5min":2.6459999999999999,"15min":3.5510000000000002},"min":{"1min":0.40400000000000003,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":15.507999999999999,"5min":51.173000000000002,"15min":218.69499999999999},"last":1.5620000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":2.1640000000000001,"15min":2.2189999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.5579999999999998,"5min":9.8879999999999999,"15min":9.8879999999999999},"last":2.8159999999999998},{"interface":"front","average":{"1min":3.2480000000000002,"5min":3.4060000000000001,"15min":3.4319999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":13.304,"5min":20.734999999999999,"15min":20.734999999999999},"last":2.0350000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":2.1869999999999998,"15min":2.1739999999999999},"min":{"1min":0.432,"5min":0.36199999999999999,"15min":0.26800000000000002},"max":{"1min":11.893000000000001,"5min":14.23,"15min":60.213000000000001},"last":1.613},{"interface":"front","average":{"1min":2.048,"5min":1.746,"15min":1.7529999999999999},"min":{"1min":0.42599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.8309999999999995,"5min":14.188000000000001,"15min":61.298999999999999},"last":2.8420000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926019,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2325052,"kb_used_data":1901428,"kb_used_omap":8,"kb_used_meta":423351,"kb_avail":91420100,"statfs":{"total":95995035648,"available":93614182400,"internally_reserved":0,"allocated":1947062272,"data_stored":1945817614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":433512033},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":5.2039999999999997,"15min":2.8119999999999998},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":7.6669999999999998,"5min":94.055999999999997,"15min":94.055999999999997},"last":3.6850000000000001},{"interface":"front","average":{"1min":3.3210000000000002,"5min":5.5010000000000003,"15min":3.1699999999999999},"min":{"1min":0.24199999999999999,"5min":0.183,"15min":0.183},"max":{"1min":9.5609999999999999,"5min":91.632999999999996,"15min":91.632999999999996},"last":1.149}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":5.9409999999999998,"15min":3.5910000000000002},"min":{"1min":0.23300000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.75,"5min":270.51100000000002,"15min":270.51100000000002},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.8109999999999999,"5min":5.5460000000000003,"15min":3.6419999999999999},"min":{"1min":0.25700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.038,"5min":269.69299999999998,"15min":269.69299999999998},"last":0.96799999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":3.9060000000000001,"5min":4.8639999999999999,"15min":4.6470000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.5890000000000004,"5min":116.733,"15min":145.72800000000001},"last":1.8129999999999999},{"interface":"front","average":{"1min":3.4449999999999998,"5min":4.9390000000000001,"15min":4.7610000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":9.6470000000000002,"5min":116.697,"15min":145.50700000000001},"last":1.5780000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.7430000000000001,"15min":2.3159999999999998},"min":{"1min":0.27600000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.3829999999999991,"5min":12.372,"15min":110.312},"last":0.94699999999999995},{"interface":"front","average":{"1min":2.27,"5min":1.659,"15min":1.964},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27600000000000002},"max":{"1min":8.7330000000000005,"5min":8.7330000000000005,"15min":110.256},"last":1.0820000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":2.181,"15min":2.4929999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.5419999999999998,"5min":64.725999999999999,"15min":131.34399999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.8380000000000001,"15min":2.569},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.4139999999999997,"5min":12.48,"15min":131.386},"last":0.87}]},{"osd":6,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.6020000000000001,"15min":1.605},"min":{"1min":0.41199999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.4600000000000009,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.91800000000000004},{"interface":"front","average":{"1min":2.73,"5min":2.0699999999999998,"15min":1.96},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":9.4429999999999996,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":1.1240000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":2.2189999999999999,"15min":2.004},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":8.6039999999999992,"5min":65.081999999999994,"15min":77.132999999999996},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.8400000000000001,"5min":1.9419999999999999,"15min":2.0990000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.3900000000000006,"5min":12.452,"15min":75.465999999999994},"last":0.98799999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598957,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2114204,"kb_used_data":1593492,"kb_used_omap":12,"kb_used_meta":520563,"kb_avail":91630948,"statfs":{"total":95995035648,"available":93830090752,"internally_reserved":0,"allocated":1631735808,"data_stored":1630516924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":533057534},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":3.6309999999999998,"15min":2.5019999999999998},"min":{"1min":0.20899999999999999,"5min":0.192,"15min":0.154},"max":{"1min":10.073,"5min":48.966999999999999,"15min":50.506999999999998},"last":76.251999999999995},{"interface":"front","average":{"1min":1.4690000000000001,"5min":3.2509999999999999,"15min":2.3969999999999998},"min":{"1min":0.20200000000000001,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":7.8840000000000003,"5min":48.783000000000001,"15min":50.802999999999997},"last":76.590000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":6.4400000000000004,"5min":5.4279999999999999,"15min":2.9529999999999998},"min":{"1min":0.247,"5min":0.21299999999999999,"15min":0.184},"max":{"1min":50.192999999999998,"5min":94.768000000000001,"15min":94.768000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":6.7729999999999997,"5min":6.0060000000000002,"15min":3.1890000000000001},"min":{"1min":0.218,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":47.987000000000002,"5min":96.808000000000007,"15min":96.808000000000007},"last":0.25700000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:04:22 2023","interfaces":[{"interface":"back","average":{"1min":4.117,"5min":3.8380000000000001,"15min":2.7970000000000002},"min":{"1min":0.307,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":13.984,"5min":44.408000000000001,"15min":50.186999999999998},"last":72.412000000000006},{"interface":"front","average":{"1min":2.2709999999999999,"5min":3.3290000000000002,"15min":2.7229999999999999},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.084,"5min":43.981999999999999,"15min":50.585000000000001},"last":72.373000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.893,"15min":1.6890000000000001},"min":{"1min":0.30499999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.9249999999999998,"5min":20.591999999999999,"15min":50.640000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.3039999999999998,"5min":1.819,"15min":1.6990000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7629999999999999,"5min":23.062000000000001,"15min":50.154000000000003},"last":0.42399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":10.351000000000001,"5min":3.9199999999999999,"15min":2.5009999999999999},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":172.505,"5min":172.505,"15min":172.505},"last":0.51400000000000001},{"interface":"front","average":{"1min":10.007,"5min":3.5630000000000002,"15min":2.347},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":172.375,"5min":172.375,"15min":172.375},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 20:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.581,"15min":1.611},"min":{"1min":0.38100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.5250000000000004,"5min":10.144,"15min":10.144},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.917,"15min":1.9510000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.1390000000000002,"5min":7.1390000000000002,"15min":7.1390000000000002},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":1.9530000000000001,"15min":1.796},"min":{"1min":0.436,"5min":0.33700000000000002,"15min":0.27400000000000002},"max":{"1min":9.7330000000000005,"5min":23.079999999999998,"15min":50.118000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.734,"5min":2.395,"15min":2.0680000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":10.17,"5min":20.614999999999998,"15min":50.664999999999999},"last":0.55000000000000004}]}]},{"osd":3,"up_from":109,"seq":468151435431,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1766824,"kb_used_data":1317600,"kb_used_omap":4,"kb_used_meta":449019,"kb_avail":91978328,"statfs":{"total":95995035648,"available":94185807872,"internally_reserved":0,"allocated":1349222400,"data_stored":1347876614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":459795565},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,0,1,3,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0600000000000001,"5min":2.399,"15min":3.8929999999999998},"min":{"1min":0.193,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":10.401,"5min":21.100999999999999,"15min":240.40600000000001},"last":7.7400000000000002},{"interface":"front","average":{"1min":4.1799999999999997,"5min":3.327,"15min":4.4989999999999997},"min":{"1min":0.23100000000000001,"5min":0.19600000000000001,"15min":0.17199999999999999},"max":{"1min":16.978000000000002,"5min":53.201000000000001,"15min":240.352},"last":2.5720000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.4569999999999999,"5min":4.1920000000000002,"15min":2.2799999999999998},"min":{"1min":0.23000000000000001,"5min":0.217,"15min":0.20000000000000001},"max":{"1min":86.337999999999994,"5min":86.337999999999994,"15min":86.337999999999994},"last":1.3500000000000001},{"interface":"front","average":{"1min":9.1300000000000008,"5min":3.7149999999999999,"15min":2.2410000000000001},"min":{"1min":0.249,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":84.299999999999997,"5min":84.299999999999997,"15min":84.299999999999997},"last":4.915}]},{"osd":2,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.2539999999999996,"5min":4.0030000000000001,"15min":5.3659999999999997},"min":{"1min":0.25800000000000001,"5min":0.255,"15min":0.219},"max":{"1min":82.111999999999995,"5min":82.111999999999995,"15min":264.97800000000001},"last":1.3129999999999999},{"interface":"front","average":{"1min":3.5790000000000002,"5min":2.7000000000000002,"15min":4.9269999999999996},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":17.657,"5min":30.768999999999998,"15min":257.40899999999999},"last":7.6349999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":13.324999999999999,"5min":4.5519999999999996,"15min":2.7669999999999999},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":109.66800000000001,"5min":109.66800000000001,"15min":109.66800000000001},"last":2.4750000000000001},{"interface":"front","average":{"1min":13.428000000000001,"5min":5.0140000000000002,"15min":3.2130000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":110.295,"5min":110.295,"15min":110.295},"last":2.5009999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.0170000000000003,"5min":4.3049999999999997,"15min":2.5310000000000001},"min":{"1min":0.33400000000000002,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":82.138000000000005,"5min":113.881,"15min":113.881},"last":2.4529999999999998},{"interface":"front","average":{"1min":6.6020000000000003,"5min":4.3559999999999999,"15min":2.4340000000000002},"min":{"1min":0.43099999999999999,"5min":0.313,"15min":0.31},"max":{"1min":82.25,"5min":114.598,"15min":114.598},"last":1.379}]},{"osd":6,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":5.0170000000000003,"15min":5.5069999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.4530000000000003,"5min":82.281999999999996,"15min":82.281999999999996},"last":1.421},{"interface":"front","average":{"1min":1.839,"5min":4.9669999999999996,"15min":5.4889999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3470000000000004,"5min":82.233000000000004,"15min":82.233000000000004},"last":2.3839999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":13.244,"5min":4.3380000000000001,"15min":2.9729999999999999},"min":{"1min":0.45200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":112.883,"5min":112.883,"15min":138.035},"last":1.395},{"interface":"front","average":{"1min":13.119999999999999,"5min":3.8069999999999999,"15min":2.7440000000000002},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":111.85599999999999,"5min":111.85599999999999,"15min":138.054},"last":2.4399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019192,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485696,"kb_used_data":1221880,"kb_used_omap":0,"kb_used_meta":263808,"kb_avail":92259456,"statfs":{"total":95995035648,"available":94473682944,"internally_reserved":0,"allocated":1251205120,"data_stored":1250358820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":270139392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":2.9470000000000001,"15min":2.9249999999999998},"min":{"1min":0.251,"5min":0.189,"15min":0.189},"max":{"1min":5.508,"5min":89.004999999999995,"15min":104.604},"last":2.9780000000000002},{"interface":"front","average":{"1min":0.86899999999999999,"5min":2.8570000000000002,"15min":2.883},"min":{"1min":0.29499999999999998,"5min":0.249,"15min":0.23400000000000001},"max":{"1min":5.4509999999999996,"5min":89.024000000000001,"15min":104.687},"last":2.9660000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":31.100000000000001,"5min":10.585000000000001,"15min":4.8559999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.216},"max":{"1min":161.613,"5min":161.613,"15min":161.613},"last":3.657},{"interface":"front","average":{"1min":31.327999999999999,"5min":10.795,"15min":4.7619999999999996},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":162.017,"5min":162.017,"15min":162.017},"last":2.258}]},{"osd":2,"last update":"Sat Sep 23 20:03:55 2023","interfaces":[{"interface":"back","average":{"1min":14.250999999999999,"5min":9.3689999999999998,"15min":5.0190000000000001},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":109.38200000000001,"5min":144.20099999999999,"15min":144.20099999999999},"last":1.798},{"interface":"front","average":{"1min":14.645,"5min":8.7840000000000007,"15min":4.7690000000000001},"min":{"1min":0.34200000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":107.74299999999999,"5min":144.184,"15min":144.184},"last":2.7269999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":14.276999999999999,"5min":7.5449999999999999,"15min":5.2300000000000004},"min":{"1min":0.376,"5min":0.255,"15min":0.255},"max":{"1min":99.930999999999997,"5min":139.976,"15min":266.54700000000003},"last":2.6389999999999998},{"interface":"front","average":{"1min":13.647,"5min":7.1440000000000001,"15min":4.1980000000000004},"min":{"1min":0.39300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":100.01600000000001,"5min":139.911,"15min":139.911},"last":3.3900000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.8640000000000001},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":4.077,"5min":5.0650000000000004,"15min":266.59899999999999},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.60799999999999998,"15min":1.9570000000000001},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.395,"5min":4.8739999999999997,"15min":266.72899999999998},"last":2.319}]},{"osd":6,"last update":"Sat Sep 23 20:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.53300000000000003,"15min":0.51300000000000001},"min":{"1min":0.31,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0410000000000004,"5min":4.0410000000000004,"15min":4.0410000000000004},"last":0.31},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.66100000000000003,"15min":0.54100000000000004},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.2439999999999998,"5min":4.2439999999999998,"15min":4.2439999999999998},"last":4.0490000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:03:47 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.81399999999999995,"15min":0.97499999999999998},"min":{"1min":0.318,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.276,"5min":7.6210000000000004,"15min":11.59},"last":1.3260000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.95099999999999996,"15min":0.999},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":1.3240000000000001,"5min":18.012,"15min":18.012},"last":1.204}]}]},{"osd":5,"up_from":38,"seq":163208757556,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1288568,"kb_used_data":1009712,"kb_used_omap":0,"kb_used_meta":278848,"kb_avail":92456584,"statfs":{"total":95995035648,"available":94675542016,"internally_reserved":0,"allocated":1033945088,"data_stored":1033103229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285540352},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.94499999999999995,"15min":1.27},"min":{"1min":0.22900000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":2.8490000000000002,"5min":9.1699999999999999,"15min":24.852},"last":5.7130000000000001},{"interface":"front","average":{"1min":1.7829999999999999,"5min":1.3149999999999999,"15min":1.353},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.216},"max":{"1min":7.4100000000000001,"5min":9.0530000000000008,"15min":24.638999999999999},"last":5.5720000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":12.311,"5min":9.4450000000000003,"15min":4.9180000000000001},"min":{"1min":0.33100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":171.47300000000001,"5min":266.197,"15min":266.197},"last":0.501},{"interface":"front","average":{"1min":12.787000000000001,"5min":9.6140000000000008,"15min":4.9509999999999996},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":171.399,"5min":266.14999999999998,"15min":266.14999999999998},"last":10.840999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:03:49 2023","interfaces":[{"interface":"back","average":{"1min":10.763,"5min":6.2130000000000001,"15min":5.5830000000000002},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":177.816,"5min":238.96199999999999,"15min":238.96199999999999},"last":3.6680000000000001},{"interface":"front","average":{"1min":10.779999999999999,"5min":6.2439999999999998,"15min":5.6189999999999998},"min":{"1min":0.33800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":177.851,"5min":238.905,"15min":238.905},"last":3.7189999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":12.673,"5min":4.883,"15min":3.5150000000000001},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":168.31399999999999,"5min":168.31399999999999,"15min":168.31399999999999},"last":4.2469999999999999},{"interface":"front","average":{"1min":10.301,"5min":4.806,"15min":3.3679999999999999},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":168.39500000000001,"5min":168.39500000000001,"15min":168.39500000000001},"last":4.3129999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.476,"15min":1.609},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.9129999999999998,"5min":191.78899999999999,"15min":191.78899999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":2.4849999999999999,"15min":1.5960000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.177,"5min":191.821,"15min":191.821},"last":0.32200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.66500000000000004,"15min":0.621},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.0679999999999996,"5min":4.0679999999999996,"15min":4.0679999999999996},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.49299999999999999,"15min":0.47199999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.3570000000000002,"5min":3.3570000000000002,"15min":3.3570000000000002},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":3.2530000000000001,"15min":1.7330000000000001},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.996,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":3.379,"15min":1.9239999999999999},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.8169999999999999,"5min":191.69,"15min":191.69},"last":0.39500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1938886656,"data_stored":1938882584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1596329984,"data_stored":1596325912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1627324416,"data_stored":1627324416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1344606208,"data_stored":1344602136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1240465408,"data_stored":1240465408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1017839616,"data_stored":1017839616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1221001216,"data_stored":1221001216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:04:46.603 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:04:46.680 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:46.706 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:46.753 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:46.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:46.841 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:46 smithi135 ceph-mon[213871]: pgmap v768: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 9102/142218 objects misplaced (6.400%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:46.842 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:46 smithi135 ceph-mon[221113]: pgmap v768: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 9102/142218 objects misplaced (6.400%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:46.858 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:46 smithi144 ceph-mon[194833]: pgmap v768: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 9102/142218 objects misplaced (6.400%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:46.997 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:47.019 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:47.143 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:47.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:47.229 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:04:47.248 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:47.278 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:04:47.308 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:47.345 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:04:47.389 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:04:47.435 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:04:47.473 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:04:47.484 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:47.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:47.504 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:04:47.505 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:04:47.510 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:04:47.546 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:04:47.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3410336278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:47.558 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:04:47.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:04:47.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:04:47.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:04:47.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:04:47.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:04:47.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:04:47.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:04:47.605 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:47.632 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:47.678 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:47.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:47.784 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:47.812 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3410336278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:47.817 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:47.874 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:47.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3410336278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:47.983 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:48.014 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:48.073 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:48.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:48.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:48.180 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:48.210 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:48.272 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:48.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:48.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:48.378 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:48.403 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:48.444 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:48.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:48.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:48.549 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:48.577 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:48.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:48 smithi144 ceph-mon[194833]: pgmap v769: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 9102/140625 objects misplaced (6.473%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:48.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:48 smithi135 ceph-mon[221113]: pgmap v769: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 9102/140625 objects misplaced (6.473%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:48.922 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:48 smithi135 ceph-mon[213871]: pgmap v769: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 9102/140625 objects misplaced (6.473%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:04:49.428 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:49.560 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:49 smithi135 ceph-mon[213871]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.560 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:49 smithi135 ceph-mon[213871]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.560 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:49 smithi135 ceph-mon[213871]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T20:04:49.560 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:49 smithi135 ceph-mon[213871]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.707 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:49.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:49.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:49.808 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:04:49.812 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:49.833 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:49 smithi135 ceph-mon[221113]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:49 smithi135 ceph-mon[221113]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:49 smithi135 ceph-mon[221113]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T20:04:49.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:49 smithi135 ceph-mon[221113]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:49 smithi144 ceph-mon[194833]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:49 smithi144 ceph-mon[194833]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:49 smithi144 ceph-mon[194833]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T20:04:49.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:49 smithi144 ceph-mon[194833]: failed to encode map e282 with expected crc 2023-09-23T20:04:49.838 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:49.883 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:49.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:49.992 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:50.136 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:50.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:50 smithi144 ceph-mon[194833]: pgmap v771: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 8207/133983 objects misplaced (6.125%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T20:04:50.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:50 smithi144 ceph-mon[194833]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:50 smithi144 ceph-mon[194833]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:50 smithi144 ceph-mon[194833]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T20:04:50.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:50 smithi144 ceph-mon[194833]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[213871]: pgmap v771: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 8207/133983 objects misplaced (6.125%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T20:04:50.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[213871]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[213871]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[213871]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T20:04:50.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[213871]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[221113]: pgmap v771: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 8207/133983 objects misplaced (6.125%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T20:04:50.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[221113]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[221113]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[221113]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T20:04:50.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:50 smithi135 ceph-mon[221113]: failed to encode map e283 with expected crc 2023-09-23T20:04:50.977 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:50.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:51.086 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:51.175 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:51.288 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:51.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:51.410 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:51.440 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:51.492 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:51.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:51.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:51.600 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:51.706 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:51.736 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:04:52.075 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:52.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:52.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:52.193 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:52.299 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:52.420 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:52.576 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:52.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:52.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:52.685 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:04:52.823 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:52 smithi135 ceph-mon[213871]: pgmap v773: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 8207/132750 objects misplaced (6.182%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:04:52.824 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:52.824 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:52.824 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:52 smithi135 ceph-mon[221113]: pgmap v773: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 8207/132750 objects misplaced (6.182%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:04:52.824 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:52.824 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:52.840 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:52 smithi144 ceph-mon[194833]: pgmap v773: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 8207/132750 objects misplaced (6.182%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:04:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:04:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:04:53.040 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:04:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:53.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:04:53.146 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:04:53.174 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:53.223 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:04:53.224 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:04:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:53.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:04:53.336 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:53.365 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:53.415 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:53.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:53.463 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:04:53.463 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1062,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":283,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":7,"osd_in_since":1695499311,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":95},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+remapped+backfilling","count":2}],"num_pgs":99,"num_pools":5,"num_objects":44250,"data_bytes":2902857675,"bytes_used":11882205184,"bytes_avail":660083044352,"bytes_total":671965249536,"misplaced_objects":8207,"misplaced_total":132750,"misplaced_ratio":0.061822975517890774,"recovering_objects_per_sec":44,"recovering_bytes_per_sec":2940047,"recovering_keys_per_sec":0,"num_objects_recovered":359,"num_bytes_recovered":23527424,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":548},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-23T20:03:30.680470+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"8f9ef775-0a40-45cd-8917-5824ea2cda14":{"message":"Global Recovery Event (95s)\n [==========================..] (remaining: 4s)","progress":0.95959597826004028,"add_to_ceph_s":true}}} 2023-09-23T20:04:53.523 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:53.553 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:53.603 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:53.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:53.649 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:04:53.708 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:53.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:04:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:04:53] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-09-23T20:04:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:53 smithi135 ceph-mon[221113]: pgmap v774: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 8001/130122 objects misplaced (6.149%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:04:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2620030421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:53.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/563509646' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:04:53.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:53 smithi135 ceph-mon[213871]: pgmap v774: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 8001/130122 objects misplaced (6.149%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:04:53.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2620030421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:53.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/563509646' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:04:53.735 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:53.778 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:53.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:53 smithi144 ceph-mon[194833]: pgmap v774: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 8001/130122 objects misplaced (6.149%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:04:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2620030421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:04:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/563509646' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:04:53.883 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:53.909 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:53.962 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:53.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:53.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:54.069 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:04:54.099 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:54.160 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:04:54.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:54.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:04:54.267 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:54.330 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:54.602 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:54.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[213871]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2620030421' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:54.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[213871]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T20:04:54.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[213871]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[213871]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.603 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[213871]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:54.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[221113]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2620030421' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:54.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[221113]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T20:04:54.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[221113]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[221113]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:54 smithi135 ceph-mon[221113]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:54 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:04:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:54 smithi144 ceph-mon[194833]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2620030421' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:04:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:54 smithi144 ceph-mon[194833]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T20:04:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:54 smithi144 ceph-mon[194833]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:54 smithi144 ceph-mon[194833]: failed to encode map e284 with expected crc 2023-09-23T20:04:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:54 smithi144 ceph-mon[194833]: failed to encode map e284 with expected crc 2023-09-23T20:04:55.701 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:55.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:55.812 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:04:55.867 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:04:55.868 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:04:55.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:55 smithi144 ceph-mon[194833]: pgmap v776: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 6701/127158 objects misplaced (5.270%); 11 MiB/s, 183 objects/s recovering 2023-09-23T20:04:55.897 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:55 smithi135 ceph-mon[221113]: pgmap v776: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 6701/127158 objects misplaced (5.270%); 11 MiB/s, 183 objects/s recovering 2023-09-23T20:04:55.900 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:55.906 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:55 smithi135 ceph-mon[213871]: pgmap v776: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 6701/127158 objects misplaced (5.270%); 11 MiB/s, 183 objects/s recovering 2023-09-23T20:04:56.687 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:04:56.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:56.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:04:56.800 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:56.831 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:56.893 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:56.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:56.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:56.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:56 smithi135 ceph-mon[221113]: pgmap v777: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 6701/125478 objects misplaced (5.340%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:04:56.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:56 smithi135 ceph-mon[213871]: pgmap v777: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 6701/125478 objects misplaced (5.340%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:04:57.000 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:57.027 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:57.081 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:57.086 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:56 smithi144 ceph-mon[194833]: pgmap v777: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 6701/125478 objects misplaced (5.340%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:04:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:57.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:57.188 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:57.286 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:57.748 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:57.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:57.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:57.859 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:04:57.980 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:58.130 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:04:58.130 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:04:58.427 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:04:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:58.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:04:58.541 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:58.572 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:58.631 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:58.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:58.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:58.737 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:04:58.765 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:58.820 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:04:58.826 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:58 smithi144 ceph-mon[194833]: pgmap v778: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 5623/117990 objects misplaced (4.766%); 12 MiB/s, 189 objects/s recovering 2023-09-23T20:04:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:58.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:04:58.928 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:04:58.959 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:59.016 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:04:59.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:59.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:04:59.064 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:04:59.064 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:04:59.065 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:04:59.065 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:04:59.065 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:04:59.065 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:04:59.090 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:04:59.121 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:59.133 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:04:59.145 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:04:59.145 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:04:59.145 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:04:59.145 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:04:59.145 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:04:59.145 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:04:59.145 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:04:59.146 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:04:59.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:58 smithi135 ceph-mon[213871]: pgmap v778: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 5623/117990 objects misplaced (4.766%); 12 MiB/s, 189 objects/s recovering 2023-09-23T20:04:59.189 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:58 smithi135 ceph-mon[221113]: pgmap v778: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 5623/117990 objects misplaced (4.766%); 12 MiB/s, 189 objects/s recovering 2023-09-23T20:04:59.222 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:59.613 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:04:59.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:04:59.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:04:59.708 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:04:59.720 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:04:59.776 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:04:59.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:04:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2513398044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:04:59.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:04:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2513398044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:00.061 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:05:00.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:04:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2513398044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:00.342 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:00.385 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:00.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:00.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:00.493 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:00.520 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:00.544 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:00.570 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:00.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:00.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:00.675 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:00.797 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:00.962 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:00.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:00.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:01.040 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:00 smithi135 ceph-mon[213871]: pgmap v779: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 5623/116235 objects misplaced (4.838%); 11 MiB/s, 173 objects/s recovering 2023-09-23T20:05:01.040 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:00 smithi135 ceph-mon[221113]: pgmap v779: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 5623/116235 objects misplaced (4.838%); 11 MiB/s, 173 objects/s recovering 2023-09-23T20:05:01.073 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:01.100 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:00 smithi144 ceph-mon[194833]: pgmap v779: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 5623/116235 objects misplaced (4.838%); 11 MiB/s, 173 objects/s recovering 2023-09-23T20:05:01.102 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:01.158 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:01.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:01.263 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:01.358 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:01.894 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:01.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:02.002 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:02.027 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:02.068 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:02.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:02.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:02.173 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:02.251 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:02.391 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:05:03.058 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:03.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:03.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:03.173 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:03.200 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:03.247 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:03.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:03.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:03.324 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:03.351 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:03.448 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:03.528 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:03.599 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:03.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:03.709 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:03.729 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:03 smithi135 ceph-mon[221113]: pgmap v780: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 5447/112851 objects misplaced (4.827%); 10 MiB/s, 167 objects/s recovering 2023-09-23T20:05:03.730 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:03 smithi135 ceph-mon[213871]: pgmap v780: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 5447/112851 objects misplaced (4.827%); 10 MiB/s, 167 objects/s recovering 2023-09-23T20:05:03.730 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:05:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:05:03] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-09-23T20:05:03.735 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:03.787 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:03.790 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:03 smithi144 ceph-mon[194833]: pgmap v780: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 5447/112851 objects misplaced (4.827%); 10 MiB/s, 167 objects/s recovering 2023-09-23T20:05:03.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:03.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:03.892 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:03.921 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:03.969 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:03.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:04.074 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:04.168 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:04.445 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:04.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:04.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:04.565 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:04.575 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:05:04.590 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:04.603 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:05:04.643 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:04.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:04.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:04.751 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:04.781 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:04.833 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:04.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:04 smithi144 ceph-mon[194833]: from='client.55276 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:04 smithi144 ceph-mon[194833]: pgmap v781: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 4722/108792 objects misplaced (4.340%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:05:04.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:04 smithi144 ceph-mon[194833]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T20:05:04.938 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:04.998 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:04 smithi135 ceph-mon[221113]: from='client.55276 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:04.998 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:04 smithi135 ceph-mon[221113]: pgmap v781: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 4722/108792 objects misplaced (4.340%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:05:04.998 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:04 smithi135 ceph-mon[221113]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T20:05:04.999 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:04 smithi135 ceph-mon[213871]: from='client.55276 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:04.999 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:04 smithi135 ceph-mon[213871]: pgmap v781: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 4722/108792 objects misplaced (4.340%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:05:04.999 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:04 smithi135 ceph-mon[213871]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T20:05:05.008 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:05.811 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:05:06.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:05 smithi144 ceph-mon[194833]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:05 smithi144 ceph-mon[194833]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:05 smithi144 ceph-mon[194833]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3869758395' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:05:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:05 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3869758395' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:05:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:05 smithi144 ceph-mon[194833]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T20:05:06.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[221113]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[221113]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[221113]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.134 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3869758395' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:05:06.135 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:06.135 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3869758395' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:05:06.135 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[221113]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T20:05:06.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[213871]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[213871]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[213871]: failed to encode map e285 with expected crc 2023-09-23T20:05:06.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3869758395' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:05:06.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:06.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3869758395' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:05:06.139 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:05 smithi135 ceph-mon[213871]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T20:05:07.012 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:07.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:07.120 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:07.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:06 smithi144 ceph-mon[194833]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:06 smithi144 ceph-mon[194833]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:06 smithi144 ceph-mon[194833]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:06 smithi144 ceph-mon[194833]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:06 smithi144 ceph-mon[194833]: pgmap v784: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 4722/107184 objects misplaced (4.406%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:05:07.167 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[221113]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.167 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[221113]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.168 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[221113]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.168 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[221113]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.168 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[221113]: pgmap v784: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 4722/107184 objects misplaced (4.406%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:05:07.168 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[213871]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.168 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[213871]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.168 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[213871]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.168 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[213871]: failed to encode map e286 with expected crc 2023-09-23T20:05:07.168 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:06 smithi135 ceph-mon[213871]: pgmap v784: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 4722/107184 objects misplaced (4.406%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:05:07.171 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:07.856 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:08.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:08.283 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:08.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:08.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:08.314 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":780,"stamp":"2023-09-23T20:05:02.745030+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2468157387,"num_objects":37617,"num_object_clones":0,"num_object_copies":112851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5447,"num_objects_unfound":0,"num_objects_dirty":37617,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":157438,"num_write_kb":6224502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10874,"num_bytes_recovered":705436149,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111651,"ondisk_log_size":111651,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10491776,"kb_used_data":7814920,"kb_used_omap":32,"kb_used_meta":2676447,"kb_avail":645724288,"statfs":{"total":671965249536,"available":661221670912,"internally_reserved":0,"allocated":8002478080,"data_stored":7994759363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":2740682246},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1,4,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-377290752,"num_objects":-5757,"num_object_clones":0,"num_object_copies":-17271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2554,"num_objects_unfound":0,"num_objects_dirty":-5757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5757,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1679,"num_bytes_recovered":110034944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004699"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":262,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564003+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T20:04:49.564003+0000","last_peered":"2023-09-23T20:04:49.564003+0000","last_clean":"2023-09-23T20:04:49.564003+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T20:04:49.564003+0000","last_undegraded":"2023-09-23T20:04:49.564003+0000","last_fullsized":"2023-09-23T20:04:49.564003+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":106,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.561982+0000","last_change":"2023-09-23T19:59:09.151117+0000","last_active":"2023-09-23T20:04:49.561982+0000","last_peered":"2023-09-23T20:04:49.561982+0000","last_clean":"2023-09-23T20:04:49.561982+0000","last_became_active":"2023-09-23T19:59:09.150795+0000","last_became_peered":"2023-09-23T19:59:09.150795+0000","last_unstale":"2023-09-23T20:04:49.561982+0000","last_undegraded":"2023-09-23T20:04:49.561982+0000","last_fullsized":"2023-09-23T20:04:49.561982+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:34.994574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":245,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907931+0000","last_change":"2023-09-23T19:59:10.156070+0000","last_active":"2023-09-23T20:04:49.907931+0000","last_peered":"2023-09-23T20:04:49.907931+0000","last_clean":"2023-09-23T20:04:49.907931+0000","last_became_active":"2023-09-23T19:59:10.155813+0000","last_became_peered":"2023-09-23T19:59:10.155813+0000","last_unstale":"2023-09-23T20:04:49.907931+0000","last_undegraded":"2023-09-23T20:04:49.907931+0000","last_fullsized":"2023-09-23T20:04:49.907931+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:42.582380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":36,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.566025+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:04:49.566025+0000","last_peered":"2023-09-23T20:04:49.566025+0000","last_clean":"2023-09-23T20:04:49.566025+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:04:49.566025+0000","last_undegraded":"2023-09-23T20:04:49.566025+0000","last_fullsized":"2023-09-23T20:04:49.566025+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":106,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.560951+0000","last_change":"2023-09-23T19:59:09.163182+0000","last_active":"2023-09-23T20:04:49.560951+0000","last_peered":"2023-09-23T20:04:49.560951+0000","last_clean":"2023-09-23T20:04:49.560951+0000","last_became_active":"2023-09-23T19:59:09.161612+0000","last_became_peered":"2023-09-23T19:59:09.161612+0000","last_unstale":"2023-09-23T20:04:49.560951+0000","last_undegraded":"2023-09-23T20:04:49.560951+0000","last_fullsized":"2023-09-23T20:04:49.560951+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:59.524152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":209,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907594+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T20:04:49.907594+0000","last_peered":"2023-09-23T20:04:49.907594+0000","last_clean":"2023-09-23T20:04:49.907594+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T20:04:49.907594+0000","last_undegraded":"2023-09-23T20:04:49.907594+0000","last_fullsized":"2023-09-23T20:04:49.907594+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:59.043547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":210,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907530+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:04:49.907530+0000","last_peered":"2023-09-23T20:04:49.907530+0000","last_clean":"2023-09-23T20:04:49.907530+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:04:49.907530+0000","last_undegraded":"2023-09-23T20:04:49.907530+0000","last_fullsized":"2023-09-23T20:04:49.907530+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":106,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.560981+0000","last_change":"2023-09-23T19:59:09.146009+0000","last_active":"2023-09-23T20:04:49.560981+0000","last_peered":"2023-09-23T20:04:49.560981+0000","last_clean":"2023-09-23T20:04:49.560981+0000","last_became_active":"2023-09-23T19:59:09.145687+0000","last_became_peered":"2023-09-23T19:59:09.145687+0000","last_unstale":"2023-09-23T20:04:49.560981+0000","last_undegraded":"2023-09-23T20:04:49.560981+0000","last_fullsized":"2023-09-23T20:04:49.560981+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.189450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":243,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907400+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T20:04:49.907400+0000","last_peered":"2023-09-23T20:04:49.907400+0000","last_clean":"2023-09-23T20:04:49.907400+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T20:04:49.907400+0000","last_undegraded":"2023-09-23T20:04:49.907400+0000","last_fullsized":"2023-09-23T20:04:49.907400+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":125,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.561296+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:04:49.561296+0000","last_peered":"2023-09-23T20:04:49.561296+0000","last_clean":"2023-09-23T20:04:49.561296+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:04:49.561296+0000","last_undegraded":"2023-09-23T20:04:49.561296+0000","last_fullsized":"2023-09-23T20:04:49.561296+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":279,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.566106+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:04:49.566106+0000","last_peered":"2023-09-23T20:04:49.566106+0000","last_clean":"2023-09-23T20:04:49.566106+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:04:49.566106+0000","last_undegraded":"2023-09-23T20:04:49.566106+0000","last_fullsized":"2023-09-23T20:04:49.566106+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":241,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.908691+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T20:04:49.908691+0000","last_peered":"2023-09-23T20:04:49.908691+0000","last_clean":"2023-09-23T20:04:49.908691+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T20:04:49.908691+0000","last_undegraded":"2023-09-23T20:04:49.908691+0000","last_fullsized":"2023-09-23T20:04:49.908691+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:51.825640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":298,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564597+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:04:49.564597+0000","last_peered":"2023-09-23T20:04:49.564597+0000","last_clean":"2023-09-23T20:04:49.564597+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:04:49.564597+0000","last_undegraded":"2023-09-23T20:04:49.564597+0000","last_fullsized":"2023-09-23T20:04:49.564597+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":260,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564425+0000","last_change":"2023-09-23T20:01:52.439375+0000","last_active":"2023-09-23T20:04:49.564425+0000","last_peered":"2023-09-23T20:04:49.564425+0000","last_clean":"2023-09-23T20:04:49.564425+0000","last_became_active":"2023-09-23T20:01:52.439074+0000","last_became_peered":"2023-09-23T20:01:52.439074+0000","last_unstale":"2023-09-23T20:04:49.564425+0000","last_undegraded":"2023-09-23T20:04:49.564425+0000","last_fullsized":"2023-09-23T20:04:49.564425+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:31.103633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":246,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.565790+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:04:49.565790+0000","last_peered":"2023-09-23T20:04:49.565790+0000","last_clean":"2023-09-23T20:04:49.565790+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:04:49.565790+0000","last_undegraded":"2023-09-23T20:04:49.565790+0000","last_fullsized":"2023-09-23T20:04:49.565790+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":106,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.562036+0000","last_change":"2023-09-23T19:59:09.195560+0000","last_active":"2023-09-23T20:04:49.562036+0000","last_peered":"2023-09-23T20:04:49.562036+0000","last_clean":"2023-09-23T20:04:49.562036+0000","last_became_active":"2023-09-23T19:59:09.195145+0000","last_became_peered":"2023-09-23T19:59:09.195145+0000","last_unstale":"2023-09-23T20:04:49.562036+0000","last_undegraded":"2023-09-23T20:04:49.562036+0000","last_fullsized":"2023-09-23T20:04:49.562036+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.190798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":249,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.565669+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:04:49.565669+0000","last_peered":"2023-09-23T20:04:49.565669+0000","last_clean":"2023-09-23T20:04:49.565669+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:04:49.565669+0000","last_undegraded":"2023-09-23T20:04:49.565669+0000","last_fullsized":"2023-09-23T20:04:49.565669+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":262,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907271+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:04:49.907271+0000","last_peered":"2023-09-23T20:04:49.907271+0000","last_clean":"2023-09-23T20:04:49.907271+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:04:49.907271+0000","last_undegraded":"2023-09-23T20:04:49.907271+0000","last_fullsized":"2023-09-23T20:04:49.907271+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":244,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.565628+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:04:49.565628+0000","last_peered":"2023-09-23T20:04:49.565628+0000","last_clean":"2023-09-23T20:04:49.565628+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:04:49.565628+0000","last_undegraded":"2023-09-23T20:04:49.565628+0000","last_fullsized":"2023-09-23T20:04:49.565628+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":249,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.562415+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:04:49.562415+0000","last_peered":"2023-09-23T20:04:49.562415+0000","last_clean":"2023-09-23T20:04:49.562415+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:04:49.562415+0000","last_undegraded":"2023-09-23T20:04:49.562415+0000","last_fullsized":"2023-09-23T20:04:49.562415+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":77,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907047+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:04:49.907047+0000","last_peered":"2023-09-23T20:04:49.907047+0000","last_clean":"2023-09-23T20:04:49.907047+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:04:49.907047+0000","last_undegraded":"2023-09-23T20:04:49.907047+0000","last_fullsized":"2023-09-23T20:04:49.907047+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":21,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.562190+0000","last_change":"2023-09-23T20:03:59.232022+0000","last_active":"2023-09-23T20:04:49.562190+0000","last_peered":"2023-09-23T20:04:49.562190+0000","last_clean":"2023-09-23T20:04:49.562190+0000","last_became_active":"2023-09-23T20:03:59.231689+0000","last_became_peered":"2023-09-23T20:03:59.231689+0000","last_unstale":"2023-09-23T20:04:49.562190+0000","last_undegraded":"2023-09-23T20:04:49.562190+0000","last_fullsized":"2023-09-23T20:04:49.562190+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:29.669102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"284'9728","reported_seq":20016,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:05:00.576219+0000","last_change":"2023-09-23T20:04:02.290062+0000","last_active":"2023-09-23T20:05:00.576219+0000","last_peered":"2023-09-23T20:05:00.576219+0000","last_clean":"2023-09-23T20:05:00.576219+0000","last_became_active":"2023-09-23T20:04:02.271287+0000","last_became_peered":"2023-09-23T20:04:02.271287+0000","last_unstale":"2023-09-23T20:05:00.576219+0000","last_undegraded":"2023-09-23T20:05:00.576219+0000","last_fullsized":"2023-09-23T20:05:00.576219+0000","mapping_epoch":276,"log_start":"248'3200","ondisk_log_start":"248'3200","created":240,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:47.344179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154402816,"num_objects":2356,"num_object_clones":0,"num_object_copies":7068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9728,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5100,"num_bytes_recovered":334233600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":245,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.566035+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:04:49.566035+0000","last_peered":"2023-09-23T20:04:49.566035+0000","last_clean":"2023-09-23T20:04:49.566035+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:04:49.566035+0000","last_undegraded":"2023-09-23T20:04:49.566035+0000","last_fullsized":"2023-09-23T20:04:49.566035+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":272,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907006+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:04:49.907006+0000","last_peered":"2023-09-23T20:04:49.907006+0000","last_clean":"2023-09-23T20:04:49.907006+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:04:49.907006+0000","last_undegraded":"2023-09-23T20:04:49.907006+0000","last_fullsized":"2023-09-23T20:04:49.907006+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"284'9837","reported_seq":9891,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:05:02.194642+0000","last_change":"2023-09-23T20:01:39.911588+0000","last_active":"2023-09-23T20:05:02.194642+0000","last_peered":"2023-09-23T20:05:02.194642+0000","last_clean":"2023-09-23T20:05:02.194642+0000","last_became_active":"2023-09-23T20:01:39.911364+0000","last_became_peered":"2023-09-23T20:01:39.911364+0000","last_unstale":"2023-09-23T20:05:02.194642+0000","last_undegraded":"2023-09-23T20:05:02.194642+0000","last_fullsized":"2023-09-23T20:05:02.194642+0000","mapping_epoch":240,"log_start":"252'4095","ondisk_log_start":"252'4095","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5742,"log_dups_size":0,"ondisk_log_size":5742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:35.715001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145031168,"num_objects":2213,"num_object_clones":0,"num_object_copies":6639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9837,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":246,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.566160+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T20:04:49.566160+0000","last_peered":"2023-09-23T20:04:49.566160+0000","last_clean":"2023-09-23T20:04:49.566160+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T20:04:49.566160+0000","last_undegraded":"2023-09-23T20:04:49.566160+0000","last_fullsized":"2023-09-23T20:04:49.566160+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":300,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564198+0000","last_change":"2023-09-23T20:03:59.253784+0000","last_active":"2023-09-23T20:04:49.564198+0000","last_peered":"2023-09-23T20:04:49.564198+0000","last_clean":"2023-09-23T20:04:49.564198+0000","last_became_active":"2023-09-23T20:03:59.253552+0000","last_became_peered":"2023-09-23T20:03:59.253552+0000","last_unstale":"2023-09-23T20:04:49.564198+0000","last_undegraded":"2023-09-23T20:04:49.564198+0000","last_fullsized":"2023-09-23T20:04:49.564198+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:56.110746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"284'9750","reported_seq":9804,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:05:00.572757+0000","last_change":"2023-09-23T20:01:39.976582+0000","last_active":"2023-09-23T20:05:00.572757+0000","last_peered":"2023-09-23T20:05:00.572757+0000","last_clean":"2023-09-23T20:05:00.572757+0000","last_became_active":"2023-09-23T20:01:39.976244+0000","last_became_peered":"2023-09-23T20:01:39.976244+0000","last_unstale":"2023-09-23T20:05:00.572757+0000","last_undegraded":"2023-09-23T20:05:00.572757+0000","last_fullsized":"2023-09-23T20:05:00.572757+0000","mapping_epoch":240,"log_start":"248'3200","ondisk_log_start":"248'3200","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.952295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150470656,"num_objects":2296,"num_object_clones":0,"num_object_copies":6888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9750,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":249,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.566009+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:04:49.566009+0000","last_peered":"2023-09-23T20:04:49.566009+0000","last_clean":"2023-09-23T20:04:49.566009+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:04:49.566009+0000","last_undegraded":"2023-09-23T20:04:49.566009+0000","last_fullsized":"2023-09-23T20:04:49.566009+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":211,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.908986+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T20:04:49.908986+0000","last_peered":"2023-09-23T20:04:49.908986+0000","last_clean":"2023-09-23T20:04:49.908986+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T20:04:49.908986+0000","last_undegraded":"2023-09-23T20:04:49.908986+0000","last_fullsized":"2023-09-23T20:04:49.908986+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.284038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"284'9971","reported_seq":10025,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:04:58.258968+0000","last_change":"2023-09-23T20:01:39.912566+0000","last_active":"2023-09-23T20:04:58.258968+0000","last_peered":"2023-09-23T20:04:58.258968+0000","last_clean":"2023-09-23T20:04:58.258968+0000","last_became_active":"2023-09-23T20:01:39.912259+0000","last_became_peered":"2023-09-23T20:01:39.912259+0000","last_unstale":"2023-09-23T20:04:58.258968+0000","last_undegraded":"2023-09-23T20:04:58.258968+0000","last_fullsized":"2023-09-23T20:04:58.258968+0000","mapping_epoch":240,"log_start":"246'1100","ondisk_log_start":"246'1100","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8871,"log_dups_size":0,"ondisk_log_size":8871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:19.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162201600,"num_objects":2475,"num_object_clones":0,"num_object_copies":7425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9971,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":302,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907311+0000","last_change":"2023-09-23T20:00:34.468296+0000","last_active":"2023-09-23T20:04:49.907311+0000","last_peered":"2023-09-23T20:04:49.907311+0000","last_clean":"2023-09-23T20:04:49.907311+0000","last_became_active":"2023-09-23T20:00:34.467957+0000","last_became_peered":"2023-09-23T20:00:34.467957+0000","last_unstale":"2023-09-23T20:04:49.907311+0000","last_undegraded":"2023-09-23T20:04:49.907311+0000","last_fullsized":"2023-09-23T20:04:49.907311+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:50.449027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":99,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907491+0000","last_change":"2023-09-23T20:01:04.470558+0000","last_active":"2023-09-23T20:04:49.907491+0000","last_peered":"2023-09-23T20:04:49.907491+0000","last_clean":"2023-09-23T20:04:49.907491+0000","last_became_active":"2023-09-23T20:01:04.470276+0000","last_became_peered":"2023-09-23T20:01:04.470276+0000","last_unstale":"2023-09-23T20:04:49.907491+0000","last_undegraded":"2023-09-23T20:04:49.907491+0000","last_fullsized":"2023-09-23T20:04:49.907491+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:17.788583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"284'9865","reported_seq":9919,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:05:02.211639+0000","last_change":"2023-09-23T20:01:39.974611+0000","last_active":"2023-09-23T20:05:02.211639+0000","last_peered":"2023-09-23T20:05:02.211639+0000","last_clean":"2023-09-23T20:05:02.211639+0000","last_became_active":"2023-09-23T20:01:39.974318+0000","last_became_peered":"2023-09-23T20:01:39.974318+0000","last_unstale":"2023-09-23T20:05:02.211639+0000","last_undegraded":"2023-09-23T20:05:02.211639+0000","last_fullsized":"2023-09-23T20:05:02.211639+0000","mapping_epoch":240,"log_start":"252'4127","ondisk_log_start":"252'4127","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:31.047765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143065088,"num_objects":2183,"num_object_clones":0,"num_object_copies":6549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9865,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":734,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.565567+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:04:49.565567+0000","last_peered":"2023-09-23T20:04:49.565567+0000","last_clean":"2023-09-23T20:04:49.565567+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:04:49.565567+0000","last_undegraded":"2023-09-23T20:04:49.565567+0000","last_fullsized":"2023-09-23T20:04:49.565567+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.561618+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:04:49.561618+0000","last_peered":"2023-09-23T20:04:49.561618+0000","last_clean":"2023-09-23T20:04:49.561618+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:04:49.561618+0000","last_undegraded":"2023-09-23T20:04:49.561618+0000","last_fullsized":"2023-09-23T20:04:49.561618+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":280,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564243+0000","last_change":"2023-09-23T19:59:09.206075+0000","last_active":"2023-09-23T20:04:49.564243+0000","last_peered":"2023-09-23T20:04:49.564243+0000","last_clean":"2023-09-23T20:04:49.564243+0000","last_became_active":"2023-09-23T19:59:09.205766+0000","last_became_peered":"2023-09-23T19:59:09.205766+0000","last_unstale":"2023-09-23T20:04:49.564243+0000","last_undegraded":"2023-09-23T20:04:49.564243+0000","last_fullsized":"2023-09-23T20:04:49.564243+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:14.627703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"284'9519","reported_seq":9573,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:04:58.202452+0000","last_change":"2023-09-23T20:01:39.976550+0000","last_active":"2023-09-23T20:04:58.202452+0000","last_peered":"2023-09-23T20:04:58.202452+0000","last_clean":"2023-09-23T20:04:58.202452+0000","last_became_active":"2023-09-23T20:01:39.976245+0000","last_became_peered":"2023-09-23T20:01:39.976245+0000","last_unstale":"2023-09-23T20:04:58.202452+0000","last_undegraded":"2023-09-23T20:04:58.202452+0000","last_fullsized":"2023-09-23T20:04:58.202452+0000","mapping_epoch":240,"log_start":"244'600","ondisk_log_start":"244'600","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8919,"log_dups_size":0,"ondisk_log_size":8919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:45.979020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153681920,"num_objects":2345,"num_object_clones":0,"num_object_copies":7035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9519,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":643,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.906930+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:04:49.906930+0000","last_peered":"2023-09-23T20:04:49.906930+0000","last_clean":"2023-09-23T20:04:49.906930+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:04:49.906930+0000","last_undegraded":"2023-09-23T20:04:49.906930+0000","last_fullsized":"2023-09-23T20:04:49.906930+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":230,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.563950+0000","last_change":"2023-09-23T19:59:09.172601+0000","last_active":"2023-09-23T20:04:49.563950+0000","last_peered":"2023-09-23T20:04:49.563950+0000","last_clean":"2023-09-23T20:04:49.563950+0000","last_became_active":"2023-09-23T19:59:09.172297+0000","last_became_peered":"2023-09-23T19:59:09.172297+0000","last_unstale":"2023-09-23T20:04:49.563950+0000","last_undegraded":"2023-09-23T20:04:49.563950+0000","last_fullsized":"2023-09-23T20:04:49.563950+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:09.528266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":260,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564149+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T20:04:49.564149+0000","last_peered":"2023-09-23T20:04:49.564149+0000","last_clean":"2023-09-23T20:04:49.564149+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T20:04:49.564149+0000","last_undegraded":"2023-09-23T20:04:49.564149+0000","last_fullsized":"2023-09-23T20:04:49.564149+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"284'9769","reported_seq":9824,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:05:00.598481+0000","last_change":"2023-09-23T20:01:39.977019+0000","last_active":"2023-09-23T20:05:00.598481+0000","last_peered":"2023-09-23T20:05:00.598481+0000","last_clean":"2023-09-23T20:05:00.598481+0000","last_became_active":"2023-09-23T20:01:39.976270+0000","last_became_peered":"2023-09-23T20:01:39.976270+0000","last_unstale":"2023-09-23T20:05:00.598481+0000","last_undegraded":"2023-09-23T20:05:00.598481+0000","last_fullsized":"2023-09-23T20:05:00.598481+0000","mapping_epoch":240,"log_start":"248'3200","ondisk_log_start":"248'3200","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.226759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152174616,"num_objects":2323,"num_object_clones":0,"num_object_copies":6969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2323,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9769,"num_write_kb":386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":244,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564029+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:04:49.564029+0000","last_peered":"2023-09-23T20:04:49.564029+0000","last_clean":"2023-09-23T20:04:49.564029+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:04:49.564029+0000","last_undegraded":"2023-09-23T20:04:49.564029+0000","last_fullsized":"2023-09-23T20:04:49.564029+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":290,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564464+0000","last_change":"2023-09-23T19:59:09.171926+0000","last_active":"2023-09-23T20:04:49.564464+0000","last_peered":"2023-09-23T20:04:49.564464+0000","last_clean":"2023-09-23T20:04:49.564464+0000","last_became_active":"2023-09-23T19:59:09.171609+0000","last_became_peered":"2023-09-23T19:59:09.171609+0000","last_unstale":"2023-09-23T20:04:49.564464+0000","last_undegraded":"2023-09-23T20:04:49.564464+0000","last_fullsized":"2023-09-23T20:04:49.564464+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:21.795329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"284'9791","reported_seq":16923,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:05:02.228535+0000","last_change":"2023-09-23T20:04:50.012534+0000","last_active":"2023-09-23T20:05:02.228535+0000","last_peered":"2023-09-23T20:05:02.228535+0000","last_clean":"2023-09-23T20:05:02.228535+0000","last_became_active":"2023-09-23T20:04:49.916541+0000","last_became_peered":"2023-09-23T20:04:49.916541+0000","last_unstale":"2023-09-23T20:05:02.228535+0000","last_undegraded":"2023-09-23T20:05:02.228535+0000","last_fullsized":"2023-09-23T20:05:02.228535+0000","mapping_epoch":282,"log_start":"252'4123","ondisk_log_start":"252'4123","created":240,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5668,"log_dups_size":0,"ondisk_log_size":5668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:58.571894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144900096,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9791,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3528,"num_bytes_recovered":231145472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":622,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907228+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:04:49.907228+0000","last_peered":"2023-09-23T20:04:49.907228+0000","last_clean":"2023-09-23T20:04:49.907228+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:04:49.907228+0000","last_undegraded":"2023-09-23T20:04:49.907228+0000","last_fullsized":"2023-09-23T20:04:49.907228+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.560727+0000","last_change":"2023-09-23T20:03:59.233742+0000","last_active":"2023-09-23T20:04:49.560727+0000","last_peered":"2023-09-23T20:04:49.560727+0000","last_clean":"2023-09-23T20:04:49.560727+0000","last_became_active":"2023-09-23T20:03:59.233209+0000","last_became_peered":"2023-09-23T20:03:59.233209+0000","last_unstale":"2023-09-23T20:04:49.560727+0000","last_undegraded":"2023-09-23T20:04:49.560727+0000","last_fullsized":"2023-09-23T20:04:49.560727+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:50.879925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":244,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907796+0000","last_change":"2023-09-23T19:59:09.153852+0000","last_active":"2023-09-23T20:04:49.907796+0000","last_peered":"2023-09-23T20:04:49.907796+0000","last_clean":"2023-09-23T20:04:49.907796+0000","last_became_active":"2023-09-23T19:59:09.153514+0000","last_became_peered":"2023-09-23T19:59:09.153514+0000","last_unstale":"2023-09-23T20:04:49.907796+0000","last_undegraded":"2023-09-23T20:04:49.907796+0000","last_fullsized":"2023-09-23T20:04:49.907796+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:45.179600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"284'9730","reported_seq":9810,"reported_epoch":284,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:04:58.515066+0000","last_change":"2023-09-23T20:04:02.297387+0000","last_active":"2023-09-23T20:04:58.515066+0000","last_peered":"2023-09-23T20:04:58.515066+0000","last_clean":"2023-09-23T20:03:58.080766+0000","last_became_active":"2023-09-23T20:04:00.582511+0000","last_became_peered":"2023-09-23T20:04:00.582511+0000","last_unstale":"2023-09-23T20:04:58.515066+0000","last_undegraded":"2023-09-23T20:04:58.515066+0000","last_fullsized":"2023-09-23T20:04:58.515066+0000","mapping_epoch":274,"log_start":"248'3200","ondisk_log_start":"248'3200","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:56.089695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160169984,"num_objects":2444,"num_object_clones":0,"num_object_copies":7332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2442,"num_objects_unfound":0,"num_objects_dirty":2444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9730,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,1,0],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":2444}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":607,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.562247+0000","last_change":"2023-09-23T20:04:02.277887+0000","last_active":"2023-09-23T20:04:49.562247+0000","last_peered":"2023-09-23T20:04:49.562247+0000","last_clean":"2023-09-23T20:04:49.562247+0000","last_became_active":"2023-09-23T20:04:02.277519+0000","last_became_peered":"2023-09-23T20:04:02.277519+0000","last_unstale":"2023-09-23T20:04:49.562247+0000","last_undegraded":"2023-09-23T20:04:49.562247+0000","last_fullsized":"2023-09-23T20:04:49.562247+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":623,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.908301+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T20:04:49.908301+0000","last_peered":"2023-09-23T20:04:49.908301+0000","last_clean":"2023-09-23T20:04:49.908301+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T20:04:49.908301+0000","last_undegraded":"2023-09-23T20:04:49.908301+0000","last_fullsized":"2023-09-23T20:04:49.908301+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907206+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:04:49.907206+0000","last_peered":"2023-09-23T20:04:49.907206+0000","last_clean":"2023-09-23T20:04:49.907206+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:04:49.907206+0000","last_undegraded":"2023-09-23T20:04:49.907206+0000","last_fullsized":"2023-09-23T20:04:49.907206+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":212,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.908771+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T20:04:49.908771+0000","last_peered":"2023-09-23T20:04:49.908771+0000","last_clean":"2023-09-23T20:04:49.908771+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T20:04:49.908771+0000","last_undegraded":"2023-09-23T20:04:49.908771+0000","last_fullsized":"2023-09-23T20:04:49.908771+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"284'9909","reported_seq":9978,"reported_epoch":284,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:05:02.198108+0000","last_change":"2023-09-23T20:04:00.730874+0000","last_active":"2023-09-23T20:05:02.198108+0000","last_peered":"2023-09-23T20:05:02.198108+0000","last_clean":"2023-09-23T20:03:58.087054+0000","last_became_active":"2023-09-23T20:04:00.560120+0000","last_became_peered":"2023-09-23T20:04:00.560120+0000","last_unstale":"2023-09-23T20:05:02.198108+0000","last_undegraded":"2023-09-23T20:05:02.198108+0000","last_fullsized":"2023-09-23T20:05:02.198108+0000","mapping_epoch":274,"log_start":"248'3300","ondisk_log_start":"248'3300","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:17.026687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152371200,"num_objects":2325,"num_object_clones":0,"num_object_copies":6975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2325,"num_objects_unfound":0,"num_objects_dirty":2325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9909,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":2325}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":595,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907065+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:04:49.907065+0000","last_peered":"2023-09-23T20:04:49.907065+0000","last_clean":"2023-09-23T20:04:49.907065+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:04:49.907065+0000","last_undegraded":"2023-09-23T20:04:49.907065+0000","last_fullsized":"2023-09-23T20:04:49.907065+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":844,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907069+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T20:04:49.907069+0000","last_peered":"2023-09-23T20:04:49.907069+0000","last_clean":"2023-09-23T20:04:49.907069+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T20:04:49.907069+0000","last_undegraded":"2023-09-23T20:04:49.907069+0000","last_fullsized":"2023-09-23T20:04:49.907069+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:58.623015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":283,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564282+0000","last_change":"2023-09-23T20:00:34.447064+0000","last_active":"2023-09-23T20:04:49.564282+0000","last_peered":"2023-09-23T20:04:49.564282+0000","last_clean":"2023-09-23T20:04:49.564282+0000","last_became_active":"2023-09-23T20:00:34.446754+0000","last_became_peered":"2023-09-23T20:00:34.446754+0000","last_unstale":"2023-09-23T20:04:49.564282+0000","last_undegraded":"2023-09-23T20:04:49.564282+0000","last_fullsized":"2023-09-23T20:04:49.564282+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:39.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":278,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.908222+0000","last_change":"2023-09-23T20:03:59.236594+0000","last_active":"2023-09-23T20:04:49.908222+0000","last_peered":"2023-09-23T20:04:49.908222+0000","last_clean":"2023-09-23T20:04:49.908222+0000","last_became_active":"2023-09-23T20:03:59.236305+0000","last_became_peered":"2023-09-23T20:03:59.236305+0000","last_unstale":"2023-09-23T20:04:49.908222+0000","last_undegraded":"2023-09-23T20:04:49.908222+0000","last_fullsized":"2023-09-23T20:04:49.908222+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:23.103726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"284'9662","reported_seq":9714,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:04:58.508793+0000","last_change":"2023-09-23T20:01:39.977018+0000","last_active":"2023-09-23T20:04:58.508793+0000","last_peered":"2023-09-23T20:04:58.508793+0000","last_clean":"2023-09-23T20:04:58.508793+0000","last_became_active":"2023-09-23T20:01:39.976359+0000","last_became_peered":"2023-09-23T20:01:39.976359+0000","last_unstale":"2023-09-23T20:04:58.508793+0000","last_undegraded":"2023-09-23T20:04:58.508793+0000","last_fullsized":"2023-09-23T20:04:58.508793+0000","mapping_epoch":240,"log_start":"248'3100","ondisk_log_start":"248'3100","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.221346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156893184,"num_objects":2394,"num_object_clones":0,"num_object_copies":7182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9662,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":519,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.565686+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:04:49.565686+0000","last_peered":"2023-09-23T20:04:49.565686+0000","last_clean":"2023-09-23T20:04:49.565686+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:04:49.565686+0000","last_undegraded":"2023-09-23T20:04:49.565686+0000","last_fullsized":"2023-09-23T20:04:49.565686+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":917,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:04:53.632702+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:04:53.632702+0000","last_peered":"2023-09-23T20:04:53.632702+0000","last_clean":"2023-09-23T20:04:53.632702+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:04:53.632702+0000","last_undegraded":"2023-09-23T20:04:53.632702+0000","last_fullsized":"2023-09-23T20:04:53.632702+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":241,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907481+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T20:04:49.907481+0000","last_peered":"2023-09-23T20:04:49.907481+0000","last_clean":"2023-09-23T20:04:49.907481+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T20:04:49.907481+0000","last_undegraded":"2023-09-23T20:04:49.907481+0000","last_fullsized":"2023-09-23T20:04:49.907481+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.564126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":256,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:04:53.631614+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T20:04:53.631614+0000","last_peered":"2023-09-23T20:04:53.631614+0000","last_clean":"2023-09-23T20:04:53.631614+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T20:04:53.631614+0000","last_undegraded":"2023-09-23T20:04:53.631614+0000","last_fullsized":"2023-09-23T20:04:53.631614+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"284'9835","reported_seq":14152,"reported_epoch":284,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:04:58.533267+0000","last_change":"2023-09-23T20:04:48.228520+0000","last_active":"2023-09-23T20:04:58.533267+0000","last_peered":"2023-09-23T20:04:58.533267+0000","last_clean":"2023-09-23T20:03:58.125973+0000","last_became_active":"2023-09-23T20:04:00.586359+0000","last_became_peered":"2023-09-23T20:04:00.586359+0000","last_unstale":"2023-09-23T20:04:58.533267+0000","last_undegraded":"2023-09-23T20:04:58.533267+0000","last_fullsized":"2023-09-23T20:04:58.533267+0000","mapping_epoch":274,"log_start":"247'2456","ondisk_log_start":"247'2456","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7379,"log_dups_size":0,"ondisk_log_size":7379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:05.232047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162725888,"num_objects":2483,"num_object_clones":0,"num_object_copies":7449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":680,"num_objects_unfound":0,"num_objects_dirty":2483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9835,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2124,"num_bytes_recovered":139198464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,5],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":2483}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1092,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907475+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:04:49.907475+0000","last_peered":"2023-09-23T20:04:49.907475+0000","last_clean":"2023-09-23T20:04:49.907475+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:04:49.907475+0000","last_undegraded":"2023-09-23T20:04:49.907475+0000","last_fullsized":"2023-09-23T20:04:49.907475+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":694,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.565596+0000","last_change":"2023-09-23T20:00:34.445036+0000","last_active":"2023-09-23T20:04:49.565596+0000","last_peered":"2023-09-23T20:04:49.565596+0000","last_clean":"2023-09-23T20:04:49.565596+0000","last_became_active":"2023-09-23T20:00:34.444689+0000","last_became_peered":"2023-09-23T20:00:34.444689+0000","last_unstale":"2023-09-23T20:04:49.565596+0000","last_undegraded":"2023-09-23T20:04:49.565596+0000","last_fullsized":"2023-09-23T20:04:49.565596+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:44.485721+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.560913+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:04:49.560913+0000","last_peered":"2023-09-23T20:04:49.560913+0000","last_clean":"2023-09-23T20:04:49.560913+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:04:49.560913+0000","last_undegraded":"2023-09-23T20:04:49.560913+0000","last_fullsized":"2023-09-23T20:04:49.560913+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":159,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564352+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:04:49.564352+0000","last_peered":"2023-09-23T20:04:49.564352+0000","last_clean":"2023-09-23T20:04:49.564352+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:04:49.564352+0000","last_undegraded":"2023-09-23T20:04:49.564352+0000","last_fullsized":"2023-09-23T20:04:49.564352+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"284'9802","reported_seq":9852,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:04:58.510098+0000","last_change":"2023-09-23T20:01:39.977010+0000","last_active":"2023-09-23T20:04:58.510098+0000","last_peered":"2023-09-23T20:04:58.510098+0000","last_clean":"2023-09-23T20:04:58.510098+0000","last_became_active":"2023-09-23T20:01:39.976722+0000","last_became_peered":"2023-09-23T20:01:39.976722+0000","last_unstale":"2023-09-23T20:04:58.510098+0000","last_undegraded":"2023-09-23T20:04:58.510098+0000","last_fullsized":"2023-09-23T20:04:58.510098+0000","mapping_epoch":240,"log_start":"247'2431","ondisk_log_start":"247'2431","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7371,"log_dups_size":0,"ondisk_log_size":7371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:13.592964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159383552,"num_objects":2432,"num_object_clones":0,"num_object_copies":7296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9802,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":624,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907608+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T20:04:49.907608+0000","last_peered":"2023-09-23T20:04:49.907608+0000","last_clean":"2023-09-23T20:04:49.907608+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T20:04:49.907608+0000","last_undegraded":"2023-09-23T20:04:49.907608+0000","last_fullsized":"2023-09-23T20:04:49.907608+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.842550+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1002,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.565540+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T20:04:49.565540+0000","last_peered":"2023-09-23T20:04:49.565540+0000","last_clean":"2023-09-23T20:04:49.565540+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T20:04:49.565540+0000","last_undegraded":"2023-09-23T20:04:49.565540+0000","last_fullsized":"2023-09-23T20:04:49.565540+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":250,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564285+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:04:49.564285+0000","last_peered":"2023-09-23T20:04:49.564285+0000","last_clean":"2023-09-23T20:04:49.564285+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:04:49.564285+0000","last_undegraded":"2023-09-23T20:04:49.564285+0000","last_fullsized":"2023-09-23T20:04:49.564285+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":246,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907473+0000","last_change":"2023-09-23T20:03:10.175462+0000","last_active":"2023-09-23T20:04:49.907473+0000","last_peered":"2023-09-23T20:04:49.907473+0000","last_clean":"2023-09-23T20:04:49.907473+0000","last_became_active":"2023-09-23T20:03:10.175250+0000","last_became_peered":"2023-09-23T20:03:10.175250+0000","last_unstale":"2023-09-23T20:04:49.907473+0000","last_undegraded":"2023-09-23T20:04:49.907473+0000","last_fullsized":"2023-09-23T20:04:49.907473+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:42.961619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"284'9510","reported_seq":9562,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:04:58.033061+0000","last_change":"2023-09-23T20:01:39.913385+0000","last_active":"2023-09-23T20:04:58.033061+0000","last_peered":"2023-09-23T20:04:58.033061+0000","last_clean":"2023-09-23T20:04:58.033061+0000","last_became_active":"2023-09-23T20:01:39.912872+0000","last_became_peered":"2023-09-23T20:01:39.912872+0000","last_unstale":"2023-09-23T20:04:58.033061+0000","last_undegraded":"2023-09-23T20:04:58.033061+0000","last_fullsized":"2023-09-23T20:04:58.033061+0000","mapping_epoch":240,"log_start":"246'1100","ondisk_log_start":"246'1100","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8410,"log_dups_size":0,"ondisk_log_size":8410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.619813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162922496,"num_objects":2486,"num_object_clones":0,"num_object_copies":7458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9510,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":599,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.562251+0000","last_change":"2023-09-23T20:04:02.277648+0000","last_active":"2023-09-23T20:04:49.562251+0000","last_peered":"2023-09-23T20:04:49.562251+0000","last_clean":"2023-09-23T20:04:49.562251+0000","last_became_active":"2023-09-23T20:04:02.277271+0000","last_became_peered":"2023-09-23T20:04:02.277271+0000","last_unstale":"2023-09-23T20:04:49.562251+0000","last_undegraded":"2023-09-23T20:04:49.562251+0000","last_fullsized":"2023-09-23T20:04:49.562251+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":362,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.566001+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:04:49.566001+0000","last_peered":"2023-09-23T20:04:49.566001+0000","last_clean":"2023-09-23T20:04:49.566001+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:04:49.566001+0000","last_undegraded":"2023-09-23T20:04:49.566001+0000","last_fullsized":"2023-09-23T20:04:49.566001+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":244,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.908385+0000","last_change":"2023-09-23T20:03:10.174842+0000","last_active":"2023-09-23T20:04:49.908385+0000","last_peered":"2023-09-23T20:04:49.908385+0000","last_clean":"2023-09-23T20:04:49.908385+0000","last_became_active":"2023-09-23T20:03:10.174511+0000","last_became_peered":"2023-09-23T20:03:10.174511+0000","last_unstale":"2023-09-23T20:04:49.908385+0000","last_undegraded":"2023-09-23T20:04:49.908385+0000","last_fullsized":"2023-09-23T20:04:49.908385+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:12.723811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":80,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907524+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:04:49.907524+0000","last_peered":"2023-09-23T20:04:49.907524+0000","last_clean":"2023-09-23T20:04:49.907524+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:04:49.907524+0000","last_undegraded":"2023-09-23T20:04:49.907524+0000","last_fullsized":"2023-09-23T20:04:49.907524+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"284'9912","reported_seq":9966,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:05:02.207204+0000","last_change":"2023-09-23T20:01:39.912135+0000","last_active":"2023-09-23T20:05:02.207204+0000","last_peered":"2023-09-23T20:05:02.207204+0000","last_clean":"2023-09-23T20:05:02.207204+0000","last_became_active":"2023-09-23T20:01:39.911834+0000","last_became_peered":"2023-09-23T20:01:39.911834+0000","last_unstale":"2023-09-23T20:05:02.207204+0000","last_undegraded":"2023-09-23T20:05:02.207204+0000","last_fullsized":"2023-09-23T20:05:02.207204+0000","mapping_epoch":240,"log_start":"253'4226","ondisk_log_start":"253'4226","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:46.725578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152567808,"num_objects":2328,"num_object_clones":0,"num_object_copies":6984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9912,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":662,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564187+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:04:49.564187+0000","last_peered":"2023-09-23T20:04:49.564187+0000","last_clean":"2023-09-23T20:04:49.564187+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:04:49.564187+0000","last_undegraded":"2023-09-23T20:04:49.564187+0000","last_fullsized":"2023-09-23T20:04:49.564187+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.562374+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:04:49.562374+0000","last_peered":"2023-09-23T20:04:49.562374+0000","last_clean":"2023-09-23T20:04:49.562374+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:04:49.562374+0000","last_undegraded":"2023-09-23T20:04:49.562374+0000","last_fullsized":"2023-09-23T20:04:49.562374+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":690,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:04:53.636217+0000","last_change":"2023-09-23T19:59:40.899753+0000","last_active":"2023-09-23T20:04:53.636217+0000","last_peered":"2023-09-23T20:04:53.636217+0000","last_clean":"2023-09-23T20:04:53.636217+0000","last_became_active":"2023-09-23T19:59:40.899147+0000","last_became_peered":"2023-09-23T19:59:40.899147+0000","last_unstale":"2023-09-23T20:04:53.636217+0000","last_undegraded":"2023-09-23T20:04:53.636217+0000","last_fullsized":"2023-09-23T20:04:53.636217+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:08.938755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":21,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.560752+0000","last_change":"2023-09-23T20:03:59.233560+0000","last_active":"2023-09-23T20:04:49.560752+0000","last_peered":"2023-09-23T20:04:49.560752+0000","last_clean":"2023-09-23T20:04:49.560752+0000","last_became_active":"2023-09-23T20:03:59.232842+0000","last_became_peered":"2023-09-23T20:03:59.232842+0000","last_unstale":"2023-09-23T20:04:49.560752+0000","last_undegraded":"2023-09-23T20:04:49.560752+0000","last_fullsized":"2023-09-23T20:04:49.560752+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:16.909413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"284'9941","reported_seq":9995,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-09-23T20:05:02.161502+0000","last_change":"2023-09-23T20:01:39.913698+0000","last_active":"2023-09-23T20:05:02.161502+0000","last_peered":"2023-09-23T20:05:02.161502+0000","last_clean":"2023-09-23T20:05:02.161502+0000","last_became_active":"2023-09-23T20:01:39.913367+0000","last_became_peered":"2023-09-23T20:01:39.913367+0000","last_unstale":"2023-09-23T20:05:02.161502+0000","last_undegraded":"2023-09-23T20:05:02.161502+0000","last_fullsized":"2023-09-23T20:05:02.161502+0000","mapping_epoch":240,"log_start":"253'4225","ondisk_log_start":"253'4225","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150405120,"num_objects":2295,"num_object_clones":0,"num_object_copies":6885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9941,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":540,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.562089+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:04:49.562089+0000","last_peered":"2023-09-23T20:04:49.562089+0000","last_clean":"2023-09-23T20:04:49.562089+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:04:49.562089+0000","last_undegraded":"2023-09-23T20:04:49.562089+0000","last_fullsized":"2023-09-23T20:04:49.562089+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":40,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.566087+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:04:49.566087+0000","last_peered":"2023-09-23T20:04:49.566087+0000","last_clean":"2023-09-23T20:04:49.566087+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:04:49.566087+0000","last_undegraded":"2023-09-23T20:04:49.566087+0000","last_fullsized":"2023-09-23T20:04:49.566087+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":87,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907429+0000","last_change":"2023-09-23T20:01:04.470744+0000","last_active":"2023-09-23T20:04:49.907429+0000","last_peered":"2023-09-23T20:04:49.907429+0000","last_clean":"2023-09-23T20:04:49.907429+0000","last_became_active":"2023-09-23T20:01:04.470302+0000","last_became_peered":"2023-09-23T20:01:04.470302+0000","last_unstale":"2023-09-23T20:04:49.907429+0000","last_undegraded":"2023-09-23T20:04:49.907429+0000","last_fullsized":"2023-09-23T20:04:49.907429+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:25.452596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":243,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564110+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:04:49.564110+0000","last_peered":"2023-09-23T20:04:49.564110+0000","last_clean":"2023-09-23T20:04:49.564110+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:04:49.564110+0000","last_undegraded":"2023-09-23T20:04:49.564110+0000","last_fullsized":"2023-09-23T20:04:49.564110+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":29,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907651+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:04:49.907651+0000","last_peered":"2023-09-23T20:04:49.907651+0000","last_clean":"2023-09-23T20:04:49.907651+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:04:49.907651+0000","last_undegraded":"2023-09-23T20:04:49.907651+0000","last_fullsized":"2023-09-23T20:04:49.907651+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":37,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907166+0000","last_change":"2023-09-23T20:03:10.178662+0000","last_active":"2023-09-23T20:04:49.907166+0000","last_peered":"2023-09-23T20:04:49.907166+0000","last_clean":"2023-09-23T20:04:49.907166+0000","last_became_active":"2023-09-23T20:03:10.178352+0000","last_became_peered":"2023-09-23T20:03:10.178352+0000","last_unstale":"2023-09-23T20:04:49.907166+0000","last_undegraded":"2023-09-23T20:04:49.907166+0000","last_fullsized":"2023-09-23T20:04:49.907166+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.469897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":245,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907261+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T20:04:49.907261+0000","last_peered":"2023-09-23T20:04:49.907261+0000","last_clean":"2023-09-23T20:04:49.907261+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T20:04:49.907261+0000","last_undegraded":"2023-09-23T20:04:49.907261+0000","last_fullsized":"2023-09-23T20:04:49.907261+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":280,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.564360+0000","last_change":"2023-09-23T19:59:09.172452+0000","last_active":"2023-09-23T20:04:49.564360+0000","last_peered":"2023-09-23T20:04:49.564360+0000","last_clean":"2023-09-23T20:04:49.564360+0000","last_became_active":"2023-09-23T19:59:09.172080+0000","last_became_peered":"2023-09-23T19:59:09.172080+0000","last_unstale":"2023-09-23T20:04:49.564360+0000","last_undegraded":"2023-09-23T20:04:49.564360+0000","last_fullsized":"2023-09-23T20:04:49.564360+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:24.687991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":249,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907188+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T20:04:49.907188+0000","last_peered":"2023-09-23T20:04:49.907188+0000","last_clean":"2023-09-23T20:04:49.907188+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T20:04:49.907188+0000","last_undegraded":"2023-09-23T20:04:49.907188+0000","last_fullsized":"2023-09-23T20:04:49.907188+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:07.560646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":280,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.563895+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:04:49.563895+0000","last_peered":"2023-09-23T20:04:49.563895+0000","last_clean":"2023-09-23T20:04:49.563895+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:04:49.563895+0000","last_undegraded":"2023-09-23T20:04:49.563895+0000","last_fullsized":"2023-09-23T20:04:49.563895+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":106,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.562629+0000","last_change":"2023-09-23T19:59:09.145224+0000","last_active":"2023-09-23T20:04:49.562629+0000","last_peered":"2023-09-23T20:04:49.562629+0000","last_clean":"2023-09-23T20:04:49.562629+0000","last_became_active":"2023-09-23T19:59:09.144819+0000","last_became_peered":"2023-09-23T19:59:09.144819+0000","last_unstale":"2023-09-23T20:04:49.562629+0000","last_undegraded":"2023-09-23T20:04:49.562629+0000","last_fullsized":"2023-09-23T20:04:49.562629+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:04.048181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":21,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.561172+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:04:49.561172+0000","last_peered":"2023-09-23T20:04:49.561172+0000","last_clean":"2023-09-23T20:04:49.561172+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:04:49.561172+0000","last_undegraded":"2023-09-23T20:04:49.561172+0000","last_fullsized":"2023-09-23T20:04:49.561172+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":103,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.907481+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:04:49.907481+0000","last_peered":"2023-09-23T20:04:49.907481+0000","last_clean":"2023-09-23T20:04:49.907481+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:04:49.907481+0000","last_undegraded":"2023-09-23T20:04:49.907481+0000","last_fullsized":"2023-09-23T20:04:49.907481+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":310,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T20:04:49.563793+0000","last_change":"2023-09-23T20:03:59.235232+0000","last_active":"2023-09-23T20:04:49.563793+0000","last_peered":"2023-09-23T20:04:49.563793+0000","last_clean":"2023-09-23T20:04:49.563793+0000","last_became_active":"2023-09-23T20:03:59.234941+0000","last_became_peered":"2023-09-23T20:03:59.234941+0000","last_unstale":"2023-09-23T20:04:49.563793+0000","last_undegraded":"2023-09-23T20:04:49.563793+0000","last_fullsized":"2023-09-23T20:04:49.563793+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:45.591449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2463367192,"num_objects":37589,"num_object_clones":0,"num_object_copies":112767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5447,"num_objects_unfound":0,"num_objects_dirty":37589,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":156531,"num_write_kb":6211777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10758,"num_bytes_recovered":704905216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7887269888,"data_stored":7887257672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108848,"ondisk_log_size":108848,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234286,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1216756,"kb_used_data":945388,"kb_used_omap":0,"kb_used_meta":271360,"kb_avail":92528396,"statfs":{"total":95995035648,"available":94749077504,"internally_reserved":0,"allocated":968077312,"data_stored":967210845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":277872640},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:20 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5750000000000002,"15min":2.0430000000000001},"min":{"1min":0.245,"5min":0.22,"15min":0.19700000000000001},"max":{"1min":41.889000000000003,"5min":60.140000000000001,"15min":62.061999999999998},"last":7.5819999999999999},{"interface":"front","average":{"1min":4.5209999999999999,"5min":2.7120000000000002,"15min":2.27},"min":{"1min":0.25900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":41.936,"5min":62.389000000000003,"15min":62.389000000000003},"last":3.6589999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:04:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2240000000000002,"5min":5.7729999999999997,"15min":3.0739999999999998},"min":{"1min":0.27100000000000002,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":56.024999999999999,"5min":104.16800000000001,"15min":104.16800000000001},"last":3.4820000000000002},{"interface":"front","average":{"1min":6.3220000000000001,"5min":6.0590000000000002,"15min":3.2149999999999999},"min":{"1min":0.34200000000000003,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":56.146000000000001,"5min":104.003,"15min":104.003},"last":3.7719999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:04:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.794,"15min":3.1379999999999999},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.6159999999999997,"5min":105.842,"15min":130.673},"last":0.251},{"interface":"front","average":{"1min":4.0819999999999999,"5min":3.4249999999999998,"15min":3.4260000000000002},"min":{"1min":0.376,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":41.968000000000004,"5min":105.66,"15min":130.71100000000001},"last":15.163}]},{"osd":3,"last update":"Sat Sep 23 20:04:25 2023","interfaces":[{"interface":"back","average":{"1min":10.352,"5min":5.5999999999999996,"15min":3.4630000000000001},"min":{"1min":0.316,"5min":0.247,"15min":0.247},"max":{"1min":80.257999999999996,"5min":146.38800000000001,"15min":146.38800000000001},"last":3.1579999999999999},{"interface":"front","average":{"1min":10.545999999999999,"5min":5.8099999999999996,"15min":3.464},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":82.281999999999996,"5min":146.851,"15min":146.851},"last":9.4049999999999994}]},{"osd":4,"last update":"Sat Sep 23 20:04:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.80600000000000005,"15min":1.891},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.714,"5min":19.215,"15min":143.255},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.605,"5min":1.1459999999999999,"15min":1.9330000000000001},"min":{"1min":0.27000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":41.822000000000003,"5min":41.822000000000003,"15min":143.41200000000001},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:04:57 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":1.091,"15min":1.4610000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25},"max":{"1min":7.4960000000000004,"5min":35.488,"15min":84.566999999999993},"last":7.4960000000000004},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.95399999999999996,"15min":1.302},"min":{"1min":0.28599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.403,"5min":34.896000000000001,"15min":81.244},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:04:32 2023","interfaces":[{"interface":"back","average":{"1min":2.6709999999999998,"5min":1.1020000000000001,"15min":0.83999999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":41.902999999999999,"5min":41.902999999999999,"15min":41.902999999999999},"last":3.6869999999999998},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.66900000000000004,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.3620000000000001,"5min":6.3620000000000001,"15min":6.3620000000000001},"last":0.38500000000000001}]}]},{"osd":6,"up_from":0,"seq":1078036791327,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762488,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1626088,"kb_used_data":1216224,"kb_used_omap":7,"kb_used_meta":409592,"kb_avail":92119064,"statfs":{"total":95995035648,"available":94329921536,"internally_reserved":0,"allocated":1245413376,"data_stored":1244149534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":419423034},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":2.2240000000000002,"15min":3.8799999999999999},"min":{"1min":0.247,"5min":0.16900000000000001,"15min":0.151},"max":{"1min":11.710000000000001,"5min":25.238,"15min":202.917},"last":0.64400000000000002},{"interface":"front","average":{"1min":3.8780000000000001,"5min":2.4460000000000002,"15min":4.0119999999999996},"min":{"1min":0.23200000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":14.839,"5min":22.337,"15min":203.34899999999999},"last":4.8159999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:04:58 2023","interfaces":[{"interface":"back","average":{"1min":4.54,"5min":5.1639999999999997,"15min":5.5119999999999996},"min":{"1min":0.23000000000000001,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":12.554,"5min":134.72499999999999,"15min":231.57499999999999},"last":7.3399999999999999},{"interface":"front","average":{"1min":4.6079999999999997,"5min":5.6429999999999998,"15min":5.5199999999999996},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":11.881,"5min":135.74600000000001,"15min":236.077},"last":0.42699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.665,"5min":2.7349999999999999,"15min":3.016},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.217},"max":{"1min":11.98,"5min":15.571,"15min":68.180999999999997},"last":25.079999999999998},{"interface":"front","average":{"1min":5.2850000000000001,"5min":2.7749999999999999,"15min":2.8439999999999999},"min":{"1min":0.36799999999999999,"5min":0.249,"15min":0.20100000000000001},"max":{"1min":20.989999999999998,"5min":22.385999999999999,"15min":66.509},"last":9.0359999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.9450000000000001,"15min":2.4159999999999999},"min":{"1min":0.33800000000000002,"5min":0.254,"15min":0.25},"max":{"1min":9.9190000000000005,"5min":35.497999999999998,"15min":161.68299999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":3.093,"5min":2.226,"15min":2.4470000000000001},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":20.847000000000001,"5min":35.521000000000001,"15min":161.286},"last":3.0419999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":2.407,"15min":3.3610000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":20.960000000000001,"5min":51.698,"15min":219.048},"last":3.0819999999999999},{"interface":"front","average":{"1min":3.6739999999999999,"5min":2.6459999999999999,"15min":3.5510000000000002},"min":{"1min":0.40400000000000003,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":15.507999999999999,"5min":51.173000000000002,"15min":218.69499999999999},"last":7.2640000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":2.1640000000000001,"15min":2.2189999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.5579999999999998,"5min":9.8879999999999999,"15min":9.8879999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":3.2480000000000002,"5min":3.4060000000000001,"15min":3.4319999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":13.304,"5min":20.734999999999999,"15min":20.734999999999999},"last":3.1019999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":2.1869999999999998,"15min":2.1739999999999999},"min":{"1min":0.432,"5min":0.36199999999999999,"15min":0.26800000000000002},"max":{"1min":11.893000000000001,"5min":14.23,"15min":60.213000000000001},"last":7.3090000000000002},{"interface":"front","average":{"1min":2.048,"5min":1.746,"15min":1.7529999999999999},"min":{"1min":0.42599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.8309999999999995,"5min":14.188000000000001,"15min":61.298999999999999},"last":0.54800000000000004}]}]},{"osd":0,"up_from":94,"seq":403726926023,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1943208,"kb_used_data":1483360,"kb_used_omap":8,"kb_used_meta":459831,"kb_avail":91801944,"statfs":{"total":95995035648,"available":94005190656,"internally_reserved":0,"allocated":1518960640,"data_stored":1517696510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":470867553},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":5.2039999999999997,"15min":2.8119999999999998},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":7.6669999999999998,"5min":94.055999999999997,"15min":94.055999999999997},"last":1.4470000000000001},{"interface":"front","average":{"1min":3.3210000000000002,"5min":5.5010000000000003,"15min":3.1699999999999999},"min":{"1min":0.24199999999999999,"5min":0.183,"15min":0.183},"max":{"1min":9.5609999999999999,"5min":91.632999999999996,"15min":91.632999999999996},"last":8.2639999999999993}]},{"osd":2,"last update":"Sat Sep 23 20:04:56 2023","interfaces":[{"interface":"back","average":{"1min":11.645,"5min":7.7329999999999997,"15min":4.3390000000000004},"min":{"1min":0.216,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":162.62899999999999,"5min":270.51100000000002,"15min":270.51100000000002},"last":1.587},{"interface":"front","average":{"1min":10.435,"5min":7.1280000000000001,"15min":4.3120000000000003},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":162.52500000000001,"5min":269.69299999999998,"15min":269.69299999999998},"last":1.1839999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":3.9060000000000001,"5min":4.8639999999999999,"15min":4.6470000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.5890000000000004,"5min":116.733,"15min":145.72800000000001},"last":2.5449999999999999},{"interface":"front","average":{"1min":3.4449999999999998,"5min":4.9390000000000001,"15min":4.7610000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":9.6470000000000002,"5min":116.697,"15min":145.50700000000001},"last":3.6389999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.7430000000000001,"15min":2.3159999999999998},"min":{"1min":0.27600000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.3829999999999991,"5min":12.372,"15min":110.312},"last":1.2250000000000001},{"interface":"front","average":{"1min":2.27,"5min":1.659,"15min":1.964},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27600000000000002},"max":{"1min":8.7330000000000005,"5min":8.7330000000000005,"15min":110.256},"last":1.6830000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":2.181,"15min":2.4929999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.5419999999999998,"5min":64.725999999999999,"15min":131.34399999999999},"last":1.387},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.8380000000000001,"15min":2.569},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.4139999999999997,"5min":12.48,"15min":131.386},"last":1.3300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.6020000000000001,"15min":1.605},"min":{"1min":0.41199999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.4600000000000009,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":1.2929999999999999},{"interface":"front","average":{"1min":2.73,"5min":2.0699999999999998,"15min":1.96},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":9.4429999999999996,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":1.7250000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":2.2189999999999999,"15min":2.004},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":8.6039999999999992,"5min":65.081999999999994,"15min":77.132999999999996},"last":1.4259999999999999},{"interface":"front","average":{"1min":1.8400000000000001,"5min":1.9419999999999999,"15min":2.0990000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.3900000000000006,"5min":12.452,"15min":75.465999999999994},"last":1.3520000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598961,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1784784,"kb_used_data":1264200,"kb_used_omap":12,"kb_used_meta":520563,"kb_avail":91960368,"statfs":{"total":95995035648,"available":94167416832,"internally_reserved":0,"allocated":1294540800,"data_stored":1293302126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":533057534},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":3.6309999999999998,"15min":2.5019999999999998},"min":{"1min":0.20899999999999999,"5min":0.192,"15min":0.154},"max":{"1min":10.073,"5min":48.966999999999999,"15min":50.506999999999998},"last":12.539},{"interface":"front","average":{"1min":1.4690000000000001,"5min":3.2509999999999999,"15min":2.3969999999999998},"min":{"1min":0.20200000000000001,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":7.8840000000000003,"5min":48.783000000000001,"15min":50.802999999999997},"last":12.557}]},{"osd":1,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":6.4400000000000004,"5min":5.4279999999999999,"15min":2.9529999999999998},"min":{"1min":0.247,"5min":0.21299999999999999,"15min":0.184},"max":{"1min":50.192999999999998,"5min":94.768000000000001,"15min":94.768000000000001},"last":3.7280000000000002},{"interface":"front","average":{"1min":6.7729999999999997,"5min":6.0060000000000002,"15min":3.1890000000000001},"min":{"1min":0.218,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":47.987000000000002,"5min":96.808000000000007,"15min":96.808000000000007},"last":2.4380000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:04:22 2023","interfaces":[{"interface":"back","average":{"1min":4.117,"5min":3.8380000000000001,"15min":2.7970000000000002},"min":{"1min":0.307,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":13.984,"5min":44.408000000000001,"15min":50.186999999999998},"last":12.472},{"interface":"front","average":{"1min":2.2709999999999999,"5min":3.3290000000000002,"15min":2.7229999999999999},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.084,"5min":43.981999999999999,"15min":50.585000000000001},"last":2.3359999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.893,"15min":1.6890000000000001},"min":{"1min":0.30499999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.9249999999999998,"5min":20.591999999999999,"15min":50.640000000000001},"last":1.974},{"interface":"front","average":{"1min":2.3039999999999998,"5min":1.819,"15min":1.6990000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7629999999999999,"5min":23.062000000000001,"15min":50.154000000000003},"last":3.8079999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":10.351000000000001,"5min":3.9199999999999999,"15min":2.5009999999999999},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":172.505,"5min":172.505,"15min":172.505},"last":1.9399999999999999},{"interface":"front","average":{"1min":10.007,"5min":3.5630000000000002,"15min":2.347},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":172.375,"5min":172.375,"15min":172.375},"last":2.194}]},{"osd":6,"last update":"Sat Sep 23 20:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.581,"15min":1.611},"min":{"1min":0.38100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.5250000000000004,"5min":10.144,"15min":10.144},"last":2.2189999999999999},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.917,"15min":1.9510000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.1390000000000002,"5min":7.1390000000000002,"15min":7.1390000000000002},"last":2.2469999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:03:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":1.9530000000000001,"15min":1.796},"min":{"1min":0.436,"5min":0.33700000000000002,"15min":0.27400000000000002},"max":{"1min":9.7330000000000005,"5min":23.079999999999998,"15min":50.118000000000002},"last":3.782},{"interface":"front","average":{"1min":1.734,"5min":2.395,"15min":2.0680000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":10.17,"5min":20.614999999999998,"15min":50.664999999999999},"last":2.2690000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435435,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1618324,"kb_used_data":1151180,"kb_used_omap":4,"kb_used_meta":467067,"kb_avail":92126828,"statfs":{"total":95995035648,"available":94337871872,"internally_reserved":0,"allocated":1178808320,"data_stored":1177443041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":478276717},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0600000000000001,"5min":2.399,"15min":3.8929999999999998},"min":{"1min":0.193,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":10.401,"5min":21.100999999999999,"15min":240.40600000000001},"last":10.143000000000001},{"interface":"front","average":{"1min":4.1799999999999997,"5min":3.327,"15min":4.4989999999999997},"min":{"1min":0.23100000000000001,"5min":0.19600000000000001,"15min":0.17199999999999999},"max":{"1min":16.978000000000002,"5min":53.201000000000001,"15min":240.352},"last":2.343}]},{"osd":1,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.4569999999999999,"5min":4.1920000000000002,"15min":2.2799999999999998},"min":{"1min":0.23000000000000001,"5min":0.217,"15min":0.20000000000000001},"max":{"1min":86.337999999999994,"5min":86.337999999999994,"15min":86.337999999999994},"last":0.879},{"interface":"front","average":{"1min":9.1300000000000008,"5min":3.7149999999999999,"15min":2.2410000000000001},"min":{"1min":0.249,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":84.299999999999997,"5min":84.299999999999997,"15min":84.299999999999997},"last":5.694}]},{"osd":2,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.2539999999999996,"5min":4.0030000000000001,"15min":5.3659999999999997},"min":{"1min":0.25800000000000001,"5min":0.255,"15min":0.219},"max":{"1min":82.111999999999995,"5min":82.111999999999995,"15min":264.97800000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":3.5790000000000002,"5min":2.7000000000000002,"15min":4.9269999999999996},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":17.657,"5min":30.768999999999998,"15min":257.40899999999999},"last":5.7089999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":13.324999999999999,"5min":4.5519999999999996,"15min":2.7669999999999999},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":109.66800000000001,"5min":109.66800000000001,"15min":109.66800000000001},"last":5.6609999999999996},{"interface":"front","average":{"1min":13.428000000000001,"5min":5.0140000000000002,"15min":3.2130000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":110.295,"5min":110.295,"15min":110.295},"last":2.3799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.0170000000000003,"5min":4.3049999999999997,"15min":2.5310000000000001},"min":{"1min":0.33400000000000002,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":82.138000000000005,"5min":113.881,"15min":113.881},"last":2.4039999999999999},{"interface":"front","average":{"1min":6.6020000000000003,"5min":4.3559999999999999,"15min":2.4340000000000002},"min":{"1min":0.43099999999999999,"5min":0.313,"15min":0.31},"max":{"1min":82.25,"5min":114.598,"15min":114.598},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":5.0170000000000003,"15min":5.5069999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.4530000000000003,"5min":82.281999999999996,"15min":82.281999999999996},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.839,"5min":4.9669999999999996,"15min":5.4889999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3470000000000004,"5min":82.233000000000004,"15min":82.233000000000004},"last":5.6790000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":13.244,"5min":4.3380000000000001,"15min":2.9729999999999999},"min":{"1min":0.45200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":112.883,"5min":112.883,"15min":138.035},"last":1.397},{"interface":"front","average":{"1min":13.119999999999999,"5min":3.8069999999999999,"15min":2.7440000000000002},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":111.85599999999999,"5min":111.85599999999999,"15min":138.054},"last":5.633}]}]},{"osd":4,"up_from":30,"seq":128849019196,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1231280,"kb_used_data":964776,"kb_used_omap":0,"kb_used_meta":266496,"kb_avail":92513872,"statfs":{"total":95995035648,"available":94734204928,"internally_reserved":0,"allocated":987930624,"data_stored":987068241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":272891904},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":2.9470000000000001,"15min":2.9249999999999998},"min":{"1min":0.251,"5min":0.189,"15min":0.189},"max":{"1min":5.508,"5min":89.004999999999995,"15min":104.604},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.86899999999999999,"5min":2.8570000000000002,"15min":2.883},"min":{"1min":0.29499999999999998,"5min":0.249,"15min":0.23400000000000001},"max":{"1min":5.4509999999999996,"5min":89.024000000000001,"15min":104.687},"last":0.54700000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":31.100000000000001,"5min":10.585000000000001,"15min":4.8559999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.216},"max":{"1min":161.613,"5min":161.613,"15min":161.613},"last":8.2810000000000006},{"interface":"front","average":{"1min":31.327999999999999,"5min":10.795,"15min":4.7619999999999996},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":162.017,"5min":162.017,"15min":162.017},"last":8.4309999999999992}]},{"osd":2,"last update":"Sat Sep 23 20:04:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9710000000000001,"5min":6.4290000000000003,"15min":5.1130000000000004},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":5.6180000000000003,"5min":116.56,"15min":144.20099999999999},"last":2.6320000000000001},{"interface":"front","average":{"1min":3.121,"5min":6.5039999999999996,"15min":4.9400000000000004},"min":{"1min":0.30099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":11.048,"5min":117.03400000000001,"15min":144.184},"last":1.196}]},{"osd":3,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":14.276999999999999,"5min":7.5449999999999999,"15min":5.2300000000000004},"min":{"1min":0.376,"5min":0.255,"15min":0.255},"max":{"1min":99.930999999999997,"5min":139.976,"15min":266.54700000000003},"last":0.52200000000000002},{"interface":"front","average":{"1min":13.647,"5min":7.1440000000000001,"15min":4.1980000000000004},"min":{"1min":0.39300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":100.01600000000001,"5min":139.911,"15min":139.911},"last":13.637}]},{"osd":5,"last update":"Sat Sep 23 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.8640000000000001},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":4.077,"5min":5.0650000000000004,"15min":266.59899999999999},"last":0.874},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.60799999999999998,"15min":1.9570000000000001},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.395,"5min":4.8739999999999997,"15min":266.72899999999998},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.53300000000000003,"15min":0.51300000000000001},"min":{"1min":0.31,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0410000000000004,"5min":4.0410000000000004,"15min":4.0410000000000004},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.66100000000000003,"15min":0.54100000000000004},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.2439999999999998,"5min":4.2439999999999998,"15min":4.2439999999999998},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:04:49 2023","interfaces":[{"interface":"back","average":{"1min":4.7199999999999998,"5min":1.4239999999999999,"15min":1.2509999999999999},"min":{"1min":0.32600000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":65.522999999999996,"5min":65.522999999999996,"15min":65.522999999999996},"last":0.64000000000000001},{"interface":"front","average":{"1min":5.5970000000000004,"5min":1.595,"15min":1.3300000000000001},"min":{"1min":0.377,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":65.301000000000002,"5min":65.301000000000002,"15min":65.301000000000002},"last":0.65200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757560,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1071336,"kb_used_data":789792,"kb_used_omap":0,"kb_used_meta":281536,"kb_avail":92673816,"statfs":{"total":95995035648,"available":94897987584,"internally_reserved":0,"allocated":808747008,"data_stored":807889066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":288292864},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.94499999999999995,"15min":1.27},"min":{"1min":0.22900000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":2.8490000000000002,"5min":9.1699999999999999,"15min":24.852},"last":0.22600000000000001},{"interface":"front","average":{"1min":1.7829999999999999,"5min":1.3149999999999999,"15min":1.353},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.216},"max":{"1min":7.4100000000000001,"5min":9.0530000000000008,"15min":24.638999999999999},"last":0.24299999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":12.311,"5min":9.4450000000000003,"15min":4.9180000000000001},"min":{"1min":0.33100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":171.47300000000001,"5min":266.197,"15min":266.197},"last":7.407},{"interface":"front","average":{"1min":12.787000000000001,"5min":9.6140000000000008,"15min":4.9509999999999996},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":171.399,"5min":266.14999999999998,"15min":266.14999999999998},"last":2.5310000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:04:51 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":3.5609999999999999,"15min":5.6239999999999997},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":3.6680000000000001,"5min":177.816,"15min":238.96199999999999},"last":5.1820000000000004},{"interface":"front","average":{"1min":1.373,"5min":3.6299999999999999,"15min":5.6719999999999997},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":4.024,"5min":177.851,"15min":238.905},"last":5.2140000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":12.673,"5min":4.883,"15min":3.5150000000000001},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":168.31399999999999,"5min":168.31399999999999,"15min":168.31399999999999},"last":0.96499999999999997},{"interface":"front","average":{"1min":10.301,"5min":4.806,"15min":3.3679999999999999},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":168.39500000000001,"5min":168.39500000000001,"15min":168.39500000000001},"last":8.7409999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.476,"15min":1.609},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.9129999999999998,"5min":191.78899999999999,"15min":191.78899999999999},"last":0.312},{"interface":"front","average":{"1min":0.59799999999999998,"5min":2.4849999999999999,"15min":1.5960000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.177,"5min":191.821,"15min":191.821},"last":0.501}]},{"osd":6,"last update":"Sat Sep 23 20:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.66500000000000004,"15min":0.621},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.0679999999999996,"5min":4.0679999999999996,"15min":4.0679999999999996},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.49299999999999999,"15min":0.47199999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.3570000000000002,"5min":3.3570000000000002,"15min":3.3570000000000002},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":2.4460000000000002,"15min":1.74},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":3.964,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.754,"5min":2.5630000000000002,"15min":1.9379999999999999},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.976,"5min":191.69,"15min":191.69},"last":0.48199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1489113088,"data_stored":1489109016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1231949824,"data_stored":1231945752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1270284288,"data_stored":1270284288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1173295104,"data_stored":1173291032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":976551936,"data_stored":976551936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":796065792,"data_stored":796065792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":950009856,"data_stored":950009856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:05:08.394 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:08.424 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:08.485 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:08.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:08.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:08.593 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:08.723 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:08.972 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:08.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:161262 7.9 7:969c945f:::benchmark_data_smithi135_404353_object64200:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:09.016699+0000", 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0066408329999999996, 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0066734940000000003, 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:09.033 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 161262 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.016699+0000", 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:09.034 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.016699+0000", 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.016698+0000", 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.016702+0000", 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6920000000000001e-06 2023-09-23T20:05:09.035 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.016712+0000", 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0039e-05 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.016719+0000", 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4489999999999996e-06 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:09.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.016819+0000", 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100546 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.016935+0000", 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000115258 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.017035+0000", 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010041 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.023010+0000", 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0059747639999999996 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:09.023018+0000", 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7829999999999999e-06 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:09.038 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:09.039 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:05:09.040 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 1 2023-09-23T20:05:09.042 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:05:09.081 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:09.110 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:09.165 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:09.172 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:08 smithi144 ceph-mon[194833]: pgmap v785: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 3668/101010 objects misplaced (3.631%); 9.9 MiB/s, 159 objects/s recovering 2023-09-23T20:05:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:09.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:09.190 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:05:09.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:08 smithi135 ceph-mon[213871]: pgmap v785: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 3668/101010 objects misplaced (3.631%); 9.9 MiB/s, 159 objects/s recovering 2023-09-23T20:05:09.227 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:08 smithi135 ceph-mon[221113]: pgmap v785: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 3668/101010 objects misplaced (3.631%); 9.9 MiB/s, 159 objects/s recovering 2023-09-23T20:05:09.231 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:05:09.270 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:05:09.275 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:09.302 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:09.315 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:05:09.354 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:09.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:09.360 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:05:09.397 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:05:09.439 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:05:09.460 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:09.482 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:05:09.486 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:05:09.486 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:05:09.486 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:05:09.486 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:05:09.486 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:05:09.486 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:05:09.486 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:05:09.487 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:05:09.489 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:09.537 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:09.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:09.642 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:09.670 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:09.732 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:09.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:09.838 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:09.915 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1695562029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:09.928 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:10.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1695562029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:10.370 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:05:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1695562029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:10.952 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:10.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:10.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:11.059 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:11.109 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:05:11.168 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:11.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:10 smithi135 ceph-mon[213871]: pgmap v786: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 3668/99570 objects misplaced (3.684%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T20:05:11.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:10 smithi135 ceph-mon[221113]: pgmap v786: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 3668/99570 objects misplaced (3.684%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T20:05:11.321 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:05:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:10 smithi144 ceph-mon[194833]: pgmap v786: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 3668/99570 objects misplaced (3.684%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T20:05:11.391 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:11.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:11.501 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:11.597 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:11.632 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:11.798 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:11.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:11.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:11.902 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:11.996 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:12.118 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:12.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:12.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:12.227 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:12.254 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:12.280 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:12.305 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:12.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:12.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:12.409 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:12.437 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:12.489 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:12.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:12.594 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:12.622 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:12.676 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:12.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:12.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:12.784 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:12.812 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:13.328 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:13.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:13.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:13.440 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:13.446 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:05:13.470 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:13.507 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:13.519 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:13.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:13.604 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:13 smithi135 ceph-mon[221113]: pgmap v787: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 3480/96303 objects misplaced (3.614%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:05:13.604 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:13 smithi135 ceph-mon[213871]: pgmap v787: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 3480/96303 objects misplaced (3.614%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:05:13.626 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:13.653 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:13.703 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:13.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:13.809 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:13 smithi144 ceph-mon[194833]: pgmap v787: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 3480/96303 objects misplaced (3.614%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:05:13.892 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:05:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:05:13] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-09-23T20:05:13.932 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:14.218 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:14.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:14.324 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:14.352 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:14.406 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:14.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:14.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:14.514 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:14.621 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:14.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:14 smithi135 ceph-mon[221113]: pgmap v788: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 2868/92238 objects misplaced (3.109%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T20:05:14.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:14 smithi135 ceph-mon[213871]: pgmap v788: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 2868/92238 objects misplaced (3.109%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T20:05:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:14 smithi144 ceph-mon[194833]: pgmap v788: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 2868/92238 objects misplaced (3.109%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T20:05:15.185 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:15.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:15.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:15.299 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:15.328 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:15.840 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2171036178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:15.914 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:05:16.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2171036178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:16.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2171036178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:17.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2171036178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:16 smithi144 ceph-mon[194833]: pgmap v790: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 2868/90771 objects misplaced (3.160%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:05:17.178 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:17.188 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:05:17.188 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:05:17.188 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.188 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2171036178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:05:17.188 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T20:05:17.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[213871]: pgmap v790: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 2868/90771 objects misplaced (3.160%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:05:17.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:17.196 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:05:17.196 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:05:17.196 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2171036178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: failed to encode map e287 with expected crc 2023-09-23T20:05:17.197 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:16 smithi135 ceph-mon[221113]: pgmap v790: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 2868/90771 objects misplaced (3.160%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:05:17.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:17.298 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:17.388 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:17.763 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:17.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:17.825 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:17.871 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:17.898 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:17.945 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:17.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:18.050 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:18.134 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:05:18.134 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:05:18.163 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:18.364 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:05:18.381 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:05:19.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:18 smithi144 ceph-mon[194833]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:19.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:18 smithi144 ceph-mon[194833]: pgmap v791: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2184/83745 objects misplaced (2.608%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:18 smithi135 ceph-mon[221113]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:18 smithi135 ceph-mon[221113]: pgmap v791: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2184/83745 objects misplaced (2.608%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:18 smithi135 ceph-mon[213871]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:18 smithi135 ceph-mon[213871]: pgmap v791: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2184/83745 objects misplaced (2.608%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:19.285 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:05:20.122 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:20.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:20.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:20.231 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:20.303 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:20.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:05:20.932 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:20.941 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:05:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:20.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:20.945 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:05:21.043 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:21.082 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:21.176 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:05:21.197 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:21.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:21.303 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:21.355 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":790,"stamp":"2023-09-23T20:05:16.753143+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1985812427,"num_objects":30257,"num_object_clones":0,"num_object_copies":90771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2868,"num_objects_unfound":0,"num_objects_dirty":30257,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":164798,"num_write_kb":6224502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12644,"num_bytes_recovered":821434869,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111511,"ondisk_log_size":111511,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9066560,"kb_used_data":6379464,"kb_used_omap":32,"kb_used_meta":2686815,"kb_avail":647149504,"statfs":{"total":671965249536,"available":662681092096,"internally_reserved":0,"allocated":6532571136,"data_stored":6524773420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":2751299078},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-358547456,"num_objects":-5471,"num_object_clones":0,"num_object_copies":-16413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1854,"num_objects_unfound":0,"num_objects_dirty":-5471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5471,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1252,"num_bytes_recovered":82051072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006952"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":264,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812494+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T20:05:04.812494+0000","last_peered":"2023-09-23T20:05:04.812494+0000","last_clean":"2023-09-23T20:05:04.812494+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T20:05:04.812494+0000","last_undegraded":"2023-09-23T20:05:04.812494+0000","last_fullsized":"2023-09-23T20:05:04.812494+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":108,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.810464+0000","last_change":"2023-09-23T19:59:09.151117+0000","last_active":"2023-09-23T20:05:04.810464+0000","last_peered":"2023-09-23T20:05:04.810464+0000","last_clean":"2023-09-23T20:05:04.810464+0000","last_became_active":"2023-09-23T19:59:09.150795+0000","last_became_peered":"2023-09-23T19:59:09.150795+0000","last_unstale":"2023-09-23T20:05:04.810464+0000","last_undegraded":"2023-09-23T20:05:04.810464+0000","last_fullsized":"2023-09-23T20:05:04.810464+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:34.994574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":247,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812642+0000","last_change":"2023-09-23T19:59:10.156070+0000","last_active":"2023-09-23T20:05:04.812642+0000","last_peered":"2023-09-23T20:05:04.812642+0000","last_clean":"2023-09-23T20:05:04.812642+0000","last_became_active":"2023-09-23T19:59:10.155813+0000","last_became_peered":"2023-09-23T19:59:10.155813+0000","last_unstale":"2023-09-23T20:05:04.812642+0000","last_undegraded":"2023-09-23T20:05:04.812642+0000","last_fullsized":"2023-09-23T20:05:04.812642+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:42.582380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":38,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799397+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:05:04.799397+0000","last_peered":"2023-09-23T20:05:04.799397+0000","last_clean":"2023-09-23T20:05:04.799397+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:05:04.799397+0000","last_undegraded":"2023-09-23T20:05:04.799397+0000","last_fullsized":"2023-09-23T20:05:04.799397+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.801341+0000","last_change":"2023-09-23T19:59:09.163182+0000","last_active":"2023-09-23T20:05:04.801341+0000","last_peered":"2023-09-23T20:05:04.801341+0000","last_clean":"2023-09-23T20:05:04.801341+0000","last_became_active":"2023-09-23T19:59:09.161612+0000","last_became_peered":"2023-09-23T19:59:09.161612+0000","last_unstale":"2023-09-23T20:05:04.801341+0000","last_undegraded":"2023-09-23T20:05:04.801341+0000","last_fullsized":"2023-09-23T20:05:04.801341+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:59.524152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":211,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806814+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T20:05:04.806814+0000","last_peered":"2023-09-23T20:05:04.806814+0000","last_clean":"2023-09-23T20:05:04.806814+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T20:05:04.806814+0000","last_undegraded":"2023-09-23T20:05:04.806814+0000","last_fullsized":"2023-09-23T20:05:04.806814+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:59.043547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":212,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806713+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:05:04.806713+0000","last_peered":"2023-09-23T20:05:04.806713+0000","last_clean":"2023-09-23T20:05:04.806713+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:05:04.806713+0000","last_undegraded":"2023-09-23T20:05:04.806713+0000","last_fullsized":"2023-09-23T20:05:04.806713+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":108,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.801400+0000","last_change":"2023-09-23T19:59:09.146009+0000","last_active":"2023-09-23T20:05:04.801400+0000","last_peered":"2023-09-23T20:05:04.801400+0000","last_clean":"2023-09-23T20:05:04.801400+0000","last_became_active":"2023-09-23T19:59:09.145687+0000","last_became_peered":"2023-09-23T19:59:09.145687+0000","last_unstale":"2023-09-23T20:05:04.801400+0000","last_undegraded":"2023-09-23T20:05:04.801400+0000","last_fullsized":"2023-09-23T20:05:04.801400+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.189450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":245,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806674+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T20:05:04.806674+0000","last_peered":"2023-09-23T20:05:04.806674+0000","last_clean":"2023-09-23T20:05:04.806674+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T20:05:04.806674+0000","last_undegraded":"2023-09-23T20:05:04.806674+0000","last_fullsized":"2023-09-23T20:05:04.806674+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":127,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.801093+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:05:04.801093+0000","last_peered":"2023-09-23T20:05:04.801093+0000","last_clean":"2023-09-23T20:05:04.801093+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:05:04.801093+0000","last_undegraded":"2023-09-23T20:05:04.801093+0000","last_fullsized":"2023-09-23T20:05:04.801093+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":281,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799425+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:05:04.799425+0000","last_peered":"2023-09-23T20:05:04.799425+0000","last_clean":"2023-09-23T20:05:04.799425+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:05:04.799425+0000","last_undegraded":"2023-09-23T20:05:04.799425+0000","last_fullsized":"2023-09-23T20:05:04.799425+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":243,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.813710+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T20:05:04.813710+0000","last_peered":"2023-09-23T20:05:04.813710+0000","last_clean":"2023-09-23T20:05:04.813710+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T20:05:04.813710+0000","last_undegraded":"2023-09-23T20:05:04.813710+0000","last_fullsized":"2023-09-23T20:05:04.813710+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:51.825640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":300,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812911+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:05:04.812911+0000","last_peered":"2023-09-23T20:05:04.812911+0000","last_clean":"2023-09-23T20:05:04.812911+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:05:04.812911+0000","last_undegraded":"2023-09-23T20:05:04.812911+0000","last_fullsized":"2023-09-23T20:05:04.812911+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":262,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812880+0000","last_change":"2023-09-23T20:01:52.439375+0000","last_active":"2023-09-23T20:05:04.812880+0000","last_peered":"2023-09-23T20:05:04.812880+0000","last_clean":"2023-09-23T20:05:04.812880+0000","last_became_active":"2023-09-23T20:01:52.439074+0000","last_became_peered":"2023-09-23T20:01:52.439074+0000","last_unstale":"2023-09-23T20:05:04.812880+0000","last_undegraded":"2023-09-23T20:05:04.812880+0000","last_fullsized":"2023-09-23T20:05:04.812880+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:31.103633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":248,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799054+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:05:04.799054+0000","last_peered":"2023-09-23T20:05:04.799054+0000","last_clean":"2023-09-23T20:05:04.799054+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:05:04.799054+0000","last_undegraded":"2023-09-23T20:05:04.799054+0000","last_fullsized":"2023-09-23T20:05:04.799054+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.803431+0000","last_change":"2023-09-23T19:59:09.195560+0000","last_active":"2023-09-23T20:05:04.803431+0000","last_peered":"2023-09-23T20:05:04.803431+0000","last_clean":"2023-09-23T20:05:04.803431+0000","last_became_active":"2023-09-23T19:59:09.195145+0000","last_became_peered":"2023-09-23T19:59:09.195145+0000","last_unstale":"2023-09-23T20:05:04.803431+0000","last_undegraded":"2023-09-23T20:05:04.803431+0000","last_fullsized":"2023-09-23T20:05:04.803431+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.190798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":251,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799341+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:05:04.799341+0000","last_peered":"2023-09-23T20:05:04.799341+0000","last_clean":"2023-09-23T20:05:04.799341+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:05:04.799341+0000","last_undegraded":"2023-09-23T20:05:04.799341+0000","last_fullsized":"2023-09-23T20:05:04.799341+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":264,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806680+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:05:04.806680+0000","last_peered":"2023-09-23T20:05:04.806680+0000","last_clean":"2023-09-23T20:05:04.806680+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:05:04.806680+0000","last_undegraded":"2023-09-23T20:05:04.806680+0000","last_fullsized":"2023-09-23T20:05:04.806680+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":246,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799087+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:05:04.799087+0000","last_peered":"2023-09-23T20:05:04.799087+0000","last_clean":"2023-09-23T20:05:04.799087+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:05:04.799087+0000","last_undegraded":"2023-09-23T20:05:04.799087+0000","last_fullsized":"2023-09-23T20:05:04.799087+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":251,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.811760+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:05:04.811760+0000","last_peered":"2023-09-23T20:05:04.811760+0000","last_clean":"2023-09-23T20:05:04.811760+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:05:04.811760+0000","last_undegraded":"2023-09-23T20:05:04.811760+0000","last_fullsized":"2023-09-23T20:05:04.811760+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":79,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806153+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:05:04.806153+0000","last_peered":"2023-09-23T20:05:04.806153+0000","last_clean":"2023-09-23T20:05:04.806153+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:05:04.806153+0000","last_undegraded":"2023-09-23T20:05:04.806153+0000","last_fullsized":"2023-09-23T20:05:04.806153+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.810569+0000","last_change":"2023-09-23T20:03:59.232022+0000","last_active":"2023-09-23T20:05:04.810569+0000","last_peered":"2023-09-23T20:05:04.810569+0000","last_clean":"2023-09-23T20:05:04.810569+0000","last_became_active":"2023-09-23T20:03:59.231689+0000","last_became_peered":"2023-09-23T20:03:59.231689+0000","last_unstale":"2023-09-23T20:05:04.810569+0000","last_undegraded":"2023-09-23T20:05:04.810569+0000","last_fullsized":"2023-09-23T20:05:04.810569+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:29.669102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"286'10249","reported_seq":20539,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:15.540515+0000","last_change":"2023-09-23T20:04:02.290062+0000","last_active":"2023-09-23T20:05:15.540515+0000","last_peered":"2023-09-23T20:05:15.540515+0000","last_clean":"2023-09-23T20:05:15.540515+0000","last_became_active":"2023-09-23T20:04:02.271287+0000","last_became_peered":"2023-09-23T20:04:02.271287+0000","last_unstale":"2023-09-23T20:05:15.540515+0000","last_undegraded":"2023-09-23T20:05:15.540515+0000","last_fullsized":"2023-09-23T20:05:15.540515+0000","mapping_epoch":276,"log_start":"250'3700","ondisk_log_start":"250'3700","created":240,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:47.344179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120258560,"num_objects":1835,"num_object_clones":0,"num_object_copies":5505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10249,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5100,"num_bytes_recovered":334233600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":247,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799398+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:05:04.799398+0000","last_peered":"2023-09-23T20:05:04.799398+0000","last_clean":"2023-09-23T20:05:04.799398+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:05:04.799398+0000","last_undegraded":"2023-09-23T20:05:04.799398+0000","last_fullsized":"2023-09-23T20:05:04.799398+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":274,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806095+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:05:04.806095+0000","last_peered":"2023-09-23T20:05:04.806095+0000","last_clean":"2023-09-23T20:05:04.806095+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:05:04.806095+0000","last_undegraded":"2023-09-23T20:05:04.806095+0000","last_fullsized":"2023-09-23T20:05:04.806095+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"286'10175","reported_seq":10231,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:12.182141+0000","last_change":"2023-09-23T20:01:39.911588+0000","last_active":"2023-09-23T20:05:12.182141+0000","last_peered":"2023-09-23T20:05:12.182141+0000","last_clean":"2023-09-23T20:05:12.182141+0000","last_became_active":"2023-09-23T20:01:39.911364+0000","last_became_peered":"2023-09-23T20:01:39.911364+0000","last_unstale":"2023-09-23T20:05:12.182141+0000","last_undegraded":"2023-09-23T20:05:12.182141+0000","last_fullsized":"2023-09-23T20:05:12.182141+0000","mapping_epoch":240,"log_start":"254'4495","ondisk_log_start":"254'4495","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:35.715001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122880000,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10175,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":248,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799526+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T20:05:04.799526+0000","last_peered":"2023-09-23T20:05:04.799526+0000","last_clean":"2023-09-23T20:05:04.799526+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T20:05:04.799526+0000","last_undegraded":"2023-09-23T20:05:04.799526+0000","last_fullsized":"2023-09-23T20:05:04.799526+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":302,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812276+0000","last_change":"2023-09-23T20:03:59.253784+0000","last_active":"2023-09-23T20:05:04.812276+0000","last_peered":"2023-09-23T20:05:04.812276+0000","last_clean":"2023-09-23T20:05:04.812276+0000","last_became_active":"2023-09-23T20:03:59.253552+0000","last_became_peered":"2023-09-23T20:03:59.253552+0000","last_unstale":"2023-09-23T20:05:04.812276+0000","last_undegraded":"2023-09-23T20:05:04.812276+0000","last_fullsized":"2023-09-23T20:05:04.812276+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:56.110746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"286'10242","reported_seq":10298,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:15.592512+0000","last_change":"2023-09-23T20:01:39.976582+0000","last_active":"2023-09-23T20:05:15.592512+0000","last_peered":"2023-09-23T20:05:15.592512+0000","last_clean":"2023-09-23T20:05:15.592512+0000","last_became_active":"2023-09-23T20:01:39.976244+0000","last_became_peered":"2023-09-23T20:01:39.976244+0000","last_unstale":"2023-09-23T20:05:15.592512+0000","last_undegraded":"2023-09-23T20:05:15.592512+0000","last_fullsized":"2023-09-23T20:05:15.592512+0000","mapping_epoch":240,"log_start":"249'3700","ondisk_log_start":"249'3700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.952295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118226944,"num_objects":1804,"num_object_clones":0,"num_object_copies":5412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10242,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":251,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799318+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:05:04.799318+0000","last_peered":"2023-09-23T20:05:04.799318+0000","last_clean":"2023-09-23T20:05:04.799318+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:05:04.799318+0000","last_undegraded":"2023-09-23T20:05:04.799318+0000","last_fullsized":"2023-09-23T20:05:04.799318+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":213,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.813941+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T20:05:04.813941+0000","last_peered":"2023-09-23T20:05:04.813941+0000","last_clean":"2023-09-23T20:05:04.813941+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T20:05:04.813941+0000","last_undegraded":"2023-09-23T20:05:04.813941+0000","last_fullsized":"2023-09-23T20:05:04.813941+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.284038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"286'10495","reported_seq":10551,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:13.220398+0000","last_change":"2023-09-23T20:01:39.912566+0000","last_active":"2023-09-23T20:05:13.220398+0000","last_peered":"2023-09-23T20:05:13.220398+0000","last_clean":"2023-09-23T20:05:13.220398+0000","last_became_active":"2023-09-23T20:01:39.912259+0000","last_became_peered":"2023-09-23T20:01:39.912259+0000","last_unstale":"2023-09-23T20:05:13.220398+0000","last_undegraded":"2023-09-23T20:05:13.220398+0000","last_fullsized":"2023-09-23T20:05:13.220398+0000","mapping_epoch":240,"log_start":"247'1600","ondisk_log_start":"247'1600","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8895,"log_dups_size":0,"ondisk_log_size":8895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:19.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127860736,"num_objects":1951,"num_object_clones":0,"num_object_copies":5853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10495,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":304,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806516+0000","last_change":"2023-09-23T20:00:34.468296+0000","last_active":"2023-09-23T20:05:04.806516+0000","last_peered":"2023-09-23T20:05:04.806516+0000","last_clean":"2023-09-23T20:05:04.806516+0000","last_became_active":"2023-09-23T20:00:34.467957+0000","last_became_peered":"2023-09-23T20:00:34.467957+0000","last_unstale":"2023-09-23T20:05:04.806516+0000","last_undegraded":"2023-09-23T20:05:04.806516+0000","last_fullsized":"2023-09-23T20:05:04.806516+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:50.449027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":101,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812870+0000","last_change":"2023-09-23T20:01:04.470558+0000","last_active":"2023-09-23T20:05:04.812870+0000","last_peered":"2023-09-23T20:05:04.812870+0000","last_clean":"2023-09-23T20:05:04.812870+0000","last_became_active":"2023-09-23T20:01:04.470276+0000","last_became_peered":"2023-09-23T20:01:04.470276+0000","last_unstale":"2023-09-23T20:05:04.812870+0000","last_undegraded":"2023-09-23T20:05:04.812870+0000","last_fullsized":"2023-09-23T20:05:04.812870+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:17.788583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"286'10201","reported_seq":10257,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:12.252866+0000","last_change":"2023-09-23T20:01:39.974611+0000","last_active":"2023-09-23T20:05:12.252866+0000","last_peered":"2023-09-23T20:05:12.252866+0000","last_clean":"2023-09-23T20:05:12.252866+0000","last_became_active":"2023-09-23T20:01:39.974318+0000","last_became_peered":"2023-09-23T20:01:39.974318+0000","last_unstale":"2023-09-23T20:05:12.252866+0000","last_undegraded":"2023-09-23T20:05:12.252866+0000","last_fullsized":"2023-09-23T20:05:12.252866+0000","mapping_epoch":240,"log_start":"254'4527","ondisk_log_start":"254'4527","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:31.047765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121044992,"num_objects":1847,"num_object_clones":0,"num_object_copies":5541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10201,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":736,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.798980+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:05:04.798980+0000","last_peered":"2023-09-23T20:05:04.798980+0000","last_clean":"2023-09-23T20:05:04.798980+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:05:04.798980+0000","last_undegraded":"2023-09-23T20:05:04.798980+0000","last_fullsized":"2023-09-23T20:05:04.798980+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":112,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.802143+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:05:04.802143+0000","last_peered":"2023-09-23T20:05:04.802143+0000","last_clean":"2023-09-23T20:05:04.802143+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:05:04.802143+0000","last_undegraded":"2023-09-23T20:05:04.802143+0000","last_fullsized":"2023-09-23T20:05:04.802143+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":282,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812809+0000","last_change":"2023-09-23T19:59:09.206075+0000","last_active":"2023-09-23T20:05:04.812809+0000","last_peered":"2023-09-23T20:05:04.812809+0000","last_clean":"2023-09-23T20:05:04.812809+0000","last_became_active":"2023-09-23T19:59:09.205766+0000","last_became_peered":"2023-09-23T19:59:09.205766+0000","last_unstale":"2023-09-23T20:05:04.812809+0000","last_undegraded":"2023-09-23T20:05:04.812809+0000","last_fullsized":"2023-09-23T20:05:04.812809+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:14.627703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"286'10059","reported_seq":10115,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:13.247097+0000","last_change":"2023-09-23T20:01:39.976550+0000","last_active":"2023-09-23T20:05:13.247097+0000","last_peered":"2023-09-23T20:05:13.247097+0000","last_clean":"2023-09-23T20:05:13.247097+0000","last_became_active":"2023-09-23T20:01:39.976245+0000","last_became_peered":"2023-09-23T20:01:39.976245+0000","last_unstale":"2023-09-23T20:05:13.247097+0000","last_undegraded":"2023-09-23T20:05:13.247097+0000","last_fullsized":"2023-09-23T20:05:13.247097+0000","mapping_epoch":240,"log_start":"246'1200","ondisk_log_start":"246'1200","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8859,"log_dups_size":0,"ondisk_log_size":8859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:45.979020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118292480,"num_objects":1805,"num_object_clones":0,"num_object_copies":5415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10059,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":645,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806004+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:05:04.806004+0000","last_peered":"2023-09-23T20:05:04.806004+0000","last_clean":"2023-09-23T20:05:04.806004+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:05:04.806004+0000","last_undegraded":"2023-09-23T20:05:04.806004+0000","last_fullsized":"2023-09-23T20:05:04.806004+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":232,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812397+0000","last_change":"2023-09-23T19:59:09.172601+0000","last_active":"2023-09-23T20:05:04.812397+0000","last_peered":"2023-09-23T20:05:04.812397+0000","last_clean":"2023-09-23T20:05:04.812397+0000","last_became_active":"2023-09-23T19:59:09.172297+0000","last_became_peered":"2023-09-23T19:59:09.172297+0000","last_unstale":"2023-09-23T20:05:04.812397+0000","last_undegraded":"2023-09-23T20:05:04.812397+0000","last_fullsized":"2023-09-23T20:05:04.812397+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:09.528266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":262,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812741+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T20:05:04.812741+0000","last_peered":"2023-09-23T20:05:04.812741+0000","last_clean":"2023-09-23T20:05:04.812741+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T20:05:04.812741+0000","last_undegraded":"2023-09-23T20:05:04.812741+0000","last_fullsized":"2023-09-23T20:05:04.812741+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"286'10261","reported_seq":10318,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:15.590148+0000","last_change":"2023-09-23T20:01:39.977019+0000","last_active":"2023-09-23T20:05:15.590148+0000","last_peered":"2023-09-23T20:05:15.590148+0000","last_clean":"2023-09-23T20:05:15.590148+0000","last_became_active":"2023-09-23T20:01:39.976270+0000","last_became_peered":"2023-09-23T20:01:39.976270+0000","last_unstale":"2023-09-23T20:05:15.590148+0000","last_undegraded":"2023-09-23T20:05:15.590148+0000","last_fullsized":"2023-09-23T20:05:15.590148+0000","mapping_epoch":240,"log_start":"249'3700","ondisk_log_start":"249'3700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.226759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119930904,"num_objects":1831,"num_object_clones":0,"num_object_copies":5493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1831,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10261,"num_write_kb":386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":246,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812128+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:05:04.812128+0000","last_peered":"2023-09-23T20:05:04.812128+0000","last_clean":"2023-09-23T20:05:04.812128+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:05:04.812128+0000","last_undegraded":"2023-09-23T20:05:04.812128+0000","last_fullsized":"2023-09-23T20:05:04.812128+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":292,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812599+0000","last_change":"2023-09-23T19:59:09.171926+0000","last_active":"2023-09-23T20:05:04.812599+0000","last_peered":"2023-09-23T20:05:04.812599+0000","last_clean":"2023-09-23T20:05:04.812599+0000","last_became_active":"2023-09-23T19:59:09.171609+0000","last_became_peered":"2023-09-23T19:59:09.171609+0000","last_unstale":"2023-09-23T20:05:04.812599+0000","last_undegraded":"2023-09-23T20:05:04.812599+0000","last_fullsized":"2023-09-23T20:05:04.812599+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:21.795329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"286'10145","reported_seq":17279,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:12.199127+0000","last_change":"2023-09-23T20:04:50.012534+0000","last_active":"2023-09-23T20:05:12.199127+0000","last_peered":"2023-09-23T20:05:12.199127+0000","last_clean":"2023-09-23T20:05:12.199127+0000","last_became_active":"2023-09-23T20:04:49.916541+0000","last_became_peered":"2023-09-23T20:04:49.916541+0000","last_unstale":"2023-09-23T20:05:12.199127+0000","last_undegraded":"2023-09-23T20:05:12.199127+0000","last_fullsized":"2023-09-23T20:05:12.199127+0000","mapping_epoch":282,"log_start":"254'4423","ondisk_log_start":"254'4423","created":240,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:58.571894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121700352,"num_objects":1857,"num_object_clones":0,"num_object_copies":5571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10145,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3528,"num_bytes_recovered":231145472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":624,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806297+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:05:04.806297+0000","last_peered":"2023-09-23T20:05:04.806297+0000","last_clean":"2023-09-23T20:05:04.806297+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:05:04.806297+0000","last_undegraded":"2023-09-23T20:05:04.806297+0000","last_fullsized":"2023-09-23T20:05:04.806297+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":23,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.801052+0000","last_change":"2023-09-23T20:03:59.233742+0000","last_active":"2023-09-23T20:05:04.801052+0000","last_peered":"2023-09-23T20:05:04.801052+0000","last_clean":"2023-09-23T20:05:04.801052+0000","last_became_active":"2023-09-23T20:03:59.233209+0000","last_became_peered":"2023-09-23T20:03:59.233209+0000","last_unstale":"2023-09-23T20:05:04.801052+0000","last_undegraded":"2023-09-23T20:05:04.801052+0000","last_fullsized":"2023-09-23T20:05:04.801052+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:50.879925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":246,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.818965+0000","last_change":"2023-09-23T19:59:09.153852+0000","last_active":"2023-09-23T20:05:04.818965+0000","last_peered":"2023-09-23T20:05:04.818965+0000","last_clean":"2023-09-23T20:05:04.818965+0000","last_became_active":"2023-09-23T19:59:09.153514+0000","last_became_peered":"2023-09-23T19:59:09.153514+0000","last_unstale":"2023-09-23T20:05:04.818965+0000","last_undegraded":"2023-09-23T20:05:04.818965+0000","last_fullsized":"2023-09-23T20:05:04.818965+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:45.179600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"286'10233","reported_seq":12568,"reported_epoch":286,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:05:13.521290+0000","last_change":"2023-09-23T20:05:04.718899+0000","last_active":"2023-09-23T20:05:13.521290+0000","last_peered":"2023-09-23T20:05:13.521290+0000","last_clean":"2023-09-23T20:03:58.080766+0000","last_became_active":"2023-09-23T20:04:00.582511+0000","last_became_peered":"2023-09-23T20:04:00.582511+0000","last_unstale":"2023-09-23T20:05:13.521290+0000","last_undegraded":"2023-09-23T20:05:13.521290+0000","last_fullsized":"2023-09-23T20:05:13.521290+0000","mapping_epoch":274,"log_start":"249'3700","ondisk_log_start":"249'3700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6533,"log_dups_size":0,"ondisk_log_size":6533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:56.089695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127205376,"num_objects":1941,"num_object_clones":0,"num_object_copies":5823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":913,"num_objects_unfound":0,"num_objects_dirty":1941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10233,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1131,"num_bytes_recovered":74055680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,1,0],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1941}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":609,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.810665+0000","last_change":"2023-09-23T20:04:02.277887+0000","last_active":"2023-09-23T20:05:04.810665+0000","last_peered":"2023-09-23T20:05:04.810665+0000","last_clean":"2023-09-23T20:05:04.810665+0000","last_became_active":"2023-09-23T20:04:02.277519+0000","last_became_peered":"2023-09-23T20:04:02.277519+0000","last_unstale":"2023-09-23T20:05:04.810665+0000","last_undegraded":"2023-09-23T20:05:04.810665+0000","last_fullsized":"2023-09-23T20:05:04.810665+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":625,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.813403+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T20:05:04.813403+0000","last_peered":"2023-09-23T20:05:04.813403+0000","last_clean":"2023-09-23T20:05:04.813403+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T20:05:04.813403+0000","last_undegraded":"2023-09-23T20:05:04.813403+0000","last_fullsized":"2023-09-23T20:05:04.813403+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":286,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806476+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:05:04.806476+0000","last_peered":"2023-09-23T20:05:04.806476+0000","last_clean":"2023-09-23T20:05:04.806476+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:05:04.806476+0000","last_undegraded":"2023-09-23T20:05:04.806476+0000","last_fullsized":"2023-09-23T20:05:04.806476+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":214,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.813769+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T20:05:04.813769+0000","last_peered":"2023-09-23T20:05:04.813769+0000","last_clean":"2023-09-23T20:05:04.813769+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T20:05:04.813769+0000","last_undegraded":"2023-09-23T20:05:04.813769+0000","last_fullsized":"2023-09-23T20:05:04.813769+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"286'10279","reported_seq":10350,"reported_epoch":286,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:05:12.378552+0000","last_change":"2023-09-23T20:04:00.730874+0000","last_active":"2023-09-23T20:05:12.378552+0000","last_peered":"2023-09-23T20:05:12.378552+0000","last_clean":"2023-09-23T20:03:58.087054+0000","last_became_active":"2023-09-23T20:04:00.560120+0000","last_became_peered":"2023-09-23T20:04:00.560120+0000","last_unstale":"2023-09-23T20:05:12.378552+0000","last_undegraded":"2023-09-23T20:05:12.378552+0000","last_fullsized":"2023-09-23T20:05:12.378552+0000","mapping_epoch":274,"log_start":"249'3700","ondisk_log_start":"249'3700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:17.026687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128122880,"num_objects":1955,"num_object_clones":0,"num_object_copies":5865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1955,"num_objects_unfound":0,"num_objects_dirty":1955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10279,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1955}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":597,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806126+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:05:04.806126+0000","last_peered":"2023-09-23T20:05:04.806126+0000","last_clean":"2023-09-23T20:05:04.806126+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:05:04.806126+0000","last_undegraded":"2023-09-23T20:05:04.806126+0000","last_fullsized":"2023-09-23T20:05:04.806126+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":846,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806162+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T20:05:04.806162+0000","last_peered":"2023-09-23T20:05:04.806162+0000","last_clean":"2023-09-23T20:05:04.806162+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T20:05:04.806162+0000","last_undegraded":"2023-09-23T20:05:04.806162+0000","last_fullsized":"2023-09-23T20:05:04.806162+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:58.623015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":285,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812432+0000","last_change":"2023-09-23T20:00:34.447064+0000","last_active":"2023-09-23T20:05:04.812432+0000","last_peered":"2023-09-23T20:05:04.812432+0000","last_clean":"2023-09-23T20:05:04.812432+0000","last_became_active":"2023-09-23T20:00:34.446754+0000","last_became_peered":"2023-09-23T20:00:34.446754+0000","last_unstale":"2023-09-23T20:05:04.812432+0000","last_undegraded":"2023-09-23T20:05:04.812432+0000","last_fullsized":"2023-09-23T20:05:04.812432+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:39.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":280,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.813330+0000","last_change":"2023-09-23T20:03:59.236594+0000","last_active":"2023-09-23T20:05:04.813330+0000","last_peered":"2023-09-23T20:05:04.813330+0000","last_clean":"2023-09-23T20:05:04.813330+0000","last_became_active":"2023-09-23T20:03:59.236305+0000","last_became_peered":"2023-09-23T20:03:59.236305+0000","last_unstale":"2023-09-23T20:05:04.813330+0000","last_undegraded":"2023-09-23T20:05:04.813330+0000","last_fullsized":"2023-09-23T20:05:04.813330+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:23.103726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"286'10191","reported_seq":10245,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:13.515909+0000","last_change":"2023-09-23T20:01:39.977018+0000","last_active":"2023-09-23T20:05:13.515909+0000","last_peered":"2023-09-23T20:05:13.515909+0000","last_clean":"2023-09-23T20:05:13.515909+0000","last_became_active":"2023-09-23T20:01:39.976359+0000","last_became_peered":"2023-09-23T20:01:39.976359+0000","last_unstale":"2023-09-23T20:05:13.515909+0000","last_undegraded":"2023-09-23T20:05:13.515909+0000","last_fullsized":"2023-09-23T20:05:13.515909+0000","mapping_epoch":240,"log_start":"249'3600","ondisk_log_start":"249'3600","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.221346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122224640,"num_objects":1865,"num_object_clones":0,"num_object_copies":5595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10191,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":521,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799202+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:05:04.799202+0000","last_peered":"2023-09-23T20:05:04.799202+0000","last_clean":"2023-09-23T20:05:04.799202+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:05:04.799202+0000","last_undegraded":"2023-09-23T20:05:04.799202+0000","last_fullsized":"2023-09-23T20:05:04.799202+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":919,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.820017+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:05:04.820017+0000","last_peered":"2023-09-23T20:05:04.820017+0000","last_clean":"2023-09-23T20:05:04.820017+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:05:04.820017+0000","last_undegraded":"2023-09-23T20:05:04.820017+0000","last_fullsized":"2023-09-23T20:05:04.820017+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":243,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.814267+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T20:05:04.814267+0000","last_peered":"2023-09-23T20:05:04.814267+0000","last_clean":"2023-09-23T20:05:04.814267+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T20:05:04.814267+0000","last_undegraded":"2023-09-23T20:05:04.814267+0000","last_fullsized":"2023-09-23T20:05:04.814267+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.564126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":258,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.819040+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T20:05:04.819040+0000","last_peered":"2023-09-23T20:05:04.819040+0000","last_clean":"2023-09-23T20:05:04.819040+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T20:05:04.819040+0000","last_undegraded":"2023-09-23T20:05:04.819040+0000","last_fullsized":"2023-09-23T20:05:04.819040+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"286'10377","reported_seq":15997,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:13.454184+0000","last_change":"2023-09-23T20:05:05.847661+0000","last_active":"2023-09-23T20:05:13.454184+0000","last_peered":"2023-09-23T20:05:13.454184+0000","last_clean":"2023-09-23T20:05:13.454184+0000","last_became_active":"2023-09-23T20:05:05.813785+0000","last_became_peered":"2023-09-23T20:05:05.813785+0000","last_unstale":"2023-09-23T20:05:13.454184+0000","last_undegraded":"2023-09-23T20:05:13.454184+0000","last_fullsized":"2023-09-23T20:05:13.454184+0000","mapping_epoch":285,"log_start":"248'3056","ondisk_log_start":"248'3056","created":240,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7321,"log_dups_size":0,"ondisk_log_size":7321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:47.265078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127205376,"num_objects":1941,"num_object_clones":0,"num_object_copies":5823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10377,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2768,"num_bytes_recovered":181403648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1094,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806414+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:05:04.806414+0000","last_peered":"2023-09-23T20:05:04.806414+0000","last_clean":"2023-09-23T20:05:04.806414+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:05:04.806414+0000","last_undegraded":"2023-09-23T20:05:04.806414+0000","last_fullsized":"2023-09-23T20:05:04.806414+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":696,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799041+0000","last_change":"2023-09-23T20:00:34.445036+0000","last_active":"2023-09-23T20:05:04.799041+0000","last_peered":"2023-09-23T20:05:04.799041+0000","last_clean":"2023-09-23T20:05:04.799041+0000","last_became_active":"2023-09-23T20:00:34.444689+0000","last_became_peered":"2023-09-23T20:00:34.444689+0000","last_unstale":"2023-09-23T20:05:04.799041+0000","last_undegraded":"2023-09-23T20:05:04.799041+0000","last_fullsized":"2023-09-23T20:05:04.799041+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:44.485721+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.803977+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:05:04.803977+0000","last_peered":"2023-09-23T20:05:04.803977+0000","last_clean":"2023-09-23T20:05:04.803977+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:05:04.803977+0000","last_undegraded":"2023-09-23T20:05:04.803977+0000","last_fullsized":"2023-09-23T20:05:04.803977+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":161,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812840+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:05:04.812840+0000","last_peered":"2023-09-23T20:05:04.812840+0000","last_clean":"2023-09-23T20:05:04.812840+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:05:04.812840+0000","last_undegraded":"2023-09-23T20:05:04.812840+0000","last_fullsized":"2023-09-23T20:05:04.812840+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"286'10359","reported_seq":10413,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:13.522290+0000","last_change":"2023-09-23T20:01:39.977010+0000","last_active":"2023-09-23T20:05:13.522290+0000","last_peered":"2023-09-23T20:05:13.522290+0000","last_clean":"2023-09-23T20:05:13.522290+0000","last_became_active":"2023-09-23T20:01:39.976722+0000","last_became_peered":"2023-09-23T20:01:39.976722+0000","last_unstale":"2023-09-23T20:05:13.522290+0000","last_undegraded":"2023-09-23T20:05:13.522290+0000","last_fullsized":"2023-09-23T20:05:13.522290+0000","mapping_epoch":240,"log_start":"248'3031","ondisk_log_start":"248'3031","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7328,"log_dups_size":0,"ondisk_log_size":7328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:13.592964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122880000,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10359,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":626,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.813014+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T20:05:04.813014+0000","last_peered":"2023-09-23T20:05:04.813014+0000","last_clean":"2023-09-23T20:05:04.813014+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T20:05:04.813014+0000","last_undegraded":"2023-09-23T20:05:04.813014+0000","last_fullsized":"2023-09-23T20:05:04.813014+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.842550+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1004,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.798978+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T20:05:04.798978+0000","last_peered":"2023-09-23T20:05:04.798978+0000","last_clean":"2023-09-23T20:05:04.798978+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T20:05:04.798978+0000","last_undegraded":"2023-09-23T20:05:04.798978+0000","last_fullsized":"2023-09-23T20:05:04.798978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":252,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812531+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:05:04.812531+0000","last_peered":"2023-09-23T20:05:04.812531+0000","last_clean":"2023-09-23T20:05:04.812531+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:05:04.812531+0000","last_undegraded":"2023-09-23T20:05:04.812531+0000","last_fullsized":"2023-09-23T20:05:04.812531+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":248,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806761+0000","last_change":"2023-09-23T20:03:10.175462+0000","last_active":"2023-09-23T20:05:04.806761+0000","last_peered":"2023-09-23T20:05:04.806761+0000","last_clean":"2023-09-23T20:05:04.806761+0000","last_became_active":"2023-09-23T20:03:10.175250+0000","last_became_peered":"2023-09-23T20:03:10.175250+0000","last_unstale":"2023-09-23T20:05:04.806761+0000","last_undegraded":"2023-09-23T20:05:04.806761+0000","last_fullsized":"2023-09-23T20:05:04.806761+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:42.961619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"286'10091","reported_seq":10145,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:13.077329+0000","last_change":"2023-09-23T20:01:39.913385+0000","last_active":"2023-09-23T20:05:13.077329+0000","last_peered":"2023-09-23T20:05:13.077329+0000","last_clean":"2023-09-23T20:05:13.077329+0000","last_became_active":"2023-09-23T20:01:39.912872+0000","last_became_peered":"2023-09-23T20:01:39.912872+0000","last_unstale":"2023-09-23T20:05:13.077329+0000","last_undegraded":"2023-09-23T20:05:13.077329+0000","last_fullsized":"2023-09-23T20:05:13.077329+0000","mapping_epoch":240,"log_start":"247'1700","ondisk_log_start":"247'1700","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8391,"log_dups_size":0,"ondisk_log_size":8391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.619813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124846080,"num_objects":1905,"num_object_clones":0,"num_object_copies":5715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10091,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":601,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.811654+0000","last_change":"2023-09-23T20:04:02.277648+0000","last_active":"2023-09-23T20:05:04.811654+0000","last_peered":"2023-09-23T20:05:04.811654+0000","last_clean":"2023-09-23T20:05:04.811654+0000","last_became_active":"2023-09-23T20:04:02.277271+0000","last_became_peered":"2023-09-23T20:04:02.277271+0000","last_unstale":"2023-09-23T20:05:04.811654+0000","last_undegraded":"2023-09-23T20:05:04.811654+0000","last_fullsized":"2023-09-23T20:05:04.811654+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":364,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.799337+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:05:04.799337+0000","last_peered":"2023-09-23T20:05:04.799337+0000","last_clean":"2023-09-23T20:05:04.799337+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:05:04.799337+0000","last_undegraded":"2023-09-23T20:05:04.799337+0000","last_fullsized":"2023-09-23T20:05:04.799337+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":246,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.813479+0000","last_change":"2023-09-23T20:03:10.174842+0000","last_active":"2023-09-23T20:05:04.813479+0000","last_peered":"2023-09-23T20:05:04.813479+0000","last_clean":"2023-09-23T20:05:04.813479+0000","last_became_active":"2023-09-23T20:03:10.174511+0000","last_became_peered":"2023-09-23T20:03:10.174511+0000","last_unstale":"2023-09-23T20:05:04.813479+0000","last_undegraded":"2023-09-23T20:05:04.813479+0000","last_fullsized":"2023-09-23T20:05:04.813479+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:12.723811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":82,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806438+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:05:04.806438+0000","last_peered":"2023-09-23T20:05:04.806438+0000","last_clean":"2023-09-23T20:05:04.806438+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:05:04.806438+0000","last_undegraded":"2023-09-23T20:05:04.806438+0000","last_fullsized":"2023-09-23T20:05:04.806438+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"286'10275","reported_seq":10330,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:12.199145+0000","last_change":"2023-09-23T20:01:39.912135+0000","last_active":"2023-09-23T20:05:12.199145+0000","last_peered":"2023-09-23T20:05:12.199145+0000","last_clean":"2023-09-23T20:05:12.199145+0000","last_became_active":"2023-09-23T20:01:39.911834+0000","last_became_peered":"2023-09-23T20:01:39.911834+0000","last_unstale":"2023-09-23T20:05:12.199145+0000","last_undegraded":"2023-09-23T20:05:12.199145+0000","last_fullsized":"2023-09-23T20:05:12.199145+0000","mapping_epoch":240,"log_start":"254'4526","ondisk_log_start":"254'4526","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:46.725578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128778240,"num_objects":1965,"num_object_clones":0,"num_object_copies":5895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10275,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":664,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812603+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:05:04.812603+0000","last_peered":"2023-09-23T20:05:04.812603+0000","last_clean":"2023-09-23T20:05:04.812603+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:05:04.812603+0000","last_undegraded":"2023-09-23T20:05:04.812603+0000","last_fullsized":"2023-09-23T20:05:04.812603+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":112,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.813739+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:05:04.813739+0000","last_peered":"2023-09-23T20:05:04.813739+0000","last_clean":"2023-09-23T20:05:04.813739+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:05:04.813739+0000","last_undegraded":"2023-09-23T20:05:04.813739+0000","last_fullsized":"2023-09-23T20:05:04.813739+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":692,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812741+0000","last_change":"2023-09-23T19:59:40.899753+0000","last_active":"2023-09-23T20:05:04.812741+0000","last_peered":"2023-09-23T20:05:04.812741+0000","last_clean":"2023-09-23T20:05:04.812741+0000","last_became_active":"2023-09-23T19:59:40.899147+0000","last_became_peered":"2023-09-23T19:59:40.899147+0000","last_unstale":"2023-09-23T20:05:04.812741+0000","last_undegraded":"2023-09-23T20:05:04.812741+0000","last_fullsized":"2023-09-23T20:05:04.812741+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:08.938755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":23,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.801126+0000","last_change":"2023-09-23T20:03:59.233560+0000","last_active":"2023-09-23T20:05:04.801126+0000","last_peered":"2023-09-23T20:05:04.801126+0000","last_clean":"2023-09-23T20:05:04.801126+0000","last_became_active":"2023-09-23T20:03:59.232842+0000","last_became_peered":"2023-09-23T20:03:59.232842+0000","last_unstale":"2023-09-23T20:05:04.801126+0000","last_undegraded":"2023-09-23T20:05:04.801126+0000","last_fullsized":"2023-09-23T20:05:04.801126+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:16.909413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"286'10259","reported_seq":10315,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:12.220545+0000","last_change":"2023-09-23T20:01:39.913698+0000","last_active":"2023-09-23T20:05:12.220545+0000","last_peered":"2023-09-23T20:05:12.220545+0000","last_clean":"2023-09-23T20:05:12.220545+0000","last_became_active":"2023-09-23T20:01:39.913367+0000","last_became_peered":"2023-09-23T20:01:39.913367+0000","last_unstale":"2023-09-23T20:05:12.220545+0000","last_undegraded":"2023-09-23T20:05:12.220545+0000","last_fullsized":"2023-09-23T20:05:12.220545+0000","mapping_epoch":240,"log_start":"254'4525","ondisk_log_start":"254'4525","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129564672,"num_objects":1977,"num_object_clones":0,"num_object_copies":5931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10259,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":542,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.803505+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:05:04.803505+0000","last_peered":"2023-09-23T20:05:04.803505+0000","last_clean":"2023-09-23T20:05:04.803505+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:05:04.803505+0000","last_undegraded":"2023-09-23T20:05:04.803505+0000","last_fullsized":"2023-09-23T20:05:04.803505+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":42,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.802292+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:05:04.802292+0000","last_peered":"2023-09-23T20:05:04.802292+0000","last_clean":"2023-09-23T20:05:04.802292+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:05:04.802292+0000","last_undegraded":"2023-09-23T20:05:04.802292+0000","last_fullsized":"2023-09-23T20:05:04.802292+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":89,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812826+0000","last_change":"2023-09-23T20:01:04.470744+0000","last_active":"2023-09-23T20:05:04.812826+0000","last_peered":"2023-09-23T20:05:04.812826+0000","last_clean":"2023-09-23T20:05:04.812826+0000","last_became_active":"2023-09-23T20:01:04.470302+0000","last_became_peered":"2023-09-23T20:01:04.470302+0000","last_unstale":"2023-09-23T20:05:04.812826+0000","last_undegraded":"2023-09-23T20:05:04.812826+0000","last_fullsized":"2023-09-23T20:05:04.812826+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:25.452596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":245,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812216+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:05:04.812216+0000","last_peered":"2023-09-23T20:05:04.812216+0000","last_clean":"2023-09-23T20:05:04.812216+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:05:04.812216+0000","last_undegraded":"2023-09-23T20:05:04.812216+0000","last_fullsized":"2023-09-23T20:05:04.812216+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":31,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.819408+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:05:04.819408+0000","last_peered":"2023-09-23T20:05:04.819408+0000","last_clean":"2023-09-23T20:05:04.819408+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:05:04.819408+0000","last_undegraded":"2023-09-23T20:05:04.819408+0000","last_fullsized":"2023-09-23T20:05:04.819408+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":39,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.819432+0000","last_change":"2023-09-23T20:03:10.178662+0000","last_active":"2023-09-23T20:05:04.819432+0000","last_peered":"2023-09-23T20:05:04.819432+0000","last_clean":"2023-09-23T20:05:04.819432+0000","last_became_active":"2023-09-23T20:03:10.178352+0000","last_became_peered":"2023-09-23T20:03:10.178352+0000","last_unstale":"2023-09-23T20:05:04.819432+0000","last_undegraded":"2023-09-23T20:05:04.819432+0000","last_fullsized":"2023-09-23T20:05:04.819432+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.469897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":247,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806436+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T20:05:04.806436+0000","last_peered":"2023-09-23T20:05:04.806436+0000","last_clean":"2023-09-23T20:05:04.806436+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T20:05:04.806436+0000","last_undegraded":"2023-09-23T20:05:04.806436+0000","last_fullsized":"2023-09-23T20:05:04.806436+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":282,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812509+0000","last_change":"2023-09-23T19:59:09.172452+0000","last_active":"2023-09-23T20:05:04.812509+0000","last_peered":"2023-09-23T20:05:04.812509+0000","last_clean":"2023-09-23T20:05:04.812509+0000","last_became_active":"2023-09-23T19:59:09.172080+0000","last_became_peered":"2023-09-23T19:59:09.172080+0000","last_unstale":"2023-09-23T20:05:04.812509+0000","last_undegraded":"2023-09-23T20:05:04.812509+0000","last_fullsized":"2023-09-23T20:05:04.812509+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:24.687991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":251,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806435+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T20:05:04.806435+0000","last_peered":"2023-09-23T20:05:04.806435+0000","last_clean":"2023-09-23T20:05:04.806435+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T20:05:04.806435+0000","last_undegraded":"2023-09-23T20:05:04.806435+0000","last_fullsized":"2023-09-23T20:05:04.806435+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:07.560646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":282,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812311+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:05:04.812311+0000","last_peered":"2023-09-23T20:05:04.812311+0000","last_clean":"2023-09-23T20:05:04.812311+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:05:04.812311+0000","last_undegraded":"2023-09-23T20:05:04.812311+0000","last_fullsized":"2023-09-23T20:05:04.812311+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":109,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:05:05.813358+0000","last_change":"2023-09-23T19:59:09.145224+0000","last_active":"2023-09-23T20:05:05.813358+0000","last_peered":"2023-09-23T20:05:05.813358+0000","last_clean":"2023-09-23T20:05:05.813358+0000","last_became_active":"2023-09-23T19:59:09.144819+0000","last_became_peered":"2023-09-23T19:59:09.144819+0000","last_unstale":"2023-09-23T20:05:05.813358+0000","last_undegraded":"2023-09-23T20:05:05.813358+0000","last_fullsized":"2023-09-23T20:05:05.813358+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:04.048181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":23,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.800959+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:05:04.800959+0000","last_peered":"2023-09-23T20:05:04.800959+0000","last_clean":"2023-09-23T20:05:04.800959+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:05:04.800959+0000","last_undegraded":"2023-09-23T20:05:04.800959+0000","last_fullsized":"2023-09-23T20:05:04.800959+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":105,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.806622+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:05:04.806622+0000","last_peered":"2023-09-23T20:05:04.806622+0000","last_clean":"2023-09-23T20:05:04.806622+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:05:04.806622+0000","last_undegraded":"2023-09-23T20:05:04.806622+0000","last_fullsized":"2023-09-23T20:05:04.806622+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":312,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T20:05:04.812049+0000","last_change":"2023-09-23T20:03:59.235232+0000","last_active":"2023-09-23T20:05:04.812049+0000","last_peered":"2023-09-23T20:05:04.812049+0000","last_clean":"2023-09-23T20:05:04.812049+0000","last_became_active":"2023-09-23T20:03:59.234941+0000","last_became_peered":"2023-09-23T20:03:59.234941+0000","last_unstale":"2023-09-23T20:05:04.812049+0000","last_undegraded":"2023-09-23T20:05:04.812049+0000","last_fullsized":"2023-09-23T20:05:04.812049+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:45.591449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1981022232,"num_objects":30229,"num_object_clones":0,"num_object_copies":90687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2868,"num_objects_unfound":0,"num_objects_dirty":30229,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163891,"num_write_kb":6211777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12528,"num_bytes_recovered":820903936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6415265792,"data_stored":6415253576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108708,"ondisk_log_size":108708,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234289,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1009932,"kb_used_data":740164,"kb_used_omap":0,"kb_used_meta":269760,"kb_avail":92735220,"statfs":{"total":95995035648,"available":94960865280,"internally_reserved":0,"allocated":757927936,"data_stored":757053345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":276234240},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:20 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5750000000000002,"15min":2.0430000000000001},"min":{"1min":0.245,"5min":0.22,"15min":0.19700000000000001},"max":{"1min":41.889000000000003,"5min":60.140000000000001,"15min":62.061999999999998},"last":2.71},{"interface":"front","average":{"1min":4.5209999999999999,"5min":2.7120000000000002,"15min":2.27},"min":{"1min":0.25900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":41.936,"5min":62.389000000000003,"15min":62.389000000000003},"last":1.2889999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:04:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2240000000000002,"5min":5.7729999999999997,"15min":3.0739999999999998},"min":{"1min":0.27100000000000002,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":56.024999999999999,"5min":104.16800000000001,"15min":104.16800000000001},"last":5.1360000000000001},{"interface":"front","average":{"1min":6.3220000000000001,"5min":6.0590000000000002,"15min":3.2149999999999999},"min":{"1min":0.34200000000000003,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":56.146000000000001,"5min":104.003,"15min":104.003},"last":0.251}]},{"osd":2,"last update":"Sat Sep 23 20:04:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.794,"15min":3.1379999999999999},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.6159999999999997,"5min":105.842,"15min":130.673},"last":0.52000000000000002},{"interface":"front","average":{"1min":4.0819999999999999,"5min":3.4249999999999998,"15min":3.4260000000000002},"min":{"1min":0.376,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":41.968000000000004,"5min":105.66,"15min":130.71100000000001},"last":3.415}]},{"osd":3,"last update":"Sat Sep 23 20:04:25 2023","interfaces":[{"interface":"back","average":{"1min":10.352,"5min":5.5999999999999996,"15min":3.4630000000000001},"min":{"1min":0.316,"5min":0.247,"15min":0.247},"max":{"1min":80.257999999999996,"5min":146.38800000000001,"15min":146.38800000000001},"last":3.008},{"interface":"front","average":{"1min":10.545999999999999,"5min":5.8099999999999996,"15min":3.464},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":82.281999999999996,"5min":146.851,"15min":146.851},"last":3.6549999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:04:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.80600000000000005,"15min":1.891},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.714,"5min":19.215,"15min":143.255},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.605,"5min":1.1459999999999999,"15min":1.9330000000000001},"min":{"1min":0.27000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":41.822000000000003,"5min":41.822000000000003,"15min":143.41200000000001},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:04:57 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":1.091,"15min":1.4610000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25},"max":{"1min":7.4960000000000004,"5min":35.488,"15min":84.566999999999993},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.95399999999999996,"15min":1.302},"min":{"1min":0.28599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.403,"5min":34.896000000000001,"15min":81.244},"last":0.39200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:04:32 2023","interfaces":[{"interface":"back","average":{"1min":2.6709999999999998,"5min":1.1020000000000001,"15min":0.83999999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":41.902999999999999,"5min":41.902999999999999,"15min":41.902999999999999},"last":0.504},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.66900000000000004,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.3620000000000001,"5min":6.3620000000000001,"15min":6.3620000000000001},"last":0.45400000000000001}]}]},{"osd":6,"up_from":0,"seq":1078036791330,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762491,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1354440,"kb_used_data":944832,"kb_used_omap":7,"kb_used_meta":409592,"kb_avail":92390712,"statfs":{"total":95995035648,"available":94608089088,"internally_reserved":0,"allocated":967507968,"data_stored":966228625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":419423034},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":2.2240000000000002,"15min":3.8799999999999999},"min":{"1min":0.247,"5min":0.16900000000000001,"15min":0.151},"max":{"1min":11.710000000000001,"5min":25.238,"15min":202.917},"last":6.3970000000000002},{"interface":"front","average":{"1min":3.8780000000000001,"5min":2.4460000000000002,"15min":4.0119999999999996},"min":{"1min":0.23200000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":14.839,"5min":22.337,"15min":203.34899999999999},"last":8.7850000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:04:58 2023","interfaces":[{"interface":"back","average":{"1min":4.54,"5min":5.1639999999999997,"15min":5.5119999999999996},"min":{"1min":0.23000000000000001,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":12.554,"5min":134.72499999999999,"15min":231.57499999999999},"last":5.8499999999999996},{"interface":"front","average":{"1min":4.6079999999999997,"5min":5.6429999999999998,"15min":5.5199999999999996},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":11.881,"5min":135.74600000000001,"15min":236.077},"last":1.643}]},{"osd":3,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.665,"5min":2.7349999999999999,"15min":3.016},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.217},"max":{"1min":11.98,"5min":15.571,"15min":68.180999999999997},"last":3.637},{"interface":"front","average":{"1min":5.2850000000000001,"5min":2.7749999999999999,"15min":2.8439999999999999},"min":{"1min":0.36799999999999999,"5min":0.249,"15min":0.20100000000000001},"max":{"1min":20.989999999999998,"5min":22.385999999999999,"15min":66.509},"last":8.8230000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.9450000000000001,"15min":2.4159999999999999},"min":{"1min":0.33800000000000002,"5min":0.254,"15min":0.25},"max":{"1min":9.9190000000000005,"5min":35.497999999999998,"15min":161.68299999999999},"last":1.6779999999999999},{"interface":"front","average":{"1min":3.093,"5min":2.226,"15min":2.4470000000000001},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":20.847000000000001,"5min":35.521000000000001,"15min":161.286},"last":8.7029999999999994}]},{"osd":5,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":2.407,"15min":3.3610000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":20.960000000000001,"5min":51.698,"15min":219.048},"last":8.6839999999999993},{"interface":"front","average":{"1min":3.6739999999999999,"5min":2.6459999999999999,"15min":3.5510000000000002},"min":{"1min":0.40400000000000003,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":15.507999999999999,"5min":51.173000000000002,"15min":218.69499999999999},"last":1.8109999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":2.1640000000000001,"15min":2.2189999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.5579999999999998,"5min":9.8879999999999999,"15min":9.8879999999999999},"last":1.7490000000000001},{"interface":"front","average":{"1min":3.2480000000000002,"5min":3.4060000000000001,"15min":3.4319999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":13.304,"5min":20.734999999999999,"15min":20.734999999999999},"last":8.6549999999999994}]},{"osd":7,"last update":"Sat Sep 23 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":2.1869999999999998,"15min":2.1739999999999999},"min":{"1min":0.432,"5min":0.36199999999999999,"15min":0.26800000000000002},"max":{"1min":11.893000000000001,"5min":14.23,"15min":60.213000000000001},"last":1.7310000000000001},{"interface":"front","average":{"1min":2.048,"5min":1.746,"15min":1.7529999999999999},"min":{"1min":0.42599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.8309999999999995,"5min":14.188000000000001,"15min":61.298999999999999},"last":1.7170000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926025,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1736768,"kb_used_data":1264632,"kb_used_omap":8,"kb_used_meta":472119,"kb_avail":92008384,"statfs":{"total":95995035648,"available":94216585216,"internally_reserved":0,"allocated":1294983168,"data_stored":1293710914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":483450465},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":5.2039999999999997,"15min":2.8119999999999998},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":7.6669999999999998,"5min":94.055999999999997,"15min":94.055999999999997},"last":4.9900000000000002},{"interface":"front","average":{"1min":3.3210000000000002,"5min":5.5010000000000003,"15min":3.1699999999999999},"min":{"1min":0.24199999999999999,"5min":0.183,"15min":0.183},"max":{"1min":9.5609999999999999,"5min":91.632999999999996,"15min":91.632999999999996},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:04:56 2023","interfaces":[{"interface":"back","average":{"1min":11.645,"5min":7.7329999999999997,"15min":4.3390000000000004},"min":{"1min":0.216,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":162.62899999999999,"5min":270.51100000000002,"15min":270.51100000000002},"last":5.7149999999999999},{"interface":"front","average":{"1min":10.435,"5min":7.1280000000000001,"15min":4.3120000000000003},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":162.52500000000001,"5min":269.69299999999998,"15min":269.69299999999998},"last":8.2089999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":3.9060000000000001,"5min":4.8639999999999999,"15min":4.6470000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.5890000000000004,"5min":116.733,"15min":145.72800000000001},"last":3.153},{"interface":"front","average":{"1min":3.4449999999999998,"5min":4.9390000000000001,"15min":4.7610000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":9.6470000000000002,"5min":116.697,"15min":145.50700000000001},"last":3.181}]},{"osd":4,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":8.9209999999999994,"5min":3.085,"15min":2.867},"min":{"1min":0.34799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":148.20599999999999,"5min":148.20599999999999,"15min":148.20599999999999},"last":8.1310000000000002},{"interface":"front","average":{"1min":8.9429999999999996,"5min":3.0649999999999999,"15min":2.5139999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":148.11000000000001,"5min":148.11000000000001,"15min":148.11000000000001},"last":0.45600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.3460000000000001,"15min":2.6230000000000002},"min":{"1min":0.38100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":14.901999999999999,"5min":64.725999999999999,"15min":131.34399999999999},"last":5.0460000000000003},{"interface":"front","average":{"1min":1.855,"5min":1.7130000000000001,"15min":2.645},"min":{"1min":0.39400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.0890000000000004,"5min":12.48,"15min":131.386},"last":8.0890000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.6020000000000001,"15min":1.605},"min":{"1min":0.41199999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.4600000000000009,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":8.1579999999999995},{"interface":"front","average":{"1min":2.73,"5min":2.0699999999999998,"15min":1.96},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":9.4429999999999996,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.4039999999999999,"15min":2.1389999999999998},"min":{"1min":0.39000000000000001,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":14.856,"5min":65.081999999999994,"15min":77.132999999999996},"last":5.0789999999999997},{"interface":"front","average":{"1min":1.899,"5min":1.8200000000000001,"15min":2.177},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.1799999999999997,"5min":12.452,"15min":75.465999999999994},"last":8.1799999999999997}]}]},{"osd":2,"up_from":104,"seq":446676598964,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1500464,"kb_used_data":983784,"kb_used_omap":12,"kb_used_meta":516467,"kb_avail":92244688,"statfs":{"total":95995035648,"available":94458560512,"internally_reserved":0,"allocated":1007394816,"data_stored":1006140641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":528863230},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":6.569,"5min":4.5190000000000001,"15min":2.9159999999999999},"min":{"1min":0.20999999999999999,"5min":0.192,"15min":0.154},"max":{"1min":76.251999999999995,"5min":76.251999999999995,"15min":76.251999999999995},"last":4.8769999999999998},{"interface":"front","average":{"1min":7.1449999999999996,"5min":4.2480000000000002,"15min":2.8479999999999999},"min":{"1min":0.215,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":76.590000000000003,"5min":76.590000000000003,"15min":76.590000000000003},"last":1.411}]},{"osd":1,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":5.3979999999999997,"15min":3.0990000000000002},"min":{"1min":0.25900000000000001,"5min":0.22600000000000001,"15min":0.184},"max":{"1min":10.355,"5min":94.768000000000001,"15min":94.768000000000001},"last":2.1840000000000002},{"interface":"front","average":{"1min":2.2010000000000001,"5min":5.9359999999999999,"15min":3.3100000000000001},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.5999999999999996,"5min":96.808000000000007,"15min":96.808000000000007},"last":4.992}]},{"osd":3,"last update":"Sat Sep 23 20:04:22 2023","interfaces":[{"interface":"back","average":{"1min":4.117,"5min":3.8380000000000001,"15min":2.7970000000000002},"min":{"1min":0.307,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":13.984,"5min":44.408000000000001,"15min":50.186999999999998},"last":4.5259999999999998},{"interface":"front","average":{"1min":2.2709999999999999,"5min":3.3290000000000002,"15min":2.7229999999999999},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.084,"5min":43.981999999999999,"15min":50.585000000000001},"last":6.4020000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":1.7989999999999999,"15min":1.764},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.2240000000000002,"5min":20.591999999999999,"15min":50.640000000000001},"last":5.0149999999999997},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.766,"15min":1.758},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.0579999999999998,"5min":23.062000000000001,"15min":50.154000000000003},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":3.593,"15min":2.5419999999999998},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.5519999999999996,"5min":172.505,"15min":172.505},"last":4.915},{"interface":"front","average":{"1min":1.885,"5min":3.4900000000000002,"15min":2.4300000000000002},"min":{"1min":0.436,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":7.0970000000000004,"5min":172.375,"15min":172.375},"last":1.204}]},{"osd":6,"last update":"Sat Sep 23 20:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.581,"15min":1.611},"min":{"1min":0.38100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.5250000000000004,"5min":10.144,"15min":10.144},"last":4.968},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.917,"15min":1.9510000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.1390000000000002,"5min":7.1390000000000002,"15min":7.1390000000000002},"last":1.236}]},{"osd":7,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.917,"15min":1.8560000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.27400000000000002},"max":{"1min":4.0789999999999997,"5min":23.079999999999998,"15min":50.118000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.0779999999999998,"15min":2.1219999999999999},"min":{"1min":0.40100000000000002,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":5.5750000000000002,"5min":20.614999999999998,"15min":50.664999999999999},"last":4.9420000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435437,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1527668,"kb_used_data":1060588,"kb_used_omap":4,"kb_used_meta":467067,"kb_avail":92217484,"statfs":{"total":95995035648,"available":94430703616,"internally_reserved":0,"allocated":1086042112,"data_stored":1084661332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":478276717},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0600000000000001,"5min":2.399,"15min":3.8929999999999998},"min":{"1min":0.193,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":10.401,"5min":21.100999999999999,"15min":240.40600000000001},"last":0.99099999999999999},{"interface":"front","average":{"1min":4.1799999999999997,"5min":3.327,"15min":4.4989999999999997},"min":{"1min":0.23100000000000001,"5min":0.19600000000000001,"15min":0.17199999999999999},"max":{"1min":16.978000000000002,"5min":53.201000000000001,"15min":240.352},"last":1.0600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.4569999999999999,"5min":4.1920000000000002,"15min":2.2799999999999998},"min":{"1min":0.23000000000000001,"5min":0.217,"15min":0.20000000000000001},"max":{"1min":86.337999999999994,"5min":86.337999999999994,"15min":86.337999999999994},"last":0.89300000000000002},{"interface":"front","average":{"1min":9.1300000000000008,"5min":3.7149999999999999,"15min":2.2410000000000001},"min":{"1min":0.249,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":84.299999999999997,"5min":84.299999999999997,"15min":84.299999999999997},"last":0.82899999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.2539999999999996,"5min":4.0030000000000001,"15min":5.3659999999999997},"min":{"1min":0.25800000000000001,"5min":0.255,"15min":0.219},"max":{"1min":82.111999999999995,"5min":82.111999999999995,"15min":264.97800000000001},"last":1.1439999999999999},{"interface":"front","average":{"1min":3.5790000000000002,"5min":2.7000000000000002,"15min":4.9269999999999996},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":17.657,"5min":30.768999999999998,"15min":257.40899999999999},"last":1.272}]},{"osd":4,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":13.324999999999999,"5min":4.5519999999999996,"15min":2.7669999999999999},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":109.66800000000001,"5min":109.66800000000001,"15min":109.66800000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":13.428000000000001,"5min":5.0140000000000002,"15min":3.2130000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":110.295,"5min":110.295,"15min":110.295},"last":0.67600000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":7.0170000000000003,"5min":4.3049999999999997,"15min":2.5310000000000001},"min":{"1min":0.33400000000000002,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":82.138000000000005,"5min":113.881,"15min":113.881},"last":0.70599999999999996},{"interface":"front","average":{"1min":6.6020000000000003,"5min":4.3559999999999999,"15min":2.4340000000000002},"min":{"1min":0.43099999999999999,"5min":0.313,"15min":0.31},"max":{"1min":82.25,"5min":114.598,"15min":114.598},"last":0.505}]},{"osd":6,"last update":"Sat Sep 23 20:04:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":5.0170000000000003,"15min":5.5069999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.4530000000000003,"5min":82.281999999999996,"15min":82.281999999999996},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.839,"5min":4.9669999999999996,"15min":5.4889999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3470000000000004,"5min":82.233000000000004,"15min":82.233000000000004},"last":0.622}]},{"osd":7,"last update":"Sat Sep 23 20:04:16 2023","interfaces":[{"interface":"back","average":{"1min":13.244,"5min":4.3380000000000001,"15min":2.9729999999999999},"min":{"1min":0.45200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":112.883,"5min":112.883,"15min":138.035},"last":0.47499999999999998},{"interface":"front","average":{"1min":13.119999999999999,"5min":3.8069999999999999,"15min":2.7440000000000002},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":111.85599999999999,"5min":111.85599999999999,"15min":138.054},"last":0.56699999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019199,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1029512,"kb_used_data":769216,"kb_used_omap":0,"kb_used_meta":260288,"kb_avail":92715640,"statfs":{"total":95995035648,"available":94940815360,"internally_reserved":0,"allocated":787677184,"data_stored":786806677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":266534912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:09 2023","interfaces":[{"interface":"back","average":{"1min":3.089,"5min":3.1459999999999999,"15min":3.0899999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.189},"max":{"1min":10.063000000000001,"5min":89.004999999999995,"15min":104.604},"last":3.3540000000000001},{"interface":"front","average":{"1min":2.8220000000000001,"5min":3.1659999999999999,"15min":3.028},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.23400000000000001},"max":{"1min":10.045999999999999,"5min":89.024000000000001,"15min":104.687},"last":3.3650000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":31.100000000000001,"5min":10.585000000000001,"15min":4.8559999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.216},"max":{"1min":161.613,"5min":161.613,"15min":161.613},"last":6.9390000000000001},{"interface":"front","average":{"1min":31.327999999999999,"5min":10.795,"15min":4.7619999999999996},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":162.017,"5min":162.017,"15min":162.017},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:04:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9710000000000001,"5min":6.4290000000000003,"15min":5.1130000000000004},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":5.6180000000000003,"5min":116.56,"15min":144.20099999999999},"last":7.4530000000000003},{"interface":"front","average":{"1min":3.121,"5min":6.5039999999999996,"15min":4.9400000000000004},"min":{"1min":0.30099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":11.048,"5min":117.03400000000001,"15min":144.184},"last":7.274}]},{"osd":3,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":14.276999999999999,"5min":7.5449999999999999,"15min":5.2300000000000004},"min":{"1min":0.376,"5min":0.255,"15min":0.255},"max":{"1min":99.930999999999997,"5min":139.976,"15min":266.54700000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":13.647,"5min":7.1440000000000001,"15min":4.1980000000000004},"min":{"1min":0.39300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":100.01600000000001,"5min":139.911,"15min":139.911},"last":7.0570000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.53100000000000003,"15min":1.8600000000000001},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":1.569,"5min":4.423,"15min":266.59899999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.64400000000000002,"15min":1.9650000000000001},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.24399999999999999},"max":{"1min":3.004,"5min":4.8739999999999997,"15min":266.72899999999998},"last":0.35099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.53300000000000003,"15min":0.51300000000000001},"min":{"1min":0.31,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0410000000000004,"5min":4.0410000000000004,"15min":4.0410000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.66100000000000003,"15min":0.54100000000000004},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.2439999999999998,"5min":4.2439999999999998,"15min":4.2439999999999998},"last":0.41299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:04:49 2023","interfaces":[{"interface":"back","average":{"1min":4.7199999999999998,"5min":1.4239999999999999,"15min":1.2509999999999999},"min":{"1min":0.32600000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":65.522999999999996,"5min":65.522999999999996,"15min":65.522999999999996},"last":0.53700000000000003},{"interface":"front","average":{"1min":5.5970000000000004,"5min":1.595,"15min":1.3300000000000001},"min":{"1min":0.377,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":65.301000000000002,"5min":65.301000000000002,"15min":65.301000000000002},"last":0.55000000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757563,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907776,"kb_used_data":616248,"kb_used_omap":0,"kb_used_meta":291520,"kb_avail":92837376,"statfs":{"total":95995035648,"available":95065473024,"internally_reserved":0,"allocated":631037952,"data_stored":630171886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":298516480},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:07 2023","interfaces":[{"interface":"back","average":{"1min":4.7629999999999999,"5min":1.597,"15min":1.548},"min":{"1min":0.22600000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":41.503,"5min":41.503,"15min":41.503},"last":0.44500000000000001},{"interface":"front","average":{"1min":4.6029999999999998,"5min":1.978,"15min":1.6220000000000001},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.216},"max":{"1min":41.465000000000003,"5min":41.465000000000003,"15min":41.465000000000003},"last":0.502}]},{"osd":1,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":12.311,"5min":9.4450000000000003,"15min":4.9180000000000001},"min":{"1min":0.33100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":171.47300000000001,"5min":266.197,"15min":266.197},"last":0.54900000000000004},{"interface":"front","average":{"1min":12.787000000000001,"5min":9.6140000000000008,"15min":4.9509999999999996},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":171.399,"5min":266.14999999999998,"15min":266.14999999999998},"last":0.48599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:04:51 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":3.5609999999999999,"15min":5.6239999999999997},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":3.6680000000000001,"5min":177.816,"15min":238.96199999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.373,"5min":3.6299999999999999,"15min":5.6719999999999997},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":4.024,"5min":177.851,"15min":238.905},"last":0.56999999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":12.673,"5min":4.883,"15min":3.5150000000000001},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":168.31399999999999,"5min":168.31399999999999,"15min":168.31399999999999},"last":201.50899999999999},{"interface":"front","average":{"1min":10.301,"5min":4.806,"15min":3.3679999999999999},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":168.39500000000001,"5min":168.39500000000001,"15min":168.39500000000001},"last":201.58099999999999}]},{"osd":4,"last update":"Sat Sep 23 20:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":2.4329999999999998,"15min":1.611},"min":{"1min":0.24099999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.786,"5min":191.78899999999999,"15min":191.78899999999999},"last":0.373},{"interface":"front","average":{"1min":0.38300000000000001,"5min":2.444,"15min":1.587},"min":{"1min":0.308,"5min":0.255,"15min":0.255},"max":{"1min":0.54200000000000004,"5min":191.821,"15min":191.821},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.66500000000000004,"15min":0.621},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.0679999999999996,"5min":4.0679999999999996,"15min":4.0679999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.49299999999999999,"15min":0.47199999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.3570000000000002,"5min":3.3570000000000002,"15min":3.3570000000000002},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":2.4460000000000002,"15min":1.74},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":3.964,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.754,"5min":2.5630000000000002,"15min":1.9379999999999999},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.976,"5min":191.69,"15min":191.69},"last":0.46100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1264324608,"data_stored":1264320536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":956764160,"data_stored":956760088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":988807168,"data_stored":988807168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1069355008,"data_stored":1069350936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":776077312,"data_stored":776077312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":621674496,"data_stored":621674496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":738263040,"data_stored":738263040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:05:21.387 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:05:21.418 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:21.686 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:21.802 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:21.811 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:21 smithi135 ceph-mon[221113]: pgmap v792: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 2184/82065 objects misplaced (2.661%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:21.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:21.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:21.812 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:21 smithi135 ceph-mon[213871]: pgmap v792: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 2184/82065 objects misplaced (2.661%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:21.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:21 smithi144 ceph-mon[194833]: pgmap v792: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 2184/82065 objects misplaced (2.661%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:21.912 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:21.943 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:22.041 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:22.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:22.147 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:22.271 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:22.297 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:05:22.334 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:05:22.381 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:05:22.381 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:22.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:22.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:22.453 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:05:22.488 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:22.489 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:05:22.527 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:05:22.537 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:22.571 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:05:22.612 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:05:22.629 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:05:22.629 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:05:22.629 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:05:22.629 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:05:22.629 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:05:22.629 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:05:22.629 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:05:22.629 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:05:22.646 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:22.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:22.656 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:22.662 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:22.754 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:22.891 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:23.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:23 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2727516128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:23 smithi135 ceph-mon[221113]: pgmap v793: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2018/79032 objects misplaced (2.553%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:23 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2727516128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:23 smithi135 ceph-mon[213871]: pgmap v793: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2018/79032 objects misplaced (2.553%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:23 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2727516128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:23 smithi144 ceph-mon[194833]: pgmap v793: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2018/79032 objects misplaced (2.553%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:05:23.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:05:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:05:23] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-09-23T20:05:24.585 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:24.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:24.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:24.690 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:24.772 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:24 smithi144 ceph-mon[194833]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:24 smithi144 ceph-mon[194833]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T20:05:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:24 smithi144 ceph-mon[194833]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:24 smithi144 ceph-mon[194833]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:24 smithi144 ceph-mon[194833]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[221113]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[221113]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T20:05:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[221113]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[221113]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[221113]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:24.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:24.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[213871]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[213871]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T20:05:24.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[213871]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[213871]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[213871]: failed to encode map e288 with expected crc 2023-09-23T20:05:24.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:24.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:25.501 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:25.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:25.601 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:25.607 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[213871]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.607 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[213871]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T20:05:25.607 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[213871]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.607 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[213871]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.607 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[213871]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[213871]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[213871]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.608 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[213871]: pgmap v796: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 1594/75438 objects misplaced (2.113%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:05:25.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[221113]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[221113]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T20:05:25.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[221113]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[221113]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[221113]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[221113]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[221113]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:25 smithi135 ceph-mon[221113]: pgmap v796: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 1594/75438 objects misplaced (2.113%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:05:25.636 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:25.686 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:25.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:25.793 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:25.871 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:25 smithi144 ceph-mon[194833]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:25 smithi144 ceph-mon[194833]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T20:05:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:25 smithi144 ceph-mon[194833]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:25 smithi144 ceph-mon[194833]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:25 smithi144 ceph-mon[194833]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:25 smithi144 ceph-mon[194833]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:25 smithi144 ceph-mon[194833]: failed to encode map e289 with expected crc 2023-09-23T20:05:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:25 smithi144 ceph-mon[194833]: pgmap v796: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 1594/75438 objects misplaced (2.113%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:05:27.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:26 smithi144 ceph-mon[194833]: pgmap v797: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 1594/73968 objects misplaced (2.155%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:05:27.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:26 smithi135 ceph-mon[221113]: pgmap v797: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 1594/73968 objects misplaced (2.155%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:05:27.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:26 smithi135 ceph-mon[213871]: pgmap v797: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 1594/73968 objects misplaced (2.155%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:05:27.306 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:05:27.689 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:27.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:27.796 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:27.826 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:27.878 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:27.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:27.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:27.987 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:28.094 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:28.735 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:28.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:28.847 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:28.876 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:28.928 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:28.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:28.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:29.037 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:29.047 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:116217 6.4 6:2512e97b:::benchmark_data_smithi135_344736_object13561:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:58:56.523222+0000", 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 392.51836051999999, 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.87162493500000005, 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.048 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116217 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523222+0000", 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.049 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523222+0000", 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523220+0000", 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523229+0000", 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4630000000000001e-06 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523245+0000", 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.66e-05 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523255+0000", 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9490000000000008e-06 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.051 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523441+0000", 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018609799999999999 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:56.523449+0000", 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7079999999999997e-06 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.372733+0000", 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.84928379899999995 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.052 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.373252+0000", 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00051908200000000003 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.387567+0000", 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014315068 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.387671+0000", 2023-09-23T20:05:29.053 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010451 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.387943+0000", 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00027156499999999998 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.391458+0000", 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00351498 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.391486+0000", 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8178e-05 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.392778+0000", 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001291857 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.392837+0000", 2023-09-23T20:05:29.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9407000000000002e-05 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:58:57.394847+0000", 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002009547 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148803 6.14 6:2da19cbe:::benchmark_data_smithi135_344736_object46147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:05:29.056 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 346.114251443, 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2584235440000002, 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148803 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927329+0000", 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927340+0000", 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0981e-05 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927360+0000", 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0913000000000001e-05 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927371+0000", 2023-09-23T20:05:29.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0467e-05 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933494+0000", 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061233260000000001 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933501+0000", 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0530000000000001e-06 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.060 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.951971+0000", 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018469316999999999 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.951983+0000", 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2286000000000001e-05 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050605+0000", 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0986220259999999 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050613+0000", 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9969999999999998e-06 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.171636+0000", 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.121022937 2023-09-23T20:05:29.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.171779+0000", 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000142738 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.172136+0000", 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00035740899999999999 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174779+0000", 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002642574 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.180158+0000", 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053795379999999997 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185597+0000", 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0054389099999999999 2023-09-23T20:05:29.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185657+0000", 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0288999999999998e-05 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185755+0000", 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7291000000000001e-05 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148808 6.4 6:27155b23:::benchmark_data_smithi135_344736_object46152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 346.10810751999998, 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2790047449999999, 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148808 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933474+0000", 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933479+0000", 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3739999999999997e-06 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933490+0000", 2023-09-23T20:05:29.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1051e-05 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933497+0000", 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.601e-06 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933549+0000", 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1745e-05 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933555+0000", 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5440000000000004e-06 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.952509+0000", 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018953932 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.952518+0000", 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.081e-06 2023-09-23T20:05:29.070 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050570+0000", 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.098051485 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050581+0000", 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0878e-05 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.071 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174805+0000", 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1242246469999999 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174952+0000", 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014714599999999999 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.175367+0000", 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000414855 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.180166+0000", 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047984739999999996 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185798+0000", 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00563285 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212221+0000", 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.026422386999999999 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212281+0000", 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9735999999999999e-05 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212480+0000", 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000199125 2023-09-23T20:05:29.074 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148812 6.1 6:81ec3361:::benchmark_data_smithi135_344736_object46156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 346.08768444399999, 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.216623062, 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.075 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148812 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.076 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953897+0000", 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953966+0000", 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9086999999999998e-05 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953978+0000", 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1731e-05 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953989+0000", 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1902000000000001e-05 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.954238+0000", 2023-09-23T20:05:29.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000248216 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.954249+0000", 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.114e-05 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049700+0000", 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.095451519 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049712+0000", 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1780000000000001e-05 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.146621+0000", 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.096909012 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.146767+0000", 2023-09-23T20:05:29.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014557700000000001 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,2,7", 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.147202+0000", 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043543599999999998 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.152224+0000", 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050223309999999997 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.081 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.152512+0000", 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000287421 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.163594+0000", 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011081946000000001 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170347+0000", 2023-09-23T20:05:29.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067530960000000001 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170399+0000", 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2540000000000002e-05 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170521+0000", 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012170700000000001 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.083 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182375 6.14 6:2a7953cb:::benchmark_data_smithi135_344736_object79719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.461225+0000", 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 295.580357042, 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7366573399999998, 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182375 2023-09-23T20:05:29.084 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461225+0000", 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461225+0000", 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461224+0000", 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461231+0000", 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6990000000000003e-06 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461271+0000", 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9549999999999999e-05 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461286+0000", 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4688999999999999e-05 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461350+0000", 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3955999999999995e-05 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461362+0000", 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2658e-05 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439124+0000", 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97776162 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439133+0000", 2023-09-23T20:05:29.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4350000000000003e-06 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.189742+0000", 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7506088659999999 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.189894+0000", 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015184899999999999 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.190331+0000", 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043668999999999999 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193071+0000", 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027404880000000001 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.196937+0000", 2023-09-23T20:05:29.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038661120000000001 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197009+0000", 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1431000000000001e-05 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197073+0000", 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4125e-05 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.091 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197883+0000", 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00080992499999999997 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182407 6.4 6:21645038:::benchmark_data_smithi135_344736_object79751:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.517127+0000", 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 295.52445569600002, 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6541434769999999, 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.092 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182407 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517127+0000", 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517127+0000", 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517125+0000", 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517132+0000", 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3899999999999998e-06 2023-09-23T20:05:29.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517143+0000", 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0864e-05 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517150+0000", 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7659999999999998e-06 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517202+0000", 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1727999999999997e-05 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517213+0000", 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1240000000000001e-05 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.438985+0000", 2023-09-23T20:05:29.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.921771746 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439002+0000", 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7062999999999999e-05 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.164600+0000", 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7255975669999999 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.164737+0000", 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013742600000000001 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.165195+0000", 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045771700000000001 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168101+0000", 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029057380000000002 2023-09-23T20:05:29.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168107+0000", 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6059999999999999e-06 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.170461+0000", 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023538439999999999 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.099 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.170504+0000", 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3439999999999997e-05 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.171270+0000", 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00076583700000000001 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182418 6.14 6:2825501c:::benchmark_data_smithi135_344736_object79762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:05:29.100 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.530422+0000", 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 295.51116030899999, 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.670691181, 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182418 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530422+0000", 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530422+0000", 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530421+0000", 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530427+0000", 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8139999999999997e-06 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530436+0000", 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5210000000000008e-06 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530442+0000", 2023-09-23T20:05:29.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2499999999999997e-06 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530499+0000", 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6876999999999998e-05 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530509+0000", 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0801e-05 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439303+0000", 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.908794137 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439313+0000", 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0480000000000004e-06 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193088+0000", 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.753775608 2023-09-23T20:05:29.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193199+0000", 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110946 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193487+0000", 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000287566 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.196996+0000", 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0035091520000000002 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197898+0000", 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00090200999999999996 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.200328+0000", 2023-09-23T20:05:29.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024299840000000001 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.200382+0000", 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4364000000000003e-05 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201113+0000", 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00073121600000000003 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.108 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182423 6.4 6:248db571:::benchmark_data_smithi135_344736_object79767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.540102+0000", 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 295.501480039, 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6532785529999998, 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182423 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.109 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540102+0000", 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540102+0000", 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540101+0000", 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540107+0000", 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7690000000000002e-06 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540120+0000", 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3185e-05 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540126+0000", 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0809999999999999e-06 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540186+0000", 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0248999999999999e-05 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540200+0000", 2023-09-23T20:05:29.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3903000000000001e-05 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439418+0000", 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.89921767600000002 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439426+0000", 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9510000000000006e-06 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.113 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168183+0000", 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.728756765 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168361+0000", 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017813099999999999 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168797+0000", 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043667499999999998 2023-09-23T20:05:29.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.192289+0000", 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.023491365 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.192331+0000", 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1913999999999999e-05 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.192365+0000", 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4589999999999999e-05 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.192420+0000", 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4608000000000003e-05 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193381+0000", 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00096113800000000005 2023-09-23T20:05:29.116 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182425 6.14 6:2b319d69:::benchmark_data_smithi135_344736_object79769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.541828+0000", 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 295.49975460000002, 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.659927615, 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.117 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182425 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541828+0000", 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541828+0000", 2023-09-23T20:05:29.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541827+0000", 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541833+0000", 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1670000000000004e-06 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541847+0000", 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3898e-05 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541854+0000", 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4549999999999998e-06 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541931+0000", 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6481000000000002e-05 2023-09-23T20:05:29.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541943+0000", 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2354999999999999e-05 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439334+0000", 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.89739137099999999 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439343+0000", 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3170000000000002e-06 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.194858+0000", 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.755515079 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.195026+0000", 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016822699999999999 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.195313+0000", 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028750100000000001 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.200270+0000", 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049562269999999997 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201130+0000", 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00086066699999999996 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201196+0000", 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5870000000000005e-05 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201249+0000", 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2873e-05 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201756+0000", 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050643599999999997 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182451 6.4 6:22222862:::benchmark_data_smithi135_344736_object79795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.584941+0000", 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.45664119899999, 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6476608669999999, 2023-09-23T20:05:29.125 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182451 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584941+0000", 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584941+0000", 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584939+0000", 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584947+0000", 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5100000000000001e-06 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584960+0000", 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3125999999999999e-05 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584968+0000", 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8150000000000006e-06 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.585034+0000", 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6517e-05 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.585045+0000", 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0963e-05 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.218776+0000", 2023-09-23T20:05:29.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6337305529999999 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.218958+0000", 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000182385 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.219392+0000", 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043395100000000002 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.229881+0000", 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010488924 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.229957+0000", 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6092000000000003e-05 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231608+0000", 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016505210000000001 2023-09-23T20:05:29.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231660+0000", 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1894999999999997e-05 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232602+0000", 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00094250300000000003 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182466 6.4 6:249455ee:::benchmark_data_smithi135_344736_object79810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.615458+0000", 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.42612494899998, 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.618027707, 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182466 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615458+0000", 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615458+0000", 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615456+0000", 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615463+0000", 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5009999999999998e-06 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615476+0000", 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2658e-05 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615484+0000", 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4610000000000008e-06 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615548+0000", 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3742000000000001e-05 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615559+0000", 2023-09-23T20:05:29.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0558e-05 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221570+0000", 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6060114219999999 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221705+0000", 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013505799999999999 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.222076+0000", 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037122899999999999 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230893+0000", 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088166259999999993 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232681+0000", 2023-09-23T20:05:29.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001787477 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232714+0000", 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3597999999999999e-05 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232762+0000", 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8362000000000002e-05 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.233485+0000", 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00072280300000000005 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182470 6.14 6:29661942:::benchmark_data_smithi135_344736_object79814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.623602+0000", 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.41798051299997, 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.58335468, 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.140 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182470 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623602+0000", 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623602+0000", 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623600+0000", 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623608+0000", 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5000000000000002e-06 2023-09-23T20:05:29.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623624+0000", 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5818e-05 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623633+0000", 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9449999999999999e-06 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623693+0000", 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0254000000000003e-05 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623705+0000", 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2446000000000001e-05 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197940+0000", 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5742344610000001 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.198079+0000", 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013971900000000001 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.198432+0000", 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00035207899999999999 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201811+0000", 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033799030000000001 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.205930+0000", 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004118109 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.205944+0000", 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4034e-05 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.205987+0000", 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3065000000000002e-05 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.206957+0000", 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00097006100000000004 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182490 6.4 6:259d21c1:::benchmark_data_smithi135_344736_object79834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.655714+0000", 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.38586837600002, 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5807021909999999, 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182490 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655714+0000", 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655714+0000", 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655712+0000", 2023-09-23T20:05:29.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655723+0000", 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1382999999999999e-05 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655747+0000", 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3969999999999999e-05 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655759+0000", 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2118e-05 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655824+0000", 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5178999999999994e-05 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655839+0000", 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4277000000000001e-05 2023-09-23T20:05:29.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.224400+0000", 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.56856145 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.224546+0000", 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000146198 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.225014+0000", 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00046763700000000002 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230900+0000", 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005886255 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.233525+0000", 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026250050000000001 2023-09-23T20:05:29.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.235463+0000", 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019373859999999999 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.235505+0000", 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2014999999999997e-05 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.236416+0000", 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00091160799999999995 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182501 6.14 6:29309ff6:::benchmark_data_smithi135_344736_object79845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.672483+0000", 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.369099124, 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5430168900000001, 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.155 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182501 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672483+0000", 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672483+0000", 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672482+0000", 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672488+0000", 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4190000000000001e-06 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672497+0000", 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4599999999999992e-06 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672503+0000", 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9000000000000003e-06 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672556+0000", 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2802e-05 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672568+0000", 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1798e-05 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201840+0000", 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5292725059999999 2023-09-23T20:05:29.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.202025+0000", 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018504700000000001 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.202485+0000", 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045962800000000001 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209642+0000", 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0071572060000000002 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209721+0000", 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8843000000000004e-05 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.214647+0000", 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049257779999999996 2023-09-23T20:05:29.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.214683+0000", 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6659999999999998e-05 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.215500+0000", 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00081710400000000003 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182504 6.14 6:2ff3c150:::benchmark_data_smithi135_344736_object79848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.675556+0000", 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.36602655600001, 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.540447256, 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182504 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675556+0000", 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675556+0000", 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675555+0000", 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675560+0000", 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.023e-06 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675569+0000", 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.081e-06 2023-09-23T20:05:29.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675574+0000", 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1660000000000002e-06 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675623+0000", 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8992999999999997e-05 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675631+0000", 2023-09-23T20:05:29.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3159999999999997e-06 2023-09-23T20:05:29.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.167 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.167 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.203977+0000", 2023-09-23T20:05:29.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.528345866 2023-09-23T20:05:29.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.168 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:28 smithi144 ceph-mon[194833]: pgmap v798: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 941/67152 objects misplaced (1.401%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:05:29.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.204148+0000", 2023-09-23T20:05:29.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000171239 2023-09-23T20:05:29.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.204475+0000", 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00032640300000000001 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209653+0000", 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051786540000000004 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209768+0000", 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011504400000000001 2023-09-23T20:05:29.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.215570+0000", 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058017670000000002 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.215621+0000", 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1153999999999998e-05 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216003+0000", 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00038182100000000002 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182506 6.14 6:2f661eda:::benchmark_data_smithi135_344736_object79850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.676137+0000", 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.365445153, 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5404231739999998, 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.171 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182506 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676137+0000", 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676137+0000", 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676136+0000", 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676142+0000", 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4990000000000002e-06 2023-09-23T20:05:29.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676155+0000", 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2882e-05 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676177+0000", 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1885000000000001e-05 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676235+0000", 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7839e-05 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676249+0000", 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4646e-05 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.207002+0000", 2023-09-23T20:05:29.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5307524340000001 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.207166+0000", 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000164854 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.207637+0000", 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00047019199999999997 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216056+0000", 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084191519999999992 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216110+0000", 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4335000000000001e-05 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216120+0000", 2023-09-23T20:05:29.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0118999999999999e-05 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216173+0000", 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2649999999999999e-05 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216561+0000", 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00038758899999999998 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.178 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182508 6.4 6:2138bf66:::benchmark_data_smithi135_344736_object79852:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.676243+0000", 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.36533925700002, 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5664108890000001, 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182508 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.179 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676243+0000", 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676243+0000", 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676242+0000", 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676261+0000", 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8660000000000001e-05 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676273+0000", 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2125e-05 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.181 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676281+0000", 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8439999999999998e-06 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676327+0000", 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6541e-05 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676344+0000", 2023-09-23T20:05:29.182 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6133e-05 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.226959+0000", 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.550615732 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.227137+0000", 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000177615 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.227621+0000", 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000484148 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.236457+0000", 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088358269999999992 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.239381+0000", 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029245650000000001 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.184 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.241583+0000", 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022013229999999998 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.241639+0000", 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6001000000000001e-05 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.185 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.242654+0000", 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010153930000000001 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182535 6.14 6:2e05be58:::benchmark_data_smithi135_344736_object79879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.733828+0000", 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.30775414800001, 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.497046002, 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.186 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182535 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733828+0000", 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.187 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733828+0000", 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733827+0000", 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733833+0000", 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8189999999999997e-06 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733840+0000", 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1180000000000003e-06 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733846+0000", 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4020000000000001e-06 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733919+0000", 2023-09-23T20:05:29.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3867000000000003e-05 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733934+0000", 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4487e-05 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209794+0000", 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.475860226 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.190 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209986+0000", 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000192134 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.210415+0000", 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00042892000000000001 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216570+0000", 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061551009999999996 2023-09-23T20:05:29.191 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221464+0000", 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048936450000000003 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.229936+0000", 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084721339999999992 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.229994+0000", 2023-09-23T20:05:29.192 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7757000000000003e-05 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230874+0000", 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00088049799999999996 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182539 6.14 6:2bbcc305:::benchmark_data_smithi135_344736_object79883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.745414+0000", 2023-09-23T20:05:29.193 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.29616810499999, 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4861406260000001, 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182539 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.194 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745414+0000", 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745414+0000", 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745412+0000", 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745420+0000", 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8339999999999999e-06 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745432+0000", 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2692e-05 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745440+0000", 2023-09-23T20:05:29.196 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5179999999999998e-06 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745501+0000", 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1546999999999999e-05 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745512+0000", 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0458999999999999e-05 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.197 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.212416+0000", 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.466904027 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.212572+0000", 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000155916 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.212947+0000", 2023-09-23T20:05:29.198 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037548600000000002 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216578+0000", 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036308780000000001 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221544+0000", 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049656229999999997 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230942+0000", 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0093981930000000009 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230998+0000", 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5649999999999997e-05 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231555+0000", 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00055735400000000003 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.200 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182551 6.14 6:293f61b4:::benchmark_data_smithi135_344736_object79895:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.810825+0000", 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 294.23075790399997, 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.421271562, 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:05:29.201 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182551 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810825+0000", 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810825+0000", 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:29.202 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810823+0000", 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810830+0000", 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.55e-06 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810846+0000", 2023-09-23T20:05:29.203 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5909e-05 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810854+0000", 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0730000000000005e-06 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810935+0000", 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1198000000000004e-05 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.204 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810950+0000", 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.558e-05 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216606+0000", 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.405656003 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216785+0000", 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000178078 2023-09-23T20:05:29.205 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.217257+0000", 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00047277099999999998 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221473+0000", 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0042157610000000002 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.226942+0000", 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0054686429999999996 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231631+0000", 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046896519999999999 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231675+0000", 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3658000000000001e-05 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:29.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232096+0000", 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00042110599999999998 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:05:29.208 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:29.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:28 smithi135 ceph-mon[221113]: pgmap v798: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 941/67152 objects misplaced (1.401%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:05:29.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:28 smithi135 ceph-mon[213871]: pgmap v798: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 941/67152 objects misplaced (1.401%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:05:29.209 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:29.314 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:29.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:29.388 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:05:29.388 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1098,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":289,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":7,"osd_in_since":1695499311,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":98},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":99,"num_pools":5,"num_objects":22384,"data_bytes":1469847499,"bytes_used":7774175232,"bytes_avail":664191074304,"bytes_total":671965249536,"misplaced_objects":941,"misplaced_total":67152,"misplaced_ratio":0.014012985465808911,"recovering_objects_per_sec":116,"recovering_bytes_per_sec":7657342,"recovering_keys_per_sec":0,"num_objects_recovered":935,"num_bytes_recovered":61276160,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":621},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-23T20:03:30.680470+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"8f9ef775-0a40-45cd-8917-5824ea2cda14":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 2s)","progress":0.97979795932769775,"add_to_ceph_s":true}}} 2023-09-23T20:05:29.419 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:29.501 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:29.885 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:05:30.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3523663607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:05:30.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:29 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:05:30.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/664783956' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:05:30.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3523663607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:05:30.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:29 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:05:30.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/664783956' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:05:30.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3523663607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:05:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:29 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:05:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/664783956' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:05:30.646 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:30.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:30.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:30.756 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:30.791 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:30.840 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:30.842 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:05:30.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:30.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[213871]: pgmap v800: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 941/65646 objects misplaced (1.433%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: failed to encode map e290 with expected crc 2023-09-23T20:05:30.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:30 smithi135 ceph-mon[221113]: pgmap v800: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 941/65646 objects misplaced (1.433%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T20:05:30.946 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:30.974 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:31.021 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:31.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:31.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:31.127 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:31.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:31.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: failed to encode map e290 with expected crc 2023-09-23T20:05:31.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:05:31.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T20:05:31.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: failed to encode map e290 with expected crc 2023-09-23T20:05:31.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: failed to encode map e290 with expected crc 2023-09-23T20:05:31.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: failed to encode map e290 with expected crc 2023-09-23T20:05:31.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: failed to encode map e290 with expected crc 2023-09-23T20:05:31.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: failed to encode map e290 with expected crc 2023-09-23T20:05:31.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:30 smithi144 ceph-mon[194833]: pgmap v800: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 941/65646 objects misplaced (1.433%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T20:05:31.172 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:31.973 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:31.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:31.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:32.100 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:32.125 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:32.167 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:32.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:32.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:32.272 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:32.355 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:32.409 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:05:32.409 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:05:32.776 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:32.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:32.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:32.888 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:33.021 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:33.253 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:33.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:33.360 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:33.388 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:33.440 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:33.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:33.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:33.546 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:33.605 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:33.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:33 smithi144 ceph-mon[194833]: pgmap v801: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.3 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 493/62634 objects misplaced (0.787%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:05:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:05:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:05:33] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-09-23T20:05:33.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:33 smithi135 ceph-mon[221113]: pgmap v801: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.3 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 493/62634 objects misplaced (0.787%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:05:33.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:33 smithi135 ceph-mon[213871]: pgmap v801: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.3 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 493/62634 objects misplaced (0.787%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:05:34.431 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:34.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:34.542 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:34.568 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:34.636 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:34.698 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:34.791 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:34.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:34.835 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:05:34.897 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:34.979 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:35.055 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:34 smithi135 ceph-mon[213871]: pgmap v802: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.2 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 493/59148 objects misplaced (0.834%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:05:35.058 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:34 smithi135 ceph-mon[221113]: pgmap v802: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.2 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 493/59148 objects misplaced (0.834%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:05:35.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:34 smithi144 ceph-mon[194833]: pgmap v802: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.2 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 493/59148 objects misplaced (0.834%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:05:35.480 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:35.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:35.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:35.590 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:35.678 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:36.413 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:36.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:36.522 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:36.548 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:36.588 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:36.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:36.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:36.695 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:36.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4058399598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4058399598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:36.723 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4058399598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:36.726 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:36.780 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:36.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:36.891 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:36.980 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:37.215 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:05:37.215 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:05:37.215 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:05:37.215 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:05:37.215 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:05:37.216 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:05:37.217 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:05:37.310 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:37.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:37.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:37.418 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:37.500 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:37 smithi135 ceph-mon[213871]: pgmap v803: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 493/57585 objects misplaced (0.856%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:05:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:37.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:37.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:37 smithi135 ceph-mon[221113]: pgmap v803: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 493/57585 objects misplaced (0.856%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:05:37.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:37.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:37 smithi144 ceph-mon[194833]: pgmap v803: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 493/57585 objects misplaced (0.856%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:05:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:38.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T20:05:38.703 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-09-23T20:05:38.703 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T20:05:38.703 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-09-23T20:05:39.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T20:05:39.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-09-23T20:05:39.227 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:39.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:39.344 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:39.370 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:39.419 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:39.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:39.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:39.526 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:39.554 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:39.605 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:39.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:39.710 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:39.754 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:05:39.754 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:05:39.813 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:39.868 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-09-23T20:05:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-09-23T20:05:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: osdmap e291: 8 total, 8 up, 7 in 2023-09-23T20:05:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[213871]: pgmap v805: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.0 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 166/50544 objects misplaced (0.328%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-09-23T20:05:39.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: osdmap e291: 8 total, 8 up, 7 in 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: failed to encode map e291 with expected crc 2023-09-23T20:05:39.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:39 smithi135 ceph-mon[221113]: pgmap v805: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.0 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 166/50544 objects misplaced (0.328%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:05:40.032 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:40.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:40.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:40.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-09-23T20:05:40.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-09-23T20:05:40.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: osdmap e291: 8 total, 8 up, 7 in 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: failed to encode map e291 with expected crc 2023-09-23T20:05:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:39 smithi144 ceph-mon[194833]: pgmap v805: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.0 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 166/50544 objects misplaced (0.328%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:05:40.142 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:40.255 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:40.465 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:40.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:40.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:40.579 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:40.673 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:40.789 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:05:40.801 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:05:40.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[221113]: failed to encode map e292 with expected crc 2023-09-23T20:05:40.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[221113]: failed to encode map e292 with expected crc 2023-09-23T20:05:40.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[221113]: failed to encode map e292 with expected crc 2023-09-23T20:05:40.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[221113]: osdmap e292: 8 total, 8 up, 7 in 2023-09-23T20:05:40.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[221113]: failed to encode map e292 with expected crc 2023-09-23T20:05:40.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:40.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:40.835 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[213871]: failed to encode map e292 with expected crc 2023-09-23T20:05:40.835 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[213871]: failed to encode map e292 with expected crc 2023-09-23T20:05:40.835 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[213871]: failed to encode map e292 with expected crc 2023-09-23T20:05:40.835 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[213871]: osdmap e292: 8 total, 8 up, 7 in 2023-09-23T20:05:40.835 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[213871]: failed to encode map e292 with expected crc 2023-09-23T20:05:40.835 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:40.835 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:41.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:40 smithi144 ceph-mon[194833]: failed to encode map e292 with expected crc 2023-09-23T20:05:41.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:40 smithi144 ceph-mon[194833]: failed to encode map e292 with expected crc 2023-09-23T20:05:41.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:40 smithi144 ceph-mon[194833]: failed to encode map e292 with expected crc 2023-09-23T20:05:41.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:40 smithi144 ceph-mon[194833]: osdmap e292: 8 total, 8 up, 7 in 2023-09-23T20:05:41.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:40 smithi144 ceph-mon[194833]: failed to encode map e292 with expected crc 2023-09-23T20:05:41.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:41.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:41.197 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:05:41.339 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:41.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:41.401 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:05:41.443 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:41.470 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:41.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: pgmap v807: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1021 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 166/48894 objects misplaced (0.340%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:05:41.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: from='client.45449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:41.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: failed to encode map e293 with expected crc 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: osdmap e293: 8 total, 8 up, 7 in 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: failed to encode map e293 with expected crc 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: failed to encode map e293 with expected crc 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: failed to encode map e293 with expected crc 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: pgmap v807: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1021 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 166/48894 objects misplaced (0.340%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:05:41.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: from='client.45449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: failed to encode map e293 with expected crc 2023-09-23T20:05:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: osdmap e293: 8 total, 8 up, 7 in 2023-09-23T20:05:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: failed to encode map e293 with expected crc 2023-09-23T20:05:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: failed to encode map e293 with expected crc 2023-09-23T20:05:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: failed to encode map e293 with expected crc 2023-09-23T20:05:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:05:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:05:41.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:41 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: pgmap v807: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1021 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 166/48894 objects misplaced (0.340%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:05:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: from='client.45449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: failed to encode map e293 with expected crc 2023-09-23T20:05:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: osdmap e293: 8 total, 8 up, 7 in 2023-09-23T20:05:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: failed to encode map e293 with expected crc 2023-09-23T20:05:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: failed to encode map e293 with expected crc 2023-09-23T20:05:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: failed to encode map e293 with expected crc 2023-09-23T20:05:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:05:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:05:42.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:41 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:42.259 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:42.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:42.287 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":807,"stamp":"2023-09-23T20:05:40.763969+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1070995403,"num_objects":16298,"num_object_clones":0,"num_object_copies":48894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":166,"num_objects_unfound":0,"num_objects_dirty":16298,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":178757,"num_write_kb":6224502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14621,"num_bytes_recovered":950999541,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111770,"ondisk_log_size":111770,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6385568,"kb_used_data":3640680,"kb_used_omap":32,"kb_used_meta":2744671,"kb_avail":649830496,"statfs":{"total":671965249536,"available":665426427904,"internally_reserved":0,"allocated":3728056320,"data_stored":3720084343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":2810543622},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,6,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-300154880,"num_objects":-4580,"num_object_clones":0,"num_object_copies":-13740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-327,"num_objects_unfound":0,"num_objects_dirty":-4580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4580,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":279,"num_bytes_recovered":18284544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004949"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":268,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.565827+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T20:05:24.565827+0000","last_peered":"2023-09-23T20:05:24.565827+0000","last_clean":"2023-09-23T20:05:24.565827+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T20:05:24.565827+0000","last_undegraded":"2023-09-23T20:05:24.565827+0000","last_fullsized":"2023-09-23T20:05:24.565827+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":112,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561103+0000","last_change":"2023-09-23T19:59:09.151117+0000","last_active":"2023-09-23T20:05:24.561103+0000","last_peered":"2023-09-23T20:05:24.561103+0000","last_clean":"2023-09-23T20:05:24.561103+0000","last_became_active":"2023-09-23T19:59:09.150795+0000","last_became_peered":"2023-09-23T19:59:09.150795+0000","last_unstale":"2023-09-23T20:05:24.561103+0000","last_undegraded":"2023-09-23T20:05:24.561103+0000","last_fullsized":"2023-09-23T20:05:24.561103+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:34.994574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":253,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.706064+0000","last_change":"2023-09-23T19:59:10.156070+0000","last_active":"2023-09-23T20:05:38.706064+0000","last_peered":"2023-09-23T20:05:38.706064+0000","last_clean":"2023-09-23T20:05:38.706064+0000","last_became_active":"2023-09-23T19:59:10.155813+0000","last_became_peered":"2023-09-23T19:59:10.155813+0000","last_unstale":"2023-09-23T20:05:38.706064+0000","last_undegraded":"2023-09-23T20:05:38.706064+0000","last_fullsized":"2023-09-23T20:05:38.706064+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:42.582380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":42,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599486+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:05:24.599486+0000","last_peered":"2023-09-23T20:05:24.599486+0000","last_clean":"2023-09-23T20:05:24.599486+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:05:24.599486+0000","last_undegraded":"2023-09-23T20:05:24.599486+0000","last_fullsized":"2023-09-23T20:05:24.599486+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":112,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.560428+0000","last_change":"2023-09-23T19:59:09.163182+0000","last_active":"2023-09-23T20:05:24.560428+0000","last_peered":"2023-09-23T20:05:24.560428+0000","last_clean":"2023-09-23T20:05:24.560428+0000","last_became_active":"2023-09-23T19:59:09.161612+0000","last_became_peered":"2023-09-23T19:59:09.161612+0000","last_unstale":"2023-09-23T20:05:24.560428+0000","last_undegraded":"2023-09-23T20:05:24.560428+0000","last_fullsized":"2023-09-23T20:05:24.560428+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:59.524152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":215,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.605595+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T20:05:24.605595+0000","last_peered":"2023-09-23T20:05:24.605595+0000","last_clean":"2023-09-23T20:05:24.605595+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T20:05:24.605595+0000","last_undegraded":"2023-09-23T20:05:24.605595+0000","last_fullsized":"2023-09-23T20:05:24.605595+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:59.043547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":216,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.605441+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:05:24.605441+0000","last_peered":"2023-09-23T20:05:24.605441+0000","last_clean":"2023-09-23T20:05:24.605441+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:05:24.605441+0000","last_undegraded":"2023-09-23T20:05:24.605441+0000","last_fullsized":"2023-09-23T20:05:24.605441+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":112,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.560504+0000","last_change":"2023-09-23T19:59:09.146009+0000","last_active":"2023-09-23T20:05:24.560504+0000","last_peered":"2023-09-23T20:05:24.560504+0000","last_clean":"2023-09-23T20:05:24.560504+0000","last_became_active":"2023-09-23T19:59:09.145687+0000","last_became_peered":"2023-09-23T19:59:09.145687+0000","last_unstale":"2023-09-23T20:05:24.560504+0000","last_undegraded":"2023-09-23T20:05:24.560504+0000","last_fullsized":"2023-09-23T20:05:24.560504+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.189450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":249,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604341+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T20:05:24.604341+0000","last_peered":"2023-09-23T20:05:24.604341+0000","last_clean":"2023-09-23T20:05:24.604341+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T20:05:24.604341+0000","last_undegraded":"2023-09-23T20:05:24.604341+0000","last_fullsized":"2023-09-23T20:05:24.604341+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":131,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.560843+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:05:24.560843+0000","last_peered":"2023-09-23T20:05:24.560843+0000","last_clean":"2023-09-23T20:05:24.560843+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:05:24.560843+0000","last_undegraded":"2023-09-23T20:05:24.560843+0000","last_fullsized":"2023-09-23T20:05:24.560843+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":285,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599657+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:05:24.599657+0000","last_peered":"2023-09-23T20:05:24.599657+0000","last_clean":"2023-09-23T20:05:24.599657+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:05:24.599657+0000","last_undegraded":"2023-09-23T20:05:24.599657+0000","last_fullsized":"2023-09-23T20:05:24.599657+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":249,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.705656+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T20:05:38.705656+0000","last_peered":"2023-09-23T20:05:38.705656+0000","last_clean":"2023-09-23T20:05:38.705656+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T20:05:38.705656+0000","last_undegraded":"2023-09-23T20:05:38.705656+0000","last_fullsized":"2023-09-23T20:05:38.705656+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:51.825640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":304,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566063+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:05:24.566063+0000","last_peered":"2023-09-23T20:05:24.566063+0000","last_clean":"2023-09-23T20:05:24.566063+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:05:24.566063+0000","last_undegraded":"2023-09-23T20:05:24.566063+0000","last_fullsized":"2023-09-23T20:05:24.566063+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":266,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566016+0000","last_change":"2023-09-23T20:01:52.439375+0000","last_active":"2023-09-23T20:05:24.566016+0000","last_peered":"2023-09-23T20:05:24.566016+0000","last_clean":"2023-09-23T20:05:24.566016+0000","last_became_active":"2023-09-23T20:01:52.439074+0000","last_became_peered":"2023-09-23T20:01:52.439074+0000","last_unstale":"2023-09-23T20:05:24.566016+0000","last_undegraded":"2023-09-23T20:05:24.566016+0000","last_fullsized":"2023-09-23T20:05:24.566016+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:31.103633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":252,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599243+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:05:24.599243+0000","last_peered":"2023-09-23T20:05:24.599243+0000","last_clean":"2023-09-23T20:05:24.599243+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:05:24.599243+0000","last_undegraded":"2023-09-23T20:05:24.599243+0000","last_fullsized":"2023-09-23T20:05:24.599243+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":112,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561634+0000","last_change":"2023-09-23T19:59:09.195560+0000","last_active":"2023-09-23T20:05:24.561634+0000","last_peered":"2023-09-23T20:05:24.561634+0000","last_clean":"2023-09-23T20:05:24.561634+0000","last_became_active":"2023-09-23T19:59:09.195145+0000","last_became_peered":"2023-09-23T19:59:09.195145+0000","last_unstale":"2023-09-23T20:05:24.561634+0000","last_undegraded":"2023-09-23T20:05:24.561634+0000","last_fullsized":"2023-09-23T20:05:24.561634+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.190798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":255,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599286+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:05:24.599286+0000","last_peered":"2023-09-23T20:05:24.599286+0000","last_clean":"2023-09-23T20:05:24.599286+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:05:24.599286+0000","last_undegraded":"2023-09-23T20:05:24.599286+0000","last_fullsized":"2023-09-23T20:05:24.599286+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":268,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604109+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:05:24.604109+0000","last_peered":"2023-09-23T20:05:24.604109+0000","last_clean":"2023-09-23T20:05:24.604109+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:05:24.604109+0000","last_undegraded":"2023-09-23T20:05:24.604109+0000","last_fullsized":"2023-09-23T20:05:24.604109+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599833+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:05:24.599833+0000","last_peered":"2023-09-23T20:05:24.599833+0000","last_clean":"2023-09-23T20:05:24.599833+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:05:24.599833+0000","last_undegraded":"2023-09-23T20:05:24.599833+0000","last_fullsized":"2023-09-23T20:05:24.599833+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":255,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561541+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:05:24.561541+0000","last_peered":"2023-09-23T20:05:24.561541+0000","last_clean":"2023-09-23T20:05:24.561541+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:05:24.561541+0000","last_undegraded":"2023-09-23T20:05:24.561541+0000","last_fullsized":"2023-09-23T20:05:24.561541+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":83,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.603842+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:05:24.603842+0000","last_peered":"2023-09-23T20:05:24.603842+0000","last_clean":"2023-09-23T20:05:24.603842+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:05:24.603842+0000","last_undegraded":"2023-09-23T20:05:24.603842+0000","last_fullsized":"2023-09-23T20:05:24.603842+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":27,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561752+0000","last_change":"2023-09-23T20:03:59.232022+0000","last_active":"2023-09-23T20:05:24.561752+0000","last_peered":"2023-09-23T20:05:24.561752+0000","last_clean":"2023-09-23T20:05:24.561752+0000","last_became_active":"2023-09-23T20:03:59.231689+0000","last_became_peered":"2023-09-23T20:03:59.231689+0000","last_unstale":"2023-09-23T20:05:24.561752+0000","last_undegraded":"2023-09-23T20:05:24.561752+0000","last_fullsized":"2023-09-23T20:05:24.561752+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:29.669102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"292'11142","reported_seq":21438,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:05:40.605034+0000","last_change":"2023-09-23T20:04:02.290062+0000","last_active":"2023-09-23T20:05:40.605034+0000","last_peered":"2023-09-23T20:05:40.605034+0000","last_clean":"2023-09-23T20:05:40.605034+0000","last_became_active":"2023-09-23T20:04:02.271287+0000","last_became_peered":"2023-09-23T20:04:02.271287+0000","last_unstale":"2023-09-23T20:05:40.605034+0000","last_undegraded":"2023-09-23T20:05:40.605034+0000","last_fullsized":"2023-09-23T20:05:40.605034+0000","mapping_epoch":276,"log_start":"255'4600","ondisk_log_start":"255'4600","created":240,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:47.344179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61734912,"num_objects":942,"num_object_clones":0,"num_object_copies":2826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11142,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5100,"num_bytes_recovered":334233600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599802+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:05:24.599802+0000","last_peered":"2023-09-23T20:05:24.599802+0000","last_clean":"2023-09-23T20:05:24.599802+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:05:24.599802+0000","last_undegraded":"2023-09-23T20:05:24.599802+0000","last_fullsized":"2023-09-23T20:05:24.599802+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":278,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.603812+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:05:24.603812+0000","last_peered":"2023-09-23T20:05:24.603812+0000","last_clean":"2023-09-23T20:05:24.603812+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:05:24.603812+0000","last_undegraded":"2023-09-23T20:05:24.603812+0000","last_fullsized":"2023-09-23T20:05:24.603812+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"290'11038","reported_seq":11098,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:37.207857+0000","last_change":"2023-09-23T20:01:39.911588+0000","last_active":"2023-09-23T20:05:37.207857+0000","last_peered":"2023-09-23T20:05:37.207857+0000","last_clean":"2023-09-23T20:05:37.207857+0000","last_became_active":"2023-09-23T20:01:39.911364+0000","last_became_peered":"2023-09-23T20:01:39.911364+0000","last_unstale":"2023-09-23T20:05:37.207857+0000","last_undegraded":"2023-09-23T20:05:37.207857+0000","last_fullsized":"2023-09-23T20:05:37.207857+0000","mapping_epoch":240,"log_start":"256'5295","ondisk_log_start":"256'5295","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:35.715001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66322432,"num_objects":1012,"num_object_clones":0,"num_object_copies":3036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11038,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":252,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599707+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T20:05:24.599707+0000","last_peered":"2023-09-23T20:05:24.599707+0000","last_clean":"2023-09-23T20:05:24.599707+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T20:05:24.599707+0000","last_undegraded":"2023-09-23T20:05:24.599707+0000","last_fullsized":"2023-09-23T20:05:24.599707+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":306,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566116+0000","last_change":"2023-09-23T20:03:59.253784+0000","last_active":"2023-09-23T20:05:24.566116+0000","last_peered":"2023-09-23T20:05:24.566116+0000","last_clean":"2023-09-23T20:05:24.566116+0000","last_became_active":"2023-09-23T20:03:59.253552+0000","last_became_peered":"2023-09-23T20:03:59.253552+0000","last_unstale":"2023-09-23T20:05:24.566116+0000","last_undegraded":"2023-09-23T20:05:24.566116+0000","last_fullsized":"2023-09-23T20:05:24.566116+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:56.110746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"292'11121","reported_seq":11183,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:05:40.726397+0000","last_change":"2023-09-23T20:01:39.976582+0000","last_active":"2023-09-23T20:05:40.726397+0000","last_peered":"2023-09-23T20:05:40.726397+0000","last_clean":"2023-09-23T20:05:40.726397+0000","last_became_active":"2023-09-23T20:01:39.976244+0000","last_became_peered":"2023-09-23T20:01:39.976244+0000","last_unstale":"2023-09-23T20:05:40.726397+0000","last_undegraded":"2023-09-23T20:05:40.726397+0000","last_fullsized":"2023-09-23T20:05:40.726397+0000","mapping_epoch":240,"log_start":"254'4500","ondisk_log_start":"254'4500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.952295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60620800,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11121,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":255,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599602+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:05:24.599602+0000","last_peered":"2023-09-23T20:05:24.599602+0000","last_clean":"2023-09-23T20:05:24.599602+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:05:24.599602+0000","last_undegraded":"2023-09-23T20:05:24.599602+0000","last_fullsized":"2023-09-23T20:05:24.599602+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":220,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.705896+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T20:05:38.705896+0000","last_peered":"2023-09-23T20:05:38.705896+0000","last_clean":"2023-09-23T20:05:38.705896+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T20:05:38.705896+0000","last_undegraded":"2023-09-23T20:05:38.705896+0000","last_fullsized":"2023-09-23T20:05:38.705896+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.284038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"290'11424","reported_seq":11484,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.267133+0000","last_change":"2023-09-23T20:01:39.912566+0000","last_active":"2023-09-23T20:05:38.267133+0000","last_peered":"2023-09-23T20:05:38.267133+0000","last_clean":"2023-09-23T20:05:38.267133+0000","last_became_active":"2023-09-23T20:01:39.912259+0000","last_became_peered":"2023-09-23T20:01:39.912259+0000","last_unstale":"2023-09-23T20:05:38.267133+0000","last_undegraded":"2023-09-23T20:05:38.267133+0000","last_fullsized":"2023-09-23T20:05:38.267133+0000","mapping_epoch":240,"log_start":"247'2600","ondisk_log_start":"247'2600","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8824,"log_dups_size":0,"ondisk_log_size":8824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:19.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66977792,"num_objects":1022,"num_object_clones":0,"num_object_copies":3066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11424,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":308,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604245+0000","last_change":"2023-09-23T20:00:34.468296+0000","last_active":"2023-09-23T20:05:24.604245+0000","last_peered":"2023-09-23T20:05:24.604245+0000","last_clean":"2023-09-23T20:05:24.604245+0000","last_became_active":"2023-09-23T20:00:34.467957+0000","last_became_peered":"2023-09-23T20:00:34.467957+0000","last_unstale":"2023-09-23T20:05:24.604245+0000","last_undegraded":"2023-09-23T20:05:24.604245+0000","last_fullsized":"2023-09-23T20:05:24.604245+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:50.449027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":108,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.707079+0000","last_change":"2023-09-23T20:01:04.470558+0000","last_active":"2023-09-23T20:05:38.707079+0000","last_peered":"2023-09-23T20:05:38.707079+0000","last_clean":"2023-09-23T20:05:38.707079+0000","last_became_active":"2023-09-23T20:01:04.470276+0000","last_became_peered":"2023-09-23T20:01:04.470276+0000","last_unstale":"2023-09-23T20:05:38.707079+0000","last_undegraded":"2023-09-23T20:05:38.707079+0000","last_fullsized":"2023-09-23T20:05:38.707079+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:17.788583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"290'11061","reported_seq":11121,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:37.292181+0000","last_change":"2023-09-23T20:01:39.974611+0000","last_active":"2023-09-23T20:05:37.292181+0000","last_peered":"2023-09-23T20:05:37.292181+0000","last_clean":"2023-09-23T20:05:37.292181+0000","last_became_active":"2023-09-23T20:01:39.974318+0000","last_became_peered":"2023-09-23T20:01:39.974318+0000","last_unstale":"2023-09-23T20:05:37.292181+0000","last_undegraded":"2023-09-23T20:05:37.292181+0000","last_fullsized":"2023-09-23T20:05:37.292181+0000","mapping_epoch":240,"log_start":"256'5327","ondisk_log_start":"256'5327","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:31.047765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64684032,"num_objects":987,"num_object_clones":0,"num_object_copies":2961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11061,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":740,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599752+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:05:24.599752+0000","last_peered":"2023-09-23T20:05:24.599752+0000","last_clean":"2023-09-23T20:05:24.599752+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:05:24.599752+0000","last_undegraded":"2023-09-23T20:05:24.599752+0000","last_fullsized":"2023-09-23T20:05:24.599752+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":116,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561098+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:05:24.561098+0000","last_peered":"2023-09-23T20:05:24.561098+0000","last_clean":"2023-09-23T20:05:24.561098+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:05:24.561098+0000","last_undegraded":"2023-09-23T20:05:24.561098+0000","last_fullsized":"2023-09-23T20:05:24.561098+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":286,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.565970+0000","last_change":"2023-09-23T19:59:09.206075+0000","last_active":"2023-09-23T20:05:24.565970+0000","last_peered":"2023-09-23T20:05:24.565970+0000","last_clean":"2023-09-23T20:05:24.565970+0000","last_became_active":"2023-09-23T19:59:09.205766+0000","last_became_peered":"2023-09-23T19:59:09.205766+0000","last_unstale":"2023-09-23T20:05:24.565970+0000","last_undegraded":"2023-09-23T20:05:24.565970+0000","last_fullsized":"2023-09-23T20:05:24.565970+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:14.627703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"290'10876","reported_seq":10936,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.268568+0000","last_change":"2023-09-23T20:01:39.976550+0000","last_active":"2023-09-23T20:05:38.268568+0000","last_peered":"2023-09-23T20:05:38.268568+0000","last_clean":"2023-09-23T20:05:38.268568+0000","last_became_active":"2023-09-23T20:01:39.976245+0000","last_became_peered":"2023-09-23T20:01:39.976245+0000","last_unstale":"2023-09-23T20:05:38.268568+0000","last_undegraded":"2023-09-23T20:05:38.268568+0000","last_fullsized":"2023-09-23T20:05:38.268568+0000","mapping_epoch":240,"log_start":"247'2000","ondisk_log_start":"247'2000","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8876,"log_dups_size":0,"ondisk_log_size":8876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:45.979020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64749568,"num_objects":988,"num_object_clones":0,"num_object_copies":2964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10876,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":649,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.603725+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:05:24.603725+0000","last_peered":"2023-09-23T20:05:24.603725+0000","last_clean":"2023-09-23T20:05:24.603725+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:05:24.603725+0000","last_undegraded":"2023-09-23T20:05:24.603725+0000","last_fullsized":"2023-09-23T20:05:24.603725+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":236,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566036+0000","last_change":"2023-09-23T19:59:09.172601+0000","last_active":"2023-09-23T20:05:24.566036+0000","last_peered":"2023-09-23T20:05:24.566036+0000","last_clean":"2023-09-23T20:05:24.566036+0000","last_became_active":"2023-09-23T19:59:09.172297+0000","last_became_peered":"2023-09-23T19:59:09.172297+0000","last_unstale":"2023-09-23T20:05:24.566036+0000","last_undegraded":"2023-09-23T20:05:24.566036+0000","last_fullsized":"2023-09-23T20:05:24.566036+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:09.528266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":266,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566122+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T20:05:24.566122+0000","last_peered":"2023-09-23T20:05:24.566122+0000","last_clean":"2023-09-23T20:05:24.566122+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T20:05:24.566122+0000","last_undegraded":"2023-09-23T20:05:24.566122+0000","last_fullsized":"2023-09-23T20:05:24.566122+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"292'11112","reported_seq":11175,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:05:40.586355+0000","last_change":"2023-09-23T20:01:39.977019+0000","last_active":"2023-09-23T20:05:40.586355+0000","last_peered":"2023-09-23T20:05:40.586355+0000","last_clean":"2023-09-23T20:05:40.586355+0000","last_became_active":"2023-09-23T20:01:39.976270+0000","last_became_peered":"2023-09-23T20:01:39.976270+0000","last_unstale":"2023-09-23T20:05:40.586355+0000","last_undegraded":"2023-09-23T20:05:40.586355+0000","last_fullsized":"2023-09-23T20:05:40.586355+0000","mapping_epoch":240,"log_start":"254'4500","ondisk_log_start":"254'4500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6612,"log_dups_size":0,"ondisk_log_size":6612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.226759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64159768,"num_objects":980,"num_object_clones":0,"num_object_copies":2940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":980,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11112,"num_write_kb":386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.565975+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:05:24.565975+0000","last_peered":"2023-09-23T20:05:24.565975+0000","last_clean":"2023-09-23T20:05:24.565975+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:05:24.565975+0000","last_undegraded":"2023-09-23T20:05:24.565975+0000","last_fullsized":"2023-09-23T20:05:24.565975+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":296,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566162+0000","last_change":"2023-09-23T19:59:09.171926+0000","last_active":"2023-09-23T20:05:24.566162+0000","last_peered":"2023-09-23T20:05:24.566162+0000","last_clean":"2023-09-23T20:05:24.566162+0000","last_became_active":"2023-09-23T19:59:09.171609+0000","last_became_peered":"2023-09-23T19:59:09.171609+0000","last_unstale":"2023-09-23T20:05:24.566162+0000","last_undegraded":"2023-09-23T20:05:24.566162+0000","last_fullsized":"2023-09-23T20:05:24.566162+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:21.795329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"290'10982","reported_seq":18119,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:37.282764+0000","last_change":"2023-09-23T20:04:50.012534+0000","last_active":"2023-09-23T20:05:37.282764+0000","last_peered":"2023-09-23T20:05:37.282764+0000","last_clean":"2023-09-23T20:05:37.282764+0000","last_became_active":"2023-09-23T20:04:49.916541+0000","last_became_peered":"2023-09-23T20:04:49.916541+0000","last_unstale":"2023-09-23T20:05:37.282764+0000","last_undegraded":"2023-09-23T20:05:37.282764+0000","last_fullsized":"2023-09-23T20:05:37.282764+0000","mapping_epoch":282,"log_start":"256'5223","ondisk_log_start":"256'5223","created":240,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:58.571894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66846720,"num_objects":1020,"num_object_clones":0,"num_object_copies":3060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10982,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3528,"num_bytes_recovered":231145472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":628,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604203+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:05:24.604203+0000","last_peered":"2023-09-23T20:05:24.604203+0000","last_clean":"2023-09-23T20:05:24.604203+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:05:24.604203+0000","last_undegraded":"2023-09-23T20:05:24.604203+0000","last_fullsized":"2023-09-23T20:05:24.604203+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":27,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.560095+0000","last_change":"2023-09-23T20:03:59.233742+0000","last_active":"2023-09-23T20:05:24.560095+0000","last_peered":"2023-09-23T20:05:24.560095+0000","last_clean":"2023-09-23T20:05:24.560095+0000","last_became_active":"2023-09-23T20:03:59.233209+0000","last_became_peered":"2023-09-23T20:03:59.233209+0000","last_unstale":"2023-09-23T20:05:24.560095+0000","last_undegraded":"2023-09-23T20:05:24.560095+0000","last_fullsized":"2023-09-23T20:05:24.560095+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:50.879925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":253,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.706768+0000","last_change":"2023-09-23T19:59:09.153852+0000","last_active":"2023-09-23T20:05:38.706768+0000","last_peered":"2023-09-23T20:05:38.706768+0000","last_clean":"2023-09-23T20:05:38.706768+0000","last_became_active":"2023-09-23T19:59:09.153514+0000","last_became_peered":"2023-09-23T19:59:09.153514+0000","last_unstale":"2023-09-23T20:05:38.706768+0000","last_undegraded":"2023-09-23T20:05:38.706768+0000","last_fullsized":"2023-09-23T20:05:38.706768+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:45.179600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"290'11110","reported_seq":15098,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.336199+0000","last_change":"2023-09-23T20:05:24.592417+0000","last_active":"2023-09-23T20:05:38.336199+0000","last_peered":"2023-09-23T20:05:38.336199+0000","last_clean":"2023-09-23T20:05:38.336199+0000","last_became_active":"2023-09-23T20:05:24.567889+0000","last_became_peered":"2023-09-23T20:05:24.567889+0000","last_unstale":"2023-09-23T20:05:38.336199+0000","last_undegraded":"2023-09-23T20:05:38.336199+0000","last_fullsized":"2023-09-23T20:05:38.336199+0000","mapping_epoch":288,"log_start":"254'4500","ondisk_log_start":"254'4500","created":240,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:28.893902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69730304,"num_objects":1064,"num_object_clones":0,"num_object_copies":3192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11110,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1949,"num_bytes_recovered":127664128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":613,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561841+0000","last_change":"2023-09-23T20:04:02.277887+0000","last_active":"2023-09-23T20:05:24.561841+0000","last_peered":"2023-09-23T20:05:24.561841+0000","last_clean":"2023-09-23T20:05:24.561841+0000","last_became_active":"2023-09-23T20:04:02.277519+0000","last_became_peered":"2023-09-23T20:04:02.277519+0000","last_unstale":"2023-09-23T20:05:24.561841+0000","last_undegraded":"2023-09-23T20:05:24.561841+0000","last_fullsized":"2023-09-23T20:05:24.561841+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":631,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.705514+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T20:05:38.705514+0000","last_peered":"2023-09-23T20:05:38.705514+0000","last_clean":"2023-09-23T20:05:38.705514+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T20:05:38.705514+0000","last_undegraded":"2023-09-23T20:05:38.705514+0000","last_fullsized":"2023-09-23T20:05:38.705514+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":290,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604059+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:05:24.604059+0000","last_peered":"2023-09-23T20:05:24.604059+0000","last_clean":"2023-09-23T20:05:24.604059+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:05:24.604059+0000","last_undegraded":"2023-09-23T20:05:24.604059+0000","last_fullsized":"2023-09-23T20:05:24.604059+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":220,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.705720+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T20:05:38.705720+0000","last_peered":"2023-09-23T20:05:38.705720+0000","last_clean":"2023-09-23T20:05:38.705720+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T20:05:38.705720+0000","last_undegraded":"2023-09-23T20:05:38.705720+0000","last_fullsized":"2023-09-23T20:05:38.705720+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"290'11144","reported_seq":13538,"reported_epoch":290,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:05:37.414070+0000","last_change":"2023-09-23T20:05:22.911086+0000","last_active":"2023-09-23T20:05:37.414070+0000","last_peered":"2023-09-23T20:05:37.414070+0000","last_clean":"2023-09-23T20:03:58.087054+0000","last_became_active":"2023-09-23T20:04:00.560120+0000","last_became_peered":"2023-09-23T20:04:00.560120+0000","last_unstale":"2023-09-23T20:05:37.414070+0000","last_undegraded":"2023-09-23T20:05:37.414070+0000","last_fullsized":"2023-09-23T20:05:37.414070+0000","mapping_epoch":274,"log_start":"254'4600","ondisk_log_start":"254'4600","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6544,"log_dups_size":0,"ondisk_log_size":6544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:17.026687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71434240,"num_objects":1090,"num_object_clones":0,"num_object_copies":3270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":166,"num_objects_unfound":0,"num_objects_dirty":1090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11144,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1160,"num_bytes_recovered":76021760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1090}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":601,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.603826+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:05:24.603826+0000","last_peered":"2023-09-23T20:05:24.603826+0000","last_clean":"2023-09-23T20:05:24.603826+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:05:24.603826+0000","last_undegraded":"2023-09-23T20:05:24.603826+0000","last_fullsized":"2023-09-23T20:05:24.603826+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":850,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.603955+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T20:05:24.603955+0000","last_peered":"2023-09-23T20:05:24.603955+0000","last_clean":"2023-09-23T20:05:24.603955+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T20:05:24.603955+0000","last_undegraded":"2023-09-23T20:05:24.603955+0000","last_fullsized":"2023-09-23T20:05:24.603955+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:58.623015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":289,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566186+0000","last_change":"2023-09-23T20:00:34.447064+0000","last_active":"2023-09-23T20:05:24.566186+0000","last_peered":"2023-09-23T20:05:24.566186+0000","last_clean":"2023-09-23T20:05:24.566186+0000","last_became_active":"2023-09-23T20:00:34.446754+0000","last_became_peered":"2023-09-23T20:00:34.446754+0000","last_unstale":"2023-09-23T20:05:24.566186+0000","last_undegraded":"2023-09-23T20:05:24.566186+0000","last_fullsized":"2023-09-23T20:05:24.566186+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:39.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":286,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.705435+0000","last_change":"2023-09-23T20:03:59.236594+0000","last_active":"2023-09-23T20:05:38.705435+0000","last_peered":"2023-09-23T20:05:38.705435+0000","last_clean":"2023-09-23T20:05:38.705435+0000","last_became_active":"2023-09-23T20:03:59.236305+0000","last_became_peered":"2023-09-23T20:03:59.236305+0000","last_unstale":"2023-09-23T20:05:38.705435+0000","last_undegraded":"2023-09-23T20:05:38.705435+0000","last_fullsized":"2023-09-23T20:05:38.705435+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:23.103726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"290'11058","reported_seq":11116,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.350005+0000","last_change":"2023-09-23T20:01:39.977018+0000","last_active":"2023-09-23T20:05:38.350005+0000","last_peered":"2023-09-23T20:05:38.350005+0000","last_clean":"2023-09-23T20:05:38.350005+0000","last_became_active":"2023-09-23T20:01:39.976359+0000","last_became_peered":"2023-09-23T20:01:39.976359+0000","last_unstale":"2023-09-23T20:05:38.350005+0000","last_undegraded":"2023-09-23T20:05:38.350005+0000","last_fullsized":"2023-09-23T20:05:38.350005+0000","mapping_epoch":240,"log_start":"254'4500","ondisk_log_start":"254'4500","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.221346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65404928,"num_objects":998,"num_object_clones":0,"num_object_copies":2994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11058,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":525,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599202+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:05:24.599202+0000","last_peered":"2023-09-23T20:05:24.599202+0000","last_clean":"2023-09-23T20:05:24.599202+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:05:24.599202+0000","last_undegraded":"2023-09-23T20:05:24.599202+0000","last_fullsized":"2023-09-23T20:05:24.599202+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":923,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604694+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:05:24.604694+0000","last_peered":"2023-09-23T20:05:24.604694+0000","last_clean":"2023-09-23T20:05:24.604694+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:05:24.604694+0000","last_undegraded":"2023-09-23T20:05:24.604694+0000","last_fullsized":"2023-09-23T20:05:24.604694+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":249,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.706113+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T20:05:38.706113+0000","last_peered":"2023-09-23T20:05:38.706113+0000","last_clean":"2023-09-23T20:05:38.706113+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T20:05:38.706113+0000","last_undegraded":"2023-09-23T20:05:38.706113+0000","last_fullsized":"2023-09-23T20:05:38.706113+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.564126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":264,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.706844+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T20:05:38.706844+0000","last_peered":"2023-09-23T20:05:38.706844+0000","last_clean":"2023-09-23T20:05:38.706844+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T20:05:38.706844+0000","last_undegraded":"2023-09-23T20:05:38.706844+0000","last_fullsized":"2023-09-23T20:05:38.706844+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"290'11290","reported_seq":16914,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.449666+0000","last_change":"2023-09-23T20:05:05.847661+0000","last_active":"2023-09-23T20:05:38.449666+0000","last_peered":"2023-09-23T20:05:38.449666+0000","last_clean":"2023-09-23T20:05:38.449666+0000","last_became_active":"2023-09-23T20:05:05.813785+0000","last_became_peered":"2023-09-23T20:05:05.813785+0000","last_unstale":"2023-09-23T20:05:38.449666+0000","last_undegraded":"2023-09-23T20:05:38.449666+0000","last_fullsized":"2023-09-23T20:05:38.449666+0000","mapping_epoch":285,"log_start":"251'3956","ondisk_log_start":"251'3956","created":240,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7334,"log_dups_size":0,"ondisk_log_size":7334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:47.265078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67371008,"num_objects":1028,"num_object_clones":0,"num_object_copies":3084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11290,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2768,"num_bytes_recovered":181403648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1098,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604471+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:05:24.604471+0000","last_peered":"2023-09-23T20:05:24.604471+0000","last_clean":"2023-09-23T20:05:24.604471+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:05:24.604471+0000","last_undegraded":"2023-09-23T20:05:24.604471+0000","last_fullsized":"2023-09-23T20:05:24.604471+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":700,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599313+0000","last_change":"2023-09-23T20:00:34.445036+0000","last_active":"2023-09-23T20:05:24.599313+0000","last_peered":"2023-09-23T20:05:24.599313+0000","last_clean":"2023-09-23T20:05:24.599313+0000","last_became_active":"2023-09-23T20:00:34.444689+0000","last_became_peered":"2023-09-23T20:00:34.444689+0000","last_unstale":"2023-09-23T20:05:24.599313+0000","last_undegraded":"2023-09-23T20:05:24.599313+0000","last_fullsized":"2023-09-23T20:05:24.599313+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:44.485721+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":27,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.560650+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:05:24.560650+0000","last_peered":"2023-09-23T20:05:24.560650+0000","last_clean":"2023-09-23T20:05:24.560650+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:05:24.560650+0000","last_undegraded":"2023-09-23T20:05:24.560650+0000","last_fullsized":"2023-09-23T20:05:24.560650+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":165,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566286+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:05:24.566286+0000","last_peered":"2023-09-23T20:05:24.566286+0000","last_clean":"2023-09-23T20:05:24.566286+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:05:24.566286+0000","last_undegraded":"2023-09-23T20:05:24.566286+0000","last_fullsized":"2023-09-23T20:05:24.566286+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"290'11221","reported_seq":11279,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.364160+0000","last_change":"2023-09-23T20:01:39.977010+0000","last_active":"2023-09-23T20:05:38.364160+0000","last_peered":"2023-09-23T20:05:38.364160+0000","last_clean":"2023-09-23T20:05:38.364160+0000","last_became_active":"2023-09-23T20:01:39.976722+0000","last_became_peered":"2023-09-23T20:01:39.976722+0000","last_unstale":"2023-09-23T20:05:38.364160+0000","last_undegraded":"2023-09-23T20:05:38.364160+0000","last_fullsized":"2023-09-23T20:05:38.364160+0000","mapping_epoch":240,"log_start":"251'3831","ondisk_log_start":"251'3831","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:13.592964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66387968,"num_objects":1013,"num_object_clones":0,"num_object_copies":3039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11221,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":633,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.707202+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T20:05:38.707202+0000","last_peered":"2023-09-23T20:05:38.707202+0000","last_clean":"2023-09-23T20:05:38.707202+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T20:05:38.707202+0000","last_undegraded":"2023-09-23T20:05:38.707202+0000","last_fullsized":"2023-09-23T20:05:38.707202+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.842550+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1008,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599231+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T20:05:24.599231+0000","last_peered":"2023-09-23T20:05:24.599231+0000","last_clean":"2023-09-23T20:05:24.599231+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T20:05:24.599231+0000","last_undegraded":"2023-09-23T20:05:24.599231+0000","last_fullsized":"2023-09-23T20:05:24.599231+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":256,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566271+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:05:24.566271+0000","last_peered":"2023-09-23T20:05:24.566271+0000","last_clean":"2023-09-23T20:05:24.566271+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:05:24.566271+0000","last_undegraded":"2023-09-23T20:05:24.566271+0000","last_fullsized":"2023-09-23T20:05:24.566271+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":252,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604420+0000","last_change":"2023-09-23T20:03:10.175462+0000","last_active":"2023-09-23T20:05:24.604420+0000","last_peered":"2023-09-23T20:05:24.604420+0000","last_clean":"2023-09-23T20:05:24.604420+0000","last_became_active":"2023-09-23T20:03:10.175250+0000","last_became_peered":"2023-09-23T20:03:10.175250+0000","last_unstale":"2023-09-23T20:05:24.604420+0000","last_undegraded":"2023-09-23T20:05:24.604420+0000","last_fullsized":"2023-09-23T20:05:24.604420+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:42.961619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"290'10975","reported_seq":11033,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.039876+0000","last_change":"2023-09-23T20:01:39.913385+0000","last_active":"2023-09-23T20:05:38.039876+0000","last_peered":"2023-09-23T20:05:38.039876+0000","last_clean":"2023-09-23T20:05:38.039876+0000","last_became_active":"2023-09-23T20:01:39.912872+0000","last_became_peered":"2023-09-23T20:01:39.912872+0000","last_unstale":"2023-09-23T20:05:38.039876+0000","last_undegraded":"2023-09-23T20:05:38.039876+0000","last_fullsized":"2023-09-23T20:05:38.039876+0000","mapping_epoch":240,"log_start":"248'2600","ondisk_log_start":"248'2600","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8375,"log_dups_size":0,"ondisk_log_size":8375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.619813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66912256,"num_objects":1021,"num_object_clones":0,"num_object_copies":3063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10975,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":605,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561230+0000","last_change":"2023-09-23T20:04:02.277648+0000","last_active":"2023-09-23T20:05:24.561230+0000","last_peered":"2023-09-23T20:05:24.561230+0000","last_clean":"2023-09-23T20:05:24.561230+0000","last_became_active":"2023-09-23T20:04:02.277271+0000","last_became_peered":"2023-09-23T20:04:02.277271+0000","last_unstale":"2023-09-23T20:05:24.561230+0000","last_undegraded":"2023-09-23T20:05:24.561230+0000","last_fullsized":"2023-09-23T20:05:24.561230+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":368,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599635+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:05:24.599635+0000","last_peered":"2023-09-23T20:05:24.599635+0000","last_clean":"2023-09-23T20:05:24.599635+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:05:24.599635+0000","last_undegraded":"2023-09-23T20:05:24.599635+0000","last_fullsized":"2023-09-23T20:05:24.599635+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":252,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.705506+0000","last_change":"2023-09-23T20:03:10.174842+0000","last_active":"2023-09-23T20:05:38.705506+0000","last_peered":"2023-09-23T20:05:38.705506+0000","last_clean":"2023-09-23T20:05:38.705506+0000","last_became_active":"2023-09-23T20:03:10.174511+0000","last_became_peered":"2023-09-23T20:03:10.174511+0000","last_unstale":"2023-09-23T20:05:38.705506+0000","last_undegraded":"2023-09-23T20:05:38.705506+0000","last_fullsized":"2023-09-23T20:05:38.705506+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:12.723811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":86,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604537+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:05:24.604537+0000","last_peered":"2023-09-23T20:05:24.604537+0000","last_clean":"2023-09-23T20:05:24.604537+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:05:24.604537+0000","last_undegraded":"2023-09-23T20:05:24.604537+0000","last_fullsized":"2023-09-23T20:05:24.604537+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"290'11184","reported_seq":11244,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:37.265428+0000","last_change":"2023-09-23T20:01:39.912135+0000","last_active":"2023-09-23T20:05:37.265428+0000","last_peered":"2023-09-23T20:05:37.265428+0000","last_clean":"2023-09-23T20:05:37.265428+0000","last_became_active":"2023-09-23T20:01:39.911834+0000","last_became_peered":"2023-09-23T20:01:39.911834+0000","last_unstale":"2023-09-23T20:05:37.265428+0000","last_undegraded":"2023-09-23T20:05:37.265428+0000","last_fullsized":"2023-09-23T20:05:37.265428+0000","mapping_epoch":240,"log_start":"256'5426","ondisk_log_start":"256'5426","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:46.725578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69206016,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11184,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":668,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566038+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:05:24.566038+0000","last_peered":"2023-09-23T20:05:24.566038+0000","last_clean":"2023-09-23T20:05:24.566038+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:05:24.566038+0000","last_undegraded":"2023-09-23T20:05:24.566038+0000","last_fullsized":"2023-09-23T20:05:24.566038+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":116,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.562207+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:05:24.562207+0000","last_peered":"2023-09-23T20:05:24.562207+0000","last_clean":"2023-09-23T20:05:24.562207+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:05:24.562207+0000","last_undegraded":"2023-09-23T20:05:24.562207+0000","last_fullsized":"2023-09-23T20:05:24.562207+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":698,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.705813+0000","last_change":"2023-09-23T19:59:40.899753+0000","last_active":"2023-09-23T20:05:38.705813+0000","last_peered":"2023-09-23T20:05:38.705813+0000","last_clean":"2023-09-23T20:05:38.705813+0000","last_became_active":"2023-09-23T19:59:40.899147+0000","last_became_peered":"2023-09-23T19:59:40.899147+0000","last_unstale":"2023-09-23T20:05:38.705813+0000","last_undegraded":"2023-09-23T20:05:38.705813+0000","last_fullsized":"2023-09-23T20:05:38.705813+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:08.938755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":27,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.560162+0000","last_change":"2023-09-23T20:03:59.233560+0000","last_active":"2023-09-23T20:05:24.560162+0000","last_peered":"2023-09-23T20:05:24.560162+0000","last_clean":"2023-09-23T20:05:24.560162+0000","last_became_active":"2023-09-23T20:03:59.232842+0000","last_became_peered":"2023-09-23T20:03:59.232842+0000","last_unstale":"2023-09-23T20:05:24.560162+0000","last_undegraded":"2023-09-23T20:05:24.560162+0000","last_fullsized":"2023-09-23T20:05:24.560162+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:16.909413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"290'11112","reported_seq":11172,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:05:37.227611+0000","last_change":"2023-09-23T20:01:39.913698+0000","last_active":"2023-09-23T20:05:37.227611+0000","last_peered":"2023-09-23T20:05:37.227611+0000","last_clean":"2023-09-23T20:05:37.227611+0000","last_became_active":"2023-09-23T20:01:39.913367+0000","last_became_peered":"2023-09-23T20:01:39.913367+0000","last_unstale":"2023-09-23T20:05:37.227611+0000","last_undegraded":"2023-09-23T20:05:37.227611+0000","last_fullsized":"2023-09-23T20:05:37.227611+0000","mapping_epoch":240,"log_start":"256'5425","ondisk_log_start":"256'5425","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73662464,"num_objects":1124,"num_object_clones":0,"num_object_copies":3372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11112,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":546,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561716+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:05:24.561716+0000","last_peered":"2023-09-23T20:05:24.561716+0000","last_clean":"2023-09-23T20:05:24.561716+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:05:24.561716+0000","last_undegraded":"2023-09-23T20:05:24.561716+0000","last_fullsized":"2023-09-23T20:05:24.561716+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":46,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.599794+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:05:24.599794+0000","last_peered":"2023-09-23T20:05:24.599794+0000","last_clean":"2023-09-23T20:05:24.599794+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:05:24.599794+0000","last_undegraded":"2023-09-23T20:05:24.599794+0000","last_fullsized":"2023-09-23T20:05:24.599794+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:05:38.705888+0000","last_change":"2023-09-23T20:01:04.470744+0000","last_active":"2023-09-23T20:05:38.705888+0000","last_peered":"2023-09-23T20:05:38.705888+0000","last_clean":"2023-09-23T20:05:38.705888+0000","last_became_active":"2023-09-23T20:01:04.470302+0000","last_became_peered":"2023-09-23T20:01:04.470302+0000","last_unstale":"2023-09-23T20:05:38.705888+0000","last_undegraded":"2023-09-23T20:05:38.705888+0000","last_fullsized":"2023-09-23T20:05:38.705888+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:25.452596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":249,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566058+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:05:24.566058+0000","last_peered":"2023-09-23T20:05:24.566058+0000","last_clean":"2023-09-23T20:05:24.566058+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:05:24.566058+0000","last_undegraded":"2023-09-23T20:05:24.566058+0000","last_fullsized":"2023-09-23T20:05:24.566058+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":35,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604763+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:05:24.604763+0000","last_peered":"2023-09-23T20:05:24.604763+0000","last_clean":"2023-09-23T20:05:24.604763+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:05:24.604763+0000","last_undegraded":"2023-09-23T20:05:24.604763+0000","last_fullsized":"2023-09-23T20:05:24.604763+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":43,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.603708+0000","last_change":"2023-09-23T20:03:10.178662+0000","last_active":"2023-09-23T20:05:24.603708+0000","last_peered":"2023-09-23T20:05:24.603708+0000","last_clean":"2023-09-23T20:05:24.603708+0000","last_became_active":"2023-09-23T20:03:10.178352+0000","last_became_peered":"2023-09-23T20:03:10.178352+0000","last_unstale":"2023-09-23T20:05:24.603708+0000","last_undegraded":"2023-09-23T20:05:24.603708+0000","last_fullsized":"2023-09-23T20:05:24.603708+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.469897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.604090+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T20:05:24.604090+0000","last_peered":"2023-09-23T20:05:24.604090+0000","last_clean":"2023-09-23T20:05:24.604090+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T20:05:24.604090+0000","last_undegraded":"2023-09-23T20:05:24.604090+0000","last_fullsized":"2023-09-23T20:05:24.604090+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":286,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.566247+0000","last_change":"2023-09-23T19:59:09.172452+0000","last_active":"2023-09-23T20:05:24.566247+0000","last_peered":"2023-09-23T20:05:24.566247+0000","last_clean":"2023-09-23T20:05:24.566247+0000","last_became_active":"2023-09-23T19:59:09.172080+0000","last_became_peered":"2023-09-23T19:59:09.172080+0000","last_unstale":"2023-09-23T20:05:24.566247+0000","last_undegraded":"2023-09-23T20:05:24.566247+0000","last_fullsized":"2023-09-23T20:05:24.566247+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:24.687991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":255,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.603972+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T20:05:24.603972+0000","last_peered":"2023-09-23T20:05:24.603972+0000","last_clean":"2023-09-23T20:05:24.603972+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T20:05:24.603972+0000","last_undegraded":"2023-09-23T20:05:24.603972+0000","last_fullsized":"2023-09-23T20:05:24.603972+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:07.560646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":286,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.565736+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:05:24.565736+0000","last_peered":"2023-09-23T20:05:24.565736+0000","last_clean":"2023-09-23T20:05:24.565736+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:05:24.565736+0000","last_undegraded":"2023-09-23T20:05:24.565736+0000","last_fullsized":"2023-09-23T20:05:24.565736+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":113,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.561729+0000","last_change":"2023-09-23T19:59:09.145224+0000","last_active":"2023-09-23T20:05:24.561729+0000","last_peered":"2023-09-23T20:05:24.561729+0000","last_clean":"2023-09-23T20:05:24.561729+0000","last_became_active":"2023-09-23T19:59:09.144819+0000","last_became_peered":"2023-09-23T19:59:09.144819+0000","last_unstale":"2023-09-23T20:05:24.561729+0000","last_undegraded":"2023-09-23T20:05:24.561729+0000","last_fullsized":"2023-09-23T20:05:24.561729+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:04.048181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":27,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.560757+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:05:24.560757+0000","last_peered":"2023-09-23T20:05:24.560757+0000","last_clean":"2023-09-23T20:05:24.560757+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:05:24.560757+0000","last_undegraded":"2023-09-23T20:05:24.560757+0000","last_fullsized":"2023-09-23T20:05:24.560757+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":109,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.605343+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:05:24.605343+0000","last_peered":"2023-09-23T20:05:24.605343+0000","last_clean":"2023-09-23T20:05:24.605343+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:05:24.605343+0000","last_undegraded":"2023-09-23T20:05:24.605343+0000","last_fullsized":"2023-09-23T20:05:24.605343+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":316,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:05:24.565618+0000","last_change":"2023-09-23T20:03:59.235232+0000","last_active":"2023-09-23T20:05:24.565618+0000","last_peered":"2023-09-23T20:05:24.565618+0000","last_clean":"2023-09-23T20:05:24.565618+0000","last_became_active":"2023-09-23T20:03:59.234941+0000","last_became_peered":"2023-09-23T20:03:59.234941+0000","last_unstale":"2023-09-23T20:05:24.565618+0000","last_undegraded":"2023-09-23T20:05:24.565618+0000","last_fullsized":"2023-09-23T20:05:24.565618+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:45.591449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1066205208,"num_objects":16270,"num_object_clones":0,"num_object_copies":48810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":166,"num_objects_unfound":0,"num_objects_dirty":16270,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":177850,"num_write_kb":6211777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14505,"num_bytes_recovered":950468608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3603509248,"data_stored":3603497032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108967,"ondisk_log_size":108967,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234294,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":679612,"kb_used_data":401780,"kb_used_omap":0,"kb_used_meta":277824,"kb_avail":93065540,"statfs":{"total":95995035648,"available":95299112960,"internally_reserved":0,"allocated":411422720,"data_stored":410531782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284491776},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":2.4969999999999999,"15min":2.1589999999999998},"min":{"1min":0.23200000000000001,"5min":0.22,"15min":0.19700000000000001},"max":{"1min":11.256,"5min":60.140000000000001,"15min":62.061999999999998},"last":0.33400000000000002},{"interface":"front","average":{"1min":3.0569999999999999,"5min":2.827,"15min":2.4319999999999999},"min":{"1min":0.23899999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":13.143000000000001,"5min":62.389000000000003,"15min":62.389000000000003},"last":0.376}]},{"osd":1,"last update":"Sat Sep 23 20:05:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":5.4340000000000002,"15min":3.2160000000000002},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":5.2939999999999996,"5min":104.16800000000001,"15min":104.16800000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.5099999999999998,"5min":5.6719999999999997,"15min":3.3439999999999999},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":15.257,"5min":104.003,"15min":104.003},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:04:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.794,"15min":3.1379999999999999},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.6159999999999997,"5min":105.842,"15min":130.673},"last":1.1699999999999999},{"interface":"front","average":{"1min":4.0819999999999999,"5min":3.4249999999999998,"15min":3.4260000000000002},"min":{"1min":0.376,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":41.968000000000004,"5min":105.66,"15min":130.71100000000001},"last":0.94699999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:05:29 2023","interfaces":[{"interface":"back","average":{"1min":3.9820000000000002,"5min":5.6399999999999997,"15min":3.673},"min":{"1min":0.36599999999999999,"5min":0.248,"15min":0.247},"max":{"1min":12.314,"5min":146.38800000000001,"15min":146.38800000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":3.6520000000000001,"5min":5.6689999999999996,"15min":3.665},"min":{"1min":0.34399999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":15.653,"5min":146.851,"15min":146.851},"last":4.2969999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:04:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.80600000000000005,"15min":1.891},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.714,"5min":19.215,"15min":143.255},"last":0.23499999999999999},{"interface":"front","average":{"1min":2.605,"5min":1.1459999999999999,"15min":1.9330000000000001},"min":{"1min":0.27000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":41.822000000000003,"5min":41.822000000000003,"15min":143.41200000000001},"last":0.35999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:04:57 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":1.091,"15min":1.4610000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25},"max":{"1min":7.4960000000000004,"5min":35.488,"15min":84.566999999999993},"last":1.4079999999999999},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.95399999999999996,"15min":1.302},"min":{"1min":0.28599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.403,"5min":34.896000000000001,"15min":81.244},"last":0.28299999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.1220000000000001,"15min":0.84699999999999998},"min":{"1min":0.32500000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.6869999999999998,"5min":41.902999999999999,"15min":41.902999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.67900000000000005,"15min":0.63900000000000001},"min":{"1min":0.36699999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.714,"5min":6.3620000000000001,"15min":6.3620000000000001},"last":0.32200000000000001}]}]},{"osd":6,"up_from":0,"seq":1078036791335,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762496,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919324,"kb_used_data":505620,"kb_used_omap":7,"kb_used_meta":413688,"kb_avail":92825828,"statfs":{"total":95995035648,"available":95053647872,"internally_reserved":0,"allocated":517754880,"data_stored":516440053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":423617338},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":4.9800000000000004,"5min":2.8370000000000002,"15min":4.1879999999999997},"min":{"1min":0.19800000000000001,"5min":0.16900000000000001,"15min":0.151},"max":{"1min":31.413,"5min":31.413,"15min":202.917},"last":144.273},{"interface":"front","average":{"1min":6.5,"5min":3.496,"15min":4.4210000000000003},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17899999999999999},"max":{"1min":28.379999999999999,"5min":28.379999999999999,"15min":203.34899999999999},"last":144.321}]},{"osd":2,"last update":"Sat Sep 23 20:04:58 2023","interfaces":[{"interface":"back","average":{"1min":4.54,"5min":5.1639999999999997,"15min":5.5119999999999996},"min":{"1min":0.23000000000000001,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":12.554,"5min":134.72499999999999,"15min":231.57499999999999},"last":0.23000000000000001},{"interface":"front","average":{"1min":4.6079999999999997,"5min":5.6429999999999998,"15min":5.5199999999999996},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":11.881,"5min":135.74600000000001,"15min":236.077},"last":0.20699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":6.9790000000000001,"5min":3.52,"15min":3.4569999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.217},"max":{"1min":25.178999999999998,"5min":25.178999999999998,"15min":68.180999999999997},"last":0.26900000000000002},{"interface":"front","average":{"1min":6.0970000000000004,"5min":3.621,"15min":3.2240000000000002},"min":{"1min":0.376,"5min":0.249,"15min":0.20100000000000001},"max":{"1min":28.437999999999999,"5min":28.437999999999999,"15min":66.509},"last":0.247}]},{"osd":4,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":2.214,"15min":2.5449999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":10.368,"5min":35.497999999999998,"15min":161.68299999999999},"last":0.318},{"interface":"front","average":{"1min":3.298,"5min":2.6429999999999998,"15min":2.6259999999999999},"min":{"1min":0.33400000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.236000000000001,"5min":35.521000000000001,"15min":161.286},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.6640000000000001,"15min":3.5209999999999999},"min":{"1min":0.373,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":13.278,"5min":51.698,"15min":219.048},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.7549999999999999,"15min":3.6659999999999999},"min":{"1min":0.34699999999999998,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":10.189,"5min":51.173000000000002,"15min":218.69499999999999},"last":0.38800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":2.1219999999999999,"15min":2.2050000000000001},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":10.4,"5min":10.4,"15min":10.4},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.3650000000000002,"5min":3.1899999999999999,"15min":3.3599999999999999},"min":{"1min":0.39600000000000002,"5min":0.371,"15min":0.371},"max":{"1min":8.6549999999999994,"5min":20.734999999999999,"15min":20.734999999999999},"last":0.377}]},{"osd":7,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":2.3839999999999999,"15min":2.3159999999999998},"min":{"1min":0.39400000000000002,"5min":0.36199999999999999,"15min":0.26800000000000002},"max":{"1min":10.162000000000001,"5min":14.23,"15min":60.213000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.6859999999999999,"5min":1.9750000000000001,"15min":1.8879999999999999},"min":{"1min":0.41299999999999998,"5min":0.33500000000000002,"15min":0.30199999999999999},"max":{"1min":10.323,"5min":14.188000000000001,"15min":61.298999999999999},"last":0.47299999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926030,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1183036,"kb_used_data":723188,"kb_used_omap":8,"kb_used_meta":459831,"kb_avail":92562116,"statfs":{"total":95995035648,"available":94783606784,"internally_reserved":0,"allocated":740544512,"data_stored":739244798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":470867553},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":6.4210000000000003,"5min":5.6749999999999998,"15min":3.218},"min":{"1min":0.224,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":48.087000000000003,"5min":94.055999999999997,"15min":94.055999999999997},"last":9.8529999999999998},{"interface":"front","average":{"1min":5.7530000000000001,"5min":5.806,"15min":3.5310000000000001},"min":{"1min":0.187,"5min":0.183,"15min":0.183},"max":{"1min":48.119999999999997,"5min":91.632999999999996,"15min":91.632999999999996},"last":19.135999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:04:56 2023","interfaces":[{"interface":"back","average":{"1min":11.645,"5min":7.7329999999999997,"15min":4.3390000000000004},"min":{"1min":0.216,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":162.62899999999999,"5min":270.51100000000002,"15min":270.51100000000002},"last":10.215999999999999},{"interface":"front","average":{"1min":10.435,"5min":7.1280000000000001,"15min":4.3120000000000003},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":162.52500000000001,"5min":269.69299999999998,"15min":269.69299999999998},"last":4.282}]},{"osd":3,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":7.0720000000000001,"5min":5.6989999999999998,"15min":5.0940000000000003},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":51.765999999999998,"5min":116.733,"15min":145.72800000000001},"last":19.728999999999999},{"interface":"front","average":{"1min":7.0499999999999998,"5min":5.8250000000000002,"15min":5.2060000000000004},"min":{"1min":0.38900000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":51.723999999999997,"5min":116.697,"15min":145.50700000000001},"last":13.846}]},{"osd":4,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":8.9209999999999994,"5min":3.085,"15min":2.867},"min":{"1min":0.34799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":148.20599999999999,"5min":148.20599999999999,"15min":148.20599999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":8.9429999999999996,"5min":3.0649999999999999,"15min":2.5139999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":148.11000000000001,"5min":148.11000000000001,"15min":148.11000000000001},"last":10.272}]},{"osd":5,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.3460000000000001,"15min":2.6230000000000002},"min":{"1min":0.38100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":14.901999999999999,"5min":64.725999999999999,"15min":131.34399999999999},"last":6.3179999999999996},{"interface":"front","average":{"1min":1.855,"5min":1.7130000000000001,"15min":2.645},"min":{"1min":0.39400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.0890000000000004,"5min":12.48,"15min":131.386},"last":0.56200000000000006}]},{"osd":6,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.6699999999999999,"15min":1.6279999999999999},"min":{"1min":0.42499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.1579999999999995,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.2210000000000001,"5min":2.133,"15min":1.9810000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":13.766999999999999,"5min":13.766999999999999,"15min":13.766999999999999},"last":13.766999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.4039999999999999,"15min":2.1389999999999998},"min":{"1min":0.39000000000000001,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":14.856,"5min":65.081999999999994,"15min":77.132999999999996},"last":6.3650000000000002},{"interface":"front","average":{"1min":1.899,"5min":1.8200000000000001,"15min":2.177},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.1799999999999997,"5min":12.452,"15min":75.465999999999994},"last":0.54200000000000004}]}]},{"osd":2,"up_from":104,"seq":446676598969,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1060720,"kb_used_data":535912,"kb_used_omap":12,"kb_used_meta":524659,"kb_avail":92684432,"statfs":{"total":95995035648,"available":94908858368,"internally_reserved":0,"allocated":548773888,"data_stored":547488575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":537251838},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":6.569,"5min":4.5190000000000001,"15min":2.9159999999999999},"min":{"1min":0.20999999999999999,"5min":0.192,"15min":0.154},"max":{"1min":76.251999999999995,"5min":76.251999999999995,"15min":76.251999999999995},"last":0.74299999999999999},{"interface":"front","average":{"1min":7.1449999999999996,"5min":4.2480000000000002,"15min":2.8479999999999999},"min":{"1min":0.215,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":76.590000000000003,"5min":76.590000000000003,"15min":76.590000000000003},"last":0.63}]},{"osd":1,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":5.3979999999999997,"15min":3.0990000000000002},"min":{"1min":0.25900000000000001,"5min":0.22600000000000001,"15min":0.184},"max":{"1min":10.355,"5min":94.768000000000001,"15min":94.768000000000001},"last":4.157},{"interface":"front","average":{"1min":2.2010000000000001,"5min":5.9359999999999999,"15min":3.3100000000000001},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.5999999999999996,"5min":96.808000000000007,"15min":96.808000000000007},"last":3.7690000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:05:25 2023","interfaces":[{"interface":"back","average":{"1min":13.644,"5min":5.7679999999999998,"15min":3.681},"min":{"1min":0.40000000000000002,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":107.381,"5min":107.381,"15min":107.381},"last":3.992},{"interface":"front","average":{"1min":11.827,"5min":4.8579999999999997,"15min":3.4849999999999999},"min":{"1min":0.45700000000000002,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":106.301,"5min":106.301,"15min":106.301},"last":9.4359999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":1.7989999999999999,"15min":1.764},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.2240000000000002,"5min":20.591999999999999,"15min":50.640000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.766,"15min":1.758},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.0579999999999998,"5min":23.062000000000001,"15min":50.154000000000003},"last":3.956}]},{"osd":5,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":3.593,"15min":2.5419999999999998},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.5519999999999996,"5min":172.505,"15min":172.505},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.885,"5min":3.4900000000000002,"15min":2.4300000000000002},"min":{"1min":0.436,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":7.0970000000000004,"5min":172.375,"15min":172.375},"last":0.70799999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.6659999999999999,"15min":1.639},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.3759999999999994,"5min":10.144,"15min":10.144},"last":0.81100000000000005},{"interface":"front","average":{"1min":2.101,"5min":1.9430000000000001,"15min":1.96},"min":{"1min":0.437,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.3879999999999999,"5min":9.3879999999999999,"15min":9.3879999999999999},"last":0.67800000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.917,"15min":1.8560000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.27400000000000002},"max":{"1min":4.0789999999999997,"5min":23.079999999999998,"15min":50.118000000000002},"last":3.9100000000000001},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.0779999999999998,"15min":2.1219999999999999},"min":{"1min":0.40100000000000002,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":5.5750000000000002,"5min":20.614999999999998,"15min":50.664999999999999},"last":0.86299999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435442,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1196916,"kb_used_data":689644,"kb_used_omap":4,"kb_used_meta":507259,"kb_avail":92548236,"statfs":{"total":95995035648,"available":94769393664,"internally_reserved":0,"allocated":706195456,"data_stored":704783538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":519433325},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,6,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":4.2999999999999998,"5min":2.6629999999999998,"15min":4.1559999999999997},"min":{"1min":0.28299999999999997,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":12.023,"5min":21.100999999999999,"15min":240.40600000000001},"last":7.7460000000000004},{"interface":"front","average":{"1min":3.5899999999999999,"5min":3.4089999999999998,"15min":4.7140000000000004},"min":{"1min":0.38600000000000001,"5min":0.19600000000000001,"15min":0.17199999999999999},"max":{"1min":12.638,"5min":53.201000000000001,"15min":240.352},"last":5.6429999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8119999999999998,"5min":4.2069999999999999,"15min":2.4420000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":8.766,"5min":86.337999999999994,"15min":86.337999999999994},"last":7.6600000000000001},{"interface":"front","average":{"1min":3.8650000000000002,"5min":3.9390000000000001,"15min":2.4729999999999999},"min":{"1min":0.35599999999999998,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":12.055999999999999,"5min":84.299999999999997,"15min":84.299999999999997},"last":0.75900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3170000000000002,"5min":3.7349999999999999,"15min":5.4950000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.219},"max":{"1min":9.3780000000000001,"5min":82.111999999999995,"15min":264.97800000000001},"last":10.222},{"interface":"front","average":{"1min":4.9980000000000002,"5min":3.1349999999999998,"15min":5.2320000000000002},"min":{"1min":0.29099999999999998,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":12.359999999999999,"5min":30.768999999999998,"15min":257.40899999999999},"last":3.4300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":4.5990000000000002,"15min":2.9079999999999999},"min":{"1min":0.33500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.51,"5min":109.66800000000001,"15min":109.66800000000001},"last":1.0620000000000001},{"interface":"front","average":{"1min":2.5659999999999998,"5min":5.0830000000000002,"15min":3.3450000000000002},"min":{"1min":0.45600000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.637,"5min":110.295,"15min":110.295},"last":0.62}]},{"osd":5,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6480000000000001,"5min":4.4269999999999996,"15min":2.6680000000000001},"min":{"1min":0.46999999999999997,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":12.662000000000001,"5min":113.881,"15min":113.881},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.6040000000000001,"5min":4.2599999999999998,"15min":2.4990000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.3929999999999998,"5min":114.598,"15min":114.598},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:05:36 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":4.1970000000000001,"15min":5.2329999999999997},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8479999999999999,"5min":82.281999999999996,"15min":82.281999999999996},"last":0.439},{"interface":"front","average":{"1min":3.2040000000000002,"5min":4.4580000000000002,"15min":5.319},"min":{"1min":0.42799999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.554,"5min":82.233000000000004,"15min":82.233000000000004},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":4.2389999999999999,"15min":3.04},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.4249999999999998,"5min":112.883,"15min":138.035},"last":0.502},{"interface":"front","average":{"1min":2.802,"5min":3.8660000000000001,"15min":2.8889999999999998},"min":{"1min":0.47799999999999998,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":10.585000000000001,"5min":111.85599999999999,"15min":138.054},"last":0.52900000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019204,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":702520,"kb_used_data":434928,"kb_used_omap":0,"kb_used_meta":267584,"kb_avail":93042632,"statfs":{"total":95995035648,"available":95275655168,"internally_reserved":0,"allocated":445366272,"data_stored":444479418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":274006016},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:09 2023","interfaces":[{"interface":"back","average":{"1min":3.089,"5min":3.1459999999999999,"15min":3.0899999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.189},"max":{"1min":10.063000000000001,"5min":89.004999999999995,"15min":104.604},"last":33.911000000000001},{"interface":"front","average":{"1min":2.8220000000000001,"5min":3.1659999999999999,"15min":3.028},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.23400000000000001},"max":{"1min":10.045999999999999,"5min":89.024000000000001,"15min":104.687},"last":33.899000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:05:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2970000000000002,"5min":10.922000000000001,"15min":5.0380000000000003},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.216},"max":{"1min":8.2810000000000006,"5min":161.613,"15min":161.613},"last":34.073},{"interface":"front","average":{"1min":2.3479999999999999,"5min":10.935,"15min":4.8810000000000002},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":8.4309999999999992,"5min":162.017,"15min":162.017},"last":35.198999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:04:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9710000000000001,"5min":6.4290000000000003,"15min":5.1130000000000004},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":5.6180000000000003,"5min":116.56,"15min":144.20099999999999},"last":0.313},{"interface":"front","average":{"1min":3.121,"5min":6.5039999999999996,"15min":4.9400000000000004},"min":{"1min":0.30099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":11.048,"5min":117.03400000000001,"15min":144.184},"last":0.32600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":6.0439999999999996,"15min":5.3209999999999997},"min":{"1min":0.33500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":12.565,"5min":139.976,"15min":266.54700000000003},"last":118.31699999999999},{"interface":"front","average":{"1min":2.8260000000000001,"5min":6.0030000000000001,"15min":4.3470000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":13.637,"5min":139.911,"15min":139.911},"last":118.395}]},{"osd":5,"last update":"Sat Sep 23 20:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.53100000000000003,"15min":1.8600000000000001},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":1.569,"5min":4.423,"15min":266.59899999999999},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.64400000000000002,"15min":1.9650000000000001},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.24399999999999999},"max":{"1min":3.004,"5min":4.8739999999999997,"15min":266.72899999999998},"last":0.29299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.53000000000000003,"15min":0.51200000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.605,"5min":4.0410000000000004,"15min":4.0410000000000004},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.66800000000000004,"15min":0.54300000000000004},"min":{"1min":0.34499999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.615,"5min":4.2439999999999998,"15min":4.2439999999999998},"last":0.34899999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:04:49 2023","interfaces":[{"interface":"back","average":{"1min":4.7199999999999998,"5min":1.4239999999999999,"15min":1.2509999999999999},"min":{"1min":0.32600000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":65.522999999999996,"5min":65.522999999999996,"15min":65.522999999999996},"last":0.379},{"interface":"front","average":{"1min":5.5970000000000004,"5min":1.595,"15min":1.3300000000000001},"min":{"1min":0.377,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":65.301000000000002,"5min":65.301000000000002,"15min":65.301000000000002},"last":0.35799999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757568,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":643440,"kb_used_data":349608,"kb_used_omap":0,"kb_used_meta":293824,"kb_avail":93101712,"statfs":{"total":95995035648,"available":95336153088,"internally_reserved":0,"allocated":357998592,"data_stored":357116179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":300875776},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:07 2023","interfaces":[{"interface":"back","average":{"1min":4.7629999999999999,"5min":1.597,"15min":1.548},"min":{"1min":0.22600000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":41.503,"5min":41.503,"15min":41.503},"last":5.5629999999999997},{"interface":"front","average":{"1min":4.6029999999999998,"5min":1.978,"15min":1.6220000000000001},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.216},"max":{"1min":41.465000000000003,"5min":41.465000000000003,"15min":41.465000000000003},"last":0.22800000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:05:19 2023","interfaces":[{"interface":"back","average":{"1min":5.6340000000000003,"5min":7.2839999999999998,"15min":5.258},"min":{"1min":0.35299999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":60.447000000000003,"5min":171.68299999999999,"15min":266.197},"last":5.5380000000000003},{"interface":"front","average":{"1min":6.3840000000000003,"5min":7.6299999999999999,"15min":5.3419999999999996},"min":{"1min":0.436,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":60.707999999999998,"5min":171.64099999999999,"15min":266.14999999999998},"last":3.391}]},{"osd":2,"last update":"Sat Sep 23 20:04:51 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":3.5609999999999999,"15min":5.6239999999999997},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":3.6680000000000001,"5min":177.816,"15min":238.96199999999999},"last":5.407},{"interface":"front","average":{"1min":1.373,"5min":3.6299999999999999,"15min":5.6719999999999997},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":4.024,"5min":177.851,"15min":238.905},"last":5.4400000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:05:19 2023","interfaces":[{"interface":"back","average":{"1min":13.840999999999999,"5min":7.2690000000000001,"15min":4.4009999999999998},"min":{"1min":0.311,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":201.50899999999999,"5min":201.50899999999999,"15min":201.50899999999999},"last":3.6480000000000001},{"interface":"front","average":{"1min":13.923,"5min":7.1669999999999998,"15min":4.2560000000000002},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":201.58099999999999,"5min":201.58099999999999,"15min":201.58099999999999},"last":6.6340000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":2.4329999999999998,"15min":1.611},"min":{"1min":0.24099999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.786,"5min":191.78899999999999,"15min":191.78899999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.38300000000000001,"5min":2.444,"15min":1.587},"min":{"1min":0.308,"5min":0.255,"15min":0.255},"max":{"1min":0.54200000000000004,"5min":191.821,"15min":191.821},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 20:05:33 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.63200000000000001,"15min":0.60999999999999999},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.92000000000000004,"5min":4.0679999999999996,"15min":4.0679999999999996},"last":1.956},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.48199999999999998,"15min":0.46800000000000003},"min":{"1min":0.32500000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.53100000000000003,"5min":3.3570000000000002,"15min":3.3570000000000002},"last":3.536}]},{"osd":7,"last update":"Sat Sep 23 20:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":2.4460000000000002,"15min":1.74},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":3.964,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.754,"5min":2.5630000000000002,"15min":1.9379999999999999},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.976,"5min":191.69,"15min":191.69},"last":0.44600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":711921664,"data_stored":711917592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":497422336,"data_stored":497418264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":531431424,"data_stored":531431424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":684789760,"data_stored":684785688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":435683328,"data_stored":435683328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":346947584,"data_stored":346947584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":395313152,"data_stored":395313152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:05:42.384 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:42.472 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:42.753 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:42.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:42.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:42.872 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:42.897 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:42.938 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:42.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:43.047 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:43.075 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:43.076 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:42 smithi144 ceph-mon[194833]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.076 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:42 smithi144 ceph-mon[194833]: osdmap e294: 8 total, 8 up, 7 in 2023-09-23T20:05:43.076 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:42 smithi144 ceph-mon[194833]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.076 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:42 smithi144 ceph-mon[194833]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.076 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:42 smithi144 ceph-mon[194833]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.076 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:42 smithi144 ceph-mon[194833]: pgmap v810: 99 pgs: 99 active+clean; 957 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 2.4 MiB/s, 38 objects/s recovering 2023-09-23T20:05:43.133 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:43.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:43.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:43.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[213871]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[213871]: osdmap e294: 8 total, 8 up, 7 in 2023-09-23T20:05:43.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[213871]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[213871]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[213871]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[213871]: pgmap v810: 99 pgs: 99 active+clean; 957 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 2.4 MiB/s, 38 objects/s recovering 2023-09-23T20:05:43.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[221113]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[221113]: osdmap e294: 8 total, 8 up, 7 in 2023-09-23T20:05:43.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[221113]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[221113]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[221113]: failed to encode map e294 with expected crc 2023-09-23T20:05:43.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:42 smithi135 ceph-mon[221113]: pgmap v810: 99 pgs: 99 active+clean; 957 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 2.4 MiB/s, 38 objects/s recovering 2023-09-23T20:05:43.237 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:43.306 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:43.447 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:43.449 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:43.457 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:43.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:43.572 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:43.660 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:43.789 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:43.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:43.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:43.900 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:43.929 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:43.976 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:05:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:05:43] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2023-09-23T20:05:43.976 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:43.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:43.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:44.085 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:44.114 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:44.162 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:44.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:44.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:44.268 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:44.296 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:44.347 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:44.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:44.452 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:44.554 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:44.820 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:44.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:44.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:44.927 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:45.023 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:45.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:44 smithi144 ceph-mon[194833]: pgmap v811: 99 pgs: 99 active+clean; 883 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:05:45.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:44 smithi135 ceph-mon[221113]: pgmap v811: 99 pgs: 99 active+clean; 883 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:05:45.230 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:44 smithi135 ceph-mon[213871]: pgmap v811: 99 pgs: 99 active+clean; 883 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:05:45.294 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:05:45.468 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:45.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:45.572 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:45.600 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:45.654 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:45.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:45.762 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:45.792 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:45.836 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:45.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:45.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:45.874 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:05:45.943 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:45.969 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/371350654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:45 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:46.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/371350654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:46.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:45 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:46.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/371350654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:46.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:45 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:05:46.737 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:46.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:46.841 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:46.868 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:46.921 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:46.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:47.026 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:47.054 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:47.102 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:47.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:47.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:46 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:46 smithi144 ceph-mon[194833]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:05:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:46 smithi144 ceph-mon[194833]: osdmap e295: 8 total, 8 up, 7 in 2023-09-23T20:05:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:46 smithi144 ceph-mon[194833]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:46 smithi144 ceph-mon[194833]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:46 smithi144 ceph-mon[194833]: pgmap v813: 99 pgs: 99 active+clean; 853 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:05:47.211 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:47.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:47.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[221113]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:05:47.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[221113]: osdmap e295: 8 total, 8 up, 7 in 2023-09-23T20:05:47.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[221113]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[221113]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[221113]: pgmap v813: 99 pgs: 99 active+clean; 853 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:05:47.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:47.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[213871]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:05:47.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[213871]: osdmap e295: 8 total, 8 up, 7 in 2023-09-23T20:05:47.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[213871]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[213871]: failed to encode map e295 with expected crc 2023-09-23T20:05:47.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:46 smithi135 ceph-mon[213871]: pgmap v813: 99 pgs: 99 active+clean; 853 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:05:47.288 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:47.731 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:47.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:47.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:47.837 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:47.864 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:47.914 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:47.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:48.019 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:48.046 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:48.091 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:48.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:48.096 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:48.136 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:05:48.142 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:05:48.142 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:05:48.195 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:48.203 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:05:48.222 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:48.250 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:05:48.271 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:48.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:48.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:48.328 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:05:48.379 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:48.382 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:05:48.405 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:48.424 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:05:48.451 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:05:48.461 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:48.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:48.501 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:05:48.519 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:05:48.519 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:05:48.519 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:05:48.520 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:05:48.520 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:05:48.520 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:05:48.520 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:05:48.520 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:05:48.553 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:48 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1351000569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:48.567 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:05:48.594 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:48.648 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:05:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:48.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:05:48.754 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:48.842 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:48 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1351000569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:48.842 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:48 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1351000569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:05:48.855 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:49.561 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:05:49.567 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:49.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:49.679 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:49.824 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:49.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:49 smithi144 ceph-mon[194833]: pgmap v814: 99 pgs: 99 active+clean; 684 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:05:49.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:49 smithi135 ceph-mon[213871]: pgmap v814: 99 pgs: 99 active+clean; 684 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:05:49.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:49 smithi135 ceph-mon[221113]: pgmap v814: 99 pgs: 99 active+clean; 684 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:05:50.309 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:05:50.472 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:05:50.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:50.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:05:50.585 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:50.692 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:50.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:50 smithi135 ceph-mon[221113]: pgmap v815: 99 pgs: 99 active+clean; 645 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:05:50.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:50 smithi135 ceph-mon[213871]: pgmap v815: 99 pgs: 99 active+clean; 645 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:05:51.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:50 smithi144 ceph-mon[194833]: pgmap v815: 99 pgs: 99 active+clean; 645 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:05:51.463 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:51.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:51.566 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:51.662 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:51.676 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:51.770 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:51.827 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:51.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:51.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:51.953 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:52.040 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:52.373 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:52.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:52.440 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:52.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:52.541 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:52.620 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:05:52.775 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:52.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:52.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:52.796 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:05:52.820 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:05:52.880 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:52.984 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:53.452 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:05:53.469 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:53.469 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:53 smithi135 ceph-mon[213871]: from='client.55327 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:53.469 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:53 smithi135 ceph-mon[213871]: pgmap v816: 99 pgs: 99 active+clean; 571 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T20:05:53.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:05:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:05:53] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-09-23T20:05:53.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:53.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:53 smithi135 ceph-mon[221113]: from='client.55327 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:53.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:53 smithi135 ceph-mon[221113]: pgmap v816: 99 pgs: 99 active+clean; 571 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T20:05:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:05:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:53 smithi144 ceph-mon[194833]: from='client.55327 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:05:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:53 smithi144 ceph-mon[194833]: pgmap v816: 99 pgs: 99 active+clean; 571 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T20:05:55.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:54 smithi144 ceph-mon[194833]: pgmap v817: 99 pgs: 99 active+clean; 483 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T20:05:55.136 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:54 smithi135 ceph-mon[221113]: pgmap v817: 99 pgs: 99 active+clean; 483 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T20:05:55.136 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:54 smithi135 ceph-mon[213871]: pgmap v817: 99 pgs: 99 active+clean; 483 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T20:05:55.160 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":815,"stamp":"2023-09-23T20:05:50.772098+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":676730827,"num_objects":10282,"num_object_clones":0,"num_object_copies":30846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10282,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":184773,"num_write_kb":6224502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14778,"num_bytes_recovered":961223157,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111254,"ondisk_log_size":111254,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5136156,"kb_used_data":2412004,"kb_used_omap":32,"kb_used_meta":2723807,"kb_avail":651079908,"statfs":{"total":671965249536,"available":666705825792,"internally_reserved":0,"allocated":2469892096,"data_stored":2461742573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":2789178886},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-394264576,"num_objects":-6016,"num_object_clones":0,"num_object_copies":-18048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-166,"num_objects_unfound":0,"num_objects_dirty":-6016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6016,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10223616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.948470"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":273,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830854+0000","last_change":"2023-09-23T19:50:23.998710+0000","last_active":"2023-09-23T20:05:41.830854+0000","last_peered":"2023-09-23T20:05:41.830854+0000","last_clean":"2023-09-23T20:05:41.830854+0000","last_became_active":"2023-09-23T19:50:23.998185+0000","last_became_peered":"2023-09-23T19:50:23.998185+0000","last_unstale":"2023-09-23T20:05:41.830854+0000","last_undegraded":"2023-09-23T20:05:41.830854+0000","last_fullsized":"2023-09-23T20:05:41.830854+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:19.067349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.827198+0000","last_change":"2023-09-23T19:59:09.151117+0000","last_active":"2023-09-23T20:05:41.827198+0000","last_peered":"2023-09-23T20:05:41.827198+0000","last_clean":"2023-09-23T20:05:41.827198+0000","last_became_active":"2023-09-23T19:59:09.150795+0000","last_became_peered":"2023-09-23T19:59:09.150795+0000","last_unstale":"2023-09-23T20:05:41.827198+0000","last_undegraded":"2023-09-23T20:05:41.827198+0000","last_fullsized":"2023-09-23T20:05:41.827198+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:34.994574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":256,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829283+0000","last_change":"2023-09-23T19:59:10.156070+0000","last_active":"2023-09-23T20:05:41.829283+0000","last_peered":"2023-09-23T20:05:41.829283+0000","last_clean":"2023-09-23T20:05:41.829283+0000","last_became_active":"2023-09-23T19:59:10.155813+0000","last_became_peered":"2023-09-23T19:59:10.155813+0000","last_unstale":"2023-09-23T20:05:41.829283+0000","last_undegraded":"2023-09-23T20:05:41.829283+0000","last_fullsized":"2023-09-23T20:05:41.829283+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:42.582380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":47,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830681+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:05:41.830681+0000","last_peered":"2023-09-23T20:05:41.830681+0000","last_clean":"2023-09-23T20:05:41.830681+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:05:41.830681+0000","last_undegraded":"2023-09-23T20:05:41.830681+0000","last_fullsized":"2023-09-23T20:05:41.830681+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":117,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.826870+0000","last_change":"2023-09-23T19:59:09.163182+0000","last_active":"2023-09-23T20:05:41.826870+0000","last_peered":"2023-09-23T20:05:41.826870+0000","last_clean":"2023-09-23T20:05:41.826870+0000","last_became_active":"2023-09-23T19:59:09.161612+0000","last_became_peered":"2023-09-23T19:59:09.161612+0000","last_unstale":"2023-09-23T20:05:41.826870+0000","last_undegraded":"2023-09-23T20:05:41.826870+0000","last_fullsized":"2023-09-23T20:05:41.826870+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:59.524152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":220,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945380+0000","last_change":"2023-09-23T19:56:39.277074+0000","last_active":"2023-09-23T20:05:41.945380+0000","last_peered":"2023-09-23T20:05:41.945380+0000","last_clean":"2023-09-23T20:05:41.945380+0000","last_became_active":"2023-09-23T19:56:39.276744+0000","last_became_peered":"2023-09-23T19:56:39.276744+0000","last_unstale":"2023-09-23T20:05:41.945380+0000","last_undegraded":"2023-09-23T20:05:41.945380+0000","last_fullsized":"2023-09-23T20:05:41.945380+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:59.043547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":221,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945333+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:05:41.945333+0000","last_peered":"2023-09-23T20:05:41.945333+0000","last_clean":"2023-09-23T20:05:41.945333+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:05:41.945333+0000","last_undegraded":"2023-09-23T20:05:41.945333+0000","last_fullsized":"2023-09-23T20:05:41.945333+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":117,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.826864+0000","last_change":"2023-09-23T19:59:09.146009+0000","last_active":"2023-09-23T20:05:41.826864+0000","last_peered":"2023-09-23T20:05:41.826864+0000","last_clean":"2023-09-23T20:05:41.826864+0000","last_became_active":"2023-09-23T19:59:09.145687+0000","last_became_peered":"2023-09-23T19:59:09.145687+0000","last_unstale":"2023-09-23T20:05:41.826864+0000","last_undegraded":"2023-09-23T20:05:41.826864+0000","last_fullsized":"2023-09-23T20:05:41.826864+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.189450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":254,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945397+0000","last_change":"2023-09-23T19:49:51.372152+0000","last_active":"2023-09-23T20:05:41.945397+0000","last_peered":"2023-09-23T20:05:41.945397+0000","last_clean":"2023-09-23T20:05:41.945397+0000","last_became_active":"2023-09-23T19:49:51.371804+0000","last_became_peered":"2023-09-23T19:49:51.371804+0000","last_unstale":"2023-09-23T20:05:41.945397+0000","last_undegraded":"2023-09-23T20:05:41.945397+0000","last_fullsized":"2023-09-23T20:05:41.945397+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:35.318585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":136,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.826950+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:05:41.826950+0000","last_peered":"2023-09-23T20:05:41.826950+0000","last_clean":"2023-09-23T20:05:41.826950+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:05:41.826950+0000","last_undegraded":"2023-09-23T20:05:41.826950+0000","last_fullsized":"2023-09-23T20:05:41.826950+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":290,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830889+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:05:41.830889+0000","last_peered":"2023-09-23T20:05:41.830889+0000","last_clean":"2023-09-23T20:05:41.830889+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:05:41.830889+0000","last_undegraded":"2023-09-23T20:05:41.830889+0000","last_fullsized":"2023-09-23T20:05:41.830889+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":252,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829421+0000","last_change":"2023-09-23T19:56:39.280536+0000","last_active":"2023-09-23T20:05:41.829421+0000","last_peered":"2023-09-23T20:05:41.829421+0000","last_clean":"2023-09-23T20:05:41.829421+0000","last_became_active":"2023-09-23T19:56:39.280075+0000","last_became_peered":"2023-09-23T19:56:39.280075+0000","last_unstale":"2023-09-23T20:05:41.829421+0000","last_undegraded":"2023-09-23T20:05:41.829421+0000","last_fullsized":"2023-09-23T20:05:41.829421+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:51.825640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":309,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831159+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:05:41.831159+0000","last_peered":"2023-09-23T20:05:41.831159+0000","last_clean":"2023-09-23T20:05:41.831159+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:05:41.831159+0000","last_undegraded":"2023-09-23T20:05:41.831159+0000","last_fullsized":"2023-09-23T20:05:41.831159+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":271,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831265+0000","last_change":"2023-09-23T20:01:52.439375+0000","last_active":"2023-09-23T20:05:41.831265+0000","last_peered":"2023-09-23T20:05:41.831265+0000","last_clean":"2023-09-23T20:05:41.831265+0000","last_became_active":"2023-09-23T20:01:52.439074+0000","last_became_peered":"2023-09-23T20:01:52.439074+0000","last_unstale":"2023-09-23T20:05:41.831265+0000","last_undegraded":"2023-09-23T20:05:41.831265+0000","last_fullsized":"2023-09-23T20:05:41.831265+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:31.103633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":257,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830443+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:05:41.830443+0000","last_peered":"2023-09-23T20:05:41.830443+0000","last_clean":"2023-09-23T20:05:41.830443+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:05:41.830443+0000","last_undegraded":"2023-09-23T20:05:41.830443+0000","last_fullsized":"2023-09-23T20:05:41.830443+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":117,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.827295+0000","last_change":"2023-09-23T19:59:09.195560+0000","last_active":"2023-09-23T20:05:41.827295+0000","last_peered":"2023-09-23T20:05:41.827295+0000","last_clean":"2023-09-23T20:05:41.827295+0000","last_became_active":"2023-09-23T19:59:09.195145+0000","last_became_peered":"2023-09-23T19:59:09.195145+0000","last_unstale":"2023-09-23T20:05:41.827295+0000","last_undegraded":"2023-09-23T20:05:41.827295+0000","last_fullsized":"2023-09-23T20:05:41.827295+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.190798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830475+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:05:41.830475+0000","last_peered":"2023-09-23T20:05:41.830475+0000","last_clean":"2023-09-23T20:05:41.830475+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:05:41.830475+0000","last_undegraded":"2023-09-23T20:05:41.830475+0000","last_fullsized":"2023-09-23T20:05:41.830475+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":273,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945266+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:05:41.945266+0000","last_peered":"2023-09-23T20:05:41.945266+0000","last_clean":"2023-09-23T20:05:41.945266+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:05:41.945266+0000","last_undegraded":"2023-09-23T20:05:41.945266+0000","last_fullsized":"2023-09-23T20:05:41.945266+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":255,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830340+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:05:41.830340+0000","last_peered":"2023-09-23T20:05:41.830340+0000","last_clean":"2023-09-23T20:05:41.830340+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:05:41.830340+0000","last_undegraded":"2023-09-23T20:05:41.830340+0000","last_fullsized":"2023-09-23T20:05:41.830340+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.827266+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:05:41.827266+0000","last_peered":"2023-09-23T20:05:41.827266+0000","last_clean":"2023-09-23T20:05:41.827266+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:05:41.827266+0000","last_undegraded":"2023-09-23T20:05:41.827266+0000","last_fullsized":"2023-09-23T20:05:41.827266+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":88,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.944755+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:05:41.944755+0000","last_peered":"2023-09-23T20:05:41.944755+0000","last_clean":"2023-09-23T20:05:41.944755+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:05:41.944755+0000","last_undegraded":"2023-09-23T20:05:41.944755+0000","last_fullsized":"2023-09-23T20:05:41.944755+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":32,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.827993+0000","last_change":"2023-09-23T20:03:59.232022+0000","last_active":"2023-09-23T20:05:41.827993+0000","last_peered":"2023-09-23T20:05:41.827993+0000","last_clean":"2023-09-23T20:05:41.827993+0000","last_became_active":"2023-09-23T20:03:59.231689+0000","last_became_peered":"2023-09-23T20:03:59.231689+0000","last_unstale":"2023-09-23T20:05:41.827993+0000","last_undegraded":"2023-09-23T20:05:41.827993+0000","last_fullsized":"2023-09-23T20:05:41.827993+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:29.669102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"295'11509","reported_seq":21808,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:50.648633+0000","last_change":"2023-09-23T20:04:02.290062+0000","last_active":"2023-09-23T20:05:50.648633+0000","last_peered":"2023-09-23T20:05:50.648633+0000","last_clean":"2023-09-23T20:05:50.648633+0000","last_became_active":"2023-09-23T20:04:02.271287+0000","last_became_peered":"2023-09-23T20:04:02.271287+0000","last_unstale":"2023-09-23T20:05:50.648633+0000","last_undegraded":"2023-09-23T20:05:50.648633+0000","last_fullsized":"2023-09-23T20:05:50.648633+0000","mapping_epoch":276,"log_start":"255'4900","ondisk_log_start":"255'4900","created":240,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:47.344179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37683200,"num_objects":575,"num_object_clones":0,"num_object_copies":1725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11509,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5100,"num_bytes_recovered":334233600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":256,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830508+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:05:41.830508+0000","last_peered":"2023-09-23T20:05:41.830508+0000","last_clean":"2023-09-23T20:05:41.830508+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:05:41.830508+0000","last_undegraded":"2023-09-23T20:05:41.830508+0000","last_fullsized":"2023-09-23T20:05:41.830508+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":283,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.944706+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:05:41.944706+0000","last_peered":"2023-09-23T20:05:41.944706+0000","last_clean":"2023-09-23T20:05:41.944706+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:05:41.944706+0000","last_undegraded":"2023-09-23T20:05:41.944706+0000","last_fullsized":"2023-09-23T20:05:41.944706+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"295'11425","reported_seq":11490,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:47.293993+0000","last_change":"2023-09-23T20:01:39.911588+0000","last_active":"2023-09-23T20:05:47.293993+0000","last_peered":"2023-09-23T20:05:47.293993+0000","last_clean":"2023-09-23T20:05:47.293993+0000","last_became_active":"2023-09-23T20:01:39.911364+0000","last_became_peered":"2023-09-23T20:01:39.911364+0000","last_unstale":"2023-09-23T20:05:47.293993+0000","last_undegraded":"2023-09-23T20:05:47.293993+0000","last_fullsized":"2023-09-23T20:05:47.293993+0000","mapping_epoch":240,"log_start":"256'5495","ondisk_log_start":"256'5495","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5930,"log_dups_size":0,"ondisk_log_size":5930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:35.715001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40960000,"num_objects":625,"num_object_clones":0,"num_object_copies":1875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11425,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":257,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830956+0000","last_change":"2023-09-23T19:49:51.385384+0000","last_active":"2023-09-23T20:05:41.830956+0000","last_peered":"2023-09-23T20:05:41.830956+0000","last_clean":"2023-09-23T20:05:41.830956+0000","last_became_active":"2023-09-23T19:49:51.384998+0000","last_became_peered":"2023-09-23T19:49:51.384998+0000","last_unstale":"2023-09-23T20:05:41.830956+0000","last_undegraded":"2023-09-23T20:05:41.830956+0000","last_fullsized":"2023-09-23T20:05:41.830956+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:58.172038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":311,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830028+0000","last_change":"2023-09-23T20:03:59.253784+0000","last_active":"2023-09-23T20:05:41.830028+0000","last_peered":"2023-09-23T20:05:41.830028+0000","last_clean":"2023-09-23T20:05:41.830028+0000","last_became_active":"2023-09-23T20:03:59.253552+0000","last_became_peered":"2023-09-23T20:03:59.253552+0000","last_unstale":"2023-09-23T20:05:41.830028+0000","last_undegraded":"2023-09-23T20:05:41.830028+0000","last_fullsized":"2023-09-23T20:05:41.830028+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:56.110746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"295'11488","reported_seq":11553,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:50.697526+0000","last_change":"2023-09-23T20:01:39.976582+0000","last_active":"2023-09-23T20:05:50.697526+0000","last_peered":"2023-09-23T20:05:50.697526+0000","last_clean":"2023-09-23T20:05:50.697526+0000","last_became_active":"2023-09-23T20:01:39.976244+0000","last_became_peered":"2023-09-23T20:01:39.976244+0000","last_unstale":"2023-09-23T20:05:50.697526+0000","last_undegraded":"2023-09-23T20:05:50.697526+0000","last_fullsized":"2023-09-23T20:05:50.697526+0000","mapping_epoch":240,"log_start":"255'4900","ondisk_log_start":"255'4900","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6588,"log_dups_size":0,"ondisk_log_size":6588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.952295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36569088,"num_objects":558,"num_object_clones":0,"num_object_copies":1674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11488,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830716+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:05:41.830716+0000","last_peered":"2023-09-23T20:05:41.830716+0000","last_clean":"2023-09-23T20:05:41.830716+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:05:41.830716+0000","last_undegraded":"2023-09-23T20:05:41.830716+0000","last_fullsized":"2023-09-23T20:05:41.830716+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":223,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829683+0000","last_change":"2023-09-23T19:56:39.274316+0000","last_active":"2023-09-23T20:05:41.829683+0000","last_peered":"2023-09-23T20:05:41.829683+0000","last_clean":"2023-09-23T20:05:41.829683+0000","last_became_active":"2023-09-23T19:56:39.273921+0000","last_became_peered":"2023-09-23T19:56:39.273921+0000","last_unstale":"2023-09-23T20:05:41.829683+0000","last_undegraded":"2023-09-23T20:05:41.829683+0000","last_fullsized":"2023-09-23T20:05:41.829683+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.284038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"295'11780","reported_seq":11845,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:48.209412+0000","last_change":"2023-09-23T20:01:39.912566+0000","last_active":"2023-09-23T20:05:48.209412+0000","last_peered":"2023-09-23T20:05:48.209412+0000","last_clean":"2023-09-23T20:05:48.209412+0000","last_became_active":"2023-09-23T20:01:39.912259+0000","last_became_peered":"2023-09-23T20:01:39.912259+0000","last_unstale":"2023-09-23T20:05:48.209412+0000","last_undegraded":"2023-09-23T20:05:48.209412+0000","last_fullsized":"2023-09-23T20:05:48.209412+0000","mapping_epoch":240,"log_start":"248'3170","ondisk_log_start":"248'3170","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8610,"log_dups_size":0,"ondisk_log_size":8610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:19.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43646976,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11780,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":313,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945128+0000","last_change":"2023-09-23T20:00:34.468296+0000","last_active":"2023-09-23T20:05:41.945128+0000","last_peered":"2023-09-23T20:05:41.945128+0000","last_clean":"2023-09-23T20:05:41.945128+0000","last_became_active":"2023-09-23T20:00:34.467957+0000","last_became_peered":"2023-09-23T20:00:34.467957+0000","last_unstale":"2023-09-23T20:05:41.945128+0000","last_undegraded":"2023-09-23T20:05:41.945128+0000","last_fullsized":"2023-09-23T20:05:41.945128+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:50.449027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":111,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829977+0000","last_change":"2023-09-23T20:01:04.470558+0000","last_active":"2023-09-23T20:05:41.829977+0000","last_peered":"2023-09-23T20:05:41.829977+0000","last_clean":"2023-09-23T20:05:41.829977+0000","last_became_active":"2023-09-23T20:01:04.470276+0000","last_became_peered":"2023-09-23T20:01:04.470276+0000","last_unstale":"2023-09-23T20:05:41.829977+0000","last_undegraded":"2023-09-23T20:05:41.829977+0000","last_fullsized":"2023-09-23T20:05:41.829977+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:17.788583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"295'11413","reported_seq":11478,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:47.220851+0000","last_change":"2023-09-23T20:01:39.974611+0000","last_active":"2023-09-23T20:05:47.220851+0000","last_peered":"2023-09-23T20:05:47.220851+0000","last_clean":"2023-09-23T20:05:47.220851+0000","last_became_active":"2023-09-23T20:01:39.974318+0000","last_became_peered":"2023-09-23T20:01:39.974318+0000","last_unstale":"2023-09-23T20:05:47.220851+0000","last_undegraded":"2023-09-23T20:05:47.220851+0000","last_fullsized":"2023-09-23T20:05:47.220851+0000","mapping_epoch":240,"log_start":"256'5527","ondisk_log_start":"256'5527","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:31.047765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41615360,"num_objects":635,"num_object_clones":0,"num_object_copies":1905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11413,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":745,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830250+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:05:41.830250+0000","last_peered":"2023-09-23T20:05:41.830250+0000","last_clean":"2023-09-23T20:05:41.830250+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:05:41.830250+0000","last_undegraded":"2023-09-23T20:05:41.830250+0000","last_fullsized":"2023-09-23T20:05:41.830250+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":121,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.827086+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:05:41.827086+0000","last_peered":"2023-09-23T20:05:41.827086+0000","last_clean":"2023-09-23T20:05:41.827086+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:05:41.827086+0000","last_undegraded":"2023-09-23T20:05:41.827086+0000","last_fullsized":"2023-09-23T20:05:41.827086+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":291,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831058+0000","last_change":"2023-09-23T19:59:09.206075+0000","last_active":"2023-09-23T20:05:41.831058+0000","last_peered":"2023-09-23T20:05:41.831058+0000","last_clean":"2023-09-23T20:05:41.831058+0000","last_became_active":"2023-09-23T19:59:09.205766+0000","last_became_peered":"2023-09-23T19:59:09.205766+0000","last_unstale":"2023-09-23T20:05:41.831058+0000","last_undegraded":"2023-09-23T20:05:41.831058+0000","last_fullsized":"2023-09-23T20:05:41.831058+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:14.627703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"295'11234","reported_seq":11299,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:48.241361+0000","last_change":"2023-09-23T20:01:39.976550+0000","last_active":"2023-09-23T20:05:48.241361+0000","last_peered":"2023-09-23T20:05:48.241361+0000","last_clean":"2023-09-23T20:05:48.241361+0000","last_became_active":"2023-09-23T20:01:39.976245+0000","last_became_peered":"2023-09-23T20:01:39.976245+0000","last_unstale":"2023-09-23T20:05:48.241361+0000","last_undegraded":"2023-09-23T20:05:48.241361+0000","last_fullsized":"2023-09-23T20:05:48.241361+0000","mapping_epoch":240,"log_start":"248'2620","ondisk_log_start":"248'2620","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8614,"log_dups_size":0,"ondisk_log_size":8614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:45.979020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41287680,"num_objects":630,"num_object_clones":0,"num_object_copies":1890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11234,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":654,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.944611+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:05:41.944611+0000","last_peered":"2023-09-23T20:05:41.944611+0000","last_clean":"2023-09-23T20:05:41.944611+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:05:41.944611+0000","last_undegraded":"2023-09-23T20:05:41.944611+0000","last_fullsized":"2023-09-23T20:05:41.944611+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":241,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830746+0000","last_change":"2023-09-23T19:59:09.172601+0000","last_active":"2023-09-23T20:05:41.830746+0000","last_peered":"2023-09-23T20:05:41.830746+0000","last_clean":"2023-09-23T20:05:41.830746+0000","last_became_active":"2023-09-23T19:59:09.172297+0000","last_became_peered":"2023-09-23T19:59:09.172297+0000","last_unstale":"2023-09-23T20:05:41.830746+0000","last_undegraded":"2023-09-23T20:05:41.830746+0000","last_fullsized":"2023-09-23T20:05:41.830746+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:09.528266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":271,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831175+0000","last_change":"2023-09-23T19:50:23.998837+0000","last_active":"2023-09-23T20:05:41.831175+0000","last_peered":"2023-09-23T20:05:41.831175+0000","last_clean":"2023-09-23T20:05:41.831175+0000","last_became_active":"2023-09-23T19:50:23.998496+0000","last_became_peered":"2023-09-23T19:50:23.998496+0000","last_unstale":"2023-09-23T20:05:41.831175+0000","last_undegraded":"2023-09-23T20:05:41.831175+0000","last_fullsized":"2023-09-23T20:05:41.831175+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:05.750058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"295'11482","reported_seq":11548,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:50.735858+0000","last_change":"2023-09-23T20:01:39.977019+0000","last_active":"2023-09-23T20:05:50.735858+0000","last_peered":"2023-09-23T20:05:50.735858+0000","last_clean":"2023-09-23T20:05:50.735858+0000","last_became_active":"2023-09-23T20:01:39.976270+0000","last_became_peered":"2023-09-23T20:01:39.976270+0000","last_unstale":"2023-09-23T20:05:50.735858+0000","last_undegraded":"2023-09-23T20:05:50.735858+0000","last_fullsized":"2023-09-23T20:05:50.735858+0000","mapping_epoch":240,"log_start":"255'4900","ondisk_log_start":"255'4900","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.226759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39911448,"num_objects":610,"num_object_clones":0,"num_object_copies":1830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":610,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11482,"num_write_kb":386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":255,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829949+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:05:41.829949+0000","last_peered":"2023-09-23T20:05:41.829949+0000","last_clean":"2023-09-23T20:05:41.829949+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:05:41.829949+0000","last_undegraded":"2023-09-23T20:05:41.829949+0000","last_fullsized":"2023-09-23T20:05:41.829949+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":301,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831193+0000","last_change":"2023-09-23T19:59:09.171926+0000","last_active":"2023-09-23T20:05:41.831193+0000","last_peered":"2023-09-23T20:05:41.831193+0000","last_clean":"2023-09-23T20:05:41.831193+0000","last_became_active":"2023-09-23T19:59:09.171609+0000","last_became_peered":"2023-09-23T19:59:09.171609+0000","last_unstale":"2023-09-23T20:05:41.831193+0000","last_undegraded":"2023-09-23T20:05:41.831193+0000","last_fullsized":"2023-09-23T20:05:41.831193+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:21.795329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"295'11348","reported_seq":18491,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:47.306660+0000","last_change":"2023-09-23T20:04:50.012534+0000","last_active":"2023-09-23T20:05:47.306660+0000","last_peered":"2023-09-23T20:05:47.306660+0000","last_clean":"2023-09-23T20:05:47.306660+0000","last_became_active":"2023-09-23T20:04:49.916541+0000","last_became_peered":"2023-09-23T20:04:49.916541+0000","last_unstale":"2023-09-23T20:05:47.306660+0000","last_undegraded":"2023-09-23T20:05:47.306660+0000","last_fullsized":"2023-09-23T20:05:47.306660+0000","mapping_epoch":282,"log_start":"256'5423","ondisk_log_start":"256'5423","created":240,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5925,"log_dups_size":0,"ondisk_log_size":5925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:58.571894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42860544,"num_objects":654,"num_object_clones":0,"num_object_copies":1962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11348,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3528,"num_bytes_recovered":231145472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":633,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.946106+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:05:41.946106+0000","last_peered":"2023-09-23T20:05:41.946106+0000","last_clean":"2023-09-23T20:05:41.946106+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:05:41.946106+0000","last_undegraded":"2023-09-23T20:05:41.946106+0000","last_fullsized":"2023-09-23T20:05:41.946106+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":32,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.826634+0000","last_change":"2023-09-23T20:03:59.233742+0000","last_active":"2023-09-23T20:05:41.826634+0000","last_peered":"2023-09-23T20:05:41.826634+0000","last_clean":"2023-09-23T20:05:41.826634+0000","last_became_active":"2023-09-23T20:03:59.233209+0000","last_became_peered":"2023-09-23T20:03:59.233209+0000","last_unstale":"2023-09-23T20:05:41.826634+0000","last_undegraded":"2023-09-23T20:05:41.826634+0000","last_fullsized":"2023-09-23T20:05:41.826634+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:50.879925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":256,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830352+0000","last_change":"2023-09-23T19:59:09.153852+0000","last_active":"2023-09-23T20:05:41.830352+0000","last_peered":"2023-09-23T20:05:41.830352+0000","last_clean":"2023-09-23T20:05:41.830352+0000","last_became_active":"2023-09-23T19:59:09.153514+0000","last_became_peered":"2023-09-23T19:59:09.153514+0000","last_unstale":"2023-09-23T20:05:41.830352+0000","last_undegraded":"2023-09-23T20:05:41.830352+0000","last_fullsized":"2023-09-23T20:05:41.830352+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:45.179600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"295'11526","reported_seq":15519,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:48.499064+0000","last_change":"2023-09-23T20:05:24.592417+0000","last_active":"2023-09-23T20:05:48.499064+0000","last_peered":"2023-09-23T20:05:48.499064+0000","last_clean":"2023-09-23T20:05:48.499064+0000","last_became_active":"2023-09-23T20:05:24.567889+0000","last_became_peered":"2023-09-23T20:05:24.567889+0000","last_unstale":"2023-09-23T20:05:48.499064+0000","last_undegraded":"2023-09-23T20:05:48.499064+0000","last_fullsized":"2023-09-23T20:05:48.499064+0000","mapping_epoch":288,"log_start":"255'5000","ondisk_log_start":"255'5000","created":240,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6526,"log_dups_size":0,"ondisk_log_size":6526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:28.893902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42467328,"num_objects":648,"num_object_clones":0,"num_object_copies":1944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11526,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1949,"num_bytes_recovered":127664128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":618,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.828083+0000","last_change":"2023-09-23T20:04:02.277887+0000","last_active":"2023-09-23T20:05:41.828083+0000","last_peered":"2023-09-23T20:05:41.828083+0000","last_clean":"2023-09-23T20:05:41.828083+0000","last_became_active":"2023-09-23T20:04:02.277519+0000","last_became_peered":"2023-09-23T20:04:02.277519+0000","last_unstale":"2023-09-23T20:05:41.828083+0000","last_undegraded":"2023-09-23T20:05:41.828083+0000","last_fullsized":"2023-09-23T20:05:41.828083+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":634,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829889+0000","last_change":"2023-09-23T19:49:51.381825+0000","last_active":"2023-09-23T20:05:41.829889+0000","last_peered":"2023-09-23T20:05:41.829889+0000","last_clean":"2023-09-23T20:05:41.829889+0000","last_became_active":"2023-09-23T19:49:51.381378+0000","last_became_peered":"2023-09-23T19:49:51.381378+0000","last_unstale":"2023-09-23T20:05:41.829889+0000","last_undegraded":"2023-09-23T20:05:41.829889+0000","last_fullsized":"2023-09-23T20:05:41.829889+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:46.577113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":295,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945178+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:05:41.945178+0000","last_peered":"2023-09-23T20:05:41.945178+0000","last_clean":"2023-09-23T20:05:41.945178+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:05:41.945178+0000","last_undegraded":"2023-09-23T20:05:41.945178+0000","last_fullsized":"2023-09-23T20:05:41.945178+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":223,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829496+0000","last_change":"2023-09-23T19:49:51.378537+0000","last_active":"2023-09-23T20:05:41.829496+0000","last_peered":"2023-09-23T20:05:41.829496+0000","last_clean":"2023-09-23T20:05:41.829496+0000","last_became_active":"2023-09-23T19:49:51.377940+0000","last_became_peered":"2023-09-23T19:49:51.377940+0000","last_unstale":"2023-09-23T20:05:41.829496+0000","last_undegraded":"2023-09-23T20:05:41.829496+0000","last_fullsized":"2023-09-23T20:05:41.829496+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:24.790601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"295'11531","reported_seq":14257,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:47.278260+0000","last_change":"2023-09-23T20:05:41.954371+0000","last_active":"2023-09-23T20:05:47.278260+0000","last_peered":"2023-09-23T20:05:47.278260+0000","last_clean":"2023-09-23T20:05:47.278260+0000","last_became_active":"2023-09-23T20:05:41.831299+0000","last_became_peered":"2023-09-23T20:05:41.831299+0000","last_unstale":"2023-09-23T20:05:47.278260+0000","last_undegraded":"2023-09-23T20:05:47.278260+0000","last_fullsized":"2023-09-23T20:05:47.278260+0000","mapping_epoch":293,"log_start":"256'5603","ondisk_log_start":"256'5603","created":240,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5928,"log_dups_size":0,"ondisk_log_size":5928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:28.941933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46071808,"num_objects":703,"num_object_clones":0,"num_object_copies":2109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11531,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1317,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":606,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.944713+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:05:41.944713+0000","last_peered":"2023-09-23T20:05:41.944713+0000","last_clean":"2023-09-23T20:05:41.944713+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:05:41.944713+0000","last_undegraded":"2023-09-23T20:05:41.944713+0000","last_fullsized":"2023-09-23T20:05:41.944713+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":855,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945965+0000","last_change":"2023-09-23T19:56:39.291605+0000","last_active":"2023-09-23T20:05:41.945965+0000","last_peered":"2023-09-23T20:05:41.945965+0000","last_clean":"2023-09-23T20:05:41.945965+0000","last_became_active":"2023-09-23T19:56:39.280029+0000","last_became_peered":"2023-09-23T19:56:39.280029+0000","last_unstale":"2023-09-23T20:05:41.945965+0000","last_undegraded":"2023-09-23T20:05:41.945965+0000","last_fullsized":"2023-09-23T20:05:41.945965+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:58.623015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":294,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831333+0000","last_change":"2023-09-23T20:00:34.447064+0000","last_active":"2023-09-23T20:05:41.831333+0000","last_peered":"2023-09-23T20:05:41.831333+0000","last_clean":"2023-09-23T20:05:41.831333+0000","last_became_active":"2023-09-23T20:00:34.446754+0000","last_became_peered":"2023-09-23T20:00:34.446754+0000","last_unstale":"2023-09-23T20:05:41.831333+0000","last_undegraded":"2023-09-23T20:05:41.831333+0000","last_fullsized":"2023-09-23T20:05:41.831333+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:39.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":289,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829832+0000","last_change":"2023-09-23T20:03:59.236594+0000","last_active":"2023-09-23T20:05:41.829832+0000","last_peered":"2023-09-23T20:05:41.829832+0000","last_clean":"2023-09-23T20:05:41.829832+0000","last_became_active":"2023-09-23T20:03:59.236305+0000","last_became_peered":"2023-09-23T20:03:59.236305+0000","last_unstale":"2023-09-23T20:05:41.829832+0000","last_undegraded":"2023-09-23T20:05:41.829832+0000","last_fullsized":"2023-09-23T20:05:41.829832+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:23.103726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"295'11435","reported_seq":11498,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:48.498965+0000","last_change":"2023-09-23T20:01:39.977018+0000","last_active":"2023-09-23T20:05:48.498965+0000","last_peered":"2023-09-23T20:05:48.498965+0000","last_clean":"2023-09-23T20:05:48.498965+0000","last_became_active":"2023-09-23T20:01:39.976359+0000","last_became_peered":"2023-09-23T20:01:39.976359+0000","last_unstale":"2023-09-23T20:05:48.498965+0000","last_undegraded":"2023-09-23T20:05:48.498965+0000","last_fullsized":"2023-09-23T20:05:48.498965+0000","mapping_epoch":240,"log_start":"255'4900","ondisk_log_start":"255'4900","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.221346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40697856,"num_objects":621,"num_object_clones":0,"num_object_copies":1863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11435,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":530,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830393+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:05:41.830393+0000","last_peered":"2023-09-23T20:05:41.830393+0000","last_clean":"2023-09-23T20:05:41.830393+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:05:41.830393+0000","last_undegraded":"2023-09-23T20:05:41.830393+0000","last_fullsized":"2023-09-23T20:05:41.830393+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":928,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.903463+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:05:41.903463+0000","last_peered":"2023-09-23T20:05:41.903463+0000","last_clean":"2023-09-23T20:05:41.903463+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:05:41.903463+0000","last_undegraded":"2023-09-23T20:05:41.903463+0000","last_fullsized":"2023-09-23T20:05:41.903463+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":253,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:45.900461+0000","last_change":"2023-09-23T19:56:39.271842+0000","last_active":"2023-09-23T20:05:45.900461+0000","last_peered":"2023-09-23T20:05:45.900461+0000","last_clean":"2023-09-23T20:05:45.900461+0000","last_became_active":"2023-09-23T19:56:39.271568+0000","last_became_peered":"2023-09-23T19:56:39.271568+0000","last_unstale":"2023-09-23T20:05:45.900461+0000","last_undegraded":"2023-09-23T20:05:45.900461+0000","last_fullsized":"2023-09-23T20:05:45.900461+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.564126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":267,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830390+0000","last_change":"2023-09-23T19:49:51.378451+0000","last_active":"2023-09-23T20:05:41.830390+0000","last_peered":"2023-09-23T20:05:41.830390+0000","last_clean":"2023-09-23T20:05:41.830390+0000","last_became_active":"2023-09-23T19:49:51.377902+0000","last_became_peered":"2023-09-23T19:49:51.377902+0000","last_unstale":"2023-09-23T20:05:41.830390+0000","last_undegraded":"2023-09-23T20:05:41.830390+0000","last_fullsized":"2023-09-23T20:05:41.830390+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:30.531769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"295'11652","reported_seq":17281,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:48.386478+0000","last_change":"2023-09-23T20:05:05.847661+0000","last_active":"2023-09-23T20:05:48.386478+0000","last_peered":"2023-09-23T20:05:48.386478+0000","last_clean":"2023-09-23T20:05:48.386478+0000","last_became_active":"2023-09-23T20:05:05.813785+0000","last_became_peered":"2023-09-23T20:05:05.813785+0000","last_unstale":"2023-09-23T20:05:48.386478+0000","last_undegraded":"2023-09-23T20:05:48.386478+0000","last_fullsized":"2023-09-23T20:05:48.386478+0000","mapping_epoch":285,"log_start":"253'4256","ondisk_log_start":"253'4256","created":240,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:47.265078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43646976,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11652,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2768,"num_bytes_recovered":181403648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1103,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.944846+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:05:41.944846+0000","last_peered":"2023-09-23T20:05:41.944846+0000","last_clean":"2023-09-23T20:05:41.944846+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:05:41.944846+0000","last_undegraded":"2023-09-23T20:05:41.944846+0000","last_fullsized":"2023-09-23T20:05:41.944846+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":705,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830292+0000","last_change":"2023-09-23T20:00:34.445036+0000","last_active":"2023-09-23T20:05:41.830292+0000","last_peered":"2023-09-23T20:05:41.830292+0000","last_clean":"2023-09-23T20:05:41.830292+0000","last_became_active":"2023-09-23T20:00:34.444689+0000","last_became_peered":"2023-09-23T20:00:34.444689+0000","last_unstale":"2023-09-23T20:05:41.830292+0000","last_undegraded":"2023-09-23T20:05:41.830292+0000","last_fullsized":"2023-09-23T20:05:41.830292+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:44.485721+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":32,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.826982+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:05:41.826982+0000","last_peered":"2023-09-23T20:05:41.826982+0000","last_clean":"2023-09-23T20:05:41.826982+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:05:41.826982+0000","last_undegraded":"2023-09-23T20:05:41.826982+0000","last_fullsized":"2023-09-23T20:05:41.826982+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":170,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831407+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:05:41.831407+0000","last_peered":"2023-09-23T20:05:41.831407+0000","last_clean":"2023-09-23T20:05:41.831407+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:05:41.831407+0000","last_undegraded":"2023-09-23T20:05:41.831407+0000","last_fullsized":"2023-09-23T20:05:41.831407+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"295'11607","reported_seq":11669,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:48.500562+0000","last_change":"2023-09-23T20:01:39.977010+0000","last_active":"2023-09-23T20:05:48.500562+0000","last_peered":"2023-09-23T20:05:48.500562+0000","last_clean":"2023-09-23T20:05:48.500562+0000","last_became_active":"2023-09-23T20:01:39.976722+0000","last_became_peered":"2023-09-23T20:01:39.976722+0000","last_unstale":"2023-09-23T20:05:48.500562+0000","last_undegraded":"2023-09-23T20:05:48.500562+0000","last_fullsized":"2023-09-23T20:05:48.500562+0000","mapping_epoch":240,"log_start":"252'4231","ondisk_log_start":"252'4231","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":7376,"log_dups_size":0,"ondisk_log_size":7376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:13.592964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41091072,"num_objects":627,"num_object_clones":0,"num_object_copies":1881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11607,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":636,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830120+0000","last_change":"2023-09-23T19:56:42.286129+0000","last_active":"2023-09-23T20:05:41.830120+0000","last_peered":"2023-09-23T20:05:41.830120+0000","last_clean":"2023-09-23T20:05:41.830120+0000","last_became_active":"2023-09-23T19:56:42.285688+0000","last_became_peered":"2023-09-23T19:56:42.285688+0000","last_unstale":"2023-09-23T20:05:41.830120+0000","last_undegraded":"2023-09-23T20:05:41.830120+0000","last_fullsized":"2023-09-23T20:05:41.830120+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.842550+0000","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1013,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830238+0000","last_change":"2023-09-23T19:49:51.378727+0000","last_active":"2023-09-23T20:05:41.830238+0000","last_peered":"2023-09-23T20:05:41.830238+0000","last_clean":"2023-09-23T20:05:41.830238+0000","last_became_active":"2023-09-23T19:49:51.378114+0000","last_became_peered":"2023-09-23T19:49:51.378114+0000","last_unstale":"2023-09-23T20:05:41.830238+0000","last_undegraded":"2023-09-23T20:05:41.830238+0000","last_fullsized":"2023-09-23T20:05:41.830238+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:35.608906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":261,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830093+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:05:41.830093+0000","last_peered":"2023-09-23T20:05:41.830093+0000","last_clean":"2023-09-23T20:05:41.830093+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:05:41.830093+0000","last_undegraded":"2023-09-23T20:05:41.830093+0000","last_fullsized":"2023-09-23T20:05:41.830093+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":257,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945469+0000","last_change":"2023-09-23T20:03:10.175462+0000","last_active":"2023-09-23T20:05:41.945469+0000","last_peered":"2023-09-23T20:05:41.945469+0000","last_clean":"2023-09-23T20:05:41.945469+0000","last_became_active":"2023-09-23T20:03:10.175250+0000","last_became_peered":"2023-09-23T20:03:10.175250+0000","last_unstale":"2023-09-23T20:05:41.945469+0000","last_undegraded":"2023-09-23T20:05:41.945469+0000","last_fullsized":"2023-09-23T20:05:41.945469+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:42.961619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"295'11352","reported_seq":11415,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:48.018406+0000","last_change":"2023-09-23T20:01:39.913385+0000","last_active":"2023-09-23T20:05:48.018406+0000","last_peered":"2023-09-23T20:05:48.018406+0000","last_clean":"2023-09-23T20:05:48.018406+0000","last_became_active":"2023-09-23T20:01:39.912872+0000","last_became_peered":"2023-09-23T20:01:39.912872+0000","last_unstale":"2023-09-23T20:05:48.018406+0000","last_undegraded":"2023-09-23T20:05:48.018406+0000","last_fullsized":"2023-09-23T20:05:48.018406+0000","mapping_epoch":240,"log_start":"249'3239","ondisk_log_start":"249'3239","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":8113,"log_dups_size":0,"ondisk_log_size":8113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.619813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42205184,"num_objects":644,"num_object_clones":0,"num_object_copies":1932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11352,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":610,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.827177+0000","last_change":"2023-09-23T20:04:02.277648+0000","last_active":"2023-09-23T20:05:41.827177+0000","last_peered":"2023-09-23T20:05:41.827177+0000","last_clean":"2023-09-23T20:05:41.827177+0000","last_became_active":"2023-09-23T20:04:02.277271+0000","last_became_peered":"2023-09-23T20:04:02.277271+0000","last_unstale":"2023-09-23T20:05:41.827177+0000","last_undegraded":"2023-09-23T20:05:41.827177+0000","last_fullsized":"2023-09-23T20:05:41.827177+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":373,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830794+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:05:41.830794+0000","last_peered":"2023-09-23T20:05:41.830794+0000","last_clean":"2023-09-23T20:05:41.830794+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:05:41.830794+0000","last_undegraded":"2023-09-23T20:05:41.830794+0000","last_fullsized":"2023-09-23T20:05:41.830794+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":255,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829923+0000","last_change":"2023-09-23T20:03:10.174842+0000","last_active":"2023-09-23T20:05:41.829923+0000","last_peered":"2023-09-23T20:05:41.829923+0000","last_clean":"2023-09-23T20:05:41.829923+0000","last_became_active":"2023-09-23T20:03:10.174511+0000","last_became_peered":"2023-09-23T20:03:10.174511+0000","last_unstale":"2023-09-23T20:05:41.829923+0000","last_undegraded":"2023-09-23T20:05:41.829923+0000","last_fullsized":"2023-09-23T20:05:41.829923+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:12.723811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":91,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.944897+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:05:41.944897+0000","last_peered":"2023-09-23T20:05:41.944897+0000","last_clean":"2023-09-23T20:05:41.944897+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:05:41.944897+0000","last_undegraded":"2023-09-23T20:05:41.944897+0000","last_fullsized":"2023-09-23T20:05:41.944897+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"295'11573","reported_seq":11637,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:47.326696+0000","last_change":"2023-09-23T20:01:39.912135+0000","last_active":"2023-09-23T20:05:47.326696+0000","last_peered":"2023-09-23T20:05:47.326696+0000","last_clean":"2023-09-23T20:05:47.326696+0000","last_became_active":"2023-09-23T20:01:39.911834+0000","last_became_peered":"2023-09-23T20:01:39.911834+0000","last_unstale":"2023-09-23T20:05:47.326696+0000","last_undegraded":"2023-09-23T20:05:47.326696+0000","last_fullsized":"2023-09-23T20:05:47.326696+0000","mapping_epoch":240,"log_start":"256'5626","ondisk_log_start":"256'5626","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:46.725578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43712512,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11573,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":673,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831058+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:05:41.831058+0000","last_peered":"2023-09-23T20:05:41.831058+0000","last_clean":"2023-09-23T20:05:41.831058+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:05:41.831058+0000","last_undegraded":"2023-09-23T20:05:41.831058+0000","last_fullsized":"2023-09-23T20:05:41.831058+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":122,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:42.329560+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:05:42.329560+0000","last_peered":"2023-09-23T20:05:42.329560+0000","last_clean":"2023-09-23T20:05:42.329560+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:05:42.329560+0000","last_undegraded":"2023-09-23T20:05:42.329560+0000","last_fullsized":"2023-09-23T20:05:42.329560+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":701,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830028+0000","last_change":"2023-09-23T19:59:40.899753+0000","last_active":"2023-09-23T20:05:41.830028+0000","last_peered":"2023-09-23T20:05:41.830028+0000","last_clean":"2023-09-23T20:05:41.830028+0000","last_became_active":"2023-09-23T19:59:40.899147+0000","last_became_peered":"2023-09-23T19:59:40.899147+0000","last_unstale":"2023-09-23T20:05:41.830028+0000","last_undegraded":"2023-09-23T20:05:41.830028+0000","last_fullsized":"2023-09-23T20:05:41.830028+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:08.938755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:43.306343+0000","last_change":"2023-09-23T20:05:39.782474+0000","last_active":"2023-09-23T20:05:43.306343+0000","last_peered":"2023-09-23T20:05:43.306343+0000","last_clean":"2023-09-23T20:05:43.306343+0000","last_became_active":"2023-09-23T20:05:39.782131+0000","last_became_peered":"2023-09-23T20:05:39.782131+0000","last_unstale":"2023-09-23T20:05:43.306343+0000","last_undegraded":"2023-09-23T20:05:43.306343+0000","last_fullsized":"2023-09-23T20:05:43.306343+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:11.924563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.2","version":"295'11511","reported_seq":11576,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:05:47.313170+0000","last_change":"2023-09-23T20:01:39.913698+0000","last_active":"2023-09-23T20:05:47.313170+0000","last_peered":"2023-09-23T20:05:47.313170+0000","last_clean":"2023-09-23T20:05:47.313170+0000","last_became_active":"2023-09-23T20:01:39.913367+0000","last_became_peered":"2023-09-23T20:01:39.913367+0000","last_unstale":"2023-09-23T20:05:47.313170+0000","last_undegraded":"2023-09-23T20:05:47.313170+0000","last_fullsized":"2023-09-23T20:05:47.313170+0000","mapping_epoch":240,"log_start":"256'5625","ondisk_log_start":"256'5625","created":240,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:01:38.898880+0000","last_clean_scrub_stamp":"2023-09-23T20:01:38.898880+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47513600,"num_objects":725,"num_object_clones":0,"num_object_copies":2175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11511,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":551,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.827378+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:05:41.827378+0000","last_peered":"2023-09-23T20:05:41.827378+0000","last_clean":"2023-09-23T20:05:41.827378+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:05:41.827378+0000","last_undegraded":"2023-09-23T20:05:41.827378+0000","last_fullsized":"2023-09-23T20:05:41.827378+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":51,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830596+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:05:41.830596+0000","last_peered":"2023-09-23T20:05:41.830596+0000","last_clean":"2023-09-23T20:05:41.830596+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:05:41.830596+0000","last_undegraded":"2023-09-23T20:05:41.830596+0000","last_fullsized":"2023-09-23T20:05:41.830596+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":99,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830104+0000","last_change":"2023-09-23T20:01:04.470744+0000","last_active":"2023-09-23T20:05:41.830104+0000","last_peered":"2023-09-23T20:05:41.830104+0000","last_clean":"2023-09-23T20:05:41.830104+0000","last_became_active":"2023-09-23T20:01:04.470302+0000","last_became_peered":"2023-09-23T20:01:04.470302+0000","last_unstale":"2023-09-23T20:05:41.830104+0000","last_undegraded":"2023-09-23T20:05:41.830104+0000","last_fullsized":"2023-09-23T20:05:41.830104+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:25.452596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":254,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829977+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:05:41.829977+0000","last_peered":"2023-09-23T20:05:41.829977+0000","last_clean":"2023-09-23T20:05:41.829977+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:05:41.829977+0000","last_undegraded":"2023-09-23T20:05:41.829977+0000","last_fullsized":"2023-09-23T20:05:41.829977+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.902162+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:05:41.902162+0000","last_peered":"2023-09-23T20:05:41.902162+0000","last_clean":"2023-09-23T20:05:41.902162+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:05:41.902162+0000","last_undegraded":"2023-09-23T20:05:41.902162+0000","last_fullsized":"2023-09-23T20:05:41.902162+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":58,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:43.306397+0000","last_change":"2023-09-23T20:05:39.767597+0000","last_active":"2023-09-23T20:05:43.306397+0000","last_peered":"2023-09-23T20:05:43.306397+0000","last_clean":"2023-09-23T20:05:43.306397+0000","last_became_active":"2023-09-23T20:05:39.767313+0000","last_became_peered":"2023-09-23T20:05:39.767313+0000","last_unstale":"2023-09-23T20:05:43.306397+0000","last_undegraded":"2023-09-23T20:05:43.306397+0000","last_fullsized":"2023-09-23T20:05:43.306397+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:09.189511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":256,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.944756+0000","last_change":"2023-09-23T19:49:51.378475+0000","last_active":"2023-09-23T20:05:41.944756+0000","last_peered":"2023-09-23T20:05:41.944756+0000","last_clean":"2023-09-23T20:05:41.944756+0000","last_became_active":"2023-09-23T19:49:51.378108+0000","last_became_peered":"2023-09-23T19:49:51.378108+0000","last_unstale":"2023-09-23T20:05:41.944756+0000","last_undegraded":"2023-09-23T20:05:41.944756+0000","last_fullsized":"2023-09-23T20:05:41.944756+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:55.956566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":291,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.831380+0000","last_change":"2023-09-23T19:59:09.172452+0000","last_active":"2023-09-23T20:05:41.831380+0000","last_peered":"2023-09-23T20:05:41.831380+0000","last_clean":"2023-09-23T20:05:41.831380+0000","last_became_active":"2023-09-23T19:59:09.172080+0000","last_became_peered":"2023-09-23T19:59:09.172080+0000","last_unstale":"2023-09-23T20:05:41.831380+0000","last_undegraded":"2023-09-23T20:05:41.831380+0000","last_fullsized":"2023-09-23T20:05:41.831380+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:24.687991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945182+0000","last_change":"2023-09-23T19:56:39.281029+0000","last_active":"2023-09-23T20:05:41.945182+0000","last_peered":"2023-09-23T20:05:41.945182+0000","last_clean":"2023-09-23T20:05:41.945182+0000","last_became_active":"2023-09-23T19:56:39.274985+0000","last_became_peered":"2023-09-23T19:56:39.274985+0000","last_unstale":"2023-09-23T20:05:41.945182+0000","last_undegraded":"2023-09-23T20:05:41.945182+0000","last_fullsized":"2023-09-23T20:05:41.945182+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:07.560646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":291,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.830717+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:05:41.830717+0000","last_peered":"2023-09-23T20:05:41.830717+0000","last_clean":"2023-09-23T20:05:41.830717+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:05:41.830717+0000","last_undegraded":"2023-09-23T20:05:41.830717+0000","last_fullsized":"2023-09-23T20:05:41.830717+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":118,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.827384+0000","last_change":"2023-09-23T19:59:09.145224+0000","last_active":"2023-09-23T20:05:41.827384+0000","last_peered":"2023-09-23T20:05:41.827384+0000","last_clean":"2023-09-23T20:05:41.827384+0000","last_became_active":"2023-09-23T19:59:09.144819+0000","last_became_peered":"2023-09-23T19:59:09.144819+0000","last_unstale":"2023-09-23T20:05:41.827384+0000","last_undegraded":"2023-09-23T20:05:41.827384+0000","last_fullsized":"2023-09-23T20:05:41.827384+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:04.048181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":32,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.826818+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:05:41.826818+0000","last_peered":"2023-09-23T20:05:41.826818+0000","last_clean":"2023-09-23T20:05:41.826818+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:05:41.826818+0000","last_undegraded":"2023-09-23T20:05:41.826818+0000","last_fullsized":"2023-09-23T20:05:41.826818+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":114,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.945257+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:05:41.945257+0000","last_peered":"2023-09-23T20:05:41.945257+0000","last_clean":"2023-09-23T20:05:41.945257+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:05:41.945257+0000","last_undegraded":"2023-09-23T20:05:41.945257+0000","last_fullsized":"2023-09-23T20:05:41.945257+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":321,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:05:41.829899+0000","last_change":"2023-09-23T20:03:59.235232+0000","last_active":"2023-09-23T20:05:41.829899+0000","last_peered":"2023-09-23T20:05:41.829899+0000","last_clean":"2023-09-23T20:05:41.829899+0000","last_became_active":"2023-09-23T20:03:59.234941+0000","last_became_peered":"2023-09-23T20:03:59.234941+0000","last_unstale":"2023-09-23T20:05:41.829899+0000","last_undegraded":"2023-09-23T20:05:41.829899+0000","last_fullsized":"2023-09-23T20:05:41.829899+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:45.591449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":671940632,"num_objects":10254,"num_object_clones":0,"num_object_copies":30762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10254,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183866,"num_write_kb":6211777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14662,"num_bytes_recovered":960692224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2339188736,"data_stored":2339176520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108451,"ondisk_log_size":108451,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234296,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":537720,"kb_used_data":266672,"kb_used_omap":0,"kb_used_meta":271040,"kb_avail":93207432,"statfs":{"total":95995035648,"available":95444410368,"internally_reserved":0,"allocated":273072128,"data_stored":272160906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":277544960},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":2.4969999999999999,"15min":2.1589999999999998},"min":{"1min":0.23200000000000001,"5min":0.22,"15min":0.19700000000000001},"max":{"1min":11.256,"5min":60.140000000000001,"15min":62.061999999999998},"last":4.9530000000000003},{"interface":"front","average":{"1min":3.0569999999999999,"5min":2.827,"15min":2.4319999999999999},"min":{"1min":0.23899999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":13.143000000000001,"5min":62.389000000000003,"15min":62.389000000000003},"last":0.44}]},{"osd":1,"last update":"Sat Sep 23 20:05:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":5.4340000000000002,"15min":3.2160000000000002},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":5.2939999999999996,"5min":104.16800000000001,"15min":104.16800000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.5099999999999998,"5min":5.6719999999999997,"15min":3.3439999999999999},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":15.257,"5min":104.003,"15min":104.003},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:05:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":3.0339999999999998,"15min":3.246},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":14.555999999999999,"5min":105.842,"15min":130.673},"last":9.3940000000000001},{"interface":"front","average":{"1min":3.7450000000000001,"5min":3.8239999999999998,"15min":3.637},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":15.163,"5min":105.66,"15min":130.71100000000001},"last":0.50900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:05:29 2023","interfaces":[{"interface":"back","average":{"1min":3.9820000000000002,"5min":5.6399999999999997,"15min":3.673},"min":{"1min":0.36599999999999999,"5min":0.248,"15min":0.247},"max":{"1min":12.314,"5min":146.38800000000001,"15min":146.38800000000001},"last":4.8639999999999999},{"interface":"front","average":{"1min":3.6520000000000001,"5min":5.6689999999999996,"15min":3.665},"min":{"1min":0.34399999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":15.653,"5min":146.851,"15min":146.851},"last":1.7809999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:04:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.80600000000000005,"15min":1.891},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.714,"5min":19.215,"15min":143.255},"last":1.0980000000000001},{"interface":"front","average":{"1min":2.605,"5min":1.1459999999999999,"15min":1.9330000000000001},"min":{"1min":0.27000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":41.822000000000003,"5min":41.822000000000003,"15min":143.41200000000001},"last":1.2430000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:04:57 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":1.091,"15min":1.4610000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25},"max":{"1min":7.4960000000000004,"5min":35.488,"15min":84.566999999999993},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.95399999999999996,"15min":1.302},"min":{"1min":0.28599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.403,"5min":34.896000000000001,"15min":81.244},"last":1.042}]},{"osd":6,"last update":"Sat Sep 23 20:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.1220000000000001,"15min":0.84699999999999998},"min":{"1min":0.32500000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.6869999999999998,"5min":41.902999999999999,"15min":41.902999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.67900000000000005,"15min":0.63900000000000001},"min":{"1min":0.36699999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.714,"5min":6.3620000000000001,"15min":6.3620000000000001},"last":1.0720000000000001}]}]},{"osd":6,"up_from":0,"seq":1078036791337,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762498,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":715144,"kb_used_data":309632,"kb_used_omap":7,"kb_used_meta":405496,"kb_avail":93030008,"statfs":{"total":95995035648,"available":95262728192,"internally_reserved":0,"allocated":317063168,"data_stored":315728870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":415228730},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":4.9800000000000004,"5min":2.8370000000000002,"15min":4.1879999999999997},"min":{"1min":0.19800000000000001,"5min":0.16900000000000001,"15min":0.151},"max":{"1min":31.413,"5min":31.413,"15min":202.917},"last":7.0620000000000003},{"interface":"front","average":{"1min":6.5,"5min":3.496,"15min":4.4210000000000003},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17899999999999999},"max":{"1min":28.379999999999999,"5min":28.379999999999999,"15min":203.34899999999999},"last":0.32200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:04:58 2023","interfaces":[{"interface":"back","average":{"1min":4.54,"5min":5.1639999999999997,"15min":5.5119999999999996},"min":{"1min":0.23000000000000001,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":12.554,"5min":134.72499999999999,"15min":231.57499999999999},"last":2.7080000000000002},{"interface":"front","average":{"1min":4.6079999999999997,"5min":5.6429999999999998,"15min":5.5199999999999996},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":11.881,"5min":135.74600000000001,"15min":236.077},"last":7.3289999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":6.9790000000000001,"5min":3.52,"15min":3.4569999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.217},"max":{"1min":25.178999999999998,"5min":25.178999999999998,"15min":68.180999999999997},"last":7.5789999999999997},{"interface":"front","average":{"1min":6.0970000000000004,"5min":3.621,"15min":3.2240000000000002},"min":{"1min":0.376,"5min":0.249,"15min":0.20100000000000001},"max":{"1min":28.437999999999999,"5min":28.437999999999999,"15min":66.509},"last":7.5270000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":2.214,"15min":2.5449999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":10.368,"5min":35.497999999999998,"15min":161.68299999999999},"last":7.1289999999999996},{"interface":"front","average":{"1min":3.298,"5min":2.6429999999999998,"15min":2.6259999999999999},"min":{"1min":0.33400000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.236000000000001,"5min":35.521000000000001,"15min":161.286},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.6640000000000001,"15min":3.5209999999999999},"min":{"1min":0.373,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":13.278,"5min":51.698,"15min":219.048},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.7549999999999999,"15min":3.6659999999999999},"min":{"1min":0.34699999999999998,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":10.189,"5min":51.173000000000002,"15min":218.69499999999999},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":2.1219999999999999,"15min":2.2050000000000001},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":10.4,"5min":10.4,"15min":10.4},"last":7.157},{"interface":"front","average":{"1min":2.3650000000000002,"5min":3.1899999999999999,"15min":3.3599999999999999},"min":{"1min":0.39600000000000002,"5min":0.371,"15min":0.371},"max":{"1min":8.6549999999999994,"5min":20.734999999999999,"15min":20.734999999999999},"last":0.35299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":2.3839999999999999,"15min":2.3159999999999998},"min":{"1min":0.39400000000000002,"5min":0.36199999999999999,"15min":0.26800000000000002},"max":{"1min":10.162000000000001,"5min":14.23,"15min":60.213000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.6859999999999999,"5min":1.9750000000000001,"15min":1.8879999999999999},"min":{"1min":0.41299999999999998,"5min":0.33500000000000002,"15min":0.30199999999999999},"max":{"1min":10.323,"5min":14.188000000000001,"15min":61.298999999999999},"last":7.1040000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926032,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972228,"kb_used_data":500092,"kb_used_omap":8,"kb_used_meta":472119,"kb_avail":92772924,"statfs":{"total":95995035648,"available":94999474176,"internally_reserved":0,"allocated":512094208,"data_stored":510763300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":483450465},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":6.4210000000000003,"5min":5.6749999999999998,"15min":3.218},"min":{"1min":0.224,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":48.087000000000003,"5min":94.055999999999997,"15min":94.055999999999997},"last":1.5549999999999999},{"interface":"front","average":{"1min":5.7530000000000001,"5min":5.806,"15min":3.5310000000000001},"min":{"1min":0.187,"5min":0.183,"15min":0.183},"max":{"1min":48.119999999999997,"5min":91.632999999999996,"15min":91.632999999999996},"last":8.3979999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:04:56 2023","interfaces":[{"interface":"back","average":{"1min":11.645,"5min":7.7329999999999997,"15min":4.3390000000000004},"min":{"1min":0.216,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":162.62899999999999,"5min":270.51100000000002,"15min":270.51100000000002},"last":2.927},{"interface":"front","average":{"1min":10.435,"5min":7.1280000000000001,"15min":4.3120000000000003},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":162.52500000000001,"5min":269.69299999999998,"15min":269.69299999999998},"last":4.3150000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":7.0720000000000001,"5min":5.6989999999999998,"15min":5.0940000000000003},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":51.765999999999998,"5min":116.733,"15min":145.72800000000001},"last":7.9669999999999996},{"interface":"front","average":{"1min":7.0499999999999998,"5min":5.8250000000000002,"15min":5.2060000000000004},"min":{"1min":0.38900000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":51.723999999999997,"5min":116.697,"15min":145.50700000000001},"last":4.4109999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":8.9209999999999994,"5min":3.085,"15min":2.867},"min":{"1min":0.34799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":148.20599999999999,"5min":148.20599999999999,"15min":148.20599999999999},"last":4.1970000000000001},{"interface":"front","average":{"1min":8.9429999999999996,"5min":3.0649999999999999,"15min":2.5139999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":148.11000000000001,"5min":148.11000000000001,"15min":148.11000000000001},"last":4.3620000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.3460000000000001,"15min":2.6230000000000002},"min":{"1min":0.38100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":14.901999999999999,"5min":64.725999999999999,"15min":131.34399999999999},"last":1.5980000000000001},{"interface":"front","average":{"1min":1.855,"5min":1.7130000000000001,"15min":2.645},"min":{"1min":0.39400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.0890000000000004,"5min":12.48,"15min":131.386},"last":4.1360000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.6699999999999999,"15min":1.6279999999999999},"min":{"1min":0.42499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.1579999999999995,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":4.2489999999999997},{"interface":"front","average":{"1min":2.2210000000000001,"5min":2.133,"15min":1.9810000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":13.766999999999999,"5min":13.766999999999999,"15min":13.766999999999999},"last":4.3869999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:05:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.4039999999999999,"15min":2.1389999999999998},"min":{"1min":0.39000000000000001,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":14.856,"5min":65.081999999999994,"15min":77.132999999999996},"last":1.627},{"interface":"front","average":{"1min":1.899,"5min":1.8200000000000001,"15min":2.177},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.1799999999999997,"5min":12.452,"15min":75.465999999999994},"last":4.2249999999999996}]}]},{"osd":2,"up_from":104,"seq":446676598971,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":859452,"kb_used_data":342900,"kb_used_omap":12,"kb_used_meta":516467,"kb_avail":92885700,"statfs":{"total":95995035648,"available":95114956800,"internally_reserved":0,"allocated":351129600,"data_stored":349809452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":528863230},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":6.569,"5min":4.5190000000000001,"15min":2.9159999999999999},"min":{"1min":0.20999999999999999,"5min":0.192,"15min":0.154},"max":{"1min":76.251999999999995,"5min":76.251999999999995,"15min":76.251999999999995},"last":1.3759999999999999},{"interface":"front","average":{"1min":7.1449999999999996,"5min":4.2480000000000002,"15min":2.8479999999999999},"min":{"1min":0.215,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":76.590000000000003,"5min":76.590000000000003,"15min":76.590000000000003},"last":9.4860000000000007}]},{"osd":1,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":5.3979999999999997,"15min":3.0990000000000002},"min":{"1min":0.25900000000000001,"5min":0.22600000000000001,"15min":0.184},"max":{"1min":10.355,"5min":94.768000000000001,"15min":94.768000000000001},"last":9.0399999999999991},{"interface":"front","average":{"1min":2.2010000000000001,"5min":5.9359999999999999,"15min":3.3100000000000001},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.5999999999999996,"5min":96.808000000000007,"15min":96.808000000000007},"last":1.347}]},{"osd":3,"last update":"Sat Sep 23 20:05:25 2023","interfaces":[{"interface":"back","average":{"1min":13.644,"5min":5.7679999999999998,"15min":3.681},"min":{"1min":0.40000000000000002,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":107.381,"5min":107.381,"15min":107.381},"last":84.207999999999998},{"interface":"front","average":{"1min":11.827,"5min":4.8579999999999997,"15min":3.4849999999999999},"min":{"1min":0.45700000000000002,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":106.301,"5min":106.301,"15min":106.301},"last":1.3979999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":1.7989999999999999,"15min":1.764},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.2240000000000002,"5min":20.591999999999999,"15min":50.640000000000001},"last":1.4870000000000001},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.766,"15min":1.758},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.0579999999999998,"5min":23.062000000000001,"15min":50.154000000000003},"last":9.0879999999999992}]},{"osd":5,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":3.593,"15min":2.5419999999999998},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.5519999999999996,"5min":172.505,"15min":172.505},"last":1.423},{"interface":"front","average":{"1min":1.885,"5min":3.4900000000000002,"15min":2.4300000000000002},"min":{"1min":0.436,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":7.0970000000000004,"5min":172.375,"15min":172.375},"last":9.6910000000000007}]},{"osd":6,"last update":"Sat Sep 23 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.6659999999999999,"15min":1.639},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.3759999999999994,"5min":10.144,"15min":10.144},"last":1.4670000000000001},{"interface":"front","average":{"1min":2.101,"5min":1.9430000000000001,"15min":1.96},"min":{"1min":0.437,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.3879999999999999,"5min":9.3879999999999999,"15min":9.3879999999999999},"last":9.7210000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:04:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.917,"15min":1.8560000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.27400000000000002},"max":{"1min":4.0789999999999997,"5min":23.079999999999998,"15min":50.118000000000002},"last":9.1170000000000009},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.0779999999999998,"15min":2.1219999999999999},"min":{"1min":0.40100000000000002,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":5.5750000000000002,"5min":20.614999999999998,"15min":50.664999999999999},"last":1.4450000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435444,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":961404,"kb_used_data":458036,"kb_used_omap":4,"kb_used_meta":503163,"kb_avail":92783748,"statfs":{"total":95995035648,"available":95010557952,"internally_reserved":0,"allocated":469028864,"data_stored":467585752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":515239021},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":4.2999999999999998,"5min":2.6629999999999998,"15min":4.1559999999999997},"min":{"1min":0.28299999999999997,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":12.023,"5min":21.100999999999999,"15min":240.40600000000001},"last":10.798},{"interface":"front","average":{"1min":3.5899999999999999,"5min":3.4089999999999998,"15min":4.7140000000000004},"min":{"1min":0.38600000000000001,"5min":0.19600000000000001,"15min":0.17199999999999999},"max":{"1min":12.638,"5min":53.201000000000001,"15min":240.352},"last":0.79000000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8119999999999998,"5min":4.2069999999999999,"15min":2.4420000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":8.766,"5min":86.337999999999994,"15min":86.337999999999994},"last":0.40200000000000002},{"interface":"front","average":{"1min":3.8650000000000002,"5min":3.9390000000000001,"15min":2.4729999999999999},"min":{"1min":0.35599999999999998,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":12.055999999999999,"5min":84.299999999999997,"15min":84.299999999999997},"last":10.754}]},{"osd":2,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3170000000000002,"5min":3.7349999999999999,"15min":5.4950000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.219},"max":{"1min":9.3780000000000001,"5min":82.111999999999995,"15min":264.97800000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":4.9980000000000002,"5min":3.1349999999999998,"15min":5.2320000000000002},"min":{"1min":0.29099999999999998,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":12.359999999999999,"5min":30.768999999999998,"15min":257.40899999999999},"last":0.72499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":4.5990000000000002,"15min":2.9079999999999999},"min":{"1min":0.33500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.51,"5min":109.66800000000001,"15min":109.66800000000001},"last":0.63},{"interface":"front","average":{"1min":2.5659999999999998,"5min":5.0830000000000002,"15min":3.3450000000000002},"min":{"1min":0.45600000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.637,"5min":110.295,"15min":110.295},"last":0.65500000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6480000000000001,"5min":4.4269999999999996,"15min":2.6680000000000001},"min":{"1min":0.46999999999999997,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":12.662000000000001,"5min":113.881,"15min":113.881},"last":0.68999999999999995},{"interface":"front","average":{"1min":1.6040000000000001,"5min":4.2599999999999998,"15min":2.4990000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.3929999999999998,"5min":114.598,"15min":114.598},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:05:36 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":4.1970000000000001,"15min":5.2329999999999997},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8479999999999999,"5min":82.281999999999996,"15min":82.281999999999996},"last":0.434},{"interface":"front","average":{"1min":3.2040000000000002,"5min":4.4580000000000002,"15min":5.319},"min":{"1min":0.42799999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.554,"5min":82.233000000000004,"15min":82.233000000000004},"last":0.67400000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:05:18 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":4.2389999999999999,"15min":3.04},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.4249999999999998,"5min":112.883,"15min":138.035},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.802,"5min":3.8660000000000001,"15min":2.8889999999999998},"min":{"1min":0.47799999999999998,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":10.585000000000001,"5min":111.85599999999999,"15min":138.054},"last":0.70199999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019206,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":569396,"kb_used_data":296428,"kb_used_omap":0,"kb_used_meta":272960,"kb_avail":93175756,"statfs":{"total":95995035648,"available":95411974144,"internally_reserved":0,"allocated":303542272,"data_stored":302635134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":279511040},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:09 2023","interfaces":[{"interface":"back","average":{"1min":3.089,"5min":3.1459999999999999,"15min":3.0899999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.189},"max":{"1min":10.063000000000001,"5min":89.004999999999995,"15min":104.604},"last":4.8520000000000003},{"interface":"front","average":{"1min":2.8220000000000001,"5min":3.1659999999999999,"15min":3.028},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.23400000000000001},"max":{"1min":10.045999999999999,"5min":89.024000000000001,"15min":104.687},"last":4.8780000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:05:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2970000000000002,"5min":10.922000000000001,"15min":5.0380000000000003},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.216},"max":{"1min":8.2810000000000006,"5min":161.613,"15min":161.613},"last":2.8980000000000001},{"interface":"front","average":{"1min":2.3479999999999999,"5min":10.935,"15min":4.8810000000000002},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":8.4309999999999992,"5min":162.017,"15min":162.017},"last":2.335}]},{"osd":2,"last update":"Sat Sep 23 20:04:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9710000000000001,"5min":6.4290000000000003,"15min":5.1130000000000004},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":5.6180000000000003,"5min":116.56,"15min":144.20099999999999},"last":1.8799999999999999},{"interface":"front","average":{"1min":3.121,"5min":6.5039999999999996,"15min":4.9400000000000004},"min":{"1min":0.30099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":11.048,"5min":117.03400000000001,"15min":144.184},"last":12.241}]},{"osd":3,"last update":"Sat Sep 23 20:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":6.0439999999999996,"15min":5.3209999999999997},"min":{"1min":0.33500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":12.565,"5min":139.976,"15min":266.54700000000003},"last":2.597},{"interface":"front","average":{"1min":2.8260000000000001,"5min":6.0030000000000001,"15min":4.3470000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":13.637,"5min":139.911,"15min":139.911},"last":1.835}]},{"osd":5,"last update":"Sat Sep 23 20:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.53100000000000003,"15min":1.8600000000000001},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":1.569,"5min":4.423,"15min":266.59899999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.64400000000000002,"15min":1.9650000000000001},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.24399999999999999},"max":{"1min":3.004,"5min":4.8739999999999997,"15min":266.72899999999998},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.53000000000000003,"15min":0.51200000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.605,"5min":4.0410000000000004,"15min":4.0410000000000004},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.66800000000000004,"15min":0.54300000000000004},"min":{"1min":0.34499999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.615,"5min":4.2439999999999998,"15min":4.2439999999999998},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:04:49 2023","interfaces":[{"interface":"back","average":{"1min":4.7199999999999998,"5min":1.4239999999999999,"15min":1.2509999999999999},"min":{"1min":0.32600000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":65.522999999999996,"5min":65.522999999999996,"15min":65.522999999999996},"last":0.41499999999999998},{"interface":"front","average":{"1min":5.5970000000000004,"5min":1.595,"15min":1.3300000000000001},"min":{"1min":0.377,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":65.301000000000002,"5min":65.301000000000002,"15min":65.301000000000002},"last":0.43099999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757570,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":520812,"kb_used_data":238244,"kb_used_omap":0,"kb_used_meta":282560,"kb_avail":93224340,"statfs":{"total":95995035648,"available":95461724160,"internally_reserved":0,"allocated":243961856,"data_stored":243059159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":289341440},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:05:07 2023","interfaces":[{"interface":"back","average":{"1min":4.7629999999999999,"5min":1.597,"15min":1.548},"min":{"1min":0.22600000000000001,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":41.503,"5min":41.503,"15min":41.503},"last":0.53300000000000003},{"interface":"front","average":{"1min":4.6029999999999998,"5min":1.978,"15min":1.6220000000000001},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.216},"max":{"1min":41.465000000000003,"5min":41.465000000000003,"15min":41.465000000000003},"last":1.73}]},{"osd":1,"last update":"Sat Sep 23 20:05:19 2023","interfaces":[{"interface":"back","average":{"1min":5.6340000000000003,"5min":7.2839999999999998,"15min":5.258},"min":{"1min":0.35299999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":60.447000000000003,"5min":171.68299999999999,"15min":266.197},"last":3.2149999999999999},{"interface":"front","average":{"1min":6.3840000000000003,"5min":7.6299999999999999,"15min":5.3419999999999996},"min":{"1min":0.436,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":60.707999999999998,"5min":171.64099999999999,"15min":266.14999999999998},"last":2.7480000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:04:51 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":3.5609999999999999,"15min":5.6239999999999997},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":3.6680000000000001,"5min":177.816,"15min":238.96199999999999},"last":3.5379999999999998},{"interface":"front","average":{"1min":1.373,"5min":3.6299999999999999,"15min":5.6719999999999997},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":4.024,"5min":177.851,"15min":238.905},"last":3.5840000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:05:19 2023","interfaces":[{"interface":"back","average":{"1min":13.840999999999999,"5min":7.2690000000000001,"15min":4.4009999999999998},"min":{"1min":0.311,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":201.50899999999999,"5min":201.50899999999999,"15min":201.50899999999999},"last":6.2999999999999998},{"interface":"front","average":{"1min":13.923,"5min":7.1669999999999998,"15min":4.2560000000000002},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":201.58099999999999,"5min":201.58099999999999,"15min":201.58099999999999},"last":5.21}]},{"osd":4,"last update":"Sat Sep 23 20:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":2.4329999999999998,"15min":1.611},"min":{"1min":0.24099999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.786,"5min":191.78899999999999,"15min":191.78899999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.38300000000000001,"5min":2.444,"15min":1.587},"min":{"1min":0.308,"5min":0.255,"15min":0.255},"max":{"1min":0.54200000000000004,"5min":191.821,"15min":191.821},"last":3.2669999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:05:33 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.63200000000000001,"15min":0.60999999999999999},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.92000000000000004,"5min":4.0679999999999996,"15min":4.0679999999999996},"last":1.7949999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.48199999999999998,"15min":0.46800000000000003},"min":{"1min":0.32500000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.53100000000000003,"5min":3.3570000000000002,"15min":3.3570000000000002},"last":3.3010000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":2.4460000000000002,"15min":1.74},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":3.964,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.379},{"interface":"front","average":{"1min":0.754,"5min":2.5630000000000002,"15min":1.9379999999999999},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.976,"5min":191.69,"15min":191.69},"last":1.766}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":483921920,"data_stored":483917848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":293933056,"data_stored":293928984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":335740928,"data_stored":335740928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":447746048,"data_stored":447741976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":294322176,"data_stored":294322176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":229179392,"data_stored":229179392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":254345216,"data_stored":254345216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:05:55.167 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:05:55.167 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:05:55.168 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T20:05:55.168 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:05:55.168 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:05:55.168 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T20:05:55.223 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:55.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:55.332 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:55.362 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:55.420 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:55.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:55.531 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:55.594 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:05:55.616 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:55.904 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:55.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:56.019 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:05:56.116 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:56.403 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:05:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:56.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:05:56.510 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:05:56.540 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:05:56.624 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:56.751 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:05:56.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:56.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:05:56.784 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220400 5.b 5:d413819b:::benchmark_data_smithi135_284570_object104616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.640447+0000", 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.14019988500002, 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32328775900000001, 2023-09-23T20:05:56.785 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220400 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.786 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640447+0000", 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640447+0000", 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640446+0000", 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640451+0000", 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4940000000000003e-06 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640463+0000", 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1299e-05 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640471+0000", 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8220000000000007e-06 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640519+0000", 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8375999999999999e-05 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640629+0000", 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011054 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640741+0000", 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111857 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650179+0000", 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094374469999999998 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963581+0000", 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31340195599999998 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963649+0000", 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8036999999999999e-05 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963693+0000", 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4162000000000002e-05 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963735+0000", 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2098999999999998e-05 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.793 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220403 5.b 5:d5744800:::benchmark_data_smithi135_284570_object104619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.640988+0000", 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.13965871100004, 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32284452200000002, 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.794 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220403 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640988+0000", 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:56.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.640993+0000", 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.604e-06 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.641005+0000", 2023-09-23T20:05:56.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1365e-05 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.641012+0000", 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1470000000000004e-06 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645957+0000", 2023-09-23T20:05:56.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049455829999999999 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.646045+0000", 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7979999999999995e-05 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.646112+0000", 2023-09-23T20:05:56.799 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6251e-05 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650257+0000", 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041449950000000003 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963620+0000", 2023-09-23T20:05:56.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31336345500000001 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963767+0000", 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000147169 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963799+0000", 2023-09-23T20:05:56.801 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2060000000000001e-05 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963833+0000", 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3733999999999999e-05 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220406 5.d 5:b474da8d:::benchmark_data_smithi135_284570_object104622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.645154+0000", 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.13549345599995, 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32365115, 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:05:56.803 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220406 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645154+0000", 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645154+0000", 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645152+0000", 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645158+0000", 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9819999999999996e-06 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645168+0000", 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7580000000000004e-06 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.806 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645175+0000", 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9009999999999997e-06 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645213+0000", 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8535999999999998e-05 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645311+0000", 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7647999999999997e-05 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.645397+0000", 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5791999999999994e-05 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650541+0000", 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051441569999999999 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968678+0000", 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.318137369 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968716+0000", 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7765999999999997e-05 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968765+0000", 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8520000000000003e-05 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.810 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968805+0000", 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0170000000000003e-05 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220411 5.6 5:65202dbd:::benchmark_data_smithi135_284570_object104627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.650884+0000", 2023-09-23T20:05:56.811 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.12976342000002, 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31285145399999997, 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220411 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.812 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650884+0000", 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650884+0000", 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650883+0000", 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650889+0000", 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7460000000000003e-06 2023-09-23T20:05:56.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650902+0000", 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2266000000000001e-05 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650936+0000", 2023-09-23T20:05:56.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4878e-05 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650982+0000", 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5259000000000002e-05 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651089+0000", 2023-09-23T20:05:56.816 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000107784 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651201+0000", 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111297 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963506+0000", 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.312305098 2023-09-23T20:05:56.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963534+0000", 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8495000000000001e-05 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963649+0000", 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011424 2023-09-23T20:05:56.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963692+0000", 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2907000000000001e-05 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963735+0000", 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.367e-05 2023-09-23T20:05:56.819 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220414 5.d 5:b1d7f748:::benchmark_data_smithi135_284570_object104630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.650962+0000", 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.12968556500005, 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.317977958, 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.820 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220414 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650962+0000", 2023-09-23T20:05:56.821 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650962+0000", 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650961+0000", 2023-09-23T20:05:56.822 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650965+0000", 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.997e-06 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650972+0000", 2023-09-23T20:05:56.823 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0190000000000001e-06 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.650977+0000", 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1340000000000003e-06 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651027+0000", 2023-09-23T20:05:56.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9509000000000003e-05 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651141+0000", 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000114848 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.651250+0000", 2023-09-23T20:05:56.825 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000108455 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968686+0000", 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31743582999999997 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968839+0000", 2023-09-23T20:05:56.826 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015357700000000001 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968869+0000", 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9839e-05 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968902+0000", 2023-09-23T20:05:56.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2449000000000001e-05 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968940+0000", 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7911999999999999e-05 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.828 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44591.0:220415 5.d 5:bcbf5941:::benchmark_data_smithi135_284570_object104631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.654307+0000", 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.12634026199999, 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33038375800000003, 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44591", 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2505707969", 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 220415 2023-09-23T20:05:56.829 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654307+0000", 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654307+0000", 2023-09-23T20:05:56.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654305+0000", 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654313+0000", 2023-09-23T20:05:56.831 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0530000000000007e-06 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654330+0000", 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6290000000000002e-05 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654339+0000", 2023-09-23T20:05:56.832 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.499e-06 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655392+0000", 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010529370000000001 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655521+0000", 2023-09-23T20:05:56.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012935900000000001 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.655599+0000", 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7442999999999997e-05 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.834 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.979556+0000", 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32395747899999999 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.979597+0000", 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1085999999999999e-05 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984634+0000", 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005036769 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984665+0000", 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0960000000000002e-05 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.836 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.984691+0000", 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5607999999999999e-05 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.44591.0:220413 5.4 e154/112 5:23b6015e:::benchmark_data_smithi135_284570_object104629:head v 154'13837, mlcod=154'13835)", 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.654925+0000", 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.12572256199996, 2023-09-23T20:05:56.837 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.308583475, 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654925+0000", 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.838 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654925+0000", 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654924+0000", 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.839 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654932+0000", 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9510000000000006e-06 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654942+0000", 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0047999999999999e-05 2023-09-23T20:05:56.840 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654947+0000", 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2089999999999996e-06 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654985+0000", 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8016999999999999e-05 2023-09-23T20:05:56.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.654999+0000", 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.454e-05 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963414+0000", 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30841476099999998 2023-09-23T20:05:56.842 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963508+0000", 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3911000000000003e-05 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.843 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:220406 5.d e154/112 ondisk, result = 0)", 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.657056+0000", 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.123591347, 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31163112799999998, 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.844 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657056+0000", 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657056+0000", 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.845 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657055+0000", 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.657061+0000", 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4999999999999996e-06 2023-09-23T20:05:56.846 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963465+0000", 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30640367400000001 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963476+0000", 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0861e-05 2023-09-23T20:05:56.847 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968664+0000", 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051881879999999998 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968672+0000", 2023-09-23T20:05:56.848 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0690000000000002e-06 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.968687+0000", 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4715000000000001e-05 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.849 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.44591.0:220411 5.6 e154/112 ondisk, result = 0)", 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:56:21.658010+0000", 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 575.12263665199998, 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30549902800000001, 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.850 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658010+0000", 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.658015+0000", 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6190000000000002e-06 2023-09-23T20:05:56.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963397+0000", 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30538196699999998 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963412+0000", 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4925e-05 2023-09-23T20:05:56.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963488+0000", 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6477000000000007e-05 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963498+0000", 2023-09-23T20:05:56.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0478e-05 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:56:21.963510+0000", 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1058e-05 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.855 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:128729 7.8 7:12241d48:::benchmark_data_smithi135_404353_object31667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:04:11.152315+0000", 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.628332172, 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30625847099999998, 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:05:56.856 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 128729 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152315+0000", 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.857 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152315+0000", 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152314+0000", 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152319+0000", 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.417e-06 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152328+0000", 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7420000000000003e-06 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.859 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152334+0000", 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9410000000000004e-06 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152366+0000", 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2159000000000002e-05 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152441+0000", 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4626e-05 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152502+0000", 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1056999999999995e-05 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458447+0000", 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30594527599999999 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458483+0000", 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6341000000000003e-05 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458512+0000", 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8365999999999999e-05 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458545+0000", 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3136999999999998e-05 2023-09-23T20:05:56.863 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458573+0000", 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8442e-05 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.864 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:128728 7.b e278/276 7:debbd576:::benchmark_data_smithi135_404353_object31666:head v 278'7955, mlcod=278'7953)", 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:04:11.152571+0000", 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.62807616000001, 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30834860400000003, 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152571+0000", 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152571+0000", 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152570+0000", 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152576+0000", 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9970000000000004e-06 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152587+0000", 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0665e-05 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152591+0000", 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8480000000000003e-06 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152629+0000", 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7957000000000003e-05 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.152642+0000", 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3492999999999999e-05 2023-09-23T20:05:56.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.460881+0000", 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30823841800000001 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.460920+0000", 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8847999999999998e-05 2023-09-23T20:05:56.870 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.35358.0:128729 7.8 e278/240 ondisk, result = 0)", 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:04:11.156018+0000", 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 105.624629348, 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30247004599999999, 2023-09-23T20:05:56.871 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.156018+0000", 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.872 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.156018+0000", 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.156017+0000", 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.156023+0000", 2023-09-23T20:05:56.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8449999999999999e-06 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458376+0000", 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30235335200000002 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458388+0000", 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2170999999999999e-05 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458474+0000", 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5298999999999998e-05 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458480+0000", 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8270000000000003e-06 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:04:11.458488+0000", 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2509999999999994e-06 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.876 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:184182 7.e 7:7e6af0dc:::benchmark_data_smithi135_404353_object87120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:48.864058+0000", 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.916589257, 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.15914485, 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.877 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 184182 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864058+0000", 2023-09-23T20:05:56.878 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864058+0000", 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864056+0000", 2023-09-23T20:05:56.879 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864066+0000", 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0591000000000001e-05 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864086+0000", 2023-09-23T20:05:56.880 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9313999999999998e-05 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864098+0000", 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2797e-05 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.870450+0000", 2023-09-23T20:05:56.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0063515289999999999 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.870561+0000", 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110759 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992958+0000", 2023-09-23T20:05:56.882 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.122396877 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002336+0000", 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0093783470000000004 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002343+0000", 2023-09-23T20:05:56.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0199999999999997e-06 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.023114+0000", 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.020770730000000001 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.023163+0000", 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9271999999999999e-05 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.023203+0000", 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9848999999999997e-05 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.885 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:184187 7.8 7:1f3b72b9:::benchmark_data_smithi135_404353_object87125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:48.864117+0000", 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.9165300810000003, 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.12885538399999999, 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.886 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 184187 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864117+0000", 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864117+0000", 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864117+0000", 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.888 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864120+0000", 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.05e-06 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864125+0000", 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6840000000000001e-06 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.889 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864130+0000", 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7759999999999997e-06 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864179+0000", 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8906000000000002e-05 2023-09-23T20:05:56.890 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864328+0000", 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000149524 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.864464+0000", 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013525200000000001 2023-09-23T20:05:56.891 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.870388+0000", 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0059246960000000001 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.870515+0000", 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012675100000000001 2023-09-23T20:05:56.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992872+0000", 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.12235726800000001 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992926+0000", 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3304999999999999e-05 2023-09-23T20:05:56.893 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992972+0000", 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6700999999999999e-05 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.894 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:184180 7.2 e295/240 7:45346858:::benchmark_data_smithi135_404353_object87118:head v 295'11580, mlcod=295'11578)", 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:48.868264+0000", 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.9123832009999999, 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13401471200000001, 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.895 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868264+0000", 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868264+0000", 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.896 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868263+0000", 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868272+0000", 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3090000000000001e-06 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.897 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868289+0000", 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7108e-05 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868299+0000", 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3929999999999997e-06 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.898 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992823+0000", 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.124524546 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992850+0000", 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6531999999999999e-05 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.899 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002259+0000", 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094086870000000006 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002279+0000", 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0066999999999999e-05 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.900 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:184188 7.a e295/240 7:57b15e6c:::benchmark_data_smithi135_404353_object87126:head v 295'11490, mlcod=295'11487)", 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:48.868320+0000", 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.9123271539999998, 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.133935425, 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.901 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868320+0000", 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868320+0000", 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868320+0000", 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868324+0000", 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1280000000000001e-06 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868331+0000", 2023-09-23T20:05:56.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7419999999999997e-06 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868337+0000", 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5169999999999999e-06 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992846+0000", 2023-09-23T20:05:56.904 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.12450938 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992868+0000", 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1149e-05 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002225+0000", 2023-09-23T20:05:56.905 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0093570049999999998 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002255+0000", 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0892999999999997e-05 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.906 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:184189 7.e 7:78b52193:::benchmark_data_smithi135_404353_object87127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:48.868351+0000", 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.9122966120000005, 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.15495473400000001, 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:05:56.907 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 184189 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868351+0000", 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.908 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868351+0000", 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868348+0000", 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.909 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868353+0000", 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9849999999999997e-06 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868364+0000", 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0329999999999999e-05 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.910 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868370+0000", 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5429999999999999e-06 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.997478+0000", 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.12910785799999999 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.911 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.997595+0000", 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117174 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.997707+0000", 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111644 2023-09-23T20:05:56.912 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.013135+0000", 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015427939999999999 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.018023+0000", 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048876960000000004 2023-09-23T20:05:56.913 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.023241+0000", 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052186180000000004 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.023274+0000", 2023-09-23T20:05:56.914 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2384e-05 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.023305+0000", 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1624000000000001e-05 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.915 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:184192 7.e 7:78ce9401:::benchmark_data_smithi135_404353_object87130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:48.868376+0000", 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.91227079, 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.15973638900000001, 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:05:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 184192 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868376+0000", 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.917 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.918 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868376+0000", 2023-09-23T20:05:56.918 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.918 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.918 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.918 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.918 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868376+0000", 2023-09-23T20:05:56.918 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:56.918 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868378+0000", 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5019999999999999e-06 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868385+0000", 2023-09-23T20:05:56.919 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4690000000000002e-06 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868390+0000", 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4070000000000001e-06 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002366+0000", 2023-09-23T20:05:56.920 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13397579500000001 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002492+0000", 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012632799999999999 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002607+0000", 2023-09-23T20:05:56.921 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000114774 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.018073+0000", 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015465882 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:05:56.922 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.023070+0000", 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049966790000000004 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.028028+0000", 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049586839999999997 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.923 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.028075+0000", 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6558999999999999e-05 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.028113+0000", 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7712000000000001e-05 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.924 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:184190 7.c e295/282 7:37fba771:::benchmark_data_smithi135_404353_object87128:head v 295'11420, mlcod=295'11418)", 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:48.868562+0000", 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.9120853059999998, 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.124297054, 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.925 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868562+0000", 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868562+0000", 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.926 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868561+0000", 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868568+0000", 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8179999999999998e-06 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.927 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868580+0000", 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1683e-05 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868586+0000", 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5289999999999997e-06 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.928 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868647+0000", 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1099999999999994e-05 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.868668+0000", 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0324000000000001e-05 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.929 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992820+0000", 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.124152051 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992859+0000", 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9249000000000003e-05 2023-09-23T20:05:56.930 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:184191 7.b e295/276 7:d6269254:::benchmark_data_smithi135_404353_object87129:head v 295'11443, mlcod=295'11441)", 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:48.871869+0000", 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 7.9087781179999999, 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13054616699999999, 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:05:56.931 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.871869+0000", 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:05:56.932 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.871869+0000", 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.871868+0000", 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.933 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.871877+0000", 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4430000000000003e-06 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992787+0000", 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.12091072999999999 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.934 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992840+0000", 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2893000000000003e-05 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992897+0000", 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6833000000000003e-05 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.935 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:48.992926+0000", 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8827e-05 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002379+0000", 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094533240000000008 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:05:56.936 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:49.002415+0000", 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6041999999999998e-05 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:05:56.937 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:05:56.938 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:56.970 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:57.023 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:57.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:57.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:57.038 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:56 smithi135 ceph-mon[213871]: pgmap v818: 99 pgs: 99 active+clean; 392 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T20:05:57.042 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:56 smithi135 ceph-mon[221113]: pgmap v818: 99 pgs: 99 active+clean; 392 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T20:05:57.126 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:57.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:56 smithi144 ceph-mon[194833]: pgmap v818: 99 pgs: 99 active+clean; 392 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T20:05:57.157 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:57.209 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:57.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:57.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:57.313 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:57.341 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:57.395 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:57.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:57.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:57.499 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:05:57.527 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:57.583 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:05:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:57.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:05:57.687 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:05:57.713 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:57.761 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:05:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:57.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:05:57.864 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:05:57.870 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:05:57.894 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:57.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2563088064' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:05:57.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2563088064' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:05:58.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2563088064' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:05:58.932 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:05:58.932 INFO:teuthology.orchestra.run.smithi144.stdout: "success": "" 2023-09-23T20:05:58.932 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:05:58.957 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:58 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:58.958 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:58 smithi144 ceph-mon[194833]: failed to encode map e296 with expected crc 2023-09-23T20:05:58.958 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2563088064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:05:58.958 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:58 smithi144 ceph-mon[194833]: osdmap e296: 8 total, 8 up, 7 in 2023-09-23T20:05:58.958 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:58 smithi144 ceph-mon[194833]: failed to encode map e296 with expected crc 2023-09-23T20:05:58.958 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:58 smithi144 ceph-mon[194833]: failed to encode map e296 with expected crc 2023-09-23T20:05:58.958 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:58 smithi144 ceph-mon[194833]: failed to encode map e296 with expected crc 2023-09-23T20:05:58.958 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:05:58 smithi144 ceph-mon[194833]: pgmap v820: 99 pgs: 99 active+clean; 303 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:05:59.056 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:05:59.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:59.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:05:59.162 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:59.193 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[213871]: failed to encode map e296 with expected crc 2023-09-23T20:05:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2563088064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:05:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[213871]: osdmap e296: 8 total, 8 up, 7 in 2023-09-23T20:05:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[213871]: failed to encode map e296 with expected crc 2023-09-23T20:05:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[213871]: failed to encode map e296 with expected crc 2023-09-23T20:05:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[213871]: failed to encode map e296 with expected crc 2023-09-23T20:05:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[213871]: pgmap v820: 99 pgs: 99 active+clean; 303 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:05:59.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:05:59.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[221113]: failed to encode map e296 with expected crc 2023-09-23T20:05:59.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2563088064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:05:59.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[221113]: osdmap e296: 8 total, 8 up, 7 in 2023-09-23T20:05:59.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[221113]: failed to encode map e296 with expected crc 2023-09-23T20:05:59.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[221113]: failed to encode map e296 with expected crc 2023-09-23T20:05:59.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[221113]: failed to encode map e296 with expected crc 2023-09-23T20:05:59.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:05:58 smithi135 ceph-mon[221113]: pgmap v820: 99 pgs: 99 active+clean; 303 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:05:59.248 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:59.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:59.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:59.352 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:59.378 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:59.430 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:59.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:59.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:59.535 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:05:59.563 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:05:59.755 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:05:59.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:05:59.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:05:59.859 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:05:59.942 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:00.030 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:06:00.160 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:00.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:00.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:00.280 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:00.391 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:00.505 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:00.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:00.574 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:06:00.605 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:06:00.610 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:00.659 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:06:00.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:00 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1368610601' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:00.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:00 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1368610601' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:00.718 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:00.718 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:06:00.762 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:06:00.823 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:06:00.828 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:00.835 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:00.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:00.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:00.846 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:06:00.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:00 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1368610601' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:00.893 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:06:00.897 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:06:00.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:06:00.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:06:00.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:06:00.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:06:00.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:06:00.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:06:00.898 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:06:00.937 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:00.965 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:01.012 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:01.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:01.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:01.118 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:01.145 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:01.199 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:01.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:01.307 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:01.337 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:01.384 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:01.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:01.489 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:01.585 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:01.620 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:01 smithi144 ceph-mon[194833]: pgmap v821: 99 pgs: 99 active+clean; 193 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T20:06:01.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:01 smithi135 ceph-mon[221113]: pgmap v821: 99 pgs: 99 active+clean; 193 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T20:06:01.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:01 smithi135 ceph-mon[213871]: pgmap v821: 99 pgs: 99 active+clean; 193 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T20:06:02.065 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:06:02.542 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:02.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:02.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:02.656 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:02.685 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:02.748 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:02.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:02.853 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:02.953 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:03.052 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:03.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:03.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:03.161 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:03.188 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:03.230 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:03.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:03.334 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:03.407 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:03.569 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:03 smithi144 ceph-mon[194833]: pgmap v822: 99 pgs: 99 active+clean; 193 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T20:06:03.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:03 smithi135 ceph-mon[213871]: pgmap v822: 99 pgs: 99 active+clean; 193 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T20:06:03.688 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:03 smithi135 ceph-mon[221113]: pgmap v822: 99 pgs: 99 active+clean; 193 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 605 op/s 2023-09-23T20:06:03.690 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:06:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:06:03] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-09-23T20:06:03.830 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:03.838 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:03.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:03.946 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:03.974 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:04.036 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:04.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:04.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:04.141 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:04.237 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:04.245 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Removed 97059 objects 2023-09-23T20:06:04.245 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Clean up completed and total clean up time :166.315 2023-09-23T20:06:04.255 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T20:06:04.256 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T20:06:04.442 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:04.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:04.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:04.545 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:04.564 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:06:04.573 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:04.824 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:06:04.824 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:06:04.824 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T20:06:04.824 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:06:04.824 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:06:04.825 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T20:06:04.988 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:06:04.988 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:06:05.130 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:04 smithi135 ceph-mon[221113]: pgmap v823: 99 pgs: 99 active+clean; 104 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T20:06:05.130 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:04 smithi135 ceph-mon[213871]: pgmap v823: 99 pgs: 99 active+clean; 104 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T20:06:05.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:04 smithi144 ceph-mon[194833]: pgmap v823: 99 pgs: 99 active+clean; 104 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 606 op/s 2023-09-23T20:06:05.165 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:05.610 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:05.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:05.713 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:05.743 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:05.837 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:05.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:05.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:05.940 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:05.971 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:06.271 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:06:06.273 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:06.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:06.285 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:06.377 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:06.406 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:06.452 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:06.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:06.555 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:06.587 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:06.730 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:06.737 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:06.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:06.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:06.842 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:06.869 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:07.018 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:06 smithi144 ceph-mon[194833]: pgmap v824: 99 pgs: 99 active+clean; 20 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 595 op/s 2023-09-23T20:06:07.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:06 smithi135 ceph-mon[221113]: pgmap v824: 99 pgs: 99 active+clean; 20 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 595 op/s 2023-09-23T20:06:07.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:06 smithi135 ceph-mon[213871]: pgmap v824: 99 pgs: 99 active+clean; 20 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 595 op/s 2023-09-23T20:06:07.244 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:06:07.244 INFO:teuthology.orchestra.run.smithi144.stdout: "success": "" 2023-09-23T20:06:07.244 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:06:07.767 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:07.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:07.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:07.893 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:06:07.910 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:07.931 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:07.963 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:07.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:07 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2107683481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:07.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:07.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:07 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2107683481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:07.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:08.008 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:08.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:08.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:08.115 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:07 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2107683481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:08.141 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:08.182 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:08.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:08.289 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:08.315 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:08.356 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:08.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:08.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:08.461 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:08.491 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:08.687 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:08.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:08.792 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:08.819 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:08.863 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:08.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:08.968 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:08.993 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:09.035 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:09.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:09.087 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:06:09.087 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:06:09.087 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[213871]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[213871]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2107683481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[213871]: osdmap e297: 8 total, 8 up, 7 in 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[213871]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[213871]: pgmap v826: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 476 op/s 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[221113]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.088 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[221113]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.089 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2107683481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:09.089 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[221113]: osdmap e297: 8 total, 8 up, 7 in 2023-09-23T20:06:09.089 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[221113]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.089 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:08 smithi135 ceph-mon[221113]: pgmap v826: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 476 op/s 2023-09-23T20:06:09.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:08 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:06:09.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:08 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:06:09.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:08 smithi144 ceph-mon[194833]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:08 smithi144 ceph-mon[194833]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:08 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2107683481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:09.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:08 smithi144 ceph-mon[194833]: osdmap e297: 8 total, 8 up, 7 in 2023-09-23T20:06:09.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:08 smithi144 ceph-mon[194833]: failed to encode map e297 with expected crc 2023-09-23T20:06:09.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:08 smithi144 ceph-mon[194833]: pgmap v826: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 476 op/s 2023-09-23T20:06:09.140 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:09.174 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:09.219 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:09.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:09.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:09.323 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:09.350 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:09.744 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:09.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:09.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:09.847 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:09.879 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:09.927 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_2' removed 2023-09-23T20:06:09.937 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:09.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:10.042 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:10.066 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:09 smithi135 ceph-mon[221113]: failed to encode map e297 with expected crc 2023-09-23T20:06:10.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/443535986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:06:10.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:09 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:06:10.067 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:09 smithi135 ceph-mon[213871]: failed to encode map e297 with expected crc 2023-09-23T20:06:10.067 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/443535986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:06:10.067 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:09 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:06:10.069 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:10.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:09 smithi144 ceph-mon[194833]: failed to encode map e297 with expected crc 2023-09-23T20:06:10.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/443535986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:06:10.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:09 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:06:10.826 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:10.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:10.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:10.877 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:06:10.897 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:06:10.916 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:06:10.930 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:10.935 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:06:10.956 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:06:10.968 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:10.975 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:06:10.996 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:06:11.018 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:06:11.020 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:06:11.020 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:06:11.020 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:06:11.020 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:06:11.020 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:06:11.020 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:06:11.020 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:06:11.020 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:06:11.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[221113]: failed to encode map e298 with expected crc 2023-09-23T20:06:11.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[221113]: failed to encode map e298 with expected crc 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[221113]: osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[221113]: pgmap v828: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3743344504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[213871]: failed to encode map e298 with expected crc 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[213871]: failed to encode map e298 with expected crc 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[213871]: osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:06:11.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[213871]: pgmap v828: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3743344504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:10 smithi144 ceph-mon[194833]: failed to encode map e298 with expected crc 2023-09-23T20:06:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:10 smithi144 ceph-mon[194833]: failed to encode map e298 with expected crc 2023-09-23T20:06:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:06:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:10 smithi144 ceph-mon[194833]: osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:06:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:10 smithi144 ceph-mon[194833]: pgmap v828: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3743344504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:11.556 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:11.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:11.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:11.660 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:11.692 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:11.741 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:11.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:11.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:11.847 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:11.875 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:12.278 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:06:12.279 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:06:12.279 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:06:12.279 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:06:12.279 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:06:12.357 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:12.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:12.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:12.409 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:06:12.410 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:06:12.410 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T20:06:12.411 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T20:06:12.462 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:12.489 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:12.720 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:06:12.967 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:06:12.967 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:06:12.967 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 4, 5, 7, 3] 2023-09-23T20:06:12.967 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd out 1 2023-09-23T20:06:13.095 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:13.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:13.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:13.199 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:13.205 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:06:13.232 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:13.290 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:13.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:13.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:13.395 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:13.421 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:13.437 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:13.470 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:13.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:13.565 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:13.576 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:13.608 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:13.658 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:13.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:13.680 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:13 smithi135 ceph-mon[221113]: pgmap v829: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:13.681 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:13 smithi135 ceph-mon[213871]: pgmap v829: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:13.763 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:13.791 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:13.833 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:13.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:13 smithi144 ceph-mon[194833]: pgmap v829: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:13.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:13.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:13.939 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:13.940 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:06:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:06:13] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-09-23T20:06:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:14.010 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:14.010 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:14.276 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:06:14.767 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:14.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:14.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:14.871 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:14.902 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:14.952 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:14.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:14.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:15.059 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:15.086 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:15.130 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:14 smithi144 ceph-mon[194833]: pgmap v830: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:15.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:15.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:15.205 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:14 smithi135 ceph-mon[213871]: pgmap v830: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:15.205 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:14 smithi135 ceph-mon[221113]: pgmap v830: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:15.238 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:15.267 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:16.469 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:16.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:16.573 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:16.602 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:17.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:16 smithi144 ceph-mon[194833]: pgmap v831: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:17.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:16 smithi135 ceph-mon[221113]: pgmap v831: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:17.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:16 smithi135 ceph-mon[213871]: pgmap v831: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:17.442 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:17.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:17.545 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:17.574 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:17.779 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:17.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:17.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:17.886 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:17.917 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:17.927 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:17.966 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:17.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:18.072 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:18.105 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:18.310 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:18.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:18.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:18.415 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:18.447 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:18.495 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:18.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:18.559 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:06:18.560 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_3' created 2023-09-23T20:06:18.600 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:18.627 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:18.669 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:18.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:18.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2926713063' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:06:18.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:06:18.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2387873483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:06:18.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:06:18.703 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2926713063' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:06:18.703 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:06:18.703 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2387873483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:06:18.703 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:06:18.775 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:18.798 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2926713063' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:06:18.798 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:06:18.798 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2387873483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:06:18.799 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:06:18.800 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:18.842 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:18.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:18.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:18.950 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:18.976 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:19.017 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:19.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:19.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:19.123 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:19.152 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:19.358 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 conmon[213845]: 2023-09-23T20:06:19.061+0000 7f9708921700 -1 mon.a@0(leader).osd e299 definitely_dead 0 2023-09-23T20:06:19.572 INFO:teuthology.orchestra.run.smithi135.stderr:marked out osd.1. 2023-09-23T20:06:19.700 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:06:19.700 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:06:19.700 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:06:19.700 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:06:19.700 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148788 6.f 6:f326eaaa:::benchmark_data_smithi135_344736_object46132:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 396.40322239800003, 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5928274390000001, 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148788 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.701 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.702 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928731+0000", 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928740+0000", 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4009999999999997e-06 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.703 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928755+0000", 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5481999999999998e-05 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928765+0000", 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0239e-05 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928822+0000", 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6518000000000003e-05 2023-09-23T20:06:19.704 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928829+0000", 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3819999999999998e-06 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944149+0000", 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015319442000000001 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.705 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944158+0000", 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3000000000000007e-06 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049126+0000", 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.104968422 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049138+0000", 2023-09-23T20:06:19.706 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1644999999999999e-05 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.265679+0000", 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2165408280000001 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.265834+0000", 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015490799999999999 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.521437+0000", 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25560343699999999 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.521561+0000", 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123342 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.708 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148791 6.f 6:fce301f6:::benchmark_data_smithi135_344736_object46135:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 396.40316453299999, 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5984101370000001, 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.709 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148791 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928790+0000", 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928794+0000", 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8800000000000001e-06 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928802+0000", 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0499999999999992e-06 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928809+0000", 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1800000000000001e-06 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928882+0000", 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3524999999999998e-05 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928887+0000", 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0010000000000001e-06 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944176+0000", 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015289166 2023-09-23T20:06:19.713 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944182+0000", 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.105e-06 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049162+0000", 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1049793050000001 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.714 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049170+0000", 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3869999999999995e-06 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268117+0000", 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2189466419999999 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:06:19.715 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268204+0000", 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7319999999999997e-05 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527103+0000", 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25889852899999999 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527201+0000", 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8562999999999998e-05 2023-09-23T20:06:19.716 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148804 6.b 6:d3899bd3:::benchmark_data_smithi135_344736_object46148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 396.39633869800002, 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1034934710000002, 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.717 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148804 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.718 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935615+0000", 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935620+0000", 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.775e-06 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935630+0000", 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9310000000000003e-06 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935637+0000", 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3440000000000003e-06 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.936071+0000", 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043330099999999998 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.936077+0000", 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6900000000000003e-06 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.941346+0000", 2023-09-23T20:06:19.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052686859999999999 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.941360+0000", 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3464e-05 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050328+0000", 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.108968468 2023-09-23T20:06:19.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050338+0000", 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0088999999999999e-05 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.003742+0000", 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.95340369300000005 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.003918+0000", 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017656799999999999 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.004058+0000", 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000139682 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.007338+0000", 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003280303 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.021476+0000", 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014137442 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.038947+0000", 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017471477999999999 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.039000+0000", 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3022000000000002e-05 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.039110+0000", 2023-09-23T20:06:19.726 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000109966 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148813 6.f 6:f77e0ce5:::benchmark_data_smithi135_344736_object46157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 396.37101256599999, 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.545090514, 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.727 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148813 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.728 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960941+0000", 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960950+0000", 2023-09-23T20:06:19.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6949999999999995e-06 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960969+0000", 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8785999999999999e-05 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960980+0000", 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0703000000000001e-05 2023-09-23T20:06:19.730 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.961043+0000", 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2600000000000004e-05 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.961057+0000", 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4535999999999999e-05 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.731 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049192+0000", 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0881350000000001 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049200+0000", 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9049999999999997e-06 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.732 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268242+0000", 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.219042129 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268348+0000", 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010592 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268839+0000", 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00049129899999999995 2023-09-23T20:06:19.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279896+0000", 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011056725 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279913+0000", 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6673000000000001e-05 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.502819+0000", 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22290622900000001 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.502884+0000", 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4616000000000006e-05 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506033+0000", 2023-09-23T20:06:19.735 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031498979999999999 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148815 6.f 6:f7552838:::benchmark_data_smithi135_344736_object46159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 395.20896145400002, 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.385714227, 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.736 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148815 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.737 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122993+0000", 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123000+0000", 2023-09-23T20:06:19.738 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9290000000000001e-06 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123009+0000", 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5689999999999994e-06 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123015+0000", 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8189999999999997e-06 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.739 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.152030+0000", 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029014671999999998 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.152042+0000", 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1756000000000001e-05 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276241+0000", 2023-09-23T20:06:19.740 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.124199189 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276452+0000", 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000210934 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276679+0000", 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00022696700000000001 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279921+0000", 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032419129999999999 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279990+0000", 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9807000000000004e-05 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506095+0000", 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22610454399999999 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506139+0000", 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3659000000000003e-05 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.508708+0000", 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002569661 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.743 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148816 6.b 6:dde61f8d:::benchmark_data_smithi135_344736_object46160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 395.20876172800001, 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.001808705, 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.744 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148816 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123194+0000", 2023-09-23T20:06:19.745 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123193+0000", 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123196+0000", 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9679999999999998e-06 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.746 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123202+0000", 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9349999999999999e-06 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123206+0000", 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9960000000000004e-06 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.747 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123247+0000", 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1238000000000001e-05 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123256+0000", 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1999999999999994e-06 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018207+0000", 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.89495181199999996 2023-09-23T20:06:19.748 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018364+0000", 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000156269 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.018501+0000", 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013728199999999999 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.749 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.033263+0000", 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014761793 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.045761+0000", 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012498489 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.124820+0000", 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.079059044999999994 2023-09-23T20:06:19.750 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.124879+0000", 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9138999999999997e-05 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.125003+0000", 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012307900000000001 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.751 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148820 6.f 6:f8320d95:::benchmark_data_smithi135_344736_object46164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 394.28037791600002, 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.478686642, 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.752 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148820 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.753 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051578+0000", 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051576+0000", 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051584+0000", 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6049999999999998e-06 2023-09-23T20:06:19.754 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051600+0000", 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5387999999999999e-05 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.051616+0000", 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6228000000000001e-05 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.755 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.100356+0000", 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.048739598000000002 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.100371+0000", 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5068999999999999e-05 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280046+0000", 2023-09-23T20:06:19.756 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17967550800000001 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280245+0000", 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019827899999999999 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.280450+0000", 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000205689 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.757 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.287010+0000", 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065595749999999998 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.287104+0000", 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4639e-05 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527260+0000", 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24015514399999999 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527323+0000", 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3256000000000006e-05 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.530264+0000", 2023-09-23T20:06:19.759 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029414630000000001 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182372 6.b 6:d373db44:::benchmark_data_smithi135_344736_object79716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 345.88216846099999, 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1349242150000001, 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:06:19.760 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182372 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.761 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449786+0000", 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449790+0000", 2023-09-23T20:06:19.762 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4469999999999999e-06 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449800+0000", 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6909999999999992e-06 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449809+0000", 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3009999999999995e-06 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.763 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449948+0000", 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013898400000000001 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.450078+0000", 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012999600000000001 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:06:19.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.450185+0000", 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010655299999999999 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.457033+0000", 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0068485000000000004 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.457047+0000", 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.376e-05 2023-09-23T20:06:19.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.567645+0000", 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1105975640000001 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.567815+0000", 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000170104 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584606+0000", 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016791006000000001 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584711+0000", 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000105554 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182400 6.b 6:ded8ad1e:::benchmark_data_smithi135_344736_object79744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 345.82806627799999, 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0838019940000001, 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182400 2023-09-23T20:06:19.768 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503889+0000", 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503888+0000", 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503895+0000", 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3239999999999997e-06 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503912+0000", 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7125999999999998e-05 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503921+0000", 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8990000000000007e-06 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503982+0000", 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0566999999999998e-05 2023-09-23T20:06:19.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.503995+0000", 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3181e-05 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568139+0000", 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0641444200000001 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568266+0000", 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012720799999999999 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.568776+0000", 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050917599999999996 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.773 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580439+0000", 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011663902 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580591+0000", 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000151371 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584754+0000", 2023-09-23T20:06:19.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041630570000000004 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587511+0000", 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027568660000000002 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587567+0000", 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6041000000000001e-05 2023-09-23T20:06:19.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587691+0000", 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012448400000000001 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182410 6.b 6:d95be848:::benchmark_data_smithi135_344736_object79754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:06:19.776 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 345.810251897, 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0661552510000001, 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182410 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521704+0000", 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521702+0000", 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:19.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521709+0000", 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.64e-06 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521725+0000", 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6078000000000001e-05 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521733+0000", 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6809999999999998e-06 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521799+0000", 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5988999999999994e-05 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.521812+0000", 2023-09-23T20:06:19.780 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3307e-05 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573040+0000", 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.051228013 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573162+0000", 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012158 2023-09-23T20:06:19.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.573254+0000", 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2051000000000003e-05 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580447+0000", 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0071931850000000004 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.782 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580639+0000", 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000191631 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584823+0000", 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041845129999999999 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587735+0000", 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029121139999999999 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587767+0000", 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1823999999999999e-05 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587859+0000", 2023-09-23T20:06:19.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1924999999999998e-05 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182448 6.b 6:d9f3c59c:::benchmark_data_smithi135_344736_object79792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 345.72491146200002, 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98102556500000004, 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.785 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182448 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607044+0000", 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607042+0000", 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607050+0000", 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6240000000000002e-06 2023-09-23T20:06:19.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607066+0000", 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5667000000000001e-05 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607074+0000", 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4460000000000001e-06 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607133+0000", 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8736999999999997e-05 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.607146+0000", 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2845e-05 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.576979+0000", 2023-09-23T20:06:19.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96983302800000004 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.577095+0000", 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011613899999999999 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.577196+0000", 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101648 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580685+0000", 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00348869 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580699+0000", 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3509999999999999e-05 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584873+0000", 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041742089999999999 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587899+0000", 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003026543 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587941+0000", 2023-09-23T20:06:19.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1307000000000003e-05 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588070+0000", 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012892900000000001 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182450 6.b 6:d37fd40c:::benchmark_data_smithi135_344736_object79794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:06:19.793 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 345.718699173, 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97506342899999998, 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182450 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.794 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613256+0000", 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613255+0000", 2023-09-23T20:06:19.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613262+0000", 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8210000000000002e-06 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613278+0000", 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5359999999999999e-05 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613286+0000", 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613340+0000", 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3922999999999999e-05 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.613349+0000", 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0340000000000002e-06 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580746+0000", 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96739630700000001 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.580902+0000", 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015640900000000001 2023-09-23T20:06:19.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.581089+0000", 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018731100000000001 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584767+0000", 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036773520000000001 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.585362+0000", 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00059505599999999995 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.587366+0000", 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020038109999999999 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588137+0000", 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00077110600000000003 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588191+0000", 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4521999999999997e-05 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.588320+0000", 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012857600000000001 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201805 6.0 6:04364655:::benchmark_data_smithi135_344736_object99149:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 314.865480494, 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.08500007, 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.802 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201805 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.803 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466479+0000", 2023-09-23T20:06:19.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3380000000000004e-06 2023-09-23T20:06:19.805 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.805 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.805 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466490+0000", 2023-09-23T20:06:19.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.129e-05 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466500+0000", 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0305999999999999e-05 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468241+0000", 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001740143 2023-09-23T20:06:19.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468253+0000", 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2268e-05 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542090+0000", 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.073837172 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542227+0000", 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013645199999999999 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.542739+0000", 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00051271599999999997 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551138+0000", 2023-09-23T20:06:19.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083988829999999993 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551197+0000", 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9363000000000001e-05 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551211+0000", 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3148e-05 2023-09-23T20:06:19.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551266+0000", 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5229000000000003e-05 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551475+0000", 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020925699999999999 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.810 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201830 6.10 6:08e5b611:::benchmark_data_smithi135_344736_object99174:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 314.86544484500001, 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1094386300000001, 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.811 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201830 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466511+0000", 2023-09-23T20:06:19.812 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466510+0000", 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466513+0000", 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4109999999999999e-06 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466519+0000", 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2500000000000003e-06 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466524+0000", 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5350000000000004e-06 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468278+0000", 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00175377 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468286+0000", 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9920000000000007e-06 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551513+0000", 2023-09-23T20:06:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0832270980000001 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.552197+0000", 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00068353800000000003 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556569+0000", 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043722819999999999 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556637+0000", 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8348000000000004e-05 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.557117+0000", 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00047992400000000001 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560978+0000", 2023-09-23T20:06:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038601740000000001 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561510+0000", 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00053195499999999997 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575543+0000", 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014033406 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575578+0000", 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5404999999999997e-05 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.575949+0000", 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037097900000000001 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201836 6.10 6:0c24ea53:::benchmark_data_smithi135_344736_object99180:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 314.86542598800003, 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1159160830000001, 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201836 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.820 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.821 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466529+0000", 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466532+0000", 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.43e-06 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466549+0000", 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7680999999999999e-05 2023-09-23T20:06:19.822 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466556+0000", 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0380000000000002e-06 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468309+0000", 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001753075 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.823 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468317+0000", 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8699999999999992e-06 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554470+0000", 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.086153084 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555087+0000", 2023-09-23T20:06:19.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000616151 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.569798+0000", 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014711695 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.569892+0000", 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3364999999999998e-05 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.825 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.570445+0000", 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00055320300000000005 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576273+0000", 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058281779999999998 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.826 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582033+0000", 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0057600059999999998 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582060+0000", 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7141999999999999e-05 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582101+0000", 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1272000000000001e-05 2023-09-23T20:06:19.827 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582446+0000", 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034424599999999998 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201842 6.0 6:03218c77:::benchmark_data_smithi135_344736_object99186:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:19.828 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 314.86539347799999, 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.088181898, 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201842 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.829 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466562+0000", 2023-09-23T20:06:19.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466565+0000", 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2610000000000002e-06 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466571+0000", 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6059999999999999e-06 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466576+0000", 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9280000000000001e-06 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468350+0000", 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017738249999999999 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468359+0000", 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4190000000000002e-06 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548143+0000", 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.079783959 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548255+0000", 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011229400000000001 2023-09-23T20:06:19.833 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.548588+0000", 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00033316999999999998 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554427+0000", 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058389430000000001 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.834 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554444+0000", 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7081000000000001e-05 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554491+0000", 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6495999999999997e-05 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554542+0000", 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1561999999999999e-05 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.554744+0000", 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000201883 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201845 6.10 6:0a5d128c:::benchmark_data_smithi135_344736_object99189:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 314.86537488900001, 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1274511190000001, 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201845 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.837 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466581+0000", 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.838 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466580+0000", 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466583+0000", 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4509999999999999e-06 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.839 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466588+0000", 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2159999999999997e-06 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466592+0000", 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4220000000000002e-06 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468381+0000", 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001788778 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.840 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468389+0000", 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9689999999999994e-06 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555129+0000", 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.086740209 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.555800+0000", 2023-09-23T20:06:19.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00067015300000000005 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.572929+0000", 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017129485999999999 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.572997+0000", 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7753e-05 2023-09-23T20:06:19.842 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.573486+0000", 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00048920300000000001 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582000+0000", 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085139759999999995 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.843 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585622+0000", 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003622489 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.593594+0000", 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0079714260000000002 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.593638+0000", 2023-09-23T20:06:19.844 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4351000000000003e-05 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594032+0000", 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00039355800000000002 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201858 6.10 6:0e829f63:::benchmark_data_smithi135_344736_object99202:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:19.845 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 314.86535954200002, 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.12779443, 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201858 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.846 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:06:19.847 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466598+0000", 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8920000000000001e-06 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466601+0000", 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0869999999999998e-06 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.848 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466604+0000", 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.596e-06 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468607+0000", 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020026620000000001 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468616+0000", 2023-09-23T20:06:19.849 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7739999999999993e-06 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560293+0000", 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0916775329999999 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560929+0000", 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00063605599999999997 2023-09-23T20:06:19.850 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576296+0000", 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01536671 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576359+0000", 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3100000000000002e-05 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576772+0000", 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00041272599999999998 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582458+0000", 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056866119999999997 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585662+0000", 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032034730000000001 2023-09-23T20:06:19.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594087+0000", 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084249849999999994 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594137+0000", 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9614000000000003e-05 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594390+0000", 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00025390100000000001 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201894 6.0 6:00f5be23:::benchmark_data_smithi135_344736_object99238:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 314.86534728700002, 2023-09-23T20:06:19.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0976265000000001, 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201894 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.855 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.856 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466608+0000", 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466610+0000", 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3309999999999998e-06 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466616+0000", 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3630000000000002e-06 2023-09-23T20:06:19.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466621+0000", 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1719999999999998e-06 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468641+0000", 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020203489999999998 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468650+0000", 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3739999999999998e-06 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551524+0000", 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0828746149999999 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551624+0000", 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100259 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.551973+0000", 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00034831199999999998 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556557+0000", 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045842579999999999 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.556597+0000", 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0182000000000003e-05 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.563996+0000", 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073992540000000001 2023-09-23T20:06:19.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.564035+0000", 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8602000000000001e-05 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.564235+0000", 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019976 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.862 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201912 6.10 6:0a9a0b94:::benchmark_data_smithi135_344736_object99256:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 314.865330368, 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1498471859999999, 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:06:19.863 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201912 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:06:19.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466628+0000", 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6249999999999999e-06 2023-09-23T20:06:19.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466633+0000", 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6210000000000001e-06 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466638+0000", 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6809999999999997e-06 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468671+0000", 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020332229999999998 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468677+0000", 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8409999999999996e-06 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561014+0000", 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0923374370000001 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561497+0000", 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00048270000000000002 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579372+0000", 2023-09-23T20:06:19.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017874660000000001 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579464+0000", 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2050000000000001e-05 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579912+0000", 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00044868700000000002 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582554+0000", 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00264197 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585730+0000", 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031761210000000001 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.870 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.615988+0000", 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030257373000000001 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.616043+0000", 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4648000000000003e-05 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:06:19.871 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.616472+0000", 2023-09-23T20:06:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00042985000000000001 2023-09-23T20:06:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:06:19.872 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:06:19.872 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: failed to encode map e299 with expected crc 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: failed to encode map e299 with expected crc 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: pgmap v833: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4284651323' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:06:19.873 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:06:19.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[221113]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:06:19.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:19.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: failed to encode map e299 with expected crc 2023-09-23T20:06:19.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: failed to encode map e299 with expected crc 2023-09-23T20:06:19.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:06:19.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:06:19.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:06:19.874 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: pgmap v833: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:19.875 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4284651323' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:06:19.875 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:06:19.875 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:19 smithi135 ceph-mon[213871]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:06:19.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:19.876 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:19.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: failed to encode map e299 with expected crc 2023-09-23T20:06:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: failed to encode map e299 with expected crc 2023-09-23T20:06:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:06:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:06:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:06:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: pgmap v833: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:06:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4284651323' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:06:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:06:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:19 smithi144 ceph-mon[194833]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:06:19.977 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:20.005 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:20.681 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.681 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.681 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:06:20.681 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: osdmap e300: 8 total, 8 up, 6 in 2023-09-23T20:06:20.681 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.681 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[213871]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.682 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: osdmap e300: 8 total, 8 up, 6 in 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:20 smithi135 ceph-mon[221113]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: osdmap e300: 8 total, 8 up, 6 in 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:20.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:20 smithi144 ceph-mon[194833]: failed to encode map e300 with expected crc 2023-09-23T20:06:21.728 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:21.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:21.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:21.832 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:21.856 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[221113]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[221113]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[221113]: osdmap e301: 8 total, 8 up, 6 in 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[221113]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[221113]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[221113]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[221113]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[221113]: pgmap v836: 99 pgs: 11 unknown, 88 active+clean; 4.6 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[213871]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.857 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[213871]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.858 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[213871]: osdmap e301: 8 total, 8 up, 6 in 2023-09-23T20:06:21.858 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[213871]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.858 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[213871]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.858 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[213871]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.858 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[213871]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.858 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:21 smithi135 ceph-mon[213871]: pgmap v836: 99 pgs: 11 unknown, 88 active+clean; 4.6 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T20:06:21.860 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:21.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:21 smithi144 ceph-mon[194833]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:21 smithi144 ceph-mon[194833]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:21 smithi144 ceph-mon[194833]: osdmap e301: 8 total, 8 up, 6 in 2023-09-23T20:06:21.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:21 smithi144 ceph-mon[194833]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:21 smithi144 ceph-mon[194833]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:21 smithi144 ceph-mon[194833]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:21 smithi144 ceph-mon[194833]: failed to encode map e301 with expected crc 2023-09-23T20:06:21.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:21 smithi144 ceph-mon[194833]: pgmap v836: 99 pgs: 11 unknown, 88 active+clean; 4.6 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T20:06:22.114 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:22.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:22.169 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T20:06:22.219 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:22.248 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:22.604 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:06:22.604 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:06:22.604 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:06:22.604 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:06:22.604 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:06:22.604 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:06:22.606 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:06:22.625 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:06:22.626 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:06:22.626 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:06:22.626 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:06:22.626 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:06:22.626 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:06:22.626 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:06:22.627 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:06:22.627 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:06:22.733 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:06:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:22 smithi144 ceph-mon[194833]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:22 smithi144 ceph-mon[194833]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:22 smithi144 ceph-mon[194833]: osdmap e302: 8 total, 8 up, 6 in 2023-09-23T20:06:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:22 smithi144 ceph-mon[194833]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:22 smithi144 ceph-mon[194833]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:22 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/932447817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:22 smithi144 ceph-mon[194833]: osdmap e303: 8 total, 8 up, 6 in 2023-09-23T20:06:22.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[221113]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[221113]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[221113]: osdmap e302: 8 total, 8 up, 6 in 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[221113]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[221113]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/932447817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[221113]: osdmap e303: 8 total, 8 up, 6 in 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[213871]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[213871]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[213871]: osdmap e302: 8 total, 8 up, 6 in 2023-09-23T20:06:22.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[213871]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[213871]: failed to encode map e302 with expected crc 2023-09-23T20:06:22.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:22.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/932447817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:22.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:22 smithi135 ceph-mon[213871]: osdmap e303: 8 total, 8 up, 6 in 2023-09-23T20:06:22.996 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:23.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:23.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:23.085 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:23.101 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:23.127 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:23.208 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:23.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:23.312 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:23.341 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:23.385 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:23.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:23.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:23.488 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:23.514 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:23.648 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:23.678 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.678 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.678 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: pgmap v839: 99 pgs: 11 unknown, 88 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T20:06:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:23 smithi144 ceph-mon[194833]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T20:06:23.711 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:23.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:23.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:23.816 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:23.844 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:23.884 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:23.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:23.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:23.903 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:06:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:06:23] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.43.0" 2023-09-23T20:06:23.903 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.903 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.903 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.903 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.903 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: pgmap v839: 99 pgs: 11 unknown, 88 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[221113]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.905 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.905 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: failed to encode map e303 with expected crc 2023-09-23T20:06:23.905 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: pgmap v839: 99 pgs: 11 unknown, 88 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T20:06:23.905 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:23 smithi135 ceph-mon[213871]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T20:06:23.987 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:24.013 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:24.054 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:24.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:24.157 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:24.189 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:24.337 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:24.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:24.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:24.441 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:24.467 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:24.510 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:24.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:24.614 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:24.646 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:24.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[213871]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[213871]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[213871]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[213871]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[213871]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[221113]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[221113]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[221113]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[221113]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:24 smithi135 ceph-mon[221113]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.792 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:24.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:24.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:24.836 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:06:24.837 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:06:24.857 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:24 smithi144 ceph-mon[194833]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.857 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:24 smithi144 ceph-mon[194833]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.857 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:24 smithi144 ceph-mon[194833]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.857 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:24 smithi144 ceph-mon[194833]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.857 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:24 smithi144 ceph-mon[194833]: failed to encode map e304 with expected crc 2023-09-23T20:06:24.896 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:24.923 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:25.002 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:06:25.119 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:25.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:25.223 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:25.251 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:25.302 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:25.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:25.406 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:25.424 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:25.432 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:25.564 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:06:25.576 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:25.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:25.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:25.634 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:06:25.681 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:25.704 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:25 smithi144 ceph-mon[194833]: pgmap v841: 99 pgs: 5 peering, 94 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 15 op/s; 1001 KiB/s, 1 objects/s recovering 2023-09-23T20:06:25.706 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:25.750 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:25.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:25.831 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:25 smithi135 ceph-mon[221113]: pgmap v841: 99 pgs: 5 peering, 94 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 15 op/s; 1001 KiB/s, 1 objects/s recovering 2023-09-23T20:06:25.832 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:25 smithi135 ceph-mon[213871]: pgmap v841: 99 pgs: 5 peering, 94 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 15 op/s; 1001 KiB/s, 1 objects/s recovering 2023-09-23T20:06:25.855 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:25.883 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:25.928 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:25.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:25.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:25.990 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:26.032 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:26.061 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:26.106 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:26.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:26.210 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:26.236 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:26.245 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3] out_osds: [6, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:06:26.245 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:06:26.279 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:26.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:26.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:26.383 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:26.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:26.591 INFO:teuthology.orchestra.run.smithi135.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T20:06:26.659 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:26.661 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1782379814' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:06:26.661 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:06:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:26.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:26.663 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1782379814' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:06:26.663 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:06:26.763 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:26.793 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:26.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1782379814' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:06:26.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:06:27.029 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:27.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:27.133 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:27.162 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:27.223 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:27.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:27.326 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:27.354 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:27.500 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:27.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:27.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:27.533 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:06:27.533 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:06:27.590 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:27 smithi144 ceph-mon[194833]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.590 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:27 smithi144 ceph-mon[194833]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:27 smithi144 ceph-mon[194833]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:27 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:06:27.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:27 smithi144 ceph-mon[194833]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T20:06:27.591 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:27 smithi144 ceph-mon[194833]: pgmap v843: 99 pgs: 5 peering, 94 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 16 KiB/s rd, 42 KiB/s wr, 12 op/s; 812 KiB/s, 1 objects/s recovering 2023-09-23T20:06:27.604 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:27.633 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:27.669 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:27.686 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:27.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:27.791 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:27.819 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:27.878 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[213871]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.878 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[213871]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[213871]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[213871]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[213871]: pgmap v843: 99 pgs: 5 peering, 94 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 16 KiB/s rd, 42 KiB/s wr, 12 op/s; 812 KiB/s, 1 objects/s recovering 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[221113]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[221113]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[221113]: failed to encode map e305 with expected crc 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[221113]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T20:06:27.879 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:27 smithi135 ceph-mon[221113]: pgmap v843: 99 pgs: 5 peering, 94 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 16 KiB/s rd, 42 KiB/s wr, 12 op/s; 812 KiB/s, 1 objects/s recovering 2023-09-23T20:06:27.915 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:27.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:27.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:28.020 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:28.048 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:28.177 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:28.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:28.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:28.232 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:06:28.281 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:28.307 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:28.350 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:28.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:28.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:28.456 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:28.482 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:28.582 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:28.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:28.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:28.593 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:28.601 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:06:28.687 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:28.697 DEBUG:teuthology.orchestra.run.smithi135:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T20:06:28.697 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:06:28.697 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:06:28.717 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:28.719 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:06:28.719 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:06:28.842 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3446149824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:28.843 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:28 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:28.928 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:hints = 1 2023-09-23T20:06:28.928 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:06:28.928 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object prefix: benchmark_data_smithi135_460343 2023-09-23T20:06:28.928 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:06:28.929 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:06:28.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3446149824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:28.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:28 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:28.977 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3446149824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:28.978 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:28 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:28.981 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:28.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:28.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:29.089 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:29.114 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:29.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:29.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[213871]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[213871]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:29.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[213871]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T20:06:29.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[213871]: pgmap v845: 99 pgs: 99 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 709 KiB/s, 1 objects/s recovering 2023-09-23T20:06:29.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[213871]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:29.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[221113]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[221113]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:29.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[221113]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T20:06:29.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[221113]: pgmap v845: 99 pgs: 99 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 709 KiB/s, 1 objects/s recovering 2023-09-23T20:06:29.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:29 smithi135 ceph-mon[221113]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.734 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:29.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:29.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:29.840 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:29.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:29 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:29.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:29 smithi144 ceph-mon[194833]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:29 smithi144 ceph-mon[194833]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:29 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:29.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:29 smithi144 ceph-mon[194833]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T20:06:29.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:29 smithi144 ceph-mon[194833]: pgmap v845: 99 pgs: 99 active+clean; 4.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 709 KiB/s, 1 objects/s recovering 2023-09-23T20:06:29.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:29 smithi144 ceph-mon[194833]: failed to encode map e306 with expected crc 2023-09-23T20:06:29.889 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:29.928 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 1 16 2469 2453 153.304 153.312 0.00618354 0.0064997 2023-09-23T20:06:29.954 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:30.141 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:30.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:30.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:30.247 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:30.294 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:30.382 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:30.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:30.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:30.420 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:06:30.421 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:06:30.451 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:06:30.488 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:30.489 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:06:30.534 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:06:30.555 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:30.573 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:06:30.615 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:06:30.656 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:06:30.690 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:06:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:30 smithi135 ceph-mon[213871]: failed to encode map e306 with expected crc 2023-09-23T20:06:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1856001480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:30.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:30 smithi135 ceph-mon[221113]: failed to encode map e306 with expected crc 2023-09-23T20:06:30.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1856001480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:30.730 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:06:30.738 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:06:30.738 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:06:30.738 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:06:30.738 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:06:30.738 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:06:30.738 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:06:30.738 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:06:30.738 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:06:30.807 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:30.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:30.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:30.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:30 smithi144 ceph-mon[194833]: failed to encode map e306 with expected crc 2023-09-23T20:06:30.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1856001480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:30.917 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:30.928 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 2 16 5705 5689 177.768 202.25 0.00460389 0.00561446 2023-09-23T20:06:30.973 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:31.871 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:06:31.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:31 smithi144 ceph-mon[194833]: pgmap v846: 99 pgs: 99 active+clean; 107 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 14 MiB/s wr, 225 op/s 2023-09-23T20:06:31.926 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:31 smithi135 ceph-mon[213871]: pgmap v846: 99 pgs: 99 active+clean; 107 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 14 MiB/s wr, 225 op/s 2023-09-23T20:06:31.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:31 smithi135 ceph-mon[221113]: pgmap v846: 99 pgs: 99 active+clean; 107 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 14 MiB/s wr, 225 op/s 2023-09-23T20:06:31.928 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 3 16 9203 9187 191.381 218.625 0.00521297 0.00521709 2023-09-23T20:06:32.076 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:06:32.724 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:32.746 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:32.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:32.853 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:32.901 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:32.928 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 4 16 12231 12215 190.844 189.25 0.00366043 0.00523242 2023-09-23T20:06:33.037 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:33.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:33.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:33.144 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:33.195 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:33.452 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:33.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:33.558 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:33.605 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:33.683 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:33.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:33.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:33.789 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:33.842 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:33 smithi135 ceph-mon[213871]: pgmap v847: 99 pgs: 99 active+clean; 107 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 13 MiB/s wr, 204 op/s 2023-09-23T20:06:33.842 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:33 smithi135 ceph-mon[221113]: pgmap v847: 99 pgs: 99 active+clean; 107 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 13 MiB/s wr, 204 op/s 2023-09-23T20:06:33.843 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:06:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:06:33] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.43.0" 2023-09-23T20:06:33.848 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:33.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:33 smithi144 ceph-mon[194833]: pgmap v847: 99 pgs: 99 active+clean; 107 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 13 MiB/s wr, 204 op/s 2023-09-23T20:06:33.929 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 5 16 15109 15093 188.647 179.875 0.00402924 0.00529439 2023-09-23T20:06:33.935 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:33.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:34.043 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:34.094 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:34.176 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:34.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:34.284 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:34.311 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:34.353 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:34.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:34.457 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:34.499 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:34.561 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:34.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:34.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:34.664 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:34.733 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:34.771 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:34.797 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:34.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:34.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:34.903 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:34.931 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 6 16 16912 16896 175.923 112.688 0.00419984 0.00567577 2023-09-23T20:06:34.968 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:35.110 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:35.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:35.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:35.129 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:34 smithi135 ceph-mon[213871]: pgmap v848: 99 pgs: 99 active+clean; 679 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:06:35.129 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:34 smithi135 ceph-mon[221113]: pgmap v848: 99 pgs: 99 active+clean; 679 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:06:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:34 smithi144 ceph-mon[194833]: pgmap v848: 99 pgs: 99 active+clean; 679 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:06:35.217 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:35.269 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:35.363 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:35.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:35.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:35.469 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:35.531 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:35.634 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:35.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:35.744 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:35.802 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:35.875 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:06:35.875 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:06:35.875 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:19030 8.4 8:2477205b:::benchmark_data_smithi135_460343_object19029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:35.867833+0000", 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0067680860000000004, 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0068076539999999998, 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 19030 2023-09-23T20:06:35.876 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.867833+0000", 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.867833+0000", 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.877 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.867832+0000", 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.867939+0000", 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000106983 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.878 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.867949+0000", 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1649999999999995e-06 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.867959+0000", 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0291e-05 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.869502+0000", 2023-09-23T20:06:35.879 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001542676 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.869591+0000", 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9254999999999998e-05 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.869719+0000", 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012776399999999999 2023-09-23T20:06:35.880 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.871886+0000", 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021679540000000001 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.871903+0000", 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6704000000000001e-05 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.881 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35643.0:19037 8.1 e306/300 8:83fd004d:::benchmark_data_smithi135_460343_object19036:head v 306'1217, mlcod=306'1215)", 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:35.871894+0000", 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.002707428, 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002849079, 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:06:35.882 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.871894+0000", 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.871894+0000", 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.883 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.871893+0000", 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.871923+0000", 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9471999999999998e-05 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.884 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.871931+0000", 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2830000000000001e-06 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.871936+0000", 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1939999999999997e-06 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872022+0000", 2023-09-23T20:06:35.885 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6433000000000004e-05 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872041+0000", 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8766999999999998e-05 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.886 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35643.0:19034 8.2 e306/299 8:43287648:::benchmark_data_smithi135_460343_object19033:head v 306'1180, mlcod=306'1177)", 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:35.872298+0000", 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0023034169999999999, 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025066060000000002, 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872298+0000", 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872298+0000", 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872297+0000", 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:06:35.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872321+0000", 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4253999999999999e-05 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872336+0000", 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4853e-05 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872342+0000", 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3790000000000003e-06 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872404+0000", 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1686000000000005e-05 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:35.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.872428+0000", 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4425000000000001e-05 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:19044 8.9 8:95719caf:::benchmark_data_smithi135_460343_object19043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:35.873066+0000", 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0015345490000000001, 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001829652, 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:35.891 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 19044 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.873066+0000", 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.892 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.873066+0000", 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.873065+0000", 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.873234+0000", 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016903799999999999 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.873245+0000", 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0448999999999999e-05 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.873251+0000", 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3999999999999997e-06 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874686+0000", 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014347089999999999 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:35.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874827+0000", 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014114800000000001 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35643.0:19038 8.2 e306/299)", 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:35.874564+0000", 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 3.6740999999999999e-05, 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00040107599999999999, 2023-09-23T20:06:35.896 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "queued for pg", 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874564+0000", 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874564+0000", 2023-09-23T20:06:35.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874564+0000", 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874588+0000", 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4496000000000001e-05 2023-09-23T20:06:35.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874603+0000", 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4521999999999999e-05 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874609+0000", 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5209999999999999e-06 2023-09-23T20:06:35.899 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35643.0:19040 8.2 e306/299)", 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:35.874895+0000", 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 4294967295.9997058, 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000127257, 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "queued for pg", 2023-09-23T20:06:35.900 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874895+0000", 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874895+0000", 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:35.901 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874894+0000", 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874917+0000", 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2741000000000001e-05 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874927+0000", 2023-09-23T20:06:35.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0205e-05 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874934+0000", 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0509999999999999e-06 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.903 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35643.0:19043 8.a e306/300)", 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:35.874052+0000", 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.000548593, 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001020434, 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "queued for pg", 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874052+0000", 2023-09-23T20:06:35.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874052+0000", 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874052+0000", 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:06:35.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874964+0000", 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000911852 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874973+0000", 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8689999999999999e-06 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:35.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:35.874979+0000", 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2979999999999999e-06 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 7 2023-09-23T20:06:35.907 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:06:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:35.908 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:35.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:35.931 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 7 16 19177 19161 171.014 141.562 0.00348427 0.00584018 2023-09-23T20:06:35.964 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:06:35.972 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:06:36.009 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:36.052 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:36.097 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:36.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:36.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:36.205 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:36.255 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:36.345 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:36.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:36.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:36.451 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:36.478 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:36.687 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:06:36.931 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 8 16 20697 20681 161.514 95 0.00378487 0.006175 2023-09-23T20:06:37.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:36 smithi144 ceph-mon[194833]: from='client.35649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:06:37.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:36 smithi144 ceph-mon[194833]: pgmap v849: 99 pgs: 99 active+clean; 944 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:06:37.203 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:06:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:36 smithi135 ceph-mon[221113]: from='client.35649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:06:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:36 smithi135 ceph-mon[221113]: pgmap v849: 99 pgs: 99 active+clean; 944 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:06:37.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:36 smithi135 ceph-mon[213871]: from='client.35649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:06:37.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:36 smithi135 ceph-mon[213871]: pgmap v849: 99 pgs: 99 active+clean; 944 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:06:37.434 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:37.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:37.542 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:37.590 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:37.650 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:37.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:37.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:37.754 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:37.800 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:37.826 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:37.931 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 9 16 22502 22486 156.103 112.812 0.00631805 0.00640099 2023-09-23T20:06:38.151 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:38.151 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:38.221 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:38.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:38.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:38.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:38.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:38.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:38.327 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:38.340 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":848,"stamp":"2023-09-23T20:06:34.805085+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":712239449,"num_objects":10822,"num_object_clones":0,"num_object_copies":32466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10822,"num_whiteouts":0,"num_read":6039,"num_read_kb":13295,"num_write":11723,"num_write_kb":703761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":4886266,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13597,"ondisk_log_size":13597,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4397636,"kb_used_data":1918996,"kb_used_omap":25,"kb_used_meta":2478566,"kb_avail":558073276,"statfs":{"total":575970213888,"available":571467034624,"internally_reserved":0,"allocated":1965051904,"data_stored":1958018638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":2538052300},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,4,4,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":707395584,"num_objects":10794,"num_object_clones":0,"num_object_copies":32382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10794,"num_write_kb":690816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.218785"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":294,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721607+0000","last_change":"2023-09-23T20:06:20.577187+0000","last_active":"2023-09-23T20:06:26.721607+0000","last_peered":"2023-09-23T20:06:26.721607+0000","last_clean":"2023-09-23T20:06:26.721607+0000","last_became_active":"2023-09-23T20:06:20.576810+0000","last_became_peered":"2023-09-23T20:06:20.576810+0000","last_unstale":"2023-09-23T20:06:26.721607+0000","last_undegraded":"2023-09-23T20:06:26.721607+0000","last_fullsized":"2023-09-23T20:06:26.721607+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:11.009284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":137,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723612+0000","last_change":"2023-09-23T20:06:20.576529+0000","last_active":"2023-09-23T20:06:26.723612+0000","last_peered":"2023-09-23T20:06:26.723612+0000","last_clean":"2023-09-23T20:06:26.723612+0000","last_became_active":"2023-09-23T20:06:20.576173+0000","last_became_peered":"2023-09-23T20:06:20.576173+0000","last_unstale":"2023-09-23T20:06:26.723612+0000","last_undegraded":"2023-09-23T20:06:26.723612+0000","last_fullsized":"2023-09-23T20:06:26.723612+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:34.172210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":158,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.722972+0000","last_change":"2023-09-23T20:06:20.572745+0000","last_active":"2023-09-23T20:06:26.722972+0000","last_peered":"2023-09-23T20:06:26.722972+0000","last_clean":"2023-09-23T20:06:26.722972+0000","last_became_active":"2023-09-23T20:06:20.572446+0000","last_became_peered":"2023-09-23T20:06:20.572446+0000","last_unstale":"2023-09-23T20:06:26.722972+0000","last_undegraded":"2023-09-23T20:06:26.722972+0000","last_fullsized":"2023-09-23T20:06:26.722972+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:39.010617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":58,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590209+0000","last_change":"2023-09-23T20:03:59.233369+0000","last_active":"2023-09-23T20:06:26.590209+0000","last_peered":"2023-09-23T20:06:26.590209+0000","last_clean":"2023-09-23T20:06:26.590209+0000","last_became_active":"2023-09-23T20:03:59.233099+0000","last_became_peered":"2023-09-23T20:03:59.233099+0000","last_unstale":"2023-09-23T20:06:26.590209+0000","last_undegraded":"2023-09-23T20:06:26.590209+0000","last_fullsized":"2023-09-23T20:06:26.590209+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.892745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":138,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:28.596612+0000","last_change":"2023-09-23T20:06:20.565724+0000","last_active":"2023-09-23T20:06:28.596612+0000","last_peered":"2023-09-23T20:06:28.596612+0000","last_clean":"2023-09-23T20:06:28.596612+0000","last_became_active":"2023-09-23T20:06:20.565423+0000","last_became_peered":"2023-09-23T20:06:20.565423+0000","last_unstale":"2023-09-23T20:06:28.596612+0000","last_undegraded":"2023-09-23T20:06:28.596612+0000","last_fullsized":"2023-09-23T20:06:28.596612+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:03.731068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":241,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590529+0000","last_change":"2023-09-23T20:06:20.566946+0000","last_active":"2023-09-23T20:06:26.590529+0000","last_peered":"2023-09-23T20:06:26.590529+0000","last_clean":"2023-09-23T20:06:26.590529+0000","last_became_active":"2023-09-23T20:06:20.566619+0000","last_became_peered":"2023-09-23T20:06:20.566619+0000","last_unstale":"2023-09-23T20:06:26.590529+0000","last_undegraded":"2023-09-23T20:06:26.590529+0000","last_fullsized":"2023-09-23T20:06:26.590529+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:00.654327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":232,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590378+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:06:26.590378+0000","last_peered":"2023-09-23T20:06:26.590378+0000","last_clean":"2023-09-23T20:06:26.590378+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:06:26.590378+0000","last_undegraded":"2023-09-23T20:06:26.590378+0000","last_fullsized":"2023-09-23T20:06:26.590378+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":138,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.722821+0000","last_change":"2023-09-23T20:06:20.571646+0000","last_active":"2023-09-23T20:06:26.722821+0000","last_peered":"2023-09-23T20:06:26.722821+0000","last_clean":"2023-09-23T20:06:26.722821+0000","last_became_active":"2023-09-23T20:06:20.571064+0000","last_became_peered":"2023-09-23T20:06:20.571064+0000","last_unstale":"2023-09-23T20:06:26.722821+0000","last_undegraded":"2023-09-23T20:06:26.722821+0000","last_fullsized":"2023-09-23T20:06:26.722821+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:51.052072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":275,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590874+0000","last_change":"2023-09-23T20:06:20.583345+0000","last_active":"2023-09-23T20:06:26.590874+0000","last_peered":"2023-09-23T20:06:26.590874+0000","last_clean":"2023-09-23T20:06:26.590874+0000","last_became_active":"2023-09-23T20:06:20.582961+0000","last_became_peered":"2023-09-23T20:06:20.582961+0000","last_unstale":"2023-09-23T20:06:26.590874+0000","last_undegraded":"2023-09-23T20:06:26.590874+0000","last_fullsized":"2023-09-23T20:06:26.590874+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:33.630514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":148,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721585+0000","last_change":"2023-09-23T20:00:34.448638+0000","last_active":"2023-09-23T20:06:26.721585+0000","last_peered":"2023-09-23T20:06:26.721585+0000","last_clean":"2023-09-23T20:06:26.721585+0000","last_became_active":"2023-09-23T20:00:34.448297+0000","last_became_peered":"2023-09-23T20:00:34.448297+0000","last_unstale":"2023-09-23T20:06:26.721585+0000","last_undegraded":"2023-09-23T20:06:26.721585+0000","last_fullsized":"2023-09-23T20:06:26.721585+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:59.509813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":301,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589746+0000","last_change":"2023-09-23T19:59:09.234637+0000","last_active":"2023-09-23T20:06:26.589746+0000","last_peered":"2023-09-23T20:06:26.589746+0000","last_clean":"2023-09-23T20:06:26.589746+0000","last_became_active":"2023-09-23T19:59:09.170560+0000","last_became_peered":"2023-09-23T19:59:09.170560+0000","last_unstale":"2023-09-23T20:06:26.589746+0000","last_undegraded":"2023-09-23T20:06:26.589746+0000","last_fullsized":"2023-09-23T20:06:26.589746+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:27.387696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.725547+0000","last_change":"2023-09-23T20:06:20.574415+0000","last_active":"2023-09-23T20:06:26.725547+0000","last_peered":"2023-09-23T20:06:26.725547+0000","last_clean":"2023-09-23T20:06:26.725547+0000","last_became_active":"2023-09-23T20:06:20.574143+0000","last_became_peered":"2023-09-23T20:06:20.574143+0000","last_unstale":"2023-09-23T20:06:26.725547+0000","last_undegraded":"2023-09-23T20:06:26.725547+0000","last_fullsized":"2023-09-23T20:06:26.725547+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:08.220945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":320,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721790+0000","last_change":"2023-09-23T20:00:34.454851+0000","last_active":"2023-09-23T20:06:26.721790+0000","last_peered":"2023-09-23T20:06:26.721790+0000","last_clean":"2023-09-23T20:06:26.721790+0000","last_became_active":"2023-09-23T20:00:34.454480+0000","last_became_peered":"2023-09-23T20:00:34.454480+0000","last_unstale":"2023-09-23T20:06:26.721790+0000","last_undegraded":"2023-09-23T20:06:26.721790+0000","last_fullsized":"2023-09-23T20:06:26.721790+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:19.116808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":292,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721706+0000","last_change":"2023-09-23T20:06:20.573095+0000","last_active":"2023-09-23T20:06:26.721706+0000","last_peered":"2023-09-23T20:06:26.721706+0000","last_clean":"2023-09-23T20:06:26.721706+0000","last_became_active":"2023-09-23T20:06:20.572706+0000","last_became_peered":"2023-09-23T20:06:20.572706+0000","last_unstale":"2023-09-23T20:06:26.721706+0000","last_undegraded":"2023-09-23T20:06:26.721706+0000","last_fullsized":"2023-09-23T20:06:26.721706+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:29.508638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":268,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589301+0000","last_change":"2023-09-23T19:50:23.999678+0000","last_active":"2023-09-23T20:06:26.589301+0000","last_peered":"2023-09-23T20:06:26.589301+0000","last_clean":"2023-09-23T20:06:26.589301+0000","last_became_active":"2023-09-23T19:50:23.999329+0000","last_became_peered":"2023-09-23T19:50:23.999329+0000","last_unstale":"2023-09-23T20:06:26.589301+0000","last_undegraded":"2023-09-23T20:06:26.589301+0000","last_fullsized":"2023-09-23T20:06:26.589301+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:01.147950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":138,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723407+0000","last_change":"2023-09-23T20:06:20.579620+0000","last_active":"2023-09-23T20:06:26.723407+0000","last_peered":"2023-09-23T20:06:26.723407+0000","last_clean":"2023-09-23T20:06:26.723407+0000","last_became_active":"2023-09-23T20:06:20.579300+0000","last_became_peered":"2023-09-23T20:06:20.579300+0000","last_unstale":"2023-09-23T20:06:26.723407+0000","last_undegraded":"2023-09-23T20:06:26.723407+0000","last_fullsized":"2023-09-23T20:06:26.723407+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:17.347357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":271,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589300+0000","last_change":"2023-09-23T20:03:59.233224+0000","last_active":"2023-09-23T20:06:26.589300+0000","last_peered":"2023-09-23T20:06:26.589300+0000","last_clean":"2023-09-23T20:06:26.589300+0000","last_became_active":"2023-09-23T20:03:59.232955+0000","last_became_peered":"2023-09-23T20:03:59.232955+0000","last_unstale":"2023-09-23T20:06:26.589300+0000","last_undegraded":"2023-09-23T20:06:26.589300+0000","last_fullsized":"2023-09-23T20:06:26.589300+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:29.451127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":284,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590382+0000","last_change":"2023-09-23T19:59:09.174097+0000","last_active":"2023-09-23T20:06:26.590382+0000","last_peered":"2023-09-23T20:06:26.590382+0000","last_clean":"2023-09-23T20:06:26.590382+0000","last_became_active":"2023-09-23T19:59:09.169984+0000","last_became_peered":"2023-09-23T19:59:09.169984+0000","last_unstale":"2023-09-23T20:06:26.590382+0000","last_undegraded":"2023-09-23T20:06:26.590382+0000","last_fullsized":"2023-09-23T20:06:26.590382+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:39.654826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":266,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589296+0000","last_change":"2023-09-23T20:01:52.443231+0000","last_active":"2023-09-23T20:06:26.589296+0000","last_peered":"2023-09-23T20:06:26.589296+0000","last_clean":"2023-09-23T20:06:26.589296+0000","last_became_active":"2023-09-23T20:01:52.442923+0000","last_became_peered":"2023-09-23T20:01:52.442923+0000","last_unstale":"2023-09-23T20:06:26.589296+0000","last_undegraded":"2023-09-23T20:06:26.589296+0000","last_fullsized":"2023-09-23T20:06:26.589296+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:56.902502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":272,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723717+0000","last_change":"2023-09-23T20:03:46.689432+0000","last_active":"2023-09-23T20:06:26.723717+0000","last_peered":"2023-09-23T20:06:26.723717+0000","last_clean":"2023-09-23T20:06:26.723717+0000","last_became_active":"2023-09-23T20:03:46.689129+0000","last_became_peered":"2023-09-23T20:03:46.689129+0000","last_unstale":"2023-09-23T20:06:26.723717+0000","last_undegraded":"2023-09-23T20:06:26.723717+0000","last_fullsized":"2023-09-23T20:06:26.723717+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.202972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":99,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589478+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:06:26.589478+0000","last_peered":"2023-09-23T20:06:26.589478+0000","last_clean":"2023-09-23T20:06:26.589478+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:06:26.589478+0000","last_undegraded":"2023-09-23T20:06:26.589478+0000","last_fullsized":"2023-09-23T20:06:26.589478+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723796+0000","last_change":"2023-09-23T20:06:20.578938+0000","last_active":"2023-09-23T20:06:26.723796+0000","last_peered":"2023-09-23T20:06:26.723796+0000","last_clean":"2023-09-23T20:06:26.723796+0000","last_became_active":"2023-09-23T20:06:20.578671+0000","last_became_peered":"2023-09-23T20:06:20.578671+0000","last_unstale":"2023-09-23T20:06:26.723796+0000","last_undegraded":"2023-09-23T20:06:26.723796+0000","last_fullsized":"2023-09-23T20:06:26.723796+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:25.970913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"306'840","reported_seq":855,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.094053+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:06:33.094053+0000","last_peered":"2023-09-23T20:06:33.094053+0000","last_clean":"2023-09-23T20:06:33.094053+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:06:33.094053+0000","last_undegraded":"2023-09-23T20:06:33.094053+0000","last_fullsized":"2023-09-23T20:06:33.094053+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":840,"log_dups_size":0,"ondisk_log_size":840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55050240,"num_objects":840,"num_object_clones":0,"num_object_copies":2520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":840,"num_write_kb":53760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":267,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589888+0000","last_change":"2023-09-23T19:48:37.571951+0000","last_active":"2023-09-23T20:06:26.589888+0000","last_peered":"2023-09-23T20:06:26.589888+0000","last_clean":"2023-09-23T20:06:26.589888+0000","last_became_active":"2023-09-23T19:48:37.571576+0000","last_became_peered":"2023-09-23T19:48:37.571576+0000","last_unstale":"2023-09-23T20:06:26.589888+0000","last_undegraded":"2023-09-23T20:06:26.589888+0000","last_fullsized":"2023-09-23T20:06:26.589888+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:05.076248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":294,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589478+0000","last_change":"2023-09-23T20:03:46.681806+0000","last_active":"2023-09-23T20:06:26.589478+0000","last_peered":"2023-09-23T20:06:26.589478+0000","last_clean":"2023-09-23T20:06:26.589478+0000","last_became_active":"2023-09-23T20:03:46.681523+0000","last_became_peered":"2023-09-23T20:03:46.681523+0000","last_unstale":"2023-09-23T20:06:26.589478+0000","last_undegraded":"2023-09-23T20:06:26.589478+0000","last_fullsized":"2023-09-23T20:06:26.589478+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.883932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"306'824","reported_seq":839,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.535856+0000","last_change":"2023-09-23T20:06:19.576604+0000","last_active":"2023-09-23T20:06:33.535856+0000","last_peered":"2023-09-23T20:06:33.535856+0000","last_clean":"2023-09-23T20:06:33.535856+0000","last_became_active":"2023-09-23T20:06:19.576347+0000","last_became_peered":"2023-09-23T20:06:19.576347+0000","last_unstale":"2023-09-23T20:06:33.535856+0000","last_undegraded":"2023-09-23T20:06:33.535856+0000","last_fullsized":"2023-09-23T20:06:33.535856+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":824,"log_dups_size":0,"ondisk_log_size":824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.600563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54001664,"num_objects":824,"num_object_clones":0,"num_object_copies":2472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":824,"num_write_kb":52736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":282,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590390+0000","last_change":"2023-09-23T20:06:20.592245+0000","last_active":"2023-09-23T20:06:26.590390+0000","last_peered":"2023-09-23T20:06:26.590390+0000","last_clean":"2023-09-23T20:06:26.590390+0000","last_became_active":"2023-09-23T20:06:20.584327+0000","last_became_peered":"2023-09-23T20:06:20.584327+0000","last_unstale":"2023-09-23T20:06:26.590390+0000","last_undegraded":"2023-09-23T20:06:26.590390+0000","last_fullsized":"2023-09-23T20:06:26.590390+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:56.391229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":332,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721413+0000","last_change":"2023-09-23T20:06:20.583378+0000","last_active":"2023-09-23T20:06:26.721413+0000","last_peered":"2023-09-23T20:06:26.721413+0000","last_clean":"2023-09-23T20:06:26.721413+0000","last_became_active":"2023-09-23T20:06:20.583058+0000","last_became_peered":"2023-09-23T20:06:20.583058+0000","last_unstale":"2023-09-23T20:06:26.721413+0000","last_undegraded":"2023-09-23T20:06:26.721413+0000","last_fullsized":"2023-09-23T20:06:26.721413+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:48.594555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"306'879","reported_seq":892,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.539794+0000","last_change":"2023-09-23T20:06:19.575535+0000","last_active":"2023-09-23T20:06:33.539794+0000","last_peered":"2023-09-23T20:06:33.539794+0000","last_clean":"2023-09-23T20:06:33.539794+0000","last_became_active":"2023-09-23T20:06:19.575236+0000","last_became_peered":"2023-09-23T20:06:19.575236+0000","last_unstale":"2023-09-23T20:06:33.539794+0000","last_undegraded":"2023-09-23T20:06:33.539794+0000","last_fullsized":"2023-09-23T20:06:33.539794+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":879,"log_dups_size":0,"ondisk_log_size":879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:54.618915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57606144,"num_objects":879,"num_object_clones":0,"num_object_copies":2637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":879,"num_write_kb":56256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":271,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589625+0000","last_change":"2023-09-23T20:01:52.445376+0000","last_active":"2023-09-23T20:06:26.589625+0000","last_peered":"2023-09-23T20:06:26.589625+0000","last_clean":"2023-09-23T20:06:26.589625+0000","last_became_active":"2023-09-23T20:01:52.439197+0000","last_became_peered":"2023-09-23T20:01:52.439197+0000","last_unstale":"2023-09-23T20:06:26.589625+0000","last_undegraded":"2023-09-23T20:06:26.589625+0000","last_fullsized":"2023-09-23T20:06:26.589625+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:09.614912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589324+0000","last_change":"2023-09-23T20:06:20.582293+0000","last_active":"2023-09-23T20:06:26.589324+0000","last_peered":"2023-09-23T20:06:26.589324+0000","last_clean":"2023-09-23T20:06:26.589324+0000","last_became_active":"2023-09-23T20:06:20.579726+0000","last_became_peered":"2023-09-23T20:06:20.579726+0000","last_unstale":"2023-09-23T20:06:26.589324+0000","last_undegraded":"2023-09-23T20:06:26.589324+0000","last_fullsized":"2023-09-23T20:06:26.589324+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:40.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"306'323","reported_seq":338,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:30.777504+0000","last_change":"2023-09-23T20:06:20.571291+0000","last_active":"2023-09-23T20:06:30.777504+0000","last_peered":"2023-09-23T20:06:30.777504+0000","last_clean":"2023-09-23T20:06:30.777504+0000","last_became_active":"2023-09-23T20:06:20.570973+0000","last_became_peered":"2023-09-23T20:06:20.570973+0000","last_unstale":"2023-09-23T20:06:30.777504+0000","last_undegraded":"2023-09-23T20:06:30.777504+0000","last_fullsized":"2023-09-23T20:06:30.777504+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:20.789340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21168128,"num_objects":323,"num_object_clones":0,"num_object_copies":969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":323,"num_write_kb":20672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":334,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590636+0000","last_change":"2023-09-23T20:06:20.573311+0000","last_active":"2023-09-23T20:06:26.590636+0000","last_peered":"2023-09-23T20:06:26.590636+0000","last_clean":"2023-09-23T20:06:26.590636+0000","last_became_active":"2023-09-23T20:06:20.572953+0000","last_became_peered":"2023-09-23T20:06:20.572953+0000","last_unstale":"2023-09-23T20:06:26.590636+0000","last_undegraded":"2023-09-23T20:06:26.590636+0000","last_fullsized":"2023-09-23T20:06:26.590636+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:03.682011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721581+0000","last_change":"2023-09-23T20:06:20.574531+0000","last_active":"2023-09-23T20:06:26.721581+0000","last_peered":"2023-09-23T20:06:26.721581+0000","last_clean":"2023-09-23T20:06:26.721581+0000","last_became_active":"2023-09-23T20:06:20.574290+0000","last_became_peered":"2023-09-23T20:06:20.574290+0000","last_unstale":"2023-09-23T20:06:26.721581+0000","last_undegraded":"2023-09-23T20:06:26.721581+0000","last_fullsized":"2023-09-23T20:06:26.721581+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:54.226550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.0","version":"306'329","reported_seq":346,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:30.773742+0000","last_change":"2023-09-23T20:06:20.571479+0000","last_active":"2023-09-23T20:06:30.773742+0000","last_peered":"2023-09-23T20:06:30.773742+0000","last_clean":"2023-09-23T20:06:30.773742+0000","last_became_active":"2023-09-23T20:06:20.570629+0000","last_became_peered":"2023-09-23T20:06:20.570629+0000","last_unstale":"2023-09-23T20:06:30.773742+0000","last_undegraded":"2023-09-23T20:06:30.773742+0000","last_fullsized":"2023-09-23T20:06:30.773742+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":329,"log_dups_size":0,"ondisk_log_size":329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:31.331098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21561344,"num_objects":329,"num_object_clones":0,"num_object_copies":987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":329,"num_write_kb":21056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":756,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589184+0000","last_change":"2023-09-23T20:03:13.240670+0000","last_active":"2023-09-23T20:06:26.589184+0000","last_peered":"2023-09-23T20:06:26.589184+0000","last_clean":"2023-09-23T20:06:26.589184+0000","last_became_active":"2023-09-23T20:03:13.240374+0000","last_became_peered":"2023-09-23T20:03:13.240374+0000","last_unstale":"2023-09-23T20:06:26.589184+0000","last_undegraded":"2023-09-23T20:06:26.589184+0000","last_fullsized":"2023-09-23T20:06:26.589184+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:15.250266+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":133,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723186+0000","last_change":"2023-09-23T19:59:09.218898+0000","last_active":"2023-09-23T20:06:26.723186+0000","last_peered":"2023-09-23T20:06:26.723186+0000","last_clean":"2023-09-23T20:06:26.723186+0000","last_became_active":"2023-09-23T19:59:09.169296+0000","last_became_peered":"2023-09-23T19:59:09.169296+0000","last_unstale":"2023-09-23T20:06:26.723186+0000","last_undegraded":"2023-09-23T20:06:26.723186+0000","last_fullsized":"2023-09-23T20:06:26.723186+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:56.994536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":312,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721777+0000","last_change":"2023-09-23T20:06:20.584753+0000","last_active":"2023-09-23T20:06:26.721777+0000","last_peered":"2023-09-23T20:06:26.721777+0000","last_clean":"2023-09-23T20:06:26.721777+0000","last_became_active":"2023-09-23T20:06:20.584406+0000","last_became_peered":"2023-09-23T20:06:20.584406+0000","last_unstale":"2023-09-23T20:06:26.721777+0000","last_undegraded":"2023-09-23T20:06:26.721777+0000","last_fullsized":"2023-09-23T20:06:26.721777+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:39.895733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"306'863","reported_seq":877,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.343263+0000","last_change":"2023-09-23T20:06:20.581915+0000","last_active":"2023-09-23T20:06:33.343263+0000","last_peered":"2023-09-23T20:06:33.343263+0000","last_clean":"2023-09-23T20:06:33.343263+0000","last_became_active":"2023-09-23T20:06:20.581361+0000","last_became_peered":"2023-09-23T20:06:20.581361+0000","last_unstale":"2023-09-23T20:06:33.343263+0000","last_undegraded":"2023-09-23T20:06:33.343263+0000","last_fullsized":"2023-09-23T20:06:33.343263+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":863,"log_dups_size":0,"ondisk_log_size":863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:34.078259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":863,"num_write_kb":55232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":665,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589593+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:06:26.589593+0000","last_peered":"2023-09-23T20:06:26.589593+0000","last_clean":"2023-09-23T20:06:26.589593+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:06:26.589593+0000","last_undegraded":"2023-09-23T20:06:26.589593+0000","last_fullsized":"2023-09-23T20:06:26.589593+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":262,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721568+0000","last_change":"2023-09-23T20:06:20.570003+0000","last_active":"2023-09-23T20:06:26.721568+0000","last_peered":"2023-09-23T20:06:26.721568+0000","last_clean":"2023-09-23T20:06:26.721568+0000","last_became_active":"2023-09-23T20:06:20.569645+0000","last_became_peered":"2023-09-23T20:06:20.569645+0000","last_unstale":"2023-09-23T20:06:26.721568+0000","last_undegraded":"2023-09-23T20:06:26.721568+0000","last_fullsized":"2023-09-23T20:06:26.721568+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:49.053826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":292,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721928+0000","last_change":"2023-09-23T20:06:20.583559+0000","last_active":"2023-09-23T20:06:26.721928+0000","last_peered":"2023-09-23T20:06:26.721928+0000","last_clean":"2023-09-23T20:06:26.721928+0000","last_became_active":"2023-09-23T20:06:20.582997+0000","last_became_peered":"2023-09-23T20:06:20.582997+0000","last_unstale":"2023-09-23T20:06:26.721928+0000","last_undegraded":"2023-09-23T20:06:26.721928+0000","last_fullsized":"2023-09-23T20:06:26.721928+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:26.720311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"306'886","reported_seq":898,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.536114+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:06:33.536114+0000","last_peered":"2023-09-23T20:06:33.536114+0000","last_clean":"2023-09-23T20:06:33.536114+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:06:33.536114+0000","last_undegraded":"2023-09-23T20:06:33.536114+0000","last_fullsized":"2023-09-23T20:06:33.536114+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":886,"log_dups_size":0,"ondisk_log_size":886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58064896,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":886,"num_write_kb":56704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":266,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721302+0000","last_change":"2023-09-23T19:50:23.998656+0000","last_active":"2023-09-23T20:06:26.721302+0000","last_peered":"2023-09-23T20:06:26.721302+0000","last_clean":"2023-09-23T20:06:26.721302+0000","last_became_active":"2023-09-23T19:50:23.998325+0000","last_became_peered":"2023-09-23T19:50:23.998325+0000","last_unstale":"2023-09-23T20:06:26.721302+0000","last_undegraded":"2023-09-23T20:06:26.721302+0000","last_fullsized":"2023-09-23T20:06:26.721302+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.824280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":322,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721938+0000","last_change":"2023-09-23T20:06:20.572571+0000","last_active":"2023-09-23T20:06:26.721938+0000","last_peered":"2023-09-23T20:06:26.721938+0000","last_clean":"2023-09-23T20:06:26.721938+0000","last_became_active":"2023-09-23T20:06:20.572233+0000","last_became_peered":"2023-09-23T20:06:20.572233+0000","last_unstale":"2023-09-23T20:06:26.721938+0000","last_undegraded":"2023-09-23T20:06:26.721938+0000","last_fullsized":"2023-09-23T20:06:26.721938+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:50.141837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"306'779","reported_seq":793,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.098320+0000","last_change":"2023-09-23T20:06:20.577649+0000","last_active":"2023-09-23T20:06:33.098320+0000","last_peered":"2023-09-23T20:06:33.098320+0000","last_clean":"2023-09-23T20:06:33.098320+0000","last_became_active":"2023-09-23T20:06:20.577149+0000","last_became_peered":"2023-09-23T20:06:20.577149+0000","last_unstale":"2023-09-23T20:06:33.098320+0000","last_undegraded":"2023-09-23T20:06:33.098320+0000","last_fullsized":"2023-09-23T20:06:33.098320+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":779,"log_dups_size":0,"ondisk_log_size":779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:20.326837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51052544,"num_objects":779,"num_object_clones":0,"num_object_copies":2337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":779,"num_write_kb":49856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":644,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.591134+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:06:26.591134+0000","last_peered":"2023-09-23T20:06:26.591134+0000","last_clean":"2023-09-23T20:06:26.591134+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:06:26.591134+0000","last_undegraded":"2023-09-23T20:06:26.591134+0000","last_fullsized":"2023-09-23T20:06:26.591134+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":53,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.722665+0000","last_change":"2023-09-23T20:06:20.577359+0000","last_active":"2023-09-23T20:06:26.722665+0000","last_peered":"2023-09-23T20:06:26.722665+0000","last_clean":"2023-09-23T20:06:26.722665+0000","last_became_active":"2023-09-23T20:06:20.576949+0000","last_became_peered":"2023-09-23T20:06:20.576949+0000","last_unstale":"2023-09-23T20:06:26.722665+0000","last_undegraded":"2023-09-23T20:06:26.722665+0000","last_fullsized":"2023-09-23T20:06:26.722665+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:03.482417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":157,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723869+0000","last_change":"2023-09-23T20:06:20.579092+0000","last_active":"2023-09-23T20:06:26.723869+0000","last_peered":"2023-09-23T20:06:26.723869+0000","last_clean":"2023-09-23T20:06:26.723869+0000","last_became_active":"2023-09-23T20:06:20.578763+0000","last_became_peered":"2023-09-23T20:06:20.578763+0000","last_unstale":"2023-09-23T20:06:26.723869+0000","last_undegraded":"2023-09-23T20:06:26.723869+0000","last_fullsized":"2023-09-23T20:06:26.723869+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:26.611172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.e","version":"306'848","reported_seq":864,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.340742+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:06:33.340742+0000","last_peered":"2023-09-23T20:06:33.340742+0000","last_clean":"2023-09-23T20:06:33.340742+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:06:33.340742+0000","last_undegraded":"2023-09-23T20:06:33.340742+0000","last_fullsized":"2023-09-23T20:06:33.340742+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":848,"log_dups_size":0,"ondisk_log_size":848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55574528,"num_objects":848,"num_object_clones":0,"num_object_copies":2544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":848,"num_write_kb":54272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":630,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721740+0000","last_change":"2023-09-23T20:04:02.277887+0000","last_active":"2023-09-23T20:06:26.721740+0000","last_peered":"2023-09-23T20:06:26.721740+0000","last_clean":"2023-09-23T20:06:26.721740+0000","last_became_active":"2023-09-23T20:04:02.277519+0000","last_became_peered":"2023-09-23T20:04:02.277519+0000","last_unstale":"2023-09-23T20:06:26.721740+0000","last_undegraded":"2023-09-23T20:06:26.721740+0000","last_fullsized":"2023-09-23T20:06:26.721740+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:45.813213+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":439,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589688+0000","last_change":"2023-09-23T20:06:23.565210+0000","last_active":"2023-09-23T20:06:26.589688+0000","last_peered":"2023-09-23T20:06:26.589688+0000","last_clean":"2023-09-23T20:06:26.589688+0000","last_became_active":"2023-09-23T20:06:23.564891+0000","last_became_peered":"2023-09-23T20:06:23.564891+0000","last_unstale":"2023-09-23T20:06:26.589688+0000","last_undegraded":"2023-09-23T20:06:26.589688+0000","last_fullsized":"2023-09-23T20:06:26.589688+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:23.121862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":306,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590342+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:06:26.590342+0000","last_peered":"2023-09-23T20:06:26.590342+0000","last_clean":"2023-09-23T20:06:26.590342+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:06:26.590342+0000","last_undegraded":"2023-09-23T20:06:26.590342+0000","last_fullsized":"2023-09-23T20:06:26.590342+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.591091+0000","last_change":"2023-09-23T20:06:20.573343+0000","last_active":"2023-09-23T20:06:26.591091+0000","last_peered":"2023-09-23T20:06:26.591091+0000","last_clean":"2023-09-23T20:06:26.591091+0000","last_became_active":"2023-09-23T20:06:20.572974+0000","last_became_peered":"2023-09-23T20:06:20.572974+0000","last_unstale":"2023-09-23T20:06:26.591091+0000","last_undegraded":"2023-09-23T20:06:26.591091+0000","last_fullsized":"2023-09-23T20:06:26.591091+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:04.435508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"306'806","reported_seq":821,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.101735+0000","last_change":"2023-09-23T20:06:19.575186+0000","last_active":"2023-09-23T20:06:33.101735+0000","last_peered":"2023-09-23T20:06:33.101735+0000","last_clean":"2023-09-23T20:06:33.101735+0000","last_became_active":"2023-09-23T20:06:19.574811+0000","last_became_peered":"2023-09-23T20:06:19.574811+0000","last_unstale":"2023-09-23T20:06:33.101735+0000","last_undegraded":"2023-09-23T20:06:33.101735+0000","last_fullsized":"2023-09-23T20:06:33.101735+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":806,"log_dups_size":0,"ondisk_log_size":806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:37.019476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52822016,"num_objects":806,"num_object_clones":0,"num_object_copies":2418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":806,"num_write_kb":51584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":617,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590078+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:06:26.590078+0000","last_peered":"2023-09-23T20:06:26.590078+0000","last_clean":"2023-09-23T20:06:26.590078+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:06:26.590078+0000","last_undegraded":"2023-09-23T20:06:26.590078+0000","last_fullsized":"2023-09-23T20:06:26.590078+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":896,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590988+0000","last_change":"2023-09-23T20:06:20.602963+0000","last_active":"2023-09-23T20:06:26.590988+0000","last_peered":"2023-09-23T20:06:26.590988+0000","last_clean":"2023-09-23T20:06:26.590988+0000","last_became_active":"2023-09-23T20:06:20.572914+0000","last_became_peered":"2023-09-23T20:06:20.572914+0000","last_unstale":"2023-09-23T20:06:26.590988+0000","last_undegraded":"2023-09-23T20:06:26.590988+0000","last_fullsized":"2023-09-23T20:06:26.590988+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:19.938048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":315,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721821+0000","last_change":"2023-09-23T20:06:20.574762+0000","last_active":"2023-09-23T20:06:26.721821+0000","last_peered":"2023-09-23T20:06:26.721821+0000","last_clean":"2023-09-23T20:06:26.721821+0000","last_became_active":"2023-09-23T20:06:20.574407+0000","last_became_peered":"2023-09-23T20:06:20.574407+0000","last_unstale":"2023-09-23T20:06:26.721821+0000","last_undegraded":"2023-09-23T20:06:26.721821+0000","last_fullsized":"2023-09-23T20:06:26.721821+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:24.255561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":275,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.725465+0000","last_change":"2023-09-23T20:06:20.587119+0000","last_active":"2023-09-23T20:06:26.725465+0000","last_peered":"2023-09-23T20:06:26.725465+0000","last_clean":"2023-09-23T20:06:26.725465+0000","last_became_active":"2023-09-23T20:06:20.586823+0000","last_became_peered":"2023-09-23T20:06:20.586823+0000","last_unstale":"2023-09-23T20:06:26.725465+0000","last_undegraded":"2023-09-23T20:06:26.725465+0000","last_fullsized":"2023-09-23T20:06:26.725465+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:30.188944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"306'334","reported_seq":351,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:30.775139+0000","last_change":"2023-09-23T20:06:20.571593+0000","last_active":"2023-09-23T20:06:30.775139+0000","last_peered":"2023-09-23T20:06:30.775139+0000","last_clean":"2023-09-23T20:06:30.775139+0000","last_became_active":"2023-09-23T20:06:20.571270+0000","last_became_peered":"2023-09-23T20:06:20.571270+0000","last_unstale":"2023-09-23T20:06:30.775139+0000","last_undegraded":"2023-09-23T20:06:30.775139+0000","last_fullsized":"2023-09-23T20:06:30.775139+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":334,"log_dups_size":0,"ondisk_log_size":334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:37.865876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21889024,"num_objects":334,"num_object_clones":0,"num_object_copies":1002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":334,"num_write_kb":21376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":541,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589227+0000","last_change":"2023-09-23T20:03:50.811904+0000","last_active":"2023-09-23T20:06:26.589227+0000","last_peered":"2023-09-23T20:06:26.589227+0000","last_clean":"2023-09-23T20:06:26.589227+0000","last_became_active":"2023-09-23T20:03:49.770120+0000","last_became_peered":"2023-09-23T20:03:49.770120+0000","last_unstale":"2023-09-23T20:06:26.589227+0000","last_undegraded":"2023-09-23T20:06:26.589227+0000","last_fullsized":"2023-09-23T20:06:26.589227+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:25.532724+0000","scrub_duration":0.0094872359999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":939,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.725391+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:06:26.725391+0000","last_peered":"2023-09-23T20:06:26.725391+0000","last_clean":"2023-09-23T20:06:26.725391+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:06:26.725391+0000","last_undegraded":"2023-09-23T20:06:26.725391+0000","last_fullsized":"2023-09-23T20:06:26.725391+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":120,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.725343+0000","last_change":"2023-09-23T20:06:20.581690+0000","last_active":"2023-09-23T20:06:26.725343+0000","last_peered":"2023-09-23T20:06:26.725343+0000","last_clean":"2023-09-23T20:06:26.725343+0000","last_became_active":"2023-09-23T20:06:20.581304+0000","last_became_peered":"2023-09-23T20:06:20.581304+0000","last_unstale":"2023-09-23T20:06:26.725343+0000","last_undegraded":"2023-09-23T20:06:26.725343+0000","last_fullsized":"2023-09-23T20:06:26.725343+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:49.134723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590456+0000","last_change":"2023-09-23T20:06:20.579459+0000","last_active":"2023-09-23T20:06:26.590456+0000","last_peered":"2023-09-23T20:06:26.590456+0000","last_clean":"2023-09-23T20:06:26.590456+0000","last_became_active":"2023-09-23T20:06:20.579120+0000","last_became_peered":"2023-09-23T20:06:20.579120+0000","last_unstale":"2023-09-23T20:06:26.590456+0000","last_undegraded":"2023-09-23T20:06:26.590456+0000","last_fullsized":"2023-09-23T20:06:26.590456+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:04.185400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"306'792","reported_seq":807,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.097643+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:06:33.097643+0000","last_peered":"2023-09-23T20:06:33.097643+0000","last_clean":"2023-09-23T20:06:33.097643+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:06:33.097643+0000","last_undegraded":"2023-09-23T20:06:33.097643+0000","last_fullsized":"2023-09-23T20:06:33.097643+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":792,"log_dups_size":0,"ondisk_log_size":792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51904512,"num_objects":792,"num_object_clones":0,"num_object_copies":2376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":792,"num_write_kb":50688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1114,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590648+0000","last_change":"2023-09-23T19:50:23.998858+0000","last_active":"2023-09-23T20:06:26.590648+0000","last_peered":"2023-09-23T20:06:26.590648+0000","last_clean":"2023-09-23T20:06:26.590648+0000","last_became_active":"2023-09-23T19:50:23.998301+0000","last_became_peered":"2023-09-23T19:50:23.998301+0000","last_unstale":"2023-09-23T20:06:26.590648+0000","last_undegraded":"2023-09-23T20:06:26.590648+0000","last_fullsized":"2023-09-23T20:06:26.590648+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:47.896459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":736,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589191+0000","last_change":"2023-09-23T20:06:23.564241+0000","last_active":"2023-09-23T20:06:26.589191+0000","last_peered":"2023-09-23T20:06:26.589191+0000","last_clean":"2023-09-23T20:06:26.589191+0000","last_became_active":"2023-09-23T20:06:23.563784+0000","last_became_peered":"2023-09-23T20:06:23.563784+0000","last_unstale":"2023-09-23T20:06:26.589191+0000","last_undegraded":"2023-09-23T20:06:26.589191+0000","last_fullsized":"2023-09-23T20:06:26.589191+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:07.702352+0000","scrub_duration":0.0051696529999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":44,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723057+0000","last_change":"2023-09-23T20:03:59.230682+0000","last_active":"2023-09-23T20:06:26.723057+0000","last_peered":"2023-09-23T20:06:26.723057+0000","last_clean":"2023-09-23T20:06:26.723057+0000","last_became_active":"2023-09-23T20:03:59.230486+0000","last_became_peered":"2023-09-23T20:03:59.230486+0000","last_unstale":"2023-09-23T20:06:26.723057+0000","last_undegraded":"2023-09-23T20:06:26.723057+0000","last_fullsized":"2023-09-23T20:06:26.723057+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:22.992793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":181,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.722068+0000","last_change":"2023-09-23T19:56:39.275031+0000","last_active":"2023-09-23T20:06:26.722068+0000","last_peered":"2023-09-23T20:06:26.722068+0000","last_clean":"2023-09-23T20:06:26.722068+0000","last_became_active":"2023-09-23T19:56:39.274791+0000","last_became_peered":"2023-09-23T19:56:39.274791+0000","last_unstale":"2023-09-23T20:06:26.722068+0000","last_undegraded":"2023-09-23T20:06:26.722068+0000","last_fullsized":"2023-09-23T20:06:26.722068+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:51.665602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"306'831","reported_seq":846,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.336802+0000","last_change":"2023-09-23T20:06:20.570618+0000","last_active":"2023-09-23T20:06:33.336802+0000","last_peered":"2023-09-23T20:06:33.336802+0000","last_clean":"2023-09-23T20:06:33.336802+0000","last_became_active":"2023-09-23T20:06:20.570313+0000","last_became_peered":"2023-09-23T20:06:20.570313+0000","last_unstale":"2023-09-23T20:06:33.336802+0000","last_undegraded":"2023-09-23T20:06:33.336802+0000","last_fullsized":"2023-09-23T20:06:33.336802+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":831,"log_dups_size":0,"ondisk_log_size":831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:48.995959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":831,"num_write_kb":53184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":441,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721582+0000","last_change":"2023-09-23T20:06:23.564775+0000","last_active":"2023-09-23T20:06:26.721582+0000","last_peered":"2023-09-23T20:06:26.721582+0000","last_clean":"2023-09-23T20:06:26.721582+0000","last_became_active":"2023-09-23T20:06:23.564409+0000","last_became_peered":"2023-09-23T20:06:23.564409+0000","last_unstale":"2023-09-23T20:06:26.721582+0000","last_undegraded":"2023-09-23T20:06:26.721582+0000","last_fullsized":"2023-09-23T20:06:26.721582+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:53:33.852619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:53:33.852619+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1068,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589154+0000","last_change":"2023-09-23T20:06:23.564034+0000","last_active":"2023-09-23T20:06:26.589154+0000","last_peered":"2023-09-23T20:06:26.589154+0000","last_clean":"2023-09-23T20:06:26.589154+0000","last_became_active":"2023-09-23T20:06:23.563320+0000","last_became_peered":"2023-09-23T20:06:23.563320+0000","last_unstale":"2023-09-23T20:06:26.589154+0000","last_undegraded":"2023-09-23T20:06:26.589154+0000","last_fullsized":"2023-09-23T20:06:26.589154+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:04.090224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":272,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721448+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:06:26.721448+0000","last_peered":"2023-09-23T20:06:26.721448+0000","last_clean":"2023-09-23T20:06:26.721448+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:06:26.721448+0000","last_undegraded":"2023-09-23T20:06:26.721448+0000","last_fullsized":"2023-09-23T20:06:26.721448+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":228,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.726072+0000","last_change":"2023-09-23T20:06:20.578344+0000","last_active":"2023-09-23T20:06:26.726072+0000","last_peered":"2023-09-23T20:06:26.726072+0000","last_clean":"2023-09-23T20:06:26.726072+0000","last_became_active":"2023-09-23T20:06:20.578026+0000","last_became_peered":"2023-09-23T20:06:20.578026+0000","last_unstale":"2023-09-23T20:06:26.726072+0000","last_undegraded":"2023-09-23T20:06:26.726072+0000","last_fullsized":"2023-09-23T20:06:26.726072+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:33.538944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.f","version":"306'323","reported_seq":340,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:30.780975+0000","last_change":"2023-09-23T20:06:20.571099+0000","last_active":"2023-09-23T20:06:30.780975+0000","last_peered":"2023-09-23T20:06:30.780975+0000","last_clean":"2023-09-23T20:06:30.780975+0000","last_became_active":"2023-09-23T20:06:20.570671+0000","last_became_peered":"2023-09-23T20:06:20.570671+0000","last_unstale":"2023-09-23T20:06:30.780975+0000","last_undegraded":"2023-09-23T20:06:30.780975+0000","last_fullsized":"2023-09-23T20:06:30.780975+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:33.235666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21168128,"num_objects":323,"num_object_clones":0,"num_object_copies":969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":323,"num_write_kb":20672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":622,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723642+0000","last_change":"2023-09-23T20:04:02.277648+0000","last_active":"2023-09-23T20:06:26.723642+0000","last_peered":"2023-09-23T20:06:26.723642+0000","last_clean":"2023-09-23T20:06:26.723642+0000","last_became_active":"2023-09-23T20:04:02.277271+0000","last_became_peered":"2023-09-23T20:04:02.277271+0000","last_unstale":"2023-09-23T20:06:26.723642+0000","last_undegraded":"2023-09-23T20:06:26.723642+0000","last_fullsized":"2023-09-23T20:06:26.723642+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:37.645184+0000","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":384,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589649+0000","last_change":"2023-09-23T19:48:37.571903+0000","last_active":"2023-09-23T20:06:26.589649+0000","last_peered":"2023-09-23T20:06:26.589649+0000","last_clean":"2023-09-23T20:06:26.589649+0000","last_became_active":"2023-09-23T19:48:37.571452+0000","last_became_peered":"2023-09-23T19:48:37.571452+0000","last_unstale":"2023-09-23T20:06:26.589649+0000","last_undegraded":"2023-09-23T20:06:26.589649+0000","last_fullsized":"2023-09-23T20:06:26.589649+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:11.484065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":140,"num_read_kb":86,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":225,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.725704+0000","last_change":"2023-09-23T20:06:20.574611+0000","last_active":"2023-09-23T20:06:26.725704+0000","last_peered":"2023-09-23T20:06:26.725704+0000","last_clean":"2023-09-23T20:06:26.725704+0000","last_became_active":"2023-09-23T20:06:20.574016+0000","last_became_peered":"2023-09-23T20:06:20.574016+0000","last_unstale":"2023-09-23T20:06:26.725704+0000","last_undegraded":"2023-09-23T20:06:26.725704+0000","last_fullsized":"2023-09-23T20:06:26.725704+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:50.774686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":102,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590723+0000","last_change":"2023-09-23T20:01:04.474034+0000","last_active":"2023-09-23T20:06:26.590723+0000","last_peered":"2023-09-23T20:06:26.590723+0000","last_clean":"2023-09-23T20:06:26.590723+0000","last_became_active":"2023-09-23T20:01:04.473108+0000","last_became_peered":"2023-09-23T20:01:04.473108+0000","last_unstale":"2023-09-23T20:06:26.590723+0000","last_undegraded":"2023-09-23T20:06:26.590723+0000","last_fullsized":"2023-09-23T20:06:26.590723+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:50.246998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"306'329","reported_seq":345,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:30.766034+0000","last_change":"2023-09-23T20:06:19.577254+0000","last_active":"2023-09-23T20:06:30.766034+0000","last_peered":"2023-09-23T20:06:30.766034+0000","last_clean":"2023-09-23T20:06:30.766034+0000","last_became_active":"2023-09-23T20:06:19.576977+0000","last_became_peered":"2023-09-23T20:06:19.576977+0000","last_unstale":"2023-09-23T20:06:30.766034+0000","last_undegraded":"2023-09-23T20:06:30.766034+0000","last_fullsized":"2023-09-23T20:06:30.766034+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":329,"log_dups_size":0,"ondisk_log_size":329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:23.789189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21561344,"num_objects":329,"num_object_clones":0,"num_object_copies":987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":329,"num_write_kb":21056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":684,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721777+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:06:26.721777+0000","last_peered":"2023-09-23T20:06:26.721777+0000","last_clean":"2023-09-23T20:06:26.721777+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:06:26.721777+0000","last_undegraded":"2023-09-23T20:06:26.721777+0000","last_fullsized":"2023-09-23T20:06:26.721777+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":133,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723896+0000","last_change":"2023-09-23T19:59:09.266748+0000","last_active":"2023-09-23T20:06:26.723896+0000","last_peered":"2023-09-23T20:06:26.723896+0000","last_clean":"2023-09-23T20:06:26.723896+0000","last_became_active":"2023-09-23T19:59:09.202029+0000","last_became_peered":"2023-09-23T19:59:09.202029+0000","last_unstale":"2023-09-23T20:06:26.723896+0000","last_undegraded":"2023-09-23T20:06:26.723896+0000","last_fullsized":"2023-09-23T20:06:26.723896+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.024089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":394,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721927+0000","last_change":"2023-09-23T20:06:23.565047+0000","last_active":"2023-09-23T20:06:26.721927+0000","last_peered":"2023-09-23T20:06:26.721927+0000","last_clean":"2023-09-23T20:06:26.721927+0000","last_became_active":"2023-09-23T20:06:23.564469+0000","last_became_peered":"2023-09-23T20:06:23.564469+0000","last_unstale":"2023-09-23T20:06:26.721927+0000","last_undegraded":"2023-09-23T20:06:26.721927+0000","last_fullsized":"2023-09-23T20:06:26.721927+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:39.290853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.725310+0000","last_change":"2023-09-23T20:05:39.782474+0000","last_active":"2023-09-23T20:06:26.725310+0000","last_peered":"2023-09-23T20:06:26.725310+0000","last_clean":"2023-09-23T20:06:26.725310+0000","last_became_active":"2023-09-23T20:05:39.782131+0000","last_became_peered":"2023-09-23T20:05:39.782131+0000","last_unstale":"2023-09-23T20:06:26.725310+0000","last_undegraded":"2023-09-23T20:06:26.725310+0000","last_fullsized":"2023-09-23T20:06:26.725310+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:11.924563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.d","version":"306'808","reported_seq":823,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:33.098276+0000","last_change":"2023-09-23T20:06:19.574603+0000","last_active":"2023-09-23T20:06:33.098276+0000","last_peered":"2023-09-23T20:06:33.098276+0000","last_clean":"2023-09-23T20:06:33.098276+0000","last_became_active":"2023-09-23T20:06:19.574038+0000","last_became_peered":"2023-09-23T20:06:19.574038+0000","last_unstale":"2023-09-23T20:06:33.098276+0000","last_undegraded":"2023-09-23T20:06:33.098276+0000","last_fullsized":"2023-09-23T20:06:33.098276+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":808,"log_dups_size":0,"ondisk_log_size":808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:03.285861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52953088,"num_objects":808,"num_object_clones":0,"num_object_copies":2424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":808,"num_write_kb":51712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":563,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723526+0000","last_change":"2023-09-23T20:01:45.744111+0000","last_active":"2023-09-23T20:06:26.723526+0000","last_peered":"2023-09-23T20:06:26.723526+0000","last_clean":"2023-09-23T20:06:26.723526+0000","last_became_active":"2023-09-23T19:59:44.955237+0000","last_became_peered":"2023-09-23T19:59:44.955237+0000","last_unstale":"2023-09-23T20:06:26.723526+0000","last_undegraded":"2023-09-23T20:06:26.723526+0000","last_fullsized":"2023-09-23T20:06:26.723526+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:45.723774+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":62,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.589941+0000","last_change":"2023-09-23T20:03:46.715124+0000","last_active":"2023-09-23T20:06:26.589941+0000","last_peered":"2023-09-23T20:06:26.589941+0000","last_clean":"2023-09-23T20:06:26.589941+0000","last_became_active":"2023-09-23T20:03:46.688460+0000","last_became_peered":"2023-09-23T20:03:46.688460+0000","last_unstale":"2023-09-23T20:06:26.589941+0000","last_undegraded":"2023-09-23T20:06:26.589941+0000","last_fullsized":"2023-09-23T20:06:26.589941+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:27.278740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":33,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.722007+0000","last_change":"2023-09-23T20:06:20.587032+0000","last_active":"2023-09-23T20:06:26.722007+0000","last_peered":"2023-09-23T20:06:26.722007+0000","last_clean":"2023-09-23T20:06:26.722007+0000","last_became_active":"2023-09-23T20:06:20.581473+0000","last_became_peered":"2023-09-23T20:06:20.581473+0000","last_unstale":"2023-09-23T20:06:26.722007+0000","last_undegraded":"2023-09-23T20:06:26.722007+0000","last_fullsized":"2023-09-23T20:06:26.722007+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:01.054114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":265,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721316+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:06:26.721316+0000","last_peered":"2023-09-23T20:06:26.721316+0000","last_clean":"2023-09-23T20:06:26.721316+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:06:26.721316+0000","last_undegraded":"2023-09-23T20:06:26.721316+0000","last_fullsized":"2023-09-23T20:06:26.721316+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":52,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.725752+0000","last_change":"2023-09-23T20:03:39.685740+0000","last_active":"2023-09-23T20:06:26.725752+0000","last_peered":"2023-09-23T20:06:26.725752+0000","last_clean":"2023-09-23T20:06:26.725752+0000","last_became_active":"2023-09-23T20:03:39.685365+0000","last_became_peered":"2023-09-23T20:03:39.685365+0000","last_unstale":"2023-09-23T20:06:26.725752+0000","last_undegraded":"2023-09-23T20:06:26.725752+0000","last_fullsized":"2023-09-23T20:06:26.725752+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:59.187694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":78,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.725916+0000","last_change":"2023-09-23T20:06:20.574625+0000","last_active":"2023-09-23T20:06:26.725916+0000","last_peered":"2023-09-23T20:06:26.725916+0000","last_clean":"2023-09-23T20:06:26.725916+0000","last_became_active":"2023-09-23T20:06:20.574289+0000","last_became_peered":"2023-09-23T20:06:20.574289+0000","last_unstale":"2023-09-23T20:06:26.725916+0000","last_undegraded":"2023-09-23T20:06:26.725916+0000","last_fullsized":"2023-09-23T20:06:26.725916+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:43.931903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":281,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590626+0000","last_change":"2023-09-23T20:06:20.594021+0000","last_active":"2023-09-23T20:06:26.590626+0000","last_peered":"2023-09-23T20:06:26.590626+0000","last_clean":"2023-09-23T20:06:26.590626+0000","last_became_active":"2023-09-23T20:06:20.582895+0000","last_became_peered":"2023-09-23T20:06:20.582895+0000","last_unstale":"2023-09-23T20:06:26.590626+0000","last_undegraded":"2023-09-23T20:06:26.590626+0000","last_fullsized":"2023-09-23T20:06:26.590626+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:44.000940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":312,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721828+0000","last_change":"2023-09-23T20:06:20.572732+0000","last_active":"2023-09-23T20:06:26.721828+0000","last_peered":"2023-09-23T20:06:26.721828+0000","last_clean":"2023-09-23T20:06:26.721828+0000","last_became_active":"2023-09-23T20:06:20.572385+0000","last_became_peered":"2023-09-23T20:06:20.572385+0000","last_unstale":"2023-09-23T20:06:26.721828+0000","last_undegraded":"2023-09-23T20:06:26.721828+0000","last_fullsized":"2023-09-23T20:06:26.721828+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:59.528899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":285,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590297+0000","last_change":"2023-09-23T20:06:20.590577+0000","last_active":"2023-09-23T20:06:26.590297+0000","last_peered":"2023-09-23T20:06:26.590297+0000","last_clean":"2023-09-23T20:06:26.590297+0000","last_became_active":"2023-09-23T20:06:20.576210+0000","last_became_peered":"2023-09-23T20:06:20.576210+0000","last_unstale":"2023-09-23T20:06:26.590297+0000","last_undegraded":"2023-09-23T20:06:26.590297+0000","last_fullsized":"2023-09-23T20:06:26.590297+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:49.974074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":302,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721519+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:06:26.721519+0000","last_peered":"2023-09-23T20:06:26.721519+0000","last_clean":"2023-09-23T20:06:26.721519+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:06:26.721519+0000","last_undegraded":"2023-09-23T20:06:26.721519+0000","last_fullsized":"2023-09-23T20:06:26.721519+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":138,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.723819+0000","last_change":"2023-09-23T20:06:20.577877+0000","last_active":"2023-09-23T20:06:26.723819+0000","last_peered":"2023-09-23T20:06:26.723819+0000","last_clean":"2023-09-23T20:06:26.723819+0000","last_became_active":"2023-09-23T20:06:20.577514+0000","last_became_peered":"2023-09-23T20:06:20.577514+0000","last_unstale":"2023-09-23T20:06:26.723819+0000","last_undegraded":"2023-09-23T20:06:26.723819+0000","last_fullsized":"2023-09-23T20:06:26.723819+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:36.817119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":44,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T20:06:28.596542+0000","last_change":"2023-09-23T20:03:59.236048+0000","last_active":"2023-09-23T20:06:28.596542+0000","last_peered":"2023-09-23T20:06:28.596542+0000","last_clean":"2023-09-23T20:06:28.596542+0000","last_became_active":"2023-09-23T20:03:59.235747+0000","last_became_peered":"2023-09-23T20:03:59.235747+0000","last_unstale":"2023-09-23T20:06:28.596542+0000","last_undegraded":"2023-09-23T20:06:28.596542+0000","last_fullsized":"2023-09-23T20:06:28.596542+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:48.969802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":125,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.590295+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:06:26.590295+0000","last_peered":"2023-09-23T20:06:26.590295+0000","last_clean":"2023-09-23T20:06:26.590295+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:06:26.590295+0000","last_undegraded":"2023-09-23T20:06:26.590295+0000","last_fullsized":"2023-09-23T20:06:26.590295+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":342,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:06:26.721365+0000","last_change":"2023-09-23T20:06:20.571830+0000","last_active":"2023-09-23T20:06:26.721365+0000","last_peered":"2023-09-23T20:06:26.721365+0000","last_clean":"2023-09-23T20:06:26.721365+0000","last_became_active":"2023-09-23T20:06:20.571511+0000","last_became_peered":"2023-09-23T20:06:20.571511+0000","last_unstale":"2023-09-23T20:06:26.721365+0000","last_undegraded":"2023-09-23T20:06:26.721365+0000","last_fullsized":"2023-09-23T20:06:26.721365+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:02.547357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":5538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":707395584,"num_objects":10794,"num_object_clones":0,"num_object_copies":32382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10794,"num_write_kb":690816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2270363648,"data_stored":2270363648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":10794,"ondisk_log_size":10794,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686386,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234305,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":543784,"kb_used_data":252576,"kb_used_omap":0,"kb_used_meta":291200,"kb_avail":93201368,"statfs":{"total":95995035648,"available":95438200832,"internally_reserved":0,"allocated":258637824,"data_stored":257701732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":298188800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:06:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":2.4049999999999998,"15min":2.21},"min":{"1min":0.315,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":4.9530000000000003,"5min":60.140000000000001,"15min":62.061999999999998},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.361,"5min":2.806,"15min":2.4849999999999999},"min":{"1min":0.26300000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.8140000000000001,"5min":62.389000000000003,"15min":62.389000000000003},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:06:26 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":5.1269999999999998,"15min":3.2839999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":11.685,"5min":104.16800000000001,"15min":104.16800000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.3540000000000001,"5min":5.4329999999999998,"15min":3.3969999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.21199999999999999},"max":{"1min":11.319000000000001,"5min":104.003,"15min":104.003},"last":0.621}]},{"osd":2,"last update":"Sat Sep 23 20:05:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":3.0339999999999998,"15min":3.246},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":14.555999999999999,"5min":105.842,"15min":130.673},"last":0.41699999999999998},{"interface":"front","average":{"1min":3.7450000000000001,"5min":3.8239999999999998,"15min":3.637},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":15.163,"5min":105.66,"15min":130.71100000000001},"last":1.0700000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:06:31 2023","interfaces":[{"interface":"back","average":{"1min":4.6630000000000003,"5min":6.1760000000000002,"15min":3.9279999999999999},"min":{"1min":0.38800000000000001,"5min":0.248,"15min":0.247},"max":{"1min":56.134999999999998,"5min":146.38800000000001,"15min":146.38800000000001},"last":0.97999999999999998},{"interface":"front","average":{"1min":4.391,"5min":6.0629999999999997,"15min":3.915},"min":{"1min":0.41199999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":57.642000000000003,"5min":146.851,"15min":146.851},"last":0.58999999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:06:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.78600000000000003,"15min":1.887},"min":{"1min":0.23499999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":1.1990000000000001,"5min":19.215,"15min":143.255},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.143,"15min":1.9259999999999999},"min":{"1min":0.307,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.2430000000000001,"5min":41.822000000000003,"15min":143.41200000000001},"last":0.45000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:06:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.0549999999999999,"15min":1.4630000000000001},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.25},"max":{"1min":2.2519999999999998,"5min":35.488,"15min":84.566999999999993},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.95899999999999996,"15min":1.2969999999999999},"min":{"1min":0.28299999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.234,"5min":34.896000000000001,"15min":81.244},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.1220000000000001,"15min":0.84699999999999998},"min":{"1min":0.32500000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.6869999999999998,"5min":41.902999999999999,"15min":41.902999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.67900000000000005,"15min":0.63900000000000001},"min":{"1min":0.36699999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.714,"5min":6.3620000000000001,"15min":6.3620000000000001},"last":0.57599999999999996}]}]},{"osd":6,"up_from":0,"seq":1078036791346,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762506,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926041,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":782092,"kb_used_data":308292,"kb_used_omap":8,"kb_used_meta":473783,"kb_avail":92963060,"statfs":{"total":95995035648,"available":95194173440,"internally_reserved":0,"allocated":315691008,"data_stored":314315355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":485154401},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":6.4210000000000003,"5min":5.6749999999999998,"15min":3.218},"min":{"1min":0.224,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":48.087000000000003,"5min":94.055999999999997,"15min":94.055999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":5.7530000000000001,"5min":5.806,"15min":3.5310000000000001},"min":{"1min":0.187,"5min":0.183,"15min":0.183},"max":{"1min":48.119999999999997,"5min":91.632999999999996,"15min":91.632999999999996},"last":2.198}]},{"osd":2,"last update":"Sat Sep 23 20:05:59 2023","interfaces":[{"interface":"back","average":{"1min":6.0030000000000001,"5min":8.3819999999999997,"15min":4.7110000000000003},"min":{"1min":0.34100000000000003,"5min":0.20899999999999999,"15min":0.19800000000000001},"max":{"1min":55.634999999999998,"5min":270.51100000000002,"15min":270.51100000000002},"last":0.54300000000000004},{"interface":"front","average":{"1min":5.5389999999999997,"5min":7.5759999999999996,"15min":4.6559999999999997},"min":{"1min":0.33600000000000002,"5min":0.23200000000000001,"15min":0.22600000000000001},"max":{"1min":54.686999999999998,"5min":269.69299999999998,"15min":269.69299999999998},"last":1.2310000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":7.0720000000000001,"5min":5.6989999999999998,"15min":5.0940000000000003},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":51.765999999999998,"5min":116.733,"15min":145.72800000000001},"last":2.9020000000000001},{"interface":"front","average":{"1min":7.0499999999999998,"5min":5.8250000000000002,"15min":5.2060000000000004},"min":{"1min":0.38900000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":51.723999999999997,"5min":116.697,"15min":145.50700000000001},"last":2.3050000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:06:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.7789999999999999,"15min":2.911},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.3129999999999997,"5min":148.20599999999999,"15min":148.20599999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":2.0899999999999999,"5min":3.0830000000000002,"15min":2.6160000000000001},"min":{"1min":0.34499999999999997,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":10.272,"5min":148.11000000000001,"15min":148.11000000000001},"last":2.2389999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:06:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":2.3170000000000002,"15min":2.694},"min":{"1min":0.35999999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.3179999999999996,"5min":64.725999999999999,"15min":131.34399999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.3149999999999999,"15min":2.6899999999999999},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.3929999999999998,"5min":8.4139999999999997,"15min":131.386},"last":0.72999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.6699999999999999,"15min":1.6279999999999999},"min":{"1min":0.42499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.1579999999999995,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.79500000000000004},{"interface":"front","average":{"1min":2.2210000000000001,"5min":2.133,"15min":1.9810000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":13.766999999999999,"5min":13.766999999999999,"15min":13.766999999999999},"last":2.2730000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:06:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":2.355,"15min":2.2090000000000001},"min":{"1min":0.38200000000000001,"5min":0.31900000000000001,"15min":0.29699999999999999},"max":{"1min":6.3650000000000002,"5min":65.081999999999994,"15min":77.132999999999996},"last":0.56599999999999995},{"interface":"front","average":{"1min":2.0950000000000002,"5min":1.647,"15min":2.2749999999999999},"min":{"1min":0.40699999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":14.916,"5min":14.916,"15min":75.465999999999994},"last":0.76500000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598980,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1064388,"kb_used_data":471612,"kb_used_omap":12,"kb_used_meta":592755,"kb_avail":92680764,"statfs":{"total":95995035648,"available":94905102336,"internally_reserved":0,"allocated":482930688,"data_stored":481564896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":606982142},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:06:01 2023","interfaces":[{"interface":"back","average":{"1min":4.0010000000000003,"5min":3.2930000000000001,"15min":3.1589999999999998},"min":{"1min":0.219,"5min":0.20200000000000001,"15min":0.154},"max":{"1min":17.116,"5min":76.251999999999995,"15min":76.251999999999995},"last":0.309},{"interface":"front","average":{"1min":4.0259999999999998,"5min":3.411,"15min":3.0920000000000001},"min":{"1min":0.34599999999999997,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":11.34,"5min":76.590000000000003,"15min":76.590000000000003},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:06:01 2023","interfaces":[{"interface":"back","average":{"1min":5.0599999999999996,"5min":5.2889999999999997,"15min":3.411},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.184},"max":{"1min":25.102,"5min":94.768000000000001,"15min":94.768000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":3.6349999999999998,"5min":4.931,"15min":3.5259999999999998},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":12.659000000000001,"5min":96.808000000000007,"15min":96.808000000000007},"last":0.45600000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:06:27 2023","interfaces":[{"interface":"back","average":{"1min":6.9550000000000001,"5min":6.3579999999999997,"15min":4.1189999999999998},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.20499999999999999},"max":{"1min":84.207999999999998,"5min":107.381,"15min":107.381},"last":1.006},{"interface":"front","average":{"1min":2.536,"5min":4.3259999999999996,"15min":3.6280000000000001},"min":{"1min":0.25800000000000001,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":9.4359999999999999,"5min":106.301,"15min":106.301},"last":0.69099999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:06:01 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.3200000000000001,"15min":1.847},"min":{"1min":0.29299999999999998,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":8.2449999999999992,"5min":10.525,"15min":50.640000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.7679999999999998,"5min":1.7649999999999999,"15min":1.901},"min":{"1min":0.38500000000000001,"5min":0.28999999999999998,"15min":0.26700000000000002},"max":{"1min":9.0879999999999992,"5min":9.7629999999999999,"15min":50.154000000000003},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:06:01 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":3.0550000000000002,"15min":2.6240000000000001},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":8.3480000000000008,"5min":172.505,"15min":172.505},"last":0.63},{"interface":"front","average":{"1min":2.9710000000000001,"5min":3.4140000000000001,"15min":2.5870000000000002},"min":{"1min":0.38800000000000001,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":9.6910000000000007,"5min":172.375,"15min":172.375},"last":0.71299999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.6659999999999999,"15min":1.639},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.3759999999999994,"5min":10.144,"15min":10.144},"last":3.5259999999999998},{"interface":"front","average":{"1min":2.101,"5min":1.9430000000000001,"15min":1.96},"min":{"1min":0.437,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.3879999999999999,"5min":9.3879999999999999,"15min":9.3879999999999999},"last":0.74199999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:06:01 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":1.889,"15min":1.9970000000000001},"min":{"1min":0.42099999999999999,"5min":0.35099999999999998,"15min":0.27400000000000002},"max":{"1min":9.1170000000000009,"5min":9.7330000000000005,"15min":50.118000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.484,"15min":2.2050000000000001},"min":{"1min":0.41599999999999998,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":8.2970000000000006,"5min":10.449999999999999,"15min":50.664999999999999},"last":3.4820000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435453,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":627084,"kb_used_data":97668,"kb_used_omap":4,"kb_used_meta":529403,"kb_avail":93118068,"statfs":{"total":95995035648,"available":95352901632,"internally_reserved":0,"allocated":100012032,"data_stored":98515878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":542108781},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:06:22 2023","interfaces":[{"interface":"back","average":{"1min":3.524,"5min":2.9649999999999999,"15min":4.3680000000000003},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":15.699,"5min":21.100999999999999,"15min":240.40600000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":3.7229999999999999,"5min":3.621,"15min":4.9379999999999997},"min":{"1min":0.22900000000000001,"5min":0.19700000000000001,"15min":0.17199999999999999},"max":{"1min":16.623000000000001,"5min":53.201000000000001,"15min":240.352},"last":0.313}]},{"osd":1,"last update":"Sat Sep 23 20:06:22 2023","interfaces":[{"interface":"back","average":{"1min":8.2650000000000006,"5min":4.9219999999999997,"15min":2.9670000000000001},"min":{"1min":0.20300000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":112.122,"5min":112.122,"15min":112.122},"last":0.66700000000000004},{"interface":"front","average":{"1min":9.1859999999999999,"5min":5.4089999999999998,"15min":3.0600000000000001},"min":{"1min":0.255,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":111.786,"5min":111.786,"15min":111.786},"last":0.73899999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:06:22 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":3.444,"15min":5.6630000000000003},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.219},"max":{"1min":10.222,"5min":82.111999999999995,"15min":264.97800000000001},"last":0.79400000000000004},{"interface":"front","average":{"1min":3.3820000000000001,"5min":3.4319999999999999,"15min":5.4299999999999997},"min":{"1min":0.30399999999999999,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":15.782,"5min":30.768999999999998,"15min":257.40899999999999},"last":0.97799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":4.6970000000000001,"15min":2.9929999999999999},"min":{"1min":0.42599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.1159999999999997,"5min":109.66800000000001,"15min":109.66800000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":2.3090000000000002,"5min":5.0199999999999996,"15min":3.46},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.9700000000000006,"5min":110.295,"15min":110.295},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 20:06:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3220000000000001,"5min":4.3099999999999996,"15min":2.782},"min":{"1min":0.34000000000000002,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":9.0009999999999994,"5min":113.881,"15min":113.881},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.73,"5min":3.8410000000000002,"15min":2.573},"min":{"1min":0.39800000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.258,"5min":114.598,"15min":114.598},"last":0.69799999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:05:36 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":4.1970000000000001,"15min":5.2329999999999997},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8479999999999999,"5min":82.281999999999996,"15min":82.281999999999996},"last":0.75600000000000001},{"interface":"front","average":{"1min":3.2040000000000002,"5min":4.4580000000000002,"15min":5.319},"min":{"1min":0.42799999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.554,"5min":82.233000000000004,"15min":82.233000000000004},"last":0.83999999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":3.831,"15min":3.1160000000000001},"min":{"1min":0.39100000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.1829999999999998,"5min":112.883,"15min":138.035},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.0499999999999998,"5min":3.9790000000000001,"15min":2.984},"min":{"1min":0.40799999999999997,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":9.0570000000000004,"5min":111.85599999999999,"15min":138.054},"last":0.80900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019215,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":692708,"kb_used_data":405532,"kb_used_omap":0,"kb_used_meta":287168,"kb_avail":93052444,"statfs":{"total":95995035648,"available":95285702656,"internally_reserved":0,"allocated":415264768,"data_stored":414332760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":294060032},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:06:11 2023","interfaces":[{"interface":"back","average":{"1min":3.569,"5min":1.9119999999999999,"15min":3.2919999999999998},"min":{"1min":0.22600000000000001,"5min":0.22,"15min":0.189},"max":{"1min":33.911000000000001,"5min":33.911000000000001,"15min":104.604},"last":0.28699999999999998},{"interface":"front","average":{"1min":3.3319999999999999,"5min":1.8520000000000001,"15min":3.21},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":33.899000000000001,"5min":33.899000000000001,"15min":104.687},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:06:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0539999999999998,"5min":10.629,"15min":5.2050000000000001},"min":{"1min":0.32500000000000001,"5min":0.23200000000000001,"15min":0.216},"max":{"1min":34.073,"5min":161.613,"15min":161.613},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.7679999999999998,"5min":10.436999999999999,"15min":5.0279999999999996},"min":{"1min":0.30599999999999999,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":35.198999999999998,"5min":162.017,"15min":162.017},"last":0.53100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:05:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8820000000000001,"5min":5.6109999999999998,"15min":5.2670000000000003},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":9.0500000000000007,"5min":116.56,"15min":144.20099999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":3.149,"5min":5.907,"15min":5.1139999999999999},"min":{"1min":0.28100000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":19.385999999999999,"5min":117.03400000000001,"15min":144.184},"last":0.501}]},{"osd":3,"last update":"Sat Sep 23 20:06:21 2023","interfaces":[{"interface":"back","average":{"1min":7.524,"5min":5.8200000000000003,"15min":5.7869999999999999},"min":{"1min":0.33500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":118.31699999999999,"5min":118.31699999999999,"15min":266.54700000000003},"last":0.77000000000000002},{"interface":"front","average":{"1min":7.5099999999999998,"5min":5.7480000000000002,"15min":4.8079999999999998},"min":{"1min":0.33000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":118.395,"5min":118.395,"15min":139.911},"last":0.42399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:06:11 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.48199999999999998,"15min":1.849},"min":{"1min":0.23999999999999999,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":0.55700000000000005,"5min":4.077,"15min":266.59899999999999},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.53100000000000003,"15min":1.954},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.24399999999999999},"max":{"1min":0.72299999999999998,"5min":3.004,"15min":266.72899999999998},"last":0.70599999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.53000000000000003,"15min":0.51200000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.605,"5min":4.0410000000000004,"15min":4.0410000000000004},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.66800000000000004,"15min":0.54300000000000004},"min":{"1min":0.34499999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.615,"5min":4.2439999999999998,"15min":4.2439999999999998},"last":0.66900000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:05:53 2023","interfaces":[{"interface":"back","average":{"1min":4.0010000000000003,"5min":2.073,"15min":1.4730000000000001},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":76.507000000000005,"5min":76.507000000000005,"15min":76.507000000000005},"last":0.87},{"interface":"front","average":{"1min":4.3099999999999996,"5min":2.3079999999999998,"15min":1.5700000000000001},"min":{"1min":0.26600000000000001,"5min":0.26100000000000001,"15min":0.25800000000000001},"max":{"1min":76.462000000000003,"5min":76.462000000000003,"15min":76.462000000000003},"last":1.1579999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757579,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":687580,"kb_used_data":383316,"kb_used_omap":0,"kb_used_meta":304256,"kb_avail":93057572,"statfs":{"total":95995035648,"available":95290953728,"internally_reserved":0,"allocated":392515584,"data_stored":391588017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":311558144},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:06:07 2023","interfaces":[{"interface":"back","average":{"1min":8.5459999999999994,"5min":2.96,"15min":2.0859999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":121.94799999999999,"5min":121.94799999999999,"15min":121.94799999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":9.1780000000000008,"5min":3.4940000000000002,"15min":2.1989999999999998},"min":{"1min":0.22800000000000001,"5min":0.22700000000000001,"15min":0.216},"max":{"1min":121.964,"5min":121.964,"15min":121.964},"last":0.41599999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:06:24 2023","interfaces":[{"interface":"back","average":{"1min":8.4260000000000002,"5min":8.6400000000000006,"15min":5.7850000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":127.41800000000001,"5min":171.68299999999999,"15min":266.197},"last":0.749},{"interface":"front","average":{"1min":8.9939999999999998,"5min":9.0809999999999995,"15min":5.907},"min":{"1min":0.33600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":127.37,"5min":171.64099999999999,"15min":266.14999999999998},"last":0.65700000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:05:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":3.609,"15min":5.7450000000000001},"min":{"1min":0.33300000000000002,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":12.420999999999999,"5min":177.816,"15min":238.96199999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":2.3769999999999998,"5min":3.6760000000000002,"15min":5.7910000000000004},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.313},"max":{"1min":12.390000000000001,"5min":177.851,"15min":238.905},"last":1.081}]},{"osd":3,"last update":"Sat Sep 23 20:06:24 2023","interfaces":[{"interface":"back","average":{"1min":8.0850000000000009,"5min":8.4239999999999995,"15min":4.9039999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":127.83799999999999,"5min":201.50899999999999,"15min":201.50899999999999},"last":1.639},{"interface":"front","average":{"1min":8.593,"5min":8.0410000000000004,"15min":4.7880000000000003},"min":{"1min":0.309,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":128.291,"5min":201.58099999999999,"15min":201.58099999999999},"last":3.0840000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:06:07 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":2.3900000000000001,"15min":1.603},"min":{"1min":0.26400000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.624,"5min":191.78899999999999,"15min":191.78899999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.56000000000000005,"5min":2.379,"15min":1.589},"min":{"1min":0.27700000000000002,"5min":0.255,"15min":0.255},"max":{"1min":3.2669999999999999,"5min":191.821,"15min":191.821},"last":0.38900000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:05:33 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.63200000000000001,"15min":0.60999999999999999},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.92000000000000004,"5min":4.0679999999999996,"15min":4.0679999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.48199999999999998,"15min":0.46800000000000003},"min":{"1min":0.32500000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.53100000000000003,"5min":3.3570000000000002,"15min":3.3570000000000002},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":2.431,"15min":1.744},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.26200000000000001},"max":{"1min":2.5659999999999998,"5min":191.84299999999999,"15min":191.84299999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.51500000000000001,"5min":2.52,"15min":1.9359999999999999},"min":{"1min":0.32700000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.766,"5min":191.69,"15min":191.69},"last":0.63100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":310902784,"data_stored":310902784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":552992768,"data_stored":552992768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":191234048,"data_stored":191234048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":455737344,"data_stored":455737344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":470417408,"data_stored":470417408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":289079296,"data_stored":289079296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:06:38.347 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-09-23T20:06:38.399 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:38.540 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:38.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:38.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:38.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:38.734 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:38.817 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:38.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:38.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:38.862 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:38.929 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:38.932 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 10 16 24434 24418 152.559 120.75 0.00682752 0.00654781 2023-09-23T20:06:38.964 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:39.030 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:39.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:39.076 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:39.134 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:39.157 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:38 smithi144 ceph-mon[194833]: pgmap v850: 99 pgs: 99 active+clean; 1.3 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-23T20:06:39.160 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:39.177 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:38 smithi135 ceph-mon[221113]: pgmap v850: 99 pgs: 99 active+clean; 1.3 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-23T20:06:39.177 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:38 smithi135 ceph-mon[213871]: pgmap v850: 99 pgs: 99 active+clean; 1.3 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-23T20:06:39.202 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:39.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:39.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:39.305 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:39.374 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:39.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3615387069' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:06:39.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:39 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:06:39.927 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:39.932 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 11 16 26260 26244 149.065 114.125 0.00984407 0.00670163 2023-09-23T20:06:39.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:39.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:39.933 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:06:40.034 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:40.088 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3615387069' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:06:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:39 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:06:40.133 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:06:40.133 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:06:40.133 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:06:40.133 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:06:40.133 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:06:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3615387069' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:06:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:39 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:06:40.933 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 12 16 28152 28136 146.497 118.25 0.00310414 0.00680007 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[213871]: pgmap v852: 99 pgs: 99 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:40 smithi135 ceph-mon[221113]: pgmap v852: 99 pgs: 99 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-09-23T20:06:41.337 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:41.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:41.346 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:06:41.346 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.347 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.347 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:06:41.347 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T20:06:41.347 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.347 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.347 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: failed to encode map e307 with expected crc 2023-09-23T20:06:41.347 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:40 smithi144 ceph-mon[194833]: pgmap v852: 99 pgs: 99 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-09-23T20:06:41.373 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:06:41.443 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:41.509 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:41.592 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:41.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:41.699 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:41.756 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:41.886 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:41.932 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 13 16 29830 29814 143.295 104.875 0.00220565 0.00695698 2023-09-23T20:06:42.172 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:41 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:06:42.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:41 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:06:42.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:41 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:06:42.313 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:42.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:42.433 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:42.509 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:42.804 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:42.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:42.909 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:42.932 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 14 16 31343 31327 139.814 94.5625 0.00563838 0.00714693 2023-09-23T20:06:42.948 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:43.039 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:06:43.058 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:06:43.077 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:06:43.098 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:06:43.211 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:06:43.258 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:06:43.298 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:06:43.331 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:06:43.343 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:06:43.344 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:06:43.344 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:06:43.344 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:06:43.344 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:06:43.344 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:06:43.344 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:06:43.344 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:06:43.534 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:43.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:43.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:43.642 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:43.729 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:43.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:43 smithi144 ceph-mon[194833]: pgmap v853: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-09-23T20:06:43.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4109193064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:43.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:43 smithi144 ceph-mon[194833]: 1.3 scrub starts 2023-09-23T20:06:43.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:43 smithi144 ceph-mon[194833]: 1.3 scrub ok 2023-09-23T20:06:43.888 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:43.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:43.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:43.932 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 15 16 32851 32835 136.777 94.25 0.0118346 0.00730722 2023-09-23T20:06:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:06:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:06:43] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-09-23T20:06:43.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:43 smithi135 ceph-mon[213871]: pgmap v853: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-09-23T20:06:43.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4109193064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:43.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:43 smithi135 ceph-mon[213871]: 1.3 scrub starts 2023-09-23T20:06:43.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:43 smithi135 ceph-mon[213871]: 1.3 scrub ok 2023-09-23T20:06:43.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:43 smithi135 ceph-mon[221113]: pgmap v853: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-09-23T20:06:43.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4109193064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:43.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:43 smithi135 ceph-mon[221113]: 1.3 scrub starts 2023-09-23T20:06:43.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:43 smithi135 ceph-mon[221113]: 1.3 scrub ok 2023-09-23T20:06:43.993 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:44.050 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:44.249 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:44.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:44.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:44.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:06:44.356 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:44.418 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:44.495 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:06:44.495 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:06:44.509 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:44.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:44.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:44.617 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:44.664 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:44.933 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 16 16 34268 34252 133.76 88.5625 0.00503671 0.00747072 2023-09-23T20:06:45.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:44 smithi144 ceph-mon[194833]: pgmap v854: 99 pgs: 99 active+clean; 1.9 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T20:06:45.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:44 smithi135 ceph-mon[213871]: pgmap v854: 99 pgs: 99 active+clean; 1.9 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T20:06:45.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:44 smithi135 ceph-mon[221113]: pgmap v854: 99 pgs: 99 active+clean; 1.9 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T20:06:45.234 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:45.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:45.341 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:45.369 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:45.490 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:45.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:45.599 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:45.651 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:45.715 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:45.823 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:06:45.934 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 17 16 35387 35371 129.998 69.9375 0.0175995 0.00768531 2023-09-23T20:06:45.967 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:45.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:45.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:46.077 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:06:46.135 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:46.231 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:06:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:46.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:06:46.343 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:46.418 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:46.438 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:46.566 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:46.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:46.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:46.674 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:46.746 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:46.760 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 1 pgp_num to 11 2023-09-23T20:06:46.934 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 18 16 36844 36828 127.834 91.0625 0.00702046 0.00781764 2023-09-23T20:06:46.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2789449774' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T20:06:46.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T20:06:46.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2789449774' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T20:06:46.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T20:06:47.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2789449774' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T20:06:47.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T20:06:47.257 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:47.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:47.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:47.367 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:47.415 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:47.436 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:06:47.935 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 19 16 38133 38117 125.346 80.5625 0.00692573 0.00797263 2023-09-23T20:06:48.030 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[221113]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.030 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T20:06:48.030 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[221113]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T20:06:48.030 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[221113]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.030 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[221113]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[221113]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[221113]: pgmap v856: 99 pgs: 99 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[213871]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[213871]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[213871]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[213871]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[213871]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:47 smithi135 ceph-mon[213871]: pgmap v856: 99 pgs: 99 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:06:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:47 smithi144 ceph-mon[194833]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:47 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T20:06:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:47 smithi144 ceph-mon[194833]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T20:06:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:47 smithi144 ceph-mon[194833]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:47 smithi144 ceph-mon[194833]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:47 smithi144 ceph-mon[194833]: failed to encode map e308 with expected crc 2023-09-23T20:06:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:47 smithi144 ceph-mon[194833]: pgmap v856: 99 pgs: 99 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:06:48.229 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:48.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:48.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:48.341 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:48.370 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:48.487 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:48.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:48.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:48.594 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:48.683 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:48.934 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:06:48.935400+0000 min lat: 0.00105689 max lat: 0.25183 avg lat: 0.00814948 2023-09-23T20:06:48.934 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:06:48.934 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 20 16 39266 39250 122.62 70.8125 0.013279 0.00814948 2023-09-23T20:06:48.985 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:48.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:49.094 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:49.130 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:06:49.130 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:06:49.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:48 smithi144 ceph-mon[194833]: pgmap v857: 99 pgs: 99 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:06:49.147 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:49.153 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:48 smithi135 ceph-mon[221113]: pgmap v857: 99 pgs: 99 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:06:49.153 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:48 smithi135 ceph-mon[213871]: pgmap v857: 99 pgs: 99 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:06:49.161 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:49.360 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:49.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:49.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:49.471 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:49.522 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:49.935 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 21 16 40681 40665 120.988 88.4375 0.00735387 0.00826061 2023-09-23T20:06:50.222 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:06:50.497 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:50.504 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3095522723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:50.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:50.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:50.606 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:50.660 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3095522723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:50.666 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3095522723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:06:50.669 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:50.703 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:06:50.940 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 22 16 41857 41841 118.815 73.5 0.0107239 0.00841095 2023-09-23T20:06:51.177 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:51.478 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3095522723' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:51.479 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:51 smithi144 ceph-mon[194833]: pgmap v859: 99 pgs: 99 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:06:51.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:06:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:06:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3095522723' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:51.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T20:06:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:51.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[213871]: pgmap v859: 99 pgs: 99 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:06:51.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:06:51.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:06:51.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3095522723' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:06:51.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T20:06:51.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: failed to encode map e309 with expected crc 2023-09-23T20:06:51.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:51.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:51.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:51 smithi135 ceph-mon[221113]: pgmap v859: 99 pgs: 99 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:06:51.748 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:51.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:51.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:51.855 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:51.889 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:51.938 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 23 16 42847 42831 116.34 61.875 0.0109044 0.00859033 2023-09-23T20:06:51.960 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:51.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:51.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:52.066 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:52.098 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:52.138 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:52.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:52.242 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:52.285 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:52.358 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:52.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:52.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:52.463 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:52.504 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:52.624 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:52.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:52.731 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:52.803 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:52.916 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:52.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:52.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:52.938 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 24 16 44131 44115 114.836 80.25 0.00730057 0.00870316 2023-09-23T20:06:52.985 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:06:53.019 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:06:53.021 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:53.059 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:06:53.087 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:53.100 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:06:53.139 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:06:53.171 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:06:53.171 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:06:53.181 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:06:53.196 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:53.196 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:53.196 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:53 smithi144 ceph-mon[194833]: pgmap v860: 99 pgs: 99 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:06:53.196 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1084976118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:53.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:53.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:53.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:53 smithi135 ceph-mon[221113]: pgmap v860: 99 pgs: 99 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:06:53.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1084976118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:53.222 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:06:53.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:53.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:06:53.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:53 smithi135 ceph-mon[213871]: pgmap v860: 99 pgs: 99 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:06:53.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1084976118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:06:53.261 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:06:53.268 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:06:53.268 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:06:53.268 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:06:53.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:06:53.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:06:53.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:06:53.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:06:53.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:06:53.601 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:53.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:53.708 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:53.737 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:53.780 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:53.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:53.883 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:53.927 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:53.927 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3] out_osds: [6, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:06:53.928 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:06:53.928 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 2, 4, 5, 7, 3] 2023-09-23T20:06:53.928 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T20:06:53.938 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 25 16 45327 45311 113.233 74.75 0.0108154 0.00882561 2023-09-23T20:06:53.974 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:06:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:06:53] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-09-23T20:06:54.203 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:06:54.210 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:54.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:54.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:54.222 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:06:54.320 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:54.345 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:54.456 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:54.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:54.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:54.567 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:54.630 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:54.775 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:54.921 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:54.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:54.938 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 26 16 46672 46656 112.111 84.0625 0.0124226 0.00891231 2023-09-23T20:06:55.035 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:06:55.085 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:55.118 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:55.199 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:54 smithi144 ceph-mon[194833]: pgmap v861: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:06:55.213 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:54 smithi135 ceph-mon[213871]: pgmap v861: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:06:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:54 smithi135 ceph-mon[221113]: pgmap v861: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:06:55.406 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:06:55.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:55.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:06:55.511 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:06:55.551 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:55.631 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:06:55.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:55.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:06:55.739 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:55.810 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:55.934 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:55.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:55.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:55.938 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 27 16 47947 47931 110.91 79.6875 0.00238998 0.00899111 2023-09-23T20:06:56.039 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:56.068 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:56.171 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:56.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:56.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:56.276 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:06:56.337 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:56.818 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:56 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:56.818 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:56 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:56.938 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 28 16 49174 49158 109.688 76.6875 0.00421487 0.00908156 2023-09-23T20:06:56.970 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:06:56.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:56 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:56.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:56 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:56.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:56 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:56.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:56 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:57.083 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:06:57.083 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:06:57.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:06:57.083 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:06:57.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:149989 6.15 6:afeafb5c:::benchmark_data_smithi135_344736_object47333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.04996451400001, 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0666702159999999, 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149989 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030922+0000", 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030930+0000", 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9789999999999993e-06 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030945+0000", 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5231999999999999e-05 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030952+0000", 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3880000000000002e-06 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.031049+0000", 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6500000000000001e-05 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.031062+0000", 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2901000000000001e-05 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090321+0000", 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.059259199 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090433+0000", 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000112318 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090708+0000", 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00027474699999999999 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094761+0000", 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040525600000000002 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094797+0000", 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6569000000000002e-05 2023-09-23T20:06:57.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097524+0000", 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002726657 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097554+0000", 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0074e-05 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097594+0000", 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9827999999999999e-05 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:149996 6.15 6:ac0d5d46:::benchmark_data_smithi135_344736_object47340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.04047049100001, 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.057323408, 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.091 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149996 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.092 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040417+0000", 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040422+0000", 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5979999999999996e-06 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040433+0000", 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0444e-05 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040445+0000", 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2872e-05 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040503+0000", 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7289e-05 2023-09-23T20:06:57.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040512+0000", 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6269999999999995e-06 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092269+0000", 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.05175639 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092337+0000", 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8631000000000001e-05 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092523+0000", 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018535200000000001 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.095904+0000", 2023-09-23T20:06:57.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033811940000000001 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097643+0000", 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001738697 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097688+0000", 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5427999999999999e-05 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097718+0000", 2023-09-23T20:06:57.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0437999999999999e-05 2023-09-23T20:06:57.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097741+0000", 2023-09-23T20:06:57.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2449000000000001e-05 2023-09-23T20:06:57.099 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.099 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.099 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.099 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150009 6.15 6:abeabe84:::benchmark_data_smithi135_344736_object47353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.099 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.050577+0000", 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.03031114800001, 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0527466489999999, 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150009 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050577+0000", 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050577+0000", 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050575+0000", 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050582+0000", 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2179999999999996e-06 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050597+0000", 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5071e-05 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050603+0000", 2023-09-23T20:06:57.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3289999999999999e-06 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050662+0000", 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9382000000000002e-05 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.050672+0000", 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1940000000000004e-06 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.093723+0000", 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.043051862 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.093790+0000", 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6422999999999997e-05 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.093945+0000", 2023-09-23T20:06:57.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000155167 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097604+0000", 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0036585900000000002 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097794+0000", 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019022700000000001 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103257+0000", 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054631289999999997 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103299+0000", 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2123000000000003e-05 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103323+0000", 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4358999999999998e-05 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150016 6.15 6:a9052062:::benchmark_data_smithi135_344736_object47360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.062231+0000", 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.01865687200001, 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0411754929999999, 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.107 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150016 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062231+0000", 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062231+0000", 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062230+0000", 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062235+0000", 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9439999999999996e-06 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062247+0000", 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.182e-05 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062252+0000", 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6800000000000001e-06 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062306+0000", 2023-09-23T20:06:57.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4709000000000001e-05 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.062319+0000", 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2625e-05 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094832+0000", 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0325127649999999 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094901+0000", 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9375999999999994e-05 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.095056+0000", 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015530000000000001 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097752+0000", 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0026950899999999998 2023-09-23T20:06:57.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097829+0000", 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7132999999999995e-05 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103361+0000", 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055324709999999997 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103386+0000", 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5309000000000001e-05 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103407+0000", 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0146e-05 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150053 6.15 6:aea43896:::benchmark_data_smithi135_344736_object47397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.110539+0000", 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 429.97034923000001, 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99774726800000002, 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150053 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.115 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110539+0000", 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110539+0000", 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110537+0000", 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110544+0000", 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2309999999999999e-06 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110561+0000", 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6568000000000001e-05 2023-09-23T20:06:57.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110568+0000", 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8569999999999997e-06 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110622+0000", 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4069e-05 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.110631+0000", 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1749999999999994e-06 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.095941+0000", 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98531004099999997 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.096043+0000", 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101562 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.096241+0000", 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019844400000000001 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097835+0000", 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0015935840000000001 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100545+0000", 2023-09-23T20:06:57.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027106220000000002 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108195+0000", 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0076494980000000002 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108241+0000", 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6554000000000002e-05 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108286+0000", 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4629999999999998e-05 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150055 6.15 6:a8b57976:::benchmark_data_smithi135_344736_object47399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.113842+0000", 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 429.96704636800001, 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.994575243, 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.122 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150055 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113842+0000", 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113842+0000", 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113840+0000", 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113846+0000", 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9869999999999996e-06 2023-09-23T20:06:57.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113859+0000", 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3286e-05 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113871+0000", 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1279e-05 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113914+0000", 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3346000000000001e-05 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.113920+0000", 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0190000000000003e-06 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097874+0000", 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98395352999999997 2023-09-23T20:06:57.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097987+0000", 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011300500000000001 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.098190+0000", 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020352200000000001 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100550+0000", 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023602110000000001 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.105582+0000", 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0050321139999999999 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108329+0000", 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027466719999999999 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108378+0000", 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9357000000000002e-05 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108417+0000", 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8427000000000003e-05 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150062 6.15 6:ab03f741:::benchmark_data_smithi135_344736_object47406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.119730+0000", 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 429.96115819699997, 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98880436100000002, 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150062 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.130 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119730+0000", 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119730+0000", 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119729+0000", 2023-09-23T20:06:57.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119733+0000", 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4789999999999998e-06 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119742+0000", 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4530000000000002e-06 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.132 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119747+0000", 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9110000000000001e-06 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119927+0000", 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018037199999999999 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.119934+0000", 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4359999999999999e-06 2023-09-23T20:06:57.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.099463+0000", 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97952930299999996 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.099526+0000", 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3118000000000001e-05 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.099680+0000", 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015395099999999999 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103415+0000", 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037350009999999999 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108153+0000", 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0047377890000000001 2023-09-23T20:06:57.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108456+0000", 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00030299400000000001 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108498+0000", 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2416000000000002e-05 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108534+0000", 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5941e-05 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150100 6.15 6:adb76daf:::benchmark_data_smithi135_344736_object47444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.166966+0000", 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 429.91392199299997, 2023-09-23T20:06:57.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.94168644899999998, 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150100 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166966+0000", 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166966+0000", 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166965+0000", 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.139 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166971+0000", 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0070000000000003e-06 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166983+0000", 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2524000000000001e-05 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.166988+0000", 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8590000000000004e-06 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.140 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.167042+0000", 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3888000000000003e-05 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.167049+0000", 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4839999999999998e-06 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100572+0000", 2023-09-23T20:06:57.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.933522352 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100640+0000", 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8392999999999994e-05 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.100820+0000", 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017973500000000001 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.103421+0000", 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0026012230000000002 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108579+0000", 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0051578789999999998 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108610+0000", 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1353999999999998e-05 2023-09-23T20:06:57.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108634+0000", 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3447000000000001e-05 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.108652+0000", 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8783e-05 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:150109 6.15 6:aac4d8bd:::benchmark_data_smithi135_344736_object47453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.177679+0000", 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 429.90320944400003, 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.93341589000000003, 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 150109 2023-09-23T20:06:57.145 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177679+0000", 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177679+0000", 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.146 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177678+0000", 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177681+0000", 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4489999999999999e-06 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177689+0000", 2023-09-23T20:06:57.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5599999999999996e-06 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177693+0000", 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7009999999999998e-06 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177725+0000", 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2536000000000002e-05 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.177731+0000", 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6370000000000004e-06 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.101724+0000", 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.92399303899999996 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.101791+0000", 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7607999999999994e-05 2023-09-23T20:06:57.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.101971+0000", 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000179233 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.105545+0000", 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003574331 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.150 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.111014+0000", 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054689830000000002 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.111041+0000", 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7325e-05 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.111072+0000", 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0772999999999998e-05 2023-09-23T20:06:57.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.111094+0000", 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2456999999999999e-05 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201802 6.9 6:91b29cdb:::benchmark_data_smithi135_344736_object99146:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 352.61455045999998, 2023-09-23T20:06:57.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1705403599999999, 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201802 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466336+0000", 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.154 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466346+0000", 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0098000000000001e-05 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466364+0000", 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7572999999999999e-05 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466371+0000", 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.238e-06 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466440+0000", 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8442000000000006e-05 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466449+0000", 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9490000000000008e-06 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469797+0000", 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033470090000000002 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469805+0000", 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2930000000000001e-06 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593229+0000", 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.123424108 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.157 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593367+0000", 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000138338 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593937+0000", 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00056937199999999996 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608226+0000", 2023-09-23T20:06:57.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014289792000000001 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614886+0000", 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0066598359999999997 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614925+0000", 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8294999999999998e-05 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636724+0000", 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021799457000000001 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636765+0000", 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1247e-05 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636878+0000", 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000112652 2023-09-23T20:06:57.160 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201804 6.19 6:9cfa48b5:::benchmark_data_smithi135_344736_object99148:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 352.61450052200001, 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1704891850000001, 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.161 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201804 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.162 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466390+0000", 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.599e-06 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.163 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466405+0000", 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5133000000000001e-05 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466410+0000", 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8169999999999998e-06 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5672000000000002e-05 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.164 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466510+0000", 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6859999999999997e-06 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.471432+0000", 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0049220280000000002 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.471445+0000", 2023-09-23T20:06:57.165 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2568000000000001e-05 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594208+0000", 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1227630580000001 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594319+0000", 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111501 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.166 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594813+0000", 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000494181 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612638+0000", 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.017824584000000001 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614937+0000", 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022991069999999999 2023-09-23T20:06:57.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614972+0000", 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4678999999999998e-05 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636699+0000", 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021726942999999999 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636748+0000", 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9106e-05 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636877+0000", 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012898200000000001 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201813 6.9 6:96974822:::benchmark_data_smithi135_344736_object99157:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:06:57.169 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 352.61446478800002, 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1705933580000001, 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201813 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:06:57.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.171 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.171 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.171 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466425+0000", 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5450000000000002e-06 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466432+0000", 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6059999999999999e-06 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466436+0000", 2023-09-23T20:06:57.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9979999999999997e-06 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466527+0000", 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0829999999999996e-05 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466531+0000", 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.765e-06 2023-09-23T20:06:57.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469820+0000", 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032892910000000002 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469824+0000", 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5639999999999998e-06 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.598827+0000", 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1290025159999999 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.598953+0000", 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012642799999999999 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.599456+0000", 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000502974 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614525+0000", 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015068381 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614956+0000", 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043108699999999997 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614987+0000", 2023-09-23T20:06:57.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1103999999999999e-05 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636915+0000", 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021927720000000001 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636954+0000", 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8930000000000002e-05 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637017+0000", 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2964000000000002e-05 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201824 6.9 6:912a71a1:::benchmark_data_smithi135_344736_object99168:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 352.61444633299999, 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1708038080000001, 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.180 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201824 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466441+0000", 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466444+0000", 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2010000000000002e-06 2023-09-23T20:06:57.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466450+0000", 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9680000000000003e-06 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466454+0000", 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4009999999999999e-06 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466551+0000", 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6948999999999996e-05 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466557+0000", 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4999999999999996e-06 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469839+0000", 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032815219999999998 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469843+0000", 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.527e-06 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602356+0000", 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1325121519999999 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602467+0000", 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111412 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602926+0000", 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000458691 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614536+0000", 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011610646000000001 2023-09-23T20:06:57.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615029+0000", 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00049241599999999997 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615073+0000", 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4329999999999997e-05 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637077+0000", 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022004391000000002 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637136+0000", 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8501000000000001e-05 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637245+0000", 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010948800000000001 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.188 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201827 6.9 6:937e540a:::benchmark_data_smithi135_344736_object99171:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 352.614429938, 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1709668390000001, 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.189 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201827 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466460+0000", 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1710000000000001e-06 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.191 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466466+0000", 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8119999999999996e-06 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466470+0000", 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8290000000000001e-06 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466732+0000", 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000262759 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466747+0000", 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4204e-05 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469858+0000", 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003111426 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469875+0000", 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6639000000000001e-05 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605181+0000", 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.135306167 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605324+0000", 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014373500000000001 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605687+0000", 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036256599999999998 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614615+0000", 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0089275680000000003 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615114+0000", 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00049912899999999996 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615172+0000", 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8260000000000001e-05 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637300+0000", 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022127563999999999 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637335+0000", 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5825999999999999e-05 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637425+0000", 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9488999999999996e-05 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201882 6.9 6:972c2747:::benchmark_data_smithi135_344736_object99226:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 352.614414469, 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.173680238, 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201882 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.198 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466473+0000", 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8220000000000001e-06 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466481+0000", 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7080000000000002e-06 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466485+0000", 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.749e-06 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466901+0000", 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000416542 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466909+0000", 2023-09-23T20:06:57.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469891+0000", 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029817060000000002 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469895+0000", 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4510000000000002e-06 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608273+0000", 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.138377135 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608376+0000", 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010315 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608741+0000", 2023-09-23T20:06:57.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036480300000000002 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614621+0000", 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0058808610000000002 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615135+0000", 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00051361899999999999 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615214+0000", 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8611999999999999e-05 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640006+0000", 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.024792756999999999 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640051+0000", 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4193999999999997e-05 2023-09-23T20:06:57.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640154+0000", 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010318 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201893 6.9 6:9279cfff:::benchmark_data_smithi135_344736_object99237:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:57.206 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 352.61438291399998, 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.173837531, 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201893 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.207 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466500+0000", 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999952 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466507+0000", 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.905e-06 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466514+0000", 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6449999999999999e-06 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466518+0000", 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.985e-06 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466950+0000", 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043239999999999999 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466957+0000", 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1590000000000004e-06 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469913+0000", 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029559510000000001 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469918+0000", 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4739999999999999e-06 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.610696+0000", 2023-09-23T20:06:57.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1407785420000001 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.610812+0000", 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115654 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.611100+0000", 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00028765000000000003 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614629+0000", 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035288590000000001 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615255+0000", 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00062691999999999999 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615334+0000", 2023-09-23T20:06:57.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8597000000000007e-05 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640192+0000", 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.024858410000000001 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640237+0000", 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4200999999999999e-05 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640343+0000", 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010594000000000001 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201908 6.9 6:9284cb28:::benchmark_data_smithi135_344736_object99252:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 352.61435980900001, 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.174007815, 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.215 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201908 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:06:57.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0990000000000001e-06 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466541+0000", 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0922e-05 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466545+0000", 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.788e-06 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.218 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466984+0000", 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000439586 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466991+0000", 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9190000000000002e-06 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469933+0000", 2023-09-23T20:06:57.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002941534 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469937+0000", 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2810000000000002e-06 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612655+0000", 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1427177049999999 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.220 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612740+0000", 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5024000000000005e-05 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612994+0000", 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00025448499999999999 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614834+0000", 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018393560000000001 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615298+0000", 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000464303 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615356+0000", 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7827e-05 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640379+0000", 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.025023627999999999 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640429+0000", 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.922e-05 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640536+0000", 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107451 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.223 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:159322 7.6 7:6a6d6713:::benchmark_data_smithi135_404353_object62260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:05:04.885510+0000", 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 112.19537839500001, 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.96888260699999995, 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 159322 2023-09-23T20:06:57.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.885510+0000", 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.885510+0000", 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.885509+0000", 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.885513+0000", 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2910000000000001e-06 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.885523+0000", 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0173999999999999e-05 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.885528+0000", 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6990000000000002e-06 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.885574+0000", 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5395000000000002e-05 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.885582+0000", 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4419999999999998e-06 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.844315+0000", 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.95873331699999997 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.844448+0000", 2023-09-23T20:06:57.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000132285 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.844591+0000", 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014340199999999999 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.847755+0000", 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003164385 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.852338+0000", 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045828010000000001 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854310+0000", 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0019718309999999998 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854353+0000", 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2818999999999998e-05 2023-09-23T20:06:57.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854392+0000", 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9359e-05 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:159329 7.6 7:613b3877:::benchmark_data_smithi135_404353_object62267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:05:04.896481+0000", 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 112.18440712100001, 2023-09-23T20:06:57.231 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.95800598199999998, 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 159329 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.896481+0000", 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.232 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.896481+0000", 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.896480+0000", 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.233 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.896486+0000", 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3500000000000002e-06 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.896499+0000", 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2979999999999999e-05 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.896506+0000", 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7209999999999997e-06 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.234 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.896554+0000", 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8798999999999998e-05 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.896565+0000", 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0699999999999999e-05 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.235 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.847789+0000", 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.95122415299999996 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.847894+0000", 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010463099999999999 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.847967+0000", 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2708999999999998e-05 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854225+0000", 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0062582439999999996 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854283+0000", 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7725e-05 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854435+0000", 2023-09-23T20:06:57.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000152714 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854464+0000", 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9139999999999999e-05 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854487+0000", 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2476999999999998e-05 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.238 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35358.0:159350 7.6 7:6149cab1:::benchmark_data_smithi135_404353_object62288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:05:04.925505+0000", 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 112.15538298, 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.93302350199999995, 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35358", 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:06:57.239 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 159350 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.925505+0000", 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.925505+0000", 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.925503+0000", 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.925510+0000", 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3799999999999999e-06 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.925525+0000", 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5631999999999999e-05 2023-09-23T20:06:57.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.925533+0000", 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0700000000000001e-06 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.925575+0000", 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2781000000000003e-05 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:06:57.242 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:04.925582+0000", 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4559999999999997e-06 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.850263+0000", 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.92468092300000004 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.850371+0000", 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107885 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.243 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.850488+0000", 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011704700000000001 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.854496+0000", 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040082470000000004 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.858437+0000", 2023-09-23T20:06:57.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0039407189999999996 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.858462+0000", 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5848999999999999e-05 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.858491+0000", 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8445e-05 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:06:57.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:06:57.246 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:05:05.858529+0000", 2023-09-23T20:06:57.246 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7622e-05 2023-09-23T20:06:57.246 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.246 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.246 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.246 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:06:57.246 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:06:57.246 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:06:57.821 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:57 smithi135 ceph-mon[213871]: pgmap v862: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:06:57.821 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:06:57.822 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:06:57.822 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:57.896 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:06:57.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:57.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:06:57.939 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 29 16 50129 50113 107.96 59.6875 0.00674939 0.00925832 2023-09-23T20:06:58.007 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:06:58.033 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:58.072 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:57 smithi144 ceph-mon[194833]: pgmap v862: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:06:58.072 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:06:58.072 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:06:58.072 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:58.085 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:06:58.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:57 smithi135 ceph-mon[221113]: pgmap v862: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:06:58.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:06:58.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:06:58.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:06:58.460 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:06:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:58.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:06:58.570 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:06:58.580 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:06:58.613 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:58.647 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:06:58.782 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:06:58.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:58.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:06:58.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:58.941 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 30 16 51257 51241 106.705 70.5 0.013033 0.00936547 2023-09-23T20:06:58.947 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:58 smithi135 ceph-mon[221113]: pgmap v863: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:06:59.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:58 smithi135 ceph-mon[213871]: pgmap v863: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:06:59.227 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:59.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:06:59.321 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:58 smithi144 ceph-mon[194833]: pgmap v863: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:06:59.332 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:06:59.433 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:59.566 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:59 smithi135 conmon[213845]: 2023-09-23T20:06:59.563+0000 7f9708921700 -1 mon.a@0(leader).osd e309 definitely_dead 0 2023-09-23T20:06:59.668 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:06:59.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:59.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:06:59.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:06:59.851 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:06:59.920 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:06:59.920 INFO:teuthology.orchestra.run.smithi135.stderr:marked out osd.7. 2023-09-23T20:06:59.943 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 31 16 52241 52225 105.243 61.5 0.00558686 0.00949311 2023-09-23T20:06:59.957 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:06:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:06:59.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:00.060 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:00.088 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:00.184 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:00.187 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2363448507' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:07:00.187 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:59 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:07:00.187 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:59 smithi144 ceph-mon[194833]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T20:07:00.188 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/109766106' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:00.188 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:06:59 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:00.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:00.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2363448507' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:07:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:07:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[221113]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T20:07:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/109766106' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2363448507' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:07:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:07:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[213871]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T20:07:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/109766106' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:00.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:06:59 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:00.289 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:00.315 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:00.625 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:07:00.625 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:07:00.625 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:07:00.625 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:07:00.915 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:00 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:00.916 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:00 smithi144 ceph-mon[194833]: failed to encode map e310 with expected crc 2023-09-23T20:07:00.916 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:00 smithi144 ceph-mon[194833]: failed to encode map e310 with expected crc 2023-09-23T20:07:00.916 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:00 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T20:07:00.916 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:00 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:07:00.916 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:00 smithi144 ceph-mon[194833]: osdmap e310: 8 total, 8 up, 5 in 2023-09-23T20:07:00.916 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:00 smithi144 ceph-mon[194833]: failed to encode map e310 with expected crc 2023-09-23T20:07:00.916 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:00 smithi144 ceph-mon[194833]: pgmap v865: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:07:00.943 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 32 16 52251 52235 101.975 0.625 0.00373164 0.00949237 2023-09-23T20:07:00.960 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:00.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:00.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:01.065 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:01.090 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:01.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:01.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[221113]: failed to encode map e310 with expected crc 2023-09-23T20:07:01.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[221113]: failed to encode map e310 with expected crc 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[221113]: osdmap e310: 8 total, 8 up, 5 in 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[221113]: failed to encode map e310 with expected crc 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[221113]: pgmap v865: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[213871]: failed to encode map e310 with expected crc 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[213871]: failed to encode map e310 with expected crc 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:07:01.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[213871]: osdmap e310: 8 total, 8 up, 5 in 2023-09-23T20:07:01.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[213871]: failed to encode map e310 with expected crc 2023-09-23T20:07:01.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:00 smithi135 ceph-mon[213871]: pgmap v865: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:07:01.527 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:07:01.589 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:01.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:01.693 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:01.720 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:01.868 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:07:01.917 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:01.943 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 33 16 52251 52235 98.886 0 - 0.00949237 2023-09-23T20:07:02.185 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.185 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.185 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.185 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.185 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: osdmap e311: 8 total, 8 up, 5 in 2023-09-23T20:07:02.185 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.186 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.186 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.186 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.186 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:01 smithi144 ceph-mon[194833]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: osdmap e311: 8 total, 8 up, 5 in 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[213871]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: osdmap e311: 8 total, 8 up, 5 in 2023-09-23T20:07:02.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:01 smithi135 ceph-mon[221113]: failed to encode map e311 with expected crc 2023-09-23T20:07:02.388 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:02.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:02.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:02.495 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:02.585 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:02.726 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:02.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:02.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:02.779 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:07:02.799 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:07:02.819 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:07:02.832 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:02.839 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:07:02.858 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:07:02.860 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:02.904 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:02.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:02.943 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 34 16 52928 52912 97.2218 21.1562 0.00596345 0.0102793 2023-09-23T20:07:02.961 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:07:02.998 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:07:03.006 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:03.041 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:07:03.066 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.066 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.066 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.066 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: osdmap e312: 8 total, 8 up, 5 in 2023-09-23T20:07:03.066 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.066 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.066 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: osdmap e313: 8 total, 8 up, 5 in 2023-09-23T20:07:03.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/332141432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:03.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[221113]: pgmap v869: 99 pgs: 2 unknown, 15 peering, 82 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 21 MiB/s wr, 331 op/s 2023-09-23T20:07:03.067 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:07:03.067 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:07:03.067 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:07:03.067 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:07:03.068 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:07:03.068 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:07:03.068 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:07:03.068 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:07:03.068 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.068 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.068 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.068 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: osdmap e312: 8 total, 8 up, 5 in 2023-09-23T20:07:03.068 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.068 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.068 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.069 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: osdmap e313: 8 total, 8 up, 5 in 2023-09-23T20:07:03.069 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.069 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/332141432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:03.069 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:02 smithi135 ceph-mon[213871]: pgmap v869: 99 pgs: 2 unknown, 15 peering, 82 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 21 MiB/s wr, 331 op/s 2023-09-23T20:07:03.071 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:03.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: osdmap e312: 8 total, 8 up, 5 in 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: failed to encode map e312 with expected crc 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: osdmap e313: 8 total, 8 up, 5 in 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/332141432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:03.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:02 smithi144 ceph-mon[194833]: pgmap v869: 99 pgs: 2 unknown, 15 peering, 82 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 21 MiB/s wr, 331 op/s 2023-09-23T20:07:03.324 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:03.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:03.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:03.427 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:03.475 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:03.556 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:03.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:03.663 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:03.715 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:03.769 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:03.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:03.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:03.835 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:07:03.835 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:07:03.873 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:03.903 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:03.943 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 35 16 53787 53771 95.9782 53.6875 0.0141749 0.0103752 2023-09-23T20:07:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:07:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:07:03] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-09-23T20:07:03.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[213871]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[213871]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[213871]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[213871]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 4 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[213871]: failed to encode map e314 with expected crc 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[213871]: osdmap e314: 8 total, 8 up, 5 in 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[213871]: failed to encode map e314 with expected crc 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[221113]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[221113]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[221113]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[221113]: failed to encode map e313 with expected crc 2023-09-23T20:07:03.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 4 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:07:03.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[221113]: failed to encode map e314 with expected crc 2023-09-23T20:07:03.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[221113]: osdmap e314: 8 total, 8 up, 5 in 2023-09-23T20:07:03.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:03 smithi135 ceph-mon[221113]: failed to encode map e314 with expected crc 2023-09-23T20:07:04.022 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:07:04.102 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:04.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:04.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:03 smithi144 ceph-mon[194833]: failed to encode map e313 with expected crc 2023-09-23T20:07:04.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:03 smithi144 ceph-mon[194833]: failed to encode map e313 with expected crc 2023-09-23T20:07:04.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:03 smithi144 ceph-mon[194833]: failed to encode map e313 with expected crc 2023-09-23T20:07:04.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:03 smithi144 ceph-mon[194833]: failed to encode map e313 with expected crc 2023-09-23T20:07:04.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:03 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 4 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:07:04.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:03 smithi144 ceph-mon[194833]: failed to encode map e314 with expected crc 2023-09-23T20:07:04.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:03 smithi144 ceph-mon[194833]: osdmap e314: 8 total, 8 up, 5 in 2023-09-23T20:07:04.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:03 smithi144 ceph-mon[194833]: failed to encode map e314 with expected crc 2023-09-23T20:07:04.206 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:04.233 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:04.290 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:04.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:04.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:04.394 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:04.433 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:04.685 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:04.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:04.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:04.763 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:07:04.790 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:04.870 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:04.930 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:04.946 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 36 16 54450 54434 94.4576 41.4375 0.0197535 0.0105792 2023-09-23T20:07:04.948 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:04.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:05.056 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:05.140 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:04 smithi135 ceph-mon[221113]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.140 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:04 smithi135 ceph-mon[221113]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.140 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:04 smithi135 ceph-mon[221113]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.140 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:04 smithi135 ceph-mon[221113]: pgmap v871: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 unknown, 19 peering, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 51 MiB/s wr, 816 op/s; 9633/159138 objects degraded (6.053%); 3252/159138 objects misplaced (2.044%); 1.0 MiB/s, 2 keys/s, 21 objects/s recovering 2023-09-23T20:07:05.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:04 smithi135 ceph-mon[213871]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:04 smithi135 ceph-mon[213871]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:04 smithi135 ceph-mon[213871]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:04 smithi135 ceph-mon[213871]: pgmap v871: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 unknown, 19 peering, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 51 MiB/s wr, 816 op/s; 9633/159138 objects degraded (6.053%); 3252/159138 objects misplaced (2.044%); 1.0 MiB/s, 2 keys/s, 21 objects/s recovering 2023-09-23T20:07:05.148 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:05.257 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:05.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:05.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:05.284 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:07:05.371 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:05.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:04 smithi144 ceph-mon[194833]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:04 smithi144 ceph-mon[194833]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:04 smithi144 ceph-mon[194833]: failed to encode map e314 with expected crc 2023-09-23T20:07:05.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:04 smithi144 ceph-mon[194833]: pgmap v871: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 unknown, 19 peering, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 51 MiB/s wr, 816 op/s; 9633/159138 objects degraded (6.053%); 3252/159138 objects misplaced (2.044%); 1.0 MiB/s, 2 keys/s, 21 objects/s recovering 2023-09-23T20:07:05.428 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:05.524 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:05.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:05.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:05.632 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:05.677 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:05.793 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:05.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:05.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:05.903 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:05.947 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 37 16 55469 55453 93.6238 63.6875 0.0140375 0.010675 2023-09-23T20:07:05.978 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:05 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 9633/159138 objects degraded (6.053%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:05.984 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:05 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 9633/159138 objects degraded (6.053%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:05.989 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:06.128 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:05 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 9633/159138 objects degraded (6.053%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:06.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:06.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:06.236 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:06.262 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:06.308 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:06.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:06.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:06.412 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:06.447 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:06.752 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:06.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:06.779 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:07:06.863 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:06.904 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:06.947 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 38 16 56122 56106 92.2345 40.8125 0.0159846 0.0108361 2023-09-23T20:07:06.971 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:06.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:06.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:07.075 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:07.169 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:06 smithi135 ceph-mon[213871]: pgmap v872: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 peering, 90 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 31 MiB/s wr, 490 op/s; 12920/162009 objects degraded (7.975%); 3252/162009 objects misplaced (2.007%); 781 KiB/s, 1 keys/s, 19 objects/s recovering 2023-09-23T20:07:07.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:06 smithi135 ceph-mon[221113]: pgmap v872: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 peering, 90 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 31 MiB/s wr, 490 op/s; 12920/162009 objects degraded (7.975%); 3252/162009 objects misplaced (2.007%); 781 KiB/s, 1 keys/s, 19 objects/s recovering 2023-09-23T20:07:07.184 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:07.285 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:07:07.285 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:07:07.310 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:06 smithi144 ceph-mon[194833]: pgmap v872: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 peering, 90 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 31 MiB/s wr, 490 op/s; 12920/162009 objects degraded (7.975%); 3252/162009 objects misplaced (2.007%); 781 KiB/s, 1 keys/s, 19 objects/s recovering 2023-09-23T20:07:07.410 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:07.414 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3] out_osds: [6, 1, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:07:07.414 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:07:07.414 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-09-23T20:07:07.414 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd in 1 2023-09-23T20:07:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:07.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:07.524 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:07.564 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:07.632 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:07.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:07.735 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:07.737 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:07.791 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:07.924 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:07.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:07.947 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 39 16 56929 56913 91.1631 50.4375 0.0310217 0.0109638 2023-09-23T20:07:08.029 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:08.070 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:08.138 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:08.145 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:07:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:08.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:08.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:07 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 3 pgs peering) 2023-09-23T20:07:08.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:08.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:07 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 3 pgs peering) 2023-09-23T20:07:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:08.248 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:07 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 3 pgs peering) 2023-09-23T20:07:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:08.384 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:08.565 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:08.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:08.673 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:08.764 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:08.903 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:08.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:08.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:08.947 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:07:08.948196+0000 min lat: 0.00105689 max lat: 2.17642 avg lat: 0.0110888 2023-09-23T20:07:08.947 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:07:08.947 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 40 16 57727 57711 90.1312 49.875 0.0135389 0.0110888 2023-09-23T20:07:09.013 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:09.024 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:07:09.096 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:09.140 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:09.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:09.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:09.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:08 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3376717624' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:09.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:08 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:09.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:08 smithi135 ceph-mon[213871]: pgmap v873: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 51 MiB/s wr, 813 op/s; 12920/170178 objects degraded (7.592%); 3025/170178 objects misplaced (1.778%); 2.7 MiB/s, 1 keys/s, 49 objects/s recovering 2023-09-23T20:07:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:08 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3376717624' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:08 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:09.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:08 smithi135 ceph-mon[221113]: pgmap v873: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 51 MiB/s wr, 813 op/s; 12920/170178 objects degraded (7.592%); 3025/170178 objects misplaced (1.778%); 2.7 MiB/s, 1 keys/s, 49 objects/s recovering 2023-09-23T20:07:09.245 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:09.307 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:08 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3376717624' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:09.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:08 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:09.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:08 smithi144 ceph-mon[194833]: pgmap v873: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 51 MiB/s wr, 813 op/s; 12920/170178 objects degraded (7.592%); 3025/170178 objects misplaced (1.778%); 2.7 MiB/s, 1 keys/s, 49 objects/s recovering 2023-09-23T20:07:09.402 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:09.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:09.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:09.513 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:09.554 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:09.633 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:09.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:09.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:09.743 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:09.768 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:09.947 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 41 16 58663 58647 89.3598 58.5 0.00737774 0.0111856 2023-09-23T20:07:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[221113]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[221113]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[221113]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[221113]: osdmap e315: 8 total, 8 up, 5 in 2023-09-23T20:07:10.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:10.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[213871]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[213871]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[213871]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:10.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:09 smithi135 ceph-mon[213871]: osdmap e315: 8 total, 8 up, 5 in 2023-09-23T20:07:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:09 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:09 smithi144 ceph-mon[194833]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:09 smithi144 ceph-mon[194833]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:09 smithi144 ceph-mon[194833]: failed to encode map e315 with expected crc 2023-09-23T20:07:10.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:09 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:10.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:09 smithi144 ceph-mon[194833]: osdmap e315: 8 total, 8 up, 5 in 2023-09-23T20:07:10.756 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:10.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:10.866 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:10.900 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:10.948 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 42 16 59738 59722 88.8305 67.1875 0.00653502 0.0112523 2023-09-23T20:07:10.979 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:10.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:10.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:11.012 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:11 smithi135 ceph-mon[213871]: pgmap v875: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 44 MiB/s wr, 702 op/s; 12920/170178 objects degraded (7.592%); 3025/170178 objects misplaced (1.778%); 2.3 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T20:07:11.087 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:11.149 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:11.209 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:11.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:11.312 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:11.343 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:11 smithi144 ceph-mon[194833]: pgmap v875: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 44 MiB/s wr, 702 op/s; 12920/170178 objects degraded (7.592%); 3025/170178 objects misplaced (1.778%); 2.3 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T20:07:11.346 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:11.414 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:11.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:11.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:11 smithi135 ceph-mon[221113]: pgmap v875: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 44 MiB/s wr, 702 op/s; 12920/170178 objects degraded (7.592%); 3025/170178 objects misplaced (1.778%); 2.3 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T20:07:11.525 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:11.550 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:11.813 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:11.892 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:11.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:11.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:11.948 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 43 16 60476 60460 87.8377 46.125 0.0163461 0.0113777 2023-09-23T20:07:12.000 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:12.030 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:12.107 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:12.109 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:12.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:12.216 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:12.295 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:12.483 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:12.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:12.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:12.596 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:12.626 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:12.682 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:12.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:12.758 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12920/170178 objects degraded (7.592%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:12.758 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12920/170178 objects degraded (7.592%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:12.790 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:12.844 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:07:12.844 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:07:12.853 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:07:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12920/170178 objects degraded (7.592%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:12.904 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:07:12.905 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:12.948 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 44 16 61346 61330 87.0773 54.375 0.0203907 0.0114785 2023-09-23T20:07:12.963 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:07:12.992 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:07:13.029 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:07:13.065 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:07:13.084 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:07:13.084 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:07:13.084 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:07:13.084 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:07:13.084 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:07:13.085 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:07:13.085 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:07:13.085 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:07:13.385 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:13.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:13.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:13.498 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:13.575 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:13.688 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3536696917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:13.688 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:13 smithi135 ceph-mon[213871]: pgmap v876: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 34 MiB/s wr, 538 op/s; 12920/174075 objects degraded (7.422%); 3025/174075 objects misplaced (1.738%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:07:13.688 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3536696917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:13.688 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:13 smithi135 ceph-mon[221113]: pgmap v876: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 34 MiB/s wr, 538 op/s; 12920/174075 objects degraded (7.422%); 3025/174075 objects misplaced (1.738%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:07:13.689 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:07:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:07:13] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-09-23T20:07:13.907 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:07:13.950 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 45 16 62230 62214 86.367 55.25 0.0481917 0.0115681 2023-09-23T20:07:14.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3536696917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:14.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:13 smithi144 ceph-mon[194833]: pgmap v876: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 34 MiB/s wr, 538 op/s; 12920/174075 objects degraded (7.422%); 3025/174075 objects misplaced (1.738%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:07:14.909 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:14 smithi135 ceph-mon[213871]: pgmap v877: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 50 MiB/s wr, 803 op/s; 12920/183258 objects degraded (7.050%); 2845/183258 objects misplaced (1.552%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:07:14.950 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 46 16 63140 63124 85.726 56.875 0.00684495 0.0116537 2023-09-23T20:07:15.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:14 smithi135 ceph-mon[221113]: pgmap v877: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 50 MiB/s wr, 803 op/s; 12920/183258 objects degraded (7.050%); 2845/183258 objects misplaced (1.552%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:07:15.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:14 smithi144 ceph-mon[194833]: pgmap v877: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 50 MiB/s wr, 803 op/s; 12920/183258 objects degraded (7.050%); 2845/183258 objects misplaced (1.552%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:07:15.582 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:15.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:15.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:15.698 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:15.764 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:15.872 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:15.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:15.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:15.950 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 47 16 63944 63928 84.9714 50.25 0.0128649 0.0117618 2023-09-23T20:07:15.984 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:16.019 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:16.100 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:16.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:16.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:16.210 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:16.272 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:16.790 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:16.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:16.901 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:16.952 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 48 16 64912 64896 84.4588 60.5 0.0139184 0.0118336 2023-09-23T20:07:16.954 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:16 smithi135 ceph-mon[213871]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 52 MiB/s wr, 838 op/s; 12920/187164 objects degraded (6.903%); 2845/187164 objects misplaced (1.520%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:07:16.954 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:16 smithi135 ceph-mon[221113]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 52 MiB/s wr, 838 op/s; 12920/187164 objects degraded (6.903%); 2845/187164 objects misplaced (1.520%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:07:16.961 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:17.115 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:17.136 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:17.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:17.247 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:17.341 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:17.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:16 smithi144 ceph-mon[194833]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 52 MiB/s wr, 838 op/s; 12920/187164 objects degraded (6.903%); 2845/187164 objects misplaced (1.520%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:07:17.446 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:07:17.446 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:07:17.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.446 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35643.0:65255 8.8 e315/311 8:1f59694f:::benchmark_data_smithi135_460343_object65254:head v 315'4058, mlcod=0'0)", 2023-09-23T20:07:17.446 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:17.439204+0000", 2023-09-23T20:07:17.446 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0056389129999999997, 2023-09-23T20:07:17.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056687100000000004, 2023-09-23T20:07:17.446 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.439204+0000", 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.439204+0000", 2023-09-23T20:07:17.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.439203+0000", 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.439244+0000", 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.138e-05 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.439263+0000", 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9210000000000001e-05 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.439273+0000", 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6849999999999996e-06 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:07:17.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.439391+0000", 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011862399999999999 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.439422+0000", 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0196999999999999e-05 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:17.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35643.0:65262 8.c e315/311 8:3796ce13:::benchmark_data_smithi135_460343_object65261:head v 315'4092, mlcod=0'0)", 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:17.444431+0000", 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.000411784, 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00051264799999999999, 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.444431+0000", 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.444431+0000", 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.444430+0000", 2023-09-23T20:07:17.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.444461+0000", 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1479999999999997e-05 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.444474+0000", 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2557e-05 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:07:17.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.444482+0000", 2023-09-23T20:07:17.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2730000000000002e-06 2023-09-23T20:07:17.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:07:17.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.444732+0000", 2023-09-23T20:07:17.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00025047600000000002 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.444756+0000", 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3215000000000001e-05 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:17.455 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:07:17.456 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 2 2023-09-23T20:07:17.456 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:07:17.514 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:17.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:17.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:17.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:17 smithi135 conmon[213845]: 2023-09-23T20:07:17.527+0000 7f9708921700 -1 mon.a@0(leader).osd e315 definitely_dead 0 2023-09-23T20:07:17.627 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:17.628 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:07:17.703 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:17.950 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2854437332' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:07:17.950 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:17 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:07:17.950 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12920/187164 objects degraded (6.903%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:17.950 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2854437332' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:07:17.950 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:17 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:07:17.951 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12920/187164 objects degraded (6.903%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:17.955 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 49 16 65656 65640 83.6794 46.5 0.0397365 0.0119365 2023-09-23T20:07:18.016 INFO:teuthology.orchestra.run.smithi135.stderr:marked in osd.1. 2023-09-23T20:07:18.229 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:18.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:18.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:18.333 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:18.365 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2854437332' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:07:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:17 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:07:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12920/187164 objects degraded (6.903%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:18.956 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 50 16 65671 65655 82.025 0.9375 0.169748 0.011948 2023-09-23T20:07:19.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[213871]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[213871]: osdmap e316: 8 total, 8 up, 6 in 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[213871]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[213871]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[213871]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[213871]: pgmap v880: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 781 op/s; 6526/193227 objects degraded (3.377%); 2626/193227 objects misplaced (1.359%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[221113]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:07:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[221113]: osdmap e316: 8 total, 8 up, 6 in 2023-09-23T20:07:19.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[221113]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[221113]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[221113]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:18 smithi135 ceph-mon[221113]: pgmap v880: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 781 op/s; 6526/193227 objects degraded (3.377%); 2626/193227 objects misplaced (1.359%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:07:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:18 smithi144 ceph-mon[194833]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:07:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:18 smithi144 ceph-mon[194833]: osdmap e316: 8 total, 8 up, 6 in 2023-09-23T20:07:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:18 smithi144 ceph-mon[194833]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:18 smithi144 ceph-mon[194833]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:18 smithi144 ceph-mon[194833]: failed to encode map e316 with expected crc 2023-09-23T20:07:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:18 smithi144 ceph-mon[194833]: pgmap v880: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 781 op/s; 6526/193227 objects degraded (3.377%); 2626/193227 objects misplaced (1.359%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:07:19.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[213871]: failed to encode map e317 with expected crc 2023-09-23T20:07:19.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[213871]: failed to encode map e317 with expected crc 2023-09-23T20:07:19.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[213871]: osdmap e317: 8 total, 8 up, 6 in 2023-09-23T20:07:19.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[213871]: failed to encode map e317 with expected crc 2023-09-23T20:07:19.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[213871]: failed to encode map e317 with expected crc 2023-09-23T20:07:19.929 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[213871]: failed to encode map e317 with expected crc 2023-09-23T20:07:19.956 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 51 16 65683 65667 80.4309 0.75 0.00877518 0.0120181 2023-09-23T20:07:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[221113]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[221113]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[221113]: osdmap e317: 8 total, 8 up, 6 in 2023-09-23T20:07:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[221113]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[221113]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:19 smithi135 ceph-mon[221113]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:19 smithi144 ceph-mon[194833]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:19 smithi144 ceph-mon[194833]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:19 smithi144 ceph-mon[194833]: osdmap e317: 8 total, 8 up, 6 in 2023-09-23T20:07:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:19 smithi144 ceph-mon[194833]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:19 smithi144 ceph-mon[194833]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:19 smithi144 ceph-mon[194833]: failed to encode map e317 with expected crc 2023-09-23T20:07:20.831 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:20.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:20.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:20.937 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:20.964 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 52 16 65803 65787 79.0198 7.5 0.0270059 0.0126485 2023-09-23T20:07:20.985 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:21.035 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:21.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:21.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[221113]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[221113]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[221113]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[221113]: osdmap e318: 8 total, 8 up, 6 in 2023-09-23T20:07:21.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[221113]: pgmap v883: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 35 MiB/s wr, 553 op/s; 6526/193227 objects degraded (3.377%); 2626/193227 objects misplaced (1.359%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:07:21.103 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[213871]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.103 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[213871]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.103 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[213871]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.103 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[213871]: osdmap e318: 8 total, 8 up, 6 in 2023-09-23T20:07:21.103 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:20 smithi135 ceph-mon[213871]: pgmap v883: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 35 MiB/s wr, 553 op/s; 6526/193227 objects degraded (3.377%); 2626/193227 objects misplaced (1.359%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:07:21.104 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:07:21.142 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:21.176 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:21.231 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:21.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:20 smithi144 ceph-mon[194833]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:20 smithi144 ceph-mon[194833]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:20 smithi144 ceph-mon[194833]: failed to encode map e318 with expected crc 2023-09-23T20:07:21.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:20 smithi144 ceph-mon[194833]: osdmap e318: 8 total, 8 up, 6 in 2023-09-23T20:07:21.238 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:20 smithi144 ceph-mon[194833]: pgmap v883: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 35 MiB/s wr, 553 op/s; 6526/193227 objects degraded (3.377%); 2626/193227 objects misplaced (1.359%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:07:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:21.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:21.339 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:21.462 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:21.962 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 53 16 66260 66244 78.0682 28.5625 0.0182824 0.0127703 2023-09-23T20:07:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:21 smithi135 ceph-mon[221113]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:21 smithi135 ceph-mon[221113]: osdmap e319: 8 total, 8 up, 6 in 2023-09-23T20:07:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:21 smithi135 ceph-mon[221113]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:21 smithi135 ceph-mon[221113]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:21 smithi135 ceph-mon[213871]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:21 smithi135 ceph-mon[213871]: osdmap e319: 8 total, 8 up, 6 in 2023-09-23T20:07:22.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:21 smithi135 ceph-mon[213871]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:21 smithi135 ceph-mon[213871]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:21 smithi144 ceph-mon[194833]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:21 smithi144 ceph-mon[194833]: osdmap e319: 8 total, 8 up, 6 in 2023-09-23T20:07:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:21 smithi144 ceph-mon[194833]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:21 smithi144 ceph-mon[194833]: failed to encode map e319 with expected crc 2023-09-23T20:07:22.405 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:22.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:22.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:22.514 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:22.612 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-09-23T20:07:22.637 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:22.884 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:22.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:22.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:22.962 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 54 16 66760 66744 77.2015 31.25 0.0344416 0.0129446 2023-09-23T20:07:22.992 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:23.019 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:23.042 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:23.070 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:23.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:23.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:23.089 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:23.177 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[221113]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[221113]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[221113]: osdmap e320: 8 total, 8 up, 6 in 2023-09-23T20:07:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[221113]: failed to encode map e318 with expected crc 2023-09-23T20:07:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[221113]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6526/193227 objects degraded (3.377%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:23.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[221113]: pgmap v886: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 6 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 12 MiB/s wr, 192 op/s; 19569/195534 objects degraded (10.008%); 5753/195534 objects misplaced (2.942%); 2.7 MiB/s, 36 objects/s recovering 2023-09-23T20:07:23.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[213871]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[213871]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[213871]: osdmap e320: 8 total, 8 up, 6 in 2023-09-23T20:07:23.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[213871]: failed to encode map e318 with expected crc 2023-09-23T20:07:23.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[213871]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:23.227 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6526/193227 objects degraded (3.377%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:23.228 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:22 smithi135 ceph-mon[213871]: pgmap v886: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 6 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 12 MiB/s wr, 192 op/s; 19569/195534 objects degraded (10.008%); 5753/195534 objects misplaced (2.942%); 2.7 MiB/s, 36 objects/s recovering 2023-09-23T20:07:23.267 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:22 smithi144 ceph-mon[194833]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:22 smithi144 ceph-mon[194833]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:22 smithi144 ceph-mon[194833]: osdmap e320: 8 total, 8 up, 6 in 2023-09-23T20:07:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:22 smithi144 ceph-mon[194833]: failed to encode map e318 with expected crc 2023-09-23T20:07:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:22 smithi144 ceph-mon[194833]: failed to encode map e320 with expected crc 2023-09-23T20:07:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6526/193227 objects degraded (3.377%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:22 smithi144 ceph-mon[194833]: pgmap v886: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 25 peering, 6 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 12 MiB/s wr, 192 op/s; 19569/195534 objects degraded (10.008%); 5753/195534 objects misplaced (2.942%); 2.7 MiB/s, 36 objects/s recovering 2023-09-23T20:07:23.515 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:23.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:23.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:23.629 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:23.736 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:23.775 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:07:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:07:23] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-09-23T20:07:23.966 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 55 16 67247 67231 76.3468 30.4375 0.0143235 0.0130921 2023-09-23T20:07:24.061 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:07:24.103 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:23 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/18945048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:07:24.104 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:23 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:07:24.106 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:23 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/18945048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:07:24.106 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:23 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:07:24.107 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:24.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:24.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:24.219 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:24.326 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:23 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/18945048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:07:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:23 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:07:24.966 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 56 16 67797 67781 75.5977 34.375 0.0197846 0.0132207 2023-09-23T20:07:25.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:24 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:24 smithi144 ceph-mon[194833]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:24 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:07:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:24 smithi144 ceph-mon[194833]: osdmap e321: 8 total, 8 up, 6 in 2023-09-23T20:07:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:24 smithi144 ceph-mon[194833]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:24 smithi144 ceph-mon[194833]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:24 smithi144 ceph-mon[194833]: pgmap v888: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 29 MiB/s wr, 458 op/s; 39288/199980 objects degraded (19.646%); 6059/199980 objects misplaced (3.030%); 6.9 MiB/s, 106 objects/s recovering 2023-09-23T20:07:25.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:25.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[213871]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:07:25.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[213871]: osdmap e321: 8 total, 8 up, 6 in 2023-09-23T20:07:25.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[213871]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[213871]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[213871]: pgmap v888: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 29 MiB/s wr, 458 op/s; 39288/199980 objects degraded (19.646%); 6059/199980 objects misplaced (3.030%); 6.9 MiB/s, 106 objects/s recovering 2023-09-23T20:07:25.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:25.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[221113]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:07:25.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[221113]: osdmap e321: 8 total, 8 up, 6 in 2023-09-23T20:07:25.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[221113]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[221113]: failed to encode map e321 with expected crc 2023-09-23T20:07:25.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:24 smithi135 ceph-mon[221113]: pgmap v888: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 29 MiB/s wr, 458 op/s; 39288/199980 objects degraded (19.646%); 6059/199980 objects misplaced (3.030%); 6.9 MiB/s, 106 objects/s recovering 2023-09-23T20:07:25.968 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 57 16 68130 68114 74.6348 20.8125 0.0211686 0.0133869 2023-09-23T20:07:26.972 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 58 16 68644 68628 73.8996 32.125 0.0343621 0.0135228 2023-09-23T20:07:27.038 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:26 smithi135 ceph-mon[213871]: pgmap v889: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 27 KiB/s rd, 30 MiB/s wr, 493 op/s; 39288/201723 objects degraded (19.476%); 5890/201723 objects misplaced (2.920%); 7.4 MiB/s, 115 objects/s recovering 2023-09-23T20:07:27.039 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:26 smithi135 ceph-mon[221113]: pgmap v889: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 27 KiB/s rd, 30 MiB/s wr, 493 op/s; 39288/201723 objects degraded (19.476%); 5890/201723 objects misplaced (2.920%); 7.4 MiB/s, 115 objects/s recovering 2023-09-23T20:07:27.049 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:27.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:27.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:27.170 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:27.272 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:27.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:26 smithi144 ceph-mon[194833]: pgmap v889: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 27 KiB/s rd, 30 MiB/s wr, 493 op/s; 39288/201723 objects degraded (19.476%); 5890/201723 objects misplaced (2.920%); 7.4 MiB/s, 115 objects/s recovering 2023-09-23T20:07:27.434 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:27.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:27.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:27.549 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:27.577 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:27.613 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1] out_osds: [6, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:07:27.613 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:07:27.613 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T20:07:27.613 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T20:07:27.626 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:27.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:27.733 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:27.746 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:07:27.746 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:07:27.746 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:07:27.746 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:07:27.746 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:07:27.838 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:27.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 39288/201723 objects degraded (19.476%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:27.977 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 39288/201723 objects degraded (19.476%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:27.978 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 59 16 69094 69078 73.1205 28.125 0.0570229 0.0136676 2023-09-23T20:07:28.103 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:07:28.143 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:07:28.168 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:28.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:28.221 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:07:28.285 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:07:28.294 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:28.320 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:07:28.357 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:07:28.378 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 39288/201723 objects degraded (19.476%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:28.413 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:07:28.531 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:07:28.531 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:07:28.531 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:07:28.532 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:07:28.532 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:07:28.532 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:07:28.532 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:07:28.532 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:07:28.532 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:07:28.949 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3079668501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:28.950 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:28 smithi135 ceph-mon[213871]: pgmap v890: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 21 KiB/s rd, 33 MiB/s wr, 537 op/s; 39288/207000 objects degraded (18.980%); 5593/207000 objects misplaced (2.702%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T20:07:28.973 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:07:28.974158+0000 min lat: 0.00105689 max lat: 2.82428 avg lat: 0.013796 2023-09-23T20:07:28.973 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:07:28.973 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 60 16 69608 69592 72.4377 32.125 0.0425109 0.013796 2023-09-23T20:07:29.073 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:07:29.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3079668501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:29.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:28 smithi135 ceph-mon[221113]: pgmap v890: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 21 KiB/s rd, 33 MiB/s wr, 537 op/s; 39288/207000 objects degraded (18.980%); 5593/207000 objects misplaced (2.702%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T20:07:29.364 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:29.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:29.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:29.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3079668501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:29.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:28 smithi144 ceph-mon[194833]: pgmap v890: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 21 KiB/s rd, 33 MiB/s wr, 537 op/s; 39288/207000 objects degraded (18.980%); 5593/207000 objects misplaced (2.702%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T20:07:29.480 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:29.510 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:29.557 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:29.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:29.663 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:29.697 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:29.744 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:29.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:29.850 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:29.881 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:29.930 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:29.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:29.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:29.976 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 61 16 70117 70101 71.7689 31.8125 0.0225245 0.0139263 2023-09-23T20:07:30.039 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:30.068 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:30.124 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:30.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:30.229 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:30.256 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:30.305 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:30.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:30.372 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:07:30.412 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:30.440 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:30.485 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:30.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:30.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:30.594 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:30.622 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:30.629 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:30.668 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:30.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:30.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:30.768 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:30.774 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:30.801 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:30.849 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:30.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:30.956 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:30.977 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 62 16 70603 70587 71.1006 30.375 0.0204928 0.0140564 2023-09-23T20:07:30.985 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:31.040 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:31.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:31.147 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:31.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:30 smithi135 ceph-mon[213871]: pgmap v891: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 19 KiB/s rd, 29 MiB/s wr, 477 op/s; 39288/207000 objects degraded (18.980%); 5593/207000 objects misplaced (2.702%); 5.9 MiB/s, 92 objects/s recovering 2023-09-23T20:07:31.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:30 smithi135 ceph-mon[221113]: pgmap v891: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 19 KiB/s rd, 29 MiB/s wr, 477 op/s; 39288/207000 objects degraded (18.980%); 5593/207000 objects misplaced (2.702%); 5.9 MiB/s, 92 objects/s recovering 2023-09-23T20:07:31.260 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:31.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:30 smithi144 ceph-mon[194833]: pgmap v891: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 19 KiB/s rd, 29 MiB/s wr, 477 op/s; 39288/207000 objects degraded (18.980%); 5593/207000 objects misplaced (2.702%); 5.9 MiB/s, 92 objects/s recovering 2023-09-23T20:07:31.543 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:31.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:31.656 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:31.683 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:31.733 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:31.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:31.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:31.839 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:31.867 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:31.927 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:31.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:31.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:31.977 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 63 16 71111 71095 70.476 31.75 0.0270443 0.0141831 2023-09-23T20:07:32.037 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:32.065 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:32.114 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:32.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:32.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:32.224 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:32.276 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:32.435 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:07:32.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 39288/207000 objects degraded (18.980%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:32.985 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 39288/207000 objects degraded (18.980%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:32.985 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 64 16 71548 71532 69.7999 27.3125 0.0570119 0.0143174 2023-09-23T20:07:33.030 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:33.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:33.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:33.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 39288/207000 objects degraded (18.980%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:33.137 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:33.181 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:07:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:07:33] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-09-23T20:07:33.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:33 smithi135 ceph-mon[213871]: pgmap v892: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 29 MiB/s wr, 472 op/s; 39288/209322 objects degraded (18.769%); 5391/209322 objects misplaced (2.575%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:07:33.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:33 smithi135 ceph-mon[221113]: pgmap v892: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 29 MiB/s wr, 472 op/s; 39288/209322 objects degraded (18.769%); 5391/209322 objects misplaced (2.575%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:07:33.979 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 65 16 72079 72063 69.2369 33.1875 0.0284363 0.0144348 2023-09-23T20:07:33.981 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:33.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:34.059 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:34.088 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:34.112 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:33 smithi144 ceph-mon[194833]: pgmap v892: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 29 MiB/s wr, 472 op/s; 39288/209322 objects degraded (18.769%); 5391/209322 objects misplaced (2.575%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:07:34.115 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:34.162 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:34.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:34.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:34.269 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:34.298 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:34.316 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:34.344 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:34.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:34.453 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:34.480 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:34.528 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:34.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:34.634 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:34.664 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:34.713 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:34.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:34.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:34.819 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:34.959 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:34.986 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 66 16 72572 72556 68.6515 30.8125 0.0237431 0.0145587 2023-09-23T20:07:35.025 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:34 smithi135 ceph-mon[221113]: pgmap v893: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 15 KiB/s rd, 28 MiB/s wr, 458 op/s; 39288/214476 objects degraded (18.318%); 5105/214476 objects misplaced (2.380%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:07:35.026 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:34 smithi135 ceph-mon[213871]: pgmap v893: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 15 KiB/s rd, 28 MiB/s wr, 458 op/s; 39288/214476 objects degraded (18.318%); 5105/214476 objects misplaced (2.380%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:07:35.235 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:35.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:35.348 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:35.372 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:34 smithi144 ceph-mon[194833]: pgmap v893: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 15 KiB/s rd, 28 MiB/s wr, 458 op/s; 39288/214476 objects degraded (18.318%); 5105/214476 objects misplaced (2.380%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:07:35.375 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:35.427 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:35.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:35.535 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:35.649 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:35.819 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:35.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:35.934 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:35.971 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:35.983 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 67 16 73006 72990 68.032 27.125 0.0517814 0.0146864 2023-09-23T20:07:35.990 INFO:teuthology.orchestra.run.smithi135.stderr:marked in osd.7. 2023-09-23T20:07:36.028 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:36.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:36.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:36.133 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:36.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:35 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1852374612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:07:36.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:35 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:07:36.158 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:35 smithi135 conmon[213845]: 2023-09-23T20:07:35.772+0000 7f9708921700 -1 mon.a@0(leader).osd e321 definitely_dead 0 2023-09-23T20:07:36.158 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:35 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1852374612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:07:36.158 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:35 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:07:36.160 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:36.221 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:36.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:36.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:36.324 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:36.353 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:35 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1852374612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:07:36.353 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:35 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:07:36.356 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:36.404 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:36.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:36.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:36.509 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:36.537 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:36.581 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:36.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:36.690 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:36.719 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: osdmap e322: 8 total, 8 up, 7 in 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/553317366' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:36.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:36.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[221113]: pgmap v895: 99 pgs: 1 remapped+peering, 8 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 30 MiB/s wr, 485 op/s; 39288/216288 objects degraded (18.165%); 2362/216288 objects misplaced (1.092%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:07:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T20:07:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: osdmap e322: 8 total, 8 up, 7 in 2023-09-23T20:07:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: failed to encode map e322 with expected crc 2023-09-23T20:07:36.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/553317366' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:36.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:36.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:36 smithi135 ceph-mon[213871]: pgmap v895: 99 pgs: 1 remapped+peering, 8 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 30 MiB/s wr, 485 op/s; 39288/216288 objects degraded (18.165%); 2362/216288 objects misplaced (1.092%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:07:36.985 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 68 16 73034 73018 67.058 1.75 0.00345774 0.0147064 2023-09-23T20:07:36.989 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:07:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: failed to encode map e322 with expected crc 2023-09-23T20:07:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T20:07:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: osdmap e322: 8 total, 8 up, 7 in 2023-09-23T20:07:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: failed to encode map e322 with expected crc 2023-09-23T20:07:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: failed to encode map e322 with expected crc 2023-09-23T20:07:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: failed to encode map e322 with expected crc 2023-09-23T20:07:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: failed to encode map e322 with expected crc 2023-09-23T20:07:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/553317366' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:36 smithi144 ceph-mon[194833]: pgmap v895: 99 pgs: 1 remapped+peering, 8 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 30 MiB/s wr, 485 op/s; 39288/216288 objects degraded (18.165%); 2362/216288 objects misplaced (1.092%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:07:37.983 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 69 16 73650 73634 66.6443 38.5 0.0322945 0.0149951 2023-09-23T20:07:38.013 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:38.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:38.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:38.125 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:38.154 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:38.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: osdmap e323: 8 total, 8 up, 7 in 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 39288/216288 objects degraded (18.165%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: osdmap e323: 8 total, 8 up, 7 in 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:37 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 39288/216288 objects degraded (18.165%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:38.226 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: osdmap e323: 8 total, 8 up, 7 in 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: failed to encode map e323 with expected crc 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:38.236 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:38.237 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:37 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 39288/216288 objects degraded (18.165%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:38.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:38.338 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:38.368 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:38.429 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:38.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:38.536 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:38.563 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:38.781 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T20:07:38.986 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 70 16 74226 74210 66.2042 36 0.0176884 0.0150987 2023-09-23T20:07:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:38 smithi144 ceph-mon[194833]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:38 smithi144 ceph-mon[194833]: osdmap e324: 8 total, 8 up, 7 in 2023-09-23T20:07:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:38 smithi144 ceph-mon[194833]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:38 smithi144 ceph-mon[194833]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:38 smithi144 ceph-mon[194833]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:38 smithi144 ceph-mon[194833]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:38 smithi144 ceph-mon[194833]: pgmap v898: 99 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 41 MiB/s wr, 662 op/s; 32571/221046 objects degraded (14.735%); 1080/221046 objects misplaced (0.489%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:07:39.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[221113]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[221113]: osdmap e324: 8 total, 8 up, 7 in 2023-09-23T20:07:39.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[221113]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[221113]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[221113]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[221113]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.460 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[221113]: pgmap v898: 99 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 41 MiB/s wr, 662 op/s; 32571/221046 objects degraded (14.735%); 1080/221046 objects misplaced (0.489%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:07:39.460 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[213871]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.461 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[213871]: osdmap e324: 8 total, 8 up, 7 in 2023-09-23T20:07:39.461 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[213871]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.461 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[213871]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.461 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[213871]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.461 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[213871]: failed to encode map e324 with expected crc 2023-09-23T20:07:39.461 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:38 smithi135 ceph-mon[213871]: pgmap v898: 99 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 41 MiB/s wr, 662 op/s; 32571/221046 objects degraded (14.735%); 1080/221046 objects misplaced (0.489%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:07:39.689 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:39.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:39.795 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:39.929 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:39.986 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 71 16 74761 74745 65.7428 33.4375 0.0321918 0.0152017 2023-09-23T20:07:40.074 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:40.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:40.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:40.183 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:40.205 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:39 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:07:40.205 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:39 smithi135 ceph-mon[221113]: failed to encode map e325 with expected crc 2023-09-23T20:07:40.205 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:39 smithi135 ceph-mon[221113]: failed to encode map e325 with expected crc 2023-09-23T20:07:40.205 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:39 smithi135 ceph-mon[221113]: osdmap e325: 8 total, 8 up, 7 in 2023-09-23T20:07:40.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:39 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:07:40.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:39 smithi135 ceph-mon[213871]: failed to encode map e325 with expected crc 2023-09-23T20:07:40.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:39 smithi135 ceph-mon[213871]: failed to encode map e325 with expected crc 2023-09-23T20:07:40.208 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:39 smithi135 ceph-mon[213871]: osdmap e325: 8 total, 8 up, 7 in 2023-09-23T20:07:40.215 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:40.269 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:40.274 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:39 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:07:40.274 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:39 smithi144 ceph-mon[194833]: failed to encode map e325 with expected crc 2023-09-23T20:07:40.274 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:39 smithi144 ceph-mon[194833]: failed to encode map e325 with expected crc 2023-09-23T20:07:40.275 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:39 smithi144 ceph-mon[194833]: osdmap e325: 8 total, 8 up, 7 in 2023-09-23T20:07:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:40.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:40.376 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:40.407 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:40.448 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:07:40.448 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:07:40.472 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:40.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:40.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:40.579 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:40.584 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:07:40.616 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:40.629 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:07:40.670 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:40.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:40.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:40.684 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:07:40.741 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:07:40.781 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:40.791 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:07:40.839 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:07:40.868 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:40.885 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:07:40.925 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:07:40.940 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:07:40.940 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:07:40.940 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:07:40.940 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:07:40.940 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:07:40.940 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:07:40.941 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:07:40.941 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:07:40.992 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 72 16 75372 75356 65.3551 38.1875 0.0260298 0.0152939 2023-09-23T20:07:40.997 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:41.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:41.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:41.105 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:41.136 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:41.193 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:41.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:41.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[213871]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[213871]: osdmap e326: 8 total, 8 up, 7 in 2023-09-23T20:07:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[213871]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[213871]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[213871]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2673072883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[213871]: pgmap v901: 99 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 25 MiB/s wr, 412 op/s; 32571/221046 objects degraded (14.735%); 1080/221046 objects misplaced (0.489%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T20:07:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[221113]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[221113]: osdmap e326: 8 total, 8 up, 7 in 2023-09-23T20:07:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[221113]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[221113]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[221113]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2673072883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:41 smithi135 ceph-mon[221113]: pgmap v901: 99 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 25 MiB/s wr, 412 op/s; 32571/221046 objects degraded (14.735%); 1080/221046 objects misplaced (0.489%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T20:07:41.298 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:41.327 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:41 smithi144 ceph-mon[194833]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.327 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:41 smithi144 ceph-mon[194833]: osdmap e326: 8 total, 8 up, 7 in 2023-09-23T20:07:41.328 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:41 smithi144 ceph-mon[194833]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.328 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:41 smithi144 ceph-mon[194833]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.328 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:41 smithi144 ceph-mon[194833]: failed to encode map e326 with expected crc 2023-09-23T20:07:41.328 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2673072883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:41.328 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:41 smithi144 ceph-mon[194833]: pgmap v901: 99 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 21 KiB/s rd, 25 MiB/s wr, 412 op/s; 32571/221046 objects degraded (14.735%); 1080/221046 objects misplaced (0.489%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T20:07:41.333 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:41.387 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:41.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:41.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:41.493 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:41.533 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:41.593 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:41.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:41.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:41.700 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:41.732 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:41.799 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:41.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:41.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:41.868 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:07:41.896 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:07:41.905 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:41.987 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:41.992 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 73 16 76032 76016 65.0252 41.25 0.0083006 0.0153678 2023-09-23T20:07:42.034 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:42.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:42.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:42.138 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:42.163 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:42.219 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:42.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:42.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:42.424 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:42.678 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:42.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32571/221046 objects degraded (14.735%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:42.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32571/221046 objects degraded (14.735%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:42.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:42.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:42.786 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:42.814 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:42.846 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:42.870 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:42.992 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 74 16 76533 76517 64.5699 31.3125 0.0295402 0.0154808 2023-09-23T20:07:43.020 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:43.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:43.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:43.133 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:43.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32571/221046 objects degraded (14.735%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:43.225 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:43.438 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:43.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:43.548 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:43.583 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:43.637 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:43.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:43.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:43.664 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:43 smithi135 ceph-mon[213871]: pgmap v902: 99 pgs: 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 25 MiB/s wr, 409 op/s; 33922/223332 objects degraded (15.189%); 1067/223332 objects misplaced (0.478%); 4.7 MiB/s, 77 objects/s recovering 2023-09-23T20:07:43.664 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:43 smithi135 ceph-mon[221113]: pgmap v902: 99 pgs: 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 25 MiB/s wr, 409 op/s; 33922/223332 objects degraded (15.189%); 1067/223332 objects misplaced (0.478%); 4.7 MiB/s, 77 objects/s recovering 2023-09-23T20:07:43.743 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:43.782 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:07:43.782 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:07:43.782 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T20:07:43.782 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:07:43.782 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:07:43.783 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T20:07:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:43.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:43 smithi144 ceph-mon[194833]: pgmap v902: 99 pgs: 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 14 KiB/s rd, 25 MiB/s wr, 409 op/s; 33922/223332 objects degraded (15.189%); 1067/223332 objects misplaced (0.478%); 4.7 MiB/s, 77 objects/s recovering 2023-09-23T20:07:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:07:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:07:43] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-09-23T20:07:43.996 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 75 16 77158 77142 64.2268 39.0625 0.01125 0.0155639 2023-09-23T20:07:44.253 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:07:44.664 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:07:44.908 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:44.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:44.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:44.996 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 76 16 77651 77635 63.7874 30.8125 0.00741595 0.0156459 2023-09-23T20:07:45.020 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:45.129 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:45.680 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:45.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:45.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:45.784 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:45.828 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:45.885 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:45.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:45.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:45.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[213871]: failed to encode map e327 with expected crc 2023-09-23T20:07:45.953 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[213871]: osdmap e327: 8 total, 8 up, 7 in 2023-09-23T20:07:45.953 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[213871]: failed to encode map e327 with expected crc 2023-09-23T20:07:45.953 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[213871]: failed to encode map e327 with expected crc 2023-09-23T20:07:45.953 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[213871]: pgmap v904: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 29 MiB/s wr, 462 op/s; 33922/229374 objects degraded (14.789%); 72/229374 objects misplaced (0.031%); 10 MiB/s, 168 objects/s recovering 2023-09-23T20:07:45.954 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[221113]: failed to encode map e327 with expected crc 2023-09-23T20:07:45.954 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[221113]: osdmap e327: 8 total, 8 up, 7 in 2023-09-23T20:07:45.954 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[221113]: failed to encode map e327 with expected crc 2023-09-23T20:07:45.955 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[221113]: failed to encode map e327 with expected crc 2023-09-23T20:07:45.955 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:45 smithi135 ceph-mon[221113]: pgmap v904: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 29 MiB/s wr, 462 op/s; 33922/229374 objects degraded (14.789%); 72/229374 objects misplaced (0.031%); 10 MiB/s, 168 objects/s recovering 2023-09-23T20:07:45.988 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:46.013 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 77 16 77928 77912 63.1832 17.3125 0.0211804 0.015821 2023-09-23T20:07:46.091 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:45 smithi144 ceph-mon[194833]: failed to encode map e327 with expected crc 2023-09-23T20:07:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:45 smithi144 ceph-mon[194833]: osdmap e327: 8 total, 8 up, 7 in 2023-09-23T20:07:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:45 smithi144 ceph-mon[194833]: failed to encode map e327 with expected crc 2023-09-23T20:07:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:45 smithi144 ceph-mon[194833]: failed to encode map e327 with expected crc 2023-09-23T20:07:46.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:45 smithi144 ceph-mon[194833]: pgmap v904: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 29 MiB/s wr, 462 op/s; 33922/229374 objects degraded (14.789%); 72/229374 objects misplaced (0.031%); 10 MiB/s, 168 objects/s recovering 2023-09-23T20:07:46.180 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:46.292 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:07:46.292 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:07:46.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.293 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35643.0:78068 8.b e328/317 8:d53a6104:::benchmark_data_smithi135_460343_object78067:head v 328'4768, mlcod=0'0)", 2023-09-23T20:07:46.293 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:46.289135+0000", 2023-09-23T20:07:46.293 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0030588389999999998, 2023-09-23T20:07:46.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030927929999999999, 2023-09-23T20:07:46.293 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:07:46.293 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:07:46.293 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:07:46.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:07:46.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289135+0000", 2023-09-23T20:07:46.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:46.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:07:46.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289135+0000", 2023-09-23T20:07:46.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:46.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:07:46.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289134+0000", 2023-09-23T20:07:46.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:07:46.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:07:46.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289159+0000", 2023-09-23T20:07:46.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5439000000000002e-05 2023-09-23T20:07:46.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:07:46.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289178+0000", 2023-09-23T20:07:46.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9286000000000001e-05 2023-09-23T20:07:46.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:07:46.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289188+0000", 2023-09-23T20:07:46.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2650000000000002e-06 2023-09-23T20:07:46.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:07:46.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289324+0000", 2023-09-23T20:07:46.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013604200000000001 2023-09-23T20:07:46.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:07:46.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289350+0000", 2023-09-23T20:07:46.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6806999999999999e-05 2023-09-23T20:07:46.300 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:46.300 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:07:46.300 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:46.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.301 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35643.0:78073 8.0 e328/317 8:02cf40e4:::benchmark_data_smithi135_460343_object78072:head v 328'4784, mlcod=306'1)", 2023-09-23T20:07:46.301 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:46.289203+0000", 2023-09-23T20:07:46.301 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0029905299999999999, 2023-09-23T20:07:46.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030963119999999999, 2023-09-23T20:07:46.301 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:07:46.301 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:07:46.301 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:07:46.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:07:46.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289203+0000", 2023-09-23T20:07:46.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:46.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:07:46.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289203+0000", 2023-09-23T20:07:46.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:46.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:07:46.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289203+0000", 2023-09-23T20:07:46.304 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:07:46.304 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.304 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.304 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:07:46.304 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289222+0000", 2023-09-23T20:07:46.304 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9613999999999999e-05 2023-09-23T20:07:46.304 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:07:46.305 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289228+0000", 2023-09-23T20:07:46.305 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0220000000000001e-06 2023-09-23T20:07:46.305 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.306 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:07:46.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289233+0000", 2023-09-23T20:07:46.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3680000000000004e-06 2023-09-23T20:07:46.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.306 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.306 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:07:46.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289980+0000", 2023-09-23T20:07:46.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00074700000000000005 2023-09-23T20:07:46.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:07:46.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.290017+0000", 2023-09-23T20:07:46.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6695e-05 2023-09-23T20:07:46.308 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:46.308 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:07:46.308 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:46.308 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.308 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.308 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35643.0:78076 8.3 e328/317)", 2023-09-23T20:07:46.308 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:46.289922+0000", 2023-09-23T20:07:46.309 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.00227188, 2023-09-23T20:07:46.309 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024337619999999999, 2023-09-23T20:07:46.309 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:07:46.309 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T20:07:46.309 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:07:46.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:07:46.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289922+0000", 2023-09-23T20:07:46.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:46.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:07:46.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289922+0000", 2023-09-23T20:07:46.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:07:46.311 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:07:46.311 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289921+0000", 2023-09-23T20:07:46.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:07:46.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.312 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:07:46.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289947+0000", 2023-09-23T20:07:46.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6364e-05 2023-09-23T20:07:46.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:07:46.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289960+0000", 2023-09-23T20:07:46.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3135e-05 2023-09-23T20:07:46.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:07:46.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:07:46.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:07:46.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:46.289968+0000", 2023-09-23T20:07:46.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3590000000000002e-06 2023-09-23T20:07:46.314 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:46.314 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:07:46.314 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:46.314 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:07:46.314 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:07:46.314 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 3 2023-09-23T20:07:46.315 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:07:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:46.316 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:46.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:46.417 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:46.450 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:46.507 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:46.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:46.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:46.611 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:46.741 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:46 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:07:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:46 smithi144 ceph-mon[194833]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:46 smithi144 ceph-mon[194833]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:46 smithi144 ceph-mon[194833]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:46 smithi144 ceph-mon[194833]: osdmap e328: 8 total, 8 up, 7 in 2023-09-23T20:07:46.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:07:46.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[213871]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[213871]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[213871]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[213871]: osdmap e328: 8 total, 8 up, 7 in 2023-09-23T20:07:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:07:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[221113]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[221113]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[221113]: failed to encode map e328 with expected crc 2023-09-23T20:07:46.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:46 smithi135 ceph-mon[221113]: osdmap e328: 8 total, 8 up, 7 in 2023-09-23T20:07:46.998 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 78 16 78465 78449 62.8037 33.5625 0.0153489 0.0159122 2023-09-23T20:07:47.240 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:47.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:47.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:47.345 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:47.445 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:47.711 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:47 smithi135 ceph-mon[213871]: pgmap v906: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 33922/231378 objects degraded (14.661%); 72/231378 objects misplaced (0.031%); 9.1 MiB/s, 147 objects/s recovering 2023-09-23T20:07:47.711 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4263777584' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:47.711 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:47 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:47.711 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 33922/229374 objects degraded (14.789%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:47.750 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:07:47.758 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:47.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:47.874 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:07:47.913 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:47.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:47 smithi135 ceph-mon[221113]: pgmap v906: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 33922/231378 objects degraded (14.661%); 72/231378 objects misplaced (0.031%); 9.1 MiB/s, 147 objects/s recovering 2023-09-23T20:07:47.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4263777584' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:47.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:47 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:47.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 33922/229374 objects degraded (14.789%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:47.983 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:07:47.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:47.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:07:47.988 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:47 smithi144 ceph-mon[194833]: pgmap v906: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 33922/231378 objects degraded (14.661%); 72/231378 objects misplaced (0.031%); 9.1 MiB/s, 147 objects/s recovering 2023-09-23T20:07:47.988 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4263777584' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:47.988 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:47 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:07:47.988 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 33922/229374 objects degraded (14.789%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:48.001 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 79 16 78967 78951 62.4038 31.375 0.0263818 0.0160175 2023-09-23T20:07:48.088 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:48.126 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:48.232 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:48.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:48.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:48.321 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:07:48.322 INFO:teuthology.orchestra.run.smithi144.stdout: "success": "" 2023-09-23T20:07:48.322 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:07:48.336 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:48.363 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:48.582 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:07:48.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:48.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[221113]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:07:48.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[221113]: osdmap e329: 8 total, 8 up, 7 in 2023-09-23T20:07:48.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[221113]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[221113]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[221113]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[221113]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[213871]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[213871]: osdmap e329: 8 total, 8 up, 7 in 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[213871]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[213871]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.937 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[213871]: failed to encode map e329 with expected crc 2023-09-23T20:07:48.938 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:48 smithi135 ceph-mon[213871]: failed to encode map e329 with expected crc 2023-09-23T20:07:49.002 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:07:49.002303+0000 min lat: 0.00105689 max lat: 2.82428 avg lat: 0.0161212 2023-09-23T20:07:49.002 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:07:49.002 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 80 16 79377 79361 61.9444 25.625 0.00803018 0.0161212 2023-09-23T20:07:49.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:48 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:49.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:48 smithi144 ceph-mon[194833]: failed to encode map e329 with expected crc 2023-09-23T20:07:49.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:48 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:07:49.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:48 smithi144 ceph-mon[194833]: osdmap e329: 8 total, 8 up, 7 in 2023-09-23T20:07:49.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:48 smithi144 ceph-mon[194833]: failed to encode map e329 with expected crc 2023-09-23T20:07:49.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:48 smithi144 ceph-mon[194833]: failed to encode map e329 with expected crc 2023-09-23T20:07:49.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:48 smithi144 ceph-mon[194833]: failed to encode map e329 with expected crc 2023-09-23T20:07:49.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:48 smithi144 ceph-mon[194833]: failed to encode map e329 with expected crc 2023-09-23T20:07:49.371 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:49.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:49.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:49.494 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:49.601 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:49.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:49 smithi144 ceph-mon[194833]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 32613/236520 objects degraded (13.789%); 581/236520 objects misplaced (0.246%); 19 MiB/s, 298 objects/s recovering 2023-09-23T20:07:49.883 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:49.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:49.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:49.895 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:49 smithi135 ceph-mon[213871]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 32613/236520 objects degraded (13.789%); 581/236520 objects misplaced (0.246%); 19 MiB/s, 298 objects/s recovering 2023-09-23T20:07:49.895 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:49 smithi135 ceph-mon[221113]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 32613/236520 objects degraded (13.789%); 581/236520 objects misplaced (0.246%); 19 MiB/s, 298 objects/s recovering 2023-09-23T20:07:49.966 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:49.995 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:07:50.002 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 81 16 79962 79946 61.6307 36.5625 0.023068 0.0162194 2023-09-23T20:07:50.032 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:50.087 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:07:50.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:50.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:07:50.145 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:07:50.185 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:07:50.194 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:50.232 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:07:50.284 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:50.289 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:07:50.332 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:07:50.382 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:07:50.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:50.428 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:07:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:50.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:50.468 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:07:50.488 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:07:50.488 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:07:50.488 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:07:50.488 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:07:50.488 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:07:50.488 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:07:50.488 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:07:50.488 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:07:50.535 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:50.619 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:50.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/49851948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:50.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/49851948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:51.002 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 82 16 80550 80534 61.3275 36.75 0.0145772 0.0162995 2023-09-23T20:07:51.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/49851948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:07:51.398 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:07:52.003 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 83 16 81032 81016 60.9512 30.125 0.0202448 0.0163959 2023-09-23T20:07:52.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:51 smithi144 ceph-mon[194833]: pgmap v909: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 32613/236520 objects degraded (13.789%); 581/236520 objects misplaced (0.246%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:07:52.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:51 smithi135 ceph-mon[221113]: pgmap v909: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 32613/236520 objects degraded (13.789%); 581/236520 objects misplaced (0.246%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:07:52.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:51 smithi135 ceph-mon[213871]: pgmap v909: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 32613/236520 objects degraded (13.789%); 581/236520 objects misplaced (0.246%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:07:52.620 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:52.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:52.724 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:52.753 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:52.809 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:52.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:52.918 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:07:52.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: osdmap e330: 8 total, 8 up, 7 in 2023-09-23T20:07:52.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:52.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:52.976 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32613/236520 objects degraded (13.789%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:52.981 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.981 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: osdmap e330: 8 total, 8 up, 7 in 2023-09-23T20:07:52.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: failed to encode map e330 with expected crc 2023-09-23T20:07:52.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:52.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:52.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:52 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32613/236520 objects degraded (13.789%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:53.003 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 84 16 81285 81269 60.4142 15.8125 0.0386328 0.0165459 2023-09-23T20:07:53.032 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:53.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: failed to encode map e330 with expected crc 2023-09-23T20:07:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: failed to encode map e330 with expected crc 2023-09-23T20:07:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: failed to encode map e330 with expected crc 2023-09-23T20:07:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: osdmap e330: 8 total, 8 up, 7 in 2023-09-23T20:07:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: failed to encode map e330 with expected crc 2023-09-23T20:07:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: failed to encode map e330 with expected crc 2023-09-23T20:07:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:07:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:52 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32613/236520 objects degraded (13.789%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:53.168 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:07:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:53.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:07:53.232 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:53.276 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:53.395 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:53.602 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:53.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:53.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:53.706 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:07:53.804 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:53.941 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:07:53.952 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[221113]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[221113]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[221113]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[221113]: osdmap e331: 8 total, 8 up, 7 in 2023-09-23T20:07:53.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[221113]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[221113]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[221113]: pgmap v912: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 424 op/s; 28463/239028 objects degraded (11.908%); 4642/239028 objects misplaced (1.942%); 9.8 MiB/s, 148 objects/s recovering 2023-09-23T20:07:53.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:53.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:07:53.955 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[213871]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.955 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[213871]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.955 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[213871]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.955 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[213871]: osdmap e331: 8 total, 8 up, 7 in 2023-09-23T20:07:53.955 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[213871]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.956 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[213871]: failed to encode map e331 with expected crc 2023-09-23T20:07:53.956 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:53 smithi135 ceph-mon[213871]: pgmap v912: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 424 op/s; 28463/239028 objects degraded (11.908%); 4642/239028 objects misplaced (1.942%); 9.8 MiB/s, 148 objects/s recovering 2023-09-23T20:07:53.956 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:07:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:07:53] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2023-09-23T20:07:54.008 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 85 16 81820 81804 60.0971 33.4375 0.0497878 0.0166337 2023-09-23T20:07:54.040 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:07:54.040 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:07:54.055 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:53 smithi144 ceph-mon[194833]: failed to encode map e331 with expected crc 2023-09-23T20:07:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:53 smithi144 ceph-mon[194833]: failed to encode map e331 with expected crc 2023-09-23T20:07:54.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:53 smithi144 ceph-mon[194833]: failed to encode map e331 with expected crc 2023-09-23T20:07:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:53 smithi144 ceph-mon[194833]: osdmap e331: 8 total, 8 up, 7 in 2023-09-23T20:07:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:53 smithi144 ceph-mon[194833]: failed to encode map e331 with expected crc 2023-09-23T20:07:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:53 smithi144 ceph-mon[194833]: failed to encode map e331 with expected crc 2023-09-23T20:07:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:53 smithi144 ceph-mon[194833]: pgmap v912: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 424 op/s; 28463/239028 objects degraded (11.908%); 4642/239028 objects misplaced (1.942%); 9.8 MiB/s, 148 objects/s recovering 2023-09-23T20:07:54.144 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:54.541 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:07:54.542 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:07:54.542 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:07:54.542 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.649144 2023-09-23T20:07:54.542 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0.6491436772730964 2023-09-23T20:07:54.954 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:54 smithi135 ceph-mon[213871]: pgmap v913: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 566 op/s; 28452/243951 objects degraded (11.663%); 4061/243951 objects misplaced (1.665%); 13 MiB/s, 207 objects/s recovering 2023-09-23T20:07:55.003 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 86 16 82428 82412 59.84 38 0.0397273 0.0167057 2023-09-23T20:07:55.013 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:07:55.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:54 smithi135 ceph-mon[221113]: pgmap v913: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 566 op/s; 28452/243951 objects degraded (11.663%); 4061/243951 objects misplaced (1.665%); 13 MiB/s, 207 objects/s recovering 2023-09-23T20:07:55.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:54 smithi144 ceph-mon[194833]: pgmap v913: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 566 op/s; 28452/243951 objects degraded (11.663%); 4061/243951 objects misplaced (1.665%); 13 MiB/s, 207 objects/s recovering 2023-09-23T20:07:55.476 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:55.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:55.584 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:55.613 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:55.661 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:55.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:55.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:55.766 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:55.791 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:56.005 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 87 16 82882 82866 59.4779 28.375 0.0506394 0.0168055 2023-09-23T20:07:56.305 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:56.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:56.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:07:56.415 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:07:56.486 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:56.566 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:56.591 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:07:56.797 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:07:56.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:56.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:07:56.908 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:07:56.934 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:56.982 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:07:56.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:56.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:07:57.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 88 16 83373 83357 59.1508 30.6875 0.0203383 0.0169001 2023-09-23T20:07:57.094 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:57.119 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:56 smithi135 ceph-mon[213871]: pgmap v914: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 382 op/s; 28452/245703 objects degraded (11.580%); 4061/245703 objects misplaced (1.653%); 5.7 MiB/s, 85 objects/s recovering 2023-09-23T20:07:57.120 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:56 smithi135 ceph-mon[221113]: pgmap v914: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 382 op/s; 28452/245703 objects degraded (11.580%); 4061/245703 objects misplaced (1.653%); 5.7 MiB/s, 85 objects/s recovering 2023-09-23T20:07:57.201 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:57.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:56 smithi144 ceph-mon[194833]: pgmap v914: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 382 op/s; 28452/245703 objects degraded (11.580%); 4061/245703 objects misplaced (1.653%); 5.7 MiB/s, 85 objects/s recovering 2023-09-23T20:07:57.529 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:57.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:57.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:57.751 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:58.005 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 89 16 83806 83790 58.7905 27.0625 0.0124871 0.0169715 2023-09-23T20:07:58.044 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 28452/245703 objects degraded (11.580%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:58.044 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/22006290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:58.044 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:57 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:58.045 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:58.045 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 28452/245703 objects degraded (11.580%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:58.045 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/22006290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:58.045 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:57 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:58.045 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 28452/245703 objects degraded (11.580%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:07:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/22006290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:57 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:07:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:58.662 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:07:59.005 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total time run: 90.0112 2023-09-23T20:07:59.005 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total writes made: 84230 2023-09-23T20:07:59.005 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Write size: 65536 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object size: 65536 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Bandwidth (MB/sec): 58.4857 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Bandwidth: 44.2731 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max bandwidth (MB/sec): 218.625 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average IOPS: 935 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev IOPS: 708.375 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max IOPS: 3498 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min IOPS: 0 2023-09-23T20:07:59.006 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average Latency(s): 0.0170935 2023-09-23T20:07:59.007 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Latency(s): 0.0546574 2023-09-23T20:07:59.007 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max latency(s): 2.82428 2023-09-23T20:07:59.007 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min latency(s): 0.00105689 2023-09-23T20:07:59.012 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:07:59.104 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:59.105 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:59.105 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:59 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:59.105 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:59 smithi144 ceph-mon[194833]: failed to encode map e332 with expected crc 2023-09-23T20:07:59.105 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:59 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:59.105 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:59 smithi144 ceph-mon[194833]: osdmap e332: 8 total, 8 up, 7 in 2023-09-23T20:07:59.105 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:07:59 smithi144 ceph-mon[194833]: pgmap v916: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 37 MiB/s wr, 598 op/s; 28452/251298 objects degraded (11.322%); 3909/251298 objects misplaced (1.556%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T20:07:59.322 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:07:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:59.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:07:59.426 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:07:59.469 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:07:59.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:59.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:59.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[221113]: failed to encode map e332 with expected crc 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[221113]: osdmap e332: 8 total, 8 up, 7 in 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[221113]: pgmap v916: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 37 MiB/s wr, 598 op/s; 28452/251298 objects degraded (11.322%); 3909/251298 objects misplaced (1.556%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[213871]: failed to encode map e332 with expected crc 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:07:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[213871]: osdmap e332: 8 total, 8 up, 7 in 2023-09-23T20:07:59.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:07:59 smithi135 ceph-mon[213871]: pgmap v916: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 37 MiB/s wr, 598 op/s; 28452/251298 objects degraded (11.322%); 3909/251298 objects misplaced (1.556%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T20:07:59.522 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:07:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:07:59.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:07:59.626 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:07:59.672 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:00 smithi144 ceph-mon[194833]: failed to encode map e332 with expected crc 2023-09-23T20:08:00.411 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:00 smithi144 ceph-mon[194833]: failed to encode map e332 with expected crc 2023-09-23T20:08:00.411 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:00 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:00.411 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:00 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:00 smithi135 ceph-mon[221113]: failed to encode map e332 with expected crc 2023-09-23T20:08:00.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:00 smithi135 ceph-mon[221113]: failed to encode map e332 with expected crc 2023-09-23T20:08:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:00 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:00 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:00 smithi135 ceph-mon[213871]: failed to encode map e332 with expected crc 2023-09-23T20:08:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:00 smithi135 ceph-mon[213871]: failed to encode map e332 with expected crc 2023-09-23T20:08:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:00 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:00 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:01.108 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:01.144 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:01.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:01.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:01.262 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:01.378 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:01 smithi135 ceph-mon[221113]: pgmap v917: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 28452/251298 objects degraded (11.322%); 3909/251298 objects misplaced (1.556%); 5.7 MiB/s, 92 objects/s recovering 2023-09-23T20:08:01.379 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:01 smithi135 ceph-mon[213871]: pgmap v917: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 28452/251298 objects degraded (11.322%); 3909/251298 objects misplaced (1.556%); 5.7 MiB/s, 92 objects/s recovering 2023-09-23T20:08:01.396 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:01.532 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:01.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:01.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:01 smithi144 ceph-mon[194833]: pgmap v917: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 28452/251298 objects degraded (11.322%); 3909/251298 objects misplaced (1.556%); 5.7 MiB/s, 92 objects/s recovering 2023-09-23T20:08:01.640 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:01.740 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:08:02.010 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:02.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:02.024 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:08:02.063 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:08:02.118 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:02.123 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:08:02.148 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:02.166 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:08:02.195 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:02.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:02.207 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:08:02.243 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:08:02.286 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:08:02.294 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:08:02.294 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:08:02.295 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:08:02.295 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:08:02.295 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:08:02.295 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:08:02.295 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:08:02.295 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:08:02.300 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:02.326 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:02.374 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:02.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:02.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:02.482 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:02.510 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:02.561 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:02.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:02.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:02.667 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:02.782 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:02.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:02.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:02.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2402817815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:02.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:02 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 28452/251298 objects degraded (11.322%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2402817815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:02 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 28452/251298 objects degraded (11.322%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:02.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:02.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:02.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2402817815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:02.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:02 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 28452/251298 objects degraded (11.322%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:03.156 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:08:03.695 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:08:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:08:03] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2023-09-23T20:08:03.771 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.3 primary-affinity to 0.649144 (8425422) 2023-09-23T20:08:03.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:03 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2354226543' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6491436772730964}]: dispatch 2023-09-23T20:08:03.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:03 smithi135 ceph-mon[221113]: pgmap v918: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 28 MiB/s wr, 487 op/s; 28452/251151 objects degraded (11.329%); 3756/251151 objects misplaced (1.496%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T20:08:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:03 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2354226543' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6491436772730964}]: dispatch 2023-09-23T20:08:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:03 smithi135 ceph-mon[213871]: pgmap v918: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 28 MiB/s wr, 487 op/s; 28452/251151 objects degraded (11.329%); 3756/251151 objects misplaced (1.496%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T20:08:04.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:03 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2354226543' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6491436772730964}]: dispatch 2023-09-23T20:08:04.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:03 smithi144 ceph-mon[194833]: pgmap v918: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 28 MiB/s wr, 487 op/s; 28452/251151 objects degraded (11.329%); 3756/251151 objects misplaced (1.496%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T20:08:05.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:04 smithi144 ceph-mon[194833]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:04 smithi144 ceph-mon[194833]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2354226543' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6491436772730964}]': finished 2023-09-23T20:08:05.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:04 smithi144 ceph-mon[194833]: osdmap e333: 8 total, 8 up, 7 in 2023-09-23T20:08:05.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:04 smithi144 ceph-mon[194833]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[221113]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[221113]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2354226543' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6491436772730964}]': finished 2023-09-23T20:08:05.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[221113]: osdmap e333: 8 total, 8 up, 7 in 2023-09-23T20:08:05.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[221113]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[213871]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[213871]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2354226543' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6491436772730964}]': finished 2023-09-23T20:08:05.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[213871]: osdmap e333: 8 total, 8 up, 7 in 2023-09-23T20:08:05.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:04 smithi135 ceph-mon[213871]: failed to encode map e333 with expected crc 2023-09-23T20:08:05.302 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:05.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:05.406 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:05.435 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:05.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[221113]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[221113]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[221113]: osdmap e334: 8 total, 8 up, 7 in 2023-09-23T20:08:05.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[221113]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[221113]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[221113]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[221113]: pgmap v921: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 164 B/s rd, 13 MiB/s wr, 541 op/s; 28452/246474 objects degraded (11.544%); 3756/246474 objects misplaced (1.524%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:08:05.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[213871]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[213871]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[213871]: osdmap e334: 8 total, 8 up, 7 in 2023-09-23T20:08:05.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[213871]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[213871]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[213871]: failed to encode map e334 with expected crc 2023-09-23T20:08:05.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:05 smithi135 ceph-mon[213871]: pgmap v921: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 164 B/s rd, 13 MiB/s wr, 541 op/s; 28452/246474 objects degraded (11.544%); 3756/246474 objects misplaced (1.524%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:08:06.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:05 smithi144 ceph-mon[194833]: failed to encode map e334 with expected crc 2023-09-23T20:08:06.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:05 smithi144 ceph-mon[194833]: failed to encode map e334 with expected crc 2023-09-23T20:08:06.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:05 smithi144 ceph-mon[194833]: osdmap e334: 8 total, 8 up, 7 in 2023-09-23T20:08:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:05 smithi144 ceph-mon[194833]: failed to encode map e334 with expected crc 2023-09-23T20:08:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:05 smithi144 ceph-mon[194833]: failed to encode map e334 with expected crc 2023-09-23T20:08:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:05 smithi144 ceph-mon[194833]: failed to encode map e334 with expected crc 2023-09-23T20:08:06.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:05 smithi144 ceph-mon[194833]: pgmap v921: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 164 B/s rd, 13 MiB/s wr, 541 op/s; 28452/246474 objects degraded (11.544%); 3756/246474 objects misplaced (1.524%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:08:06.798 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:06.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:06.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:06.905 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:06.937 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:06.962 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[213871]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.962 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[213871]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.962 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[213871]: osdmap e335: 8 total, 8 up, 7 in 2023-09-23T20:08:06.962 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[213871]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.963 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[213871]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.963 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[221113]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.963 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[221113]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.963 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[221113]: osdmap e335: 8 total, 8 up, 7 in 2023-09-23T20:08:06.963 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[221113]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.963 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:06 smithi135 ceph-mon[221113]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.996 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:06.999 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:06 smithi144 ceph-mon[194833]: failed to encode map e335 with expected crc 2023-09-23T20:08:06.999 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:06 smithi144 ceph-mon[194833]: failed to encode map e335 with expected crc 2023-09-23T20:08:07.000 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:06 smithi144 ceph-mon[194833]: osdmap e335: 8 total, 8 up, 7 in 2023-09-23T20:08:07.000 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:06 smithi144 ceph-mon[194833]: failed to encode map e335 with expected crc 2023-09-23T20:08:07.000 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:06 smithi144 ceph-mon[194833]: failed to encode map e335 with expected crc 2023-09-23T20:08:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:07.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:07.102 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:07.129 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:07.179 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:07.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:07.284 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:07.391 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:07.413 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:07.540 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:07.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:07.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:07.568 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148803 6.14 6:2da19cbe:::benchmark_data_smithi135_344736_object46147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 504.62469362600001, 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2584235440000002, 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.569 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148803 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:08:07.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927331+0000", 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927329+0000", 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:08:07.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927340+0000", 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0981e-05 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927360+0000", 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0913000000000001e-05 2023-09-23T20:08:07.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.927371+0000", 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0467e-05 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933494+0000", 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061233260000000001 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.573 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933501+0000", 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0530000000000001e-06 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.951971+0000", 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018469316999999999 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.951983+0000", 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2286000000000001e-05 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050605+0000", 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0986220259999999 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050613+0000", 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9969999999999998e-06 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.171636+0000", 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.121022937 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.171779+0000", 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000142738 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.172136+0000", 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00035740899999999999 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174779+0000", 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002642574 2023-09-23T20:08:07.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.180158+0000", 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053795379999999997 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185597+0000", 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0054389099999999999 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.578 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185657+0000", 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0288999999999998e-05 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185755+0000", 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7291000000000001e-05 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148808 6.4 6:27155b23:::benchmark_data_smithi135_344736_object46152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 504.61854970299999, 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2790047449999999, 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148808 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.580 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933475+0000", 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933474+0000", 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933479+0000", 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3739999999999997e-06 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933490+0000", 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1051e-05 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933497+0000", 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.601e-06 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933549+0000", 2023-09-23T20:08:07.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1745e-05 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.933555+0000", 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5440000000000004e-06 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.952509+0000", 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018953932 2023-09-23T20:08:07.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.952518+0000", 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.081e-06 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050570+0000", 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.098051485 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050581+0000", 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0878e-05 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174805+0000", 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1242246469999999 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.174952+0000", 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014714599999999999 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.175367+0000", 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000414855 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.587 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.180166+0000", 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047984739999999996 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.185798+0000", 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00563285 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212221+0000", 2023-09-23T20:08:07.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.026422386999999999 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212281+0000", 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9735999999999999e-05 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.212480+0000", 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000199125 2023-09-23T20:08:07.589 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148812 6.1 6:81ec3361:::benchmark_data_smithi135_344736_object46156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 504.598126627, 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.216623062, 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.590 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148812 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.591 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953898+0000", 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953897+0000", 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953966+0000", 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9086999999999998e-05 2023-09-23T20:08:07.592 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953978+0000", 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1731e-05 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.953989+0000", 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1902000000000001e-05 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.954238+0000", 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000248216 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.954249+0000", 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.114e-05 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049700+0000", 2023-09-23T20:08:07.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.095451519 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049712+0000", 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1780000000000001e-05 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.146621+0000", 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.096909012 2023-09-23T20:08:07.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.146767+0000", 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014557700000000001 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,2,7", 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.147202+0000", 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043543599999999998 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.596 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.152224+0000", 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050223309999999997 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.152512+0000", 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000287421 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.163594+0000", 2023-09-23T20:08:07.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011081946000000001 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170347+0000", 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067530960000000001 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170399+0000", 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2540000000000002e-05 2023-09-23T20:08:07.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.170521+0000", 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012170700000000001 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182375 6.14 6:2a7953cb:::benchmark_data_smithi135_344736_object79719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:08:07.599 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.461225+0000", 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 454.09079922500001, 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7366573399999998, 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182375 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461225+0000", 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461225+0000", 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461224+0000", 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461231+0000", 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6990000000000003e-06 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461271+0000", 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9549999999999999e-05 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461286+0000", 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4688999999999999e-05 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461350+0000", 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3955999999999995e-05 2023-09-23T20:08:07.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.461362+0000", 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2658e-05 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439124+0000", 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97776162 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439133+0000", 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4350000000000003e-06 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.189742+0000", 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7506088659999999 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.189894+0000", 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015184899999999999 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.190331+0000", 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043668999999999999 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193071+0000", 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027404880000000001 2023-09-23T20:08:07.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.196937+0000", 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038661120000000001 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197009+0000", 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1431000000000001e-05 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197073+0000", 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4125e-05 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197883+0000", 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00080992499999999997 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182407 6.4 6:21645038:::benchmark_data_smithi135_344736_object79751:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.517127+0000", 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 454.03489787900003, 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6541434769999999, 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182407 2023-09-23T20:08:07.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517127+0000", 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517127+0000", 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517125+0000", 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517132+0000", 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3899999999999998e-06 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517143+0000", 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0864e-05 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517150+0000", 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7659999999999998e-06 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517202+0000", 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1727999999999997e-05 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.517213+0000", 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1240000000000001e-05 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.438985+0000", 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.921771746 2023-09-23T20:08:07.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439002+0000", 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7062999999999999e-05 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.164600+0000", 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7255975669999999 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.164737+0000", 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013742600000000001 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.165195+0000", 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045771700000000001 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168101+0000", 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029057380000000002 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168107+0000", 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6059999999999999e-06 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.170461+0000", 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023538439999999999 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.170504+0000", 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3439999999999997e-05 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.171270+0000", 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00076583700000000001 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182418 6.14 6:2825501c:::benchmark_data_smithi135_344736_object79762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.530422+0000", 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 454.021602492, 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.670691181, 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.618 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182418 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530422+0000", 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530422+0000", 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530421+0000", 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530427+0000", 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8139999999999997e-06 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530436+0000", 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5210000000000008e-06 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530442+0000", 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2499999999999997e-06 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530499+0000", 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6876999999999998e-05 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.530509+0000", 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0801e-05 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439303+0000", 2023-09-23T20:08:07.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.908794137 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439313+0000", 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0480000000000004e-06 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193088+0000", 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.753775608 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193199+0000", 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110946 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193487+0000", 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000287566 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.196996+0000", 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0035091520000000002 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197898+0000", 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00090200999999999996 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.625 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.200328+0000", 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024299840000000001 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.200382+0000", 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4364000000000003e-05 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201113+0000", 2023-09-23T20:08:07.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00073121600000000003 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182423 6.4 6:248db571:::benchmark_data_smithi135_344736_object79767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.540102+0000", 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 454.01192222200001, 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6532785529999998, 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.627 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182423 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540102+0000", 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540102+0000", 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540101+0000", 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540107+0000", 2023-09-23T20:08:07.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7690000000000002e-06 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540120+0000", 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3185e-05 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540126+0000", 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0809999999999999e-06 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540186+0000", 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0248999999999999e-05 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.540200+0000", 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3903000000000001e-05 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439418+0000", 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.89921767600000002 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439426+0000", 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9510000000000006e-06 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168183+0000", 2023-09-23T20:08:07.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.728756765 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168361+0000", 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017813099999999999 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.168797+0000", 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043667499999999998 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.192289+0000", 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.023491365 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.192331+0000", 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1913999999999999e-05 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.192365+0000", 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4589999999999999e-05 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.192420+0000", 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4608000000000003e-05 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.193381+0000", 2023-09-23T20:08:07.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00096113800000000005 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182425 6.14 6:2b319d69:::benchmark_data_smithi135_344736_object79769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e223)", 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.541828+0000", 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 454.01019678300003, 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.659927615, 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.636 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182425 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541828+0000", 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541828+0000", 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541827+0000", 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541833+0000", 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1670000000000004e-06 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541847+0000", 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3898e-05 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541854+0000", 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4549999999999998e-06 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541931+0000", 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6481000000000002e-05 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.541943+0000", 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2354999999999999e-05 2023-09-23T20:08:07.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439334+0000", 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.89739137099999999 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.439343+0000", 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3170000000000002e-06 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.194858+0000", 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.755515079 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.195026+0000", 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016822699999999999 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.195313+0000", 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028750100000000001 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.200270+0000", 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049562269999999997 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201130+0000", 2023-09-23T20:08:07.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00086066699999999996 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201196+0000", 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5870000000000005e-05 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201249+0000", 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2873e-05 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201756+0000", 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050643599999999997 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182451 6.4 6:22222862:::benchmark_data_smithi135_344736_object79795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.584941+0000", 2023-09-23T20:08:07.645 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.967083382, 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6476608669999999, 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182451 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584941+0000", 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584941+0000", 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584939+0000", 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:08:07.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584947+0000", 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5100000000000001e-06 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584960+0000", 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3125999999999999e-05 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584968+0000", 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8150000000000006e-06 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.585034+0000", 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6517e-05 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.585045+0000", 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0963e-05 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.218776+0000", 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6337305529999999 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.218958+0000", 2023-09-23T20:08:07.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000182385 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.219392+0000", 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043395100000000002 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.229881+0000", 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010488924 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.229957+0000", 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6092000000000003e-05 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231608+0000", 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016505210000000001 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231660+0000", 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1894999999999997e-05 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232602+0000", 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00094250300000000003 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182466 6.4 6:249455ee:::benchmark_data_smithi135_344736_object79810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.615458+0000", 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.93656713199999, 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.618027707, 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182466 2023-09-23T20:08:07.654 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615458+0000", 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615458+0000", 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.655 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615456+0000", 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615463+0000", 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5009999999999998e-06 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615476+0000", 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2658e-05 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615484+0000", 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4610000000000008e-06 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615548+0000", 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3742000000000001e-05 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.615559+0000", 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0558e-05 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221570+0000", 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6060114219999999 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221705+0000", 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013505799999999999 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.222076+0000", 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037122899999999999 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230893+0000", 2023-09-23T20:08:07.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088166259999999993 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232681+0000", 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001787477 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232714+0000", 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3597999999999999e-05 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232762+0000", 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8362000000000002e-05 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.233485+0000", 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00072280300000000005 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.661 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182470 6.14 6:29661942:::benchmark_data_smithi135_344736_object79814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.623602+0000", 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.92842269599998, 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.58335468, 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.662 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182470 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623602+0000", 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623602+0000", 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623600+0000", 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623608+0000", 2023-09-23T20:08:07.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5000000000000002e-06 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623624+0000", 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5818e-05 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623633+0000", 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9449999999999999e-06 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623693+0000", 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0254000000000003e-05 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.623705+0000", 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2446000000000001e-05 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.197940+0000", 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5742344610000001 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.198079+0000", 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013971900000000001 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.198432+0000", 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00035207899999999999 2023-09-23T20:08:07.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201811+0000", 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033799030000000001 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.205930+0000", 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004118109 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.205944+0000", 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4034e-05 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.205987+0000", 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3065000000000002e-05 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.206957+0000", 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00097006100000000004 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182490 6.4 6:259d21c1:::benchmark_data_smithi135_344736_object79834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.655714+0000", 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.89631055900003, 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5807021909999999, 2023-09-23T20:08:07.670 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182490 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655714+0000", 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655714+0000", 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655712+0000", 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655723+0000", 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1382999999999999e-05 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655747+0000", 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3969999999999999e-05 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655759+0000", 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2118e-05 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655824+0000", 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5178999999999994e-05 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.655839+0000", 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4277000000000001e-05 2023-09-23T20:08:07.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.224400+0000", 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.56856145 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.224546+0000", 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000146198 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.225014+0000", 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00046763700000000002 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230900+0000", 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005886255 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.233525+0000", 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026250050000000001 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.235463+0000", 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019373859999999999 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.235505+0000", 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2014999999999997e-05 2023-09-23T20:08:07.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.236416+0000", 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00091160799999999995 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182501 6.14 6:29309ff6:::benchmark_data_smithi135_344736_object79845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.672483+0000", 2023-09-23T20:08:07.678 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.87954130700001, 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5430168900000001, 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182501 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672483+0000", 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672483+0000", 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672482+0000", 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:07.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672488+0000", 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4190000000000001e-06 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672497+0000", 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4599999999999992e-06 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672503+0000", 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9000000000000003e-06 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672556+0000", 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2802e-05 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.672568+0000", 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1798e-05 2023-09-23T20:08:07.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.201840+0000", 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5292725059999999 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.202025+0000", 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018504700000000001 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.202485+0000", 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045962800000000001 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209642+0000", 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0071572060000000002 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.684 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209721+0000", 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8843000000000004e-05 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.214647+0000", 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049257779999999996 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.214683+0000", 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6659999999999998e-05 2023-09-23T20:08:07.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.215500+0000", 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00081710400000000003 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182504 6.14 6:2ff3c150:::benchmark_data_smithi135_344736_object79848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.686 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.675556+0000", 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.87646873900002, 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.540447256, 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182504 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675556+0000", 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675556+0000", 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675555+0000", 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675560+0000", 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.023e-06 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675569+0000", 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.081e-06 2023-09-23T20:08:07.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675574+0000", 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1660000000000002e-06 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675623+0000", 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8992999999999997e-05 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.675631+0000", 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3159999999999997e-06 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.203977+0000", 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.528345866 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.691 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.204148+0000", 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000171239 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.204475+0000", 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00032640300000000001 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209653+0000", 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051786540000000004 2023-09-23T20:08:07.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209768+0000", 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011504400000000001 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.215570+0000", 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0058017670000000002 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.215621+0000", 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1153999999999998e-05 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216003+0000", 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00038182100000000002 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182506 6.14 6:2f661eda:::benchmark_data_smithi135_344736_object79850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.676137+0000", 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.87588733600001, 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5404231739999998, 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.695 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182506 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676137+0000", 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676137+0000", 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676136+0000", 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676142+0000", 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4990000000000002e-06 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676155+0000", 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2882e-05 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676177+0000", 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1885000000000001e-05 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676235+0000", 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7839e-05 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676249+0000", 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4646e-05 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.207002+0000", 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5307524340000001 2023-09-23T20:08:07.699 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.207166+0000", 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000164854 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.207637+0000", 2023-09-23T20:08:07.700 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00047019199999999997 2023-09-23T20:08:07.700 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:07.701 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.701 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216056+0000", 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084191519999999992 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216110+0000", 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4335000000000001e-05 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.702 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216120+0000", 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0118999999999999e-05 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216173+0000", 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2649999999999999e-05 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.703 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216561+0000", 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00038758899999999998 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182508 6.4 6:2138bf66:::benchmark_data_smithi135_344736_object79852:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.676243+0000", 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.87578144000003, 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5664108890000001, 2023-09-23T20:08:07.704 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182508 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676243+0000", 2023-09-23T20:08:07.705 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676243+0000", 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676242+0000", 2023-09-23T20:08:07.706 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676261+0000", 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8660000000000001e-05 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676273+0000", 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2125e-05 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676281+0000", 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8439999999999998e-06 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676327+0000", 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6541e-05 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.708 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.676344+0000", 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6133e-05 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.226959+0000", 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.550615732 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.709 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.227137+0000", 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000177615 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.227621+0000", 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000484148 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.236457+0000", 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088358269999999992 2023-09-23T20:08:07.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.239381+0000", 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029245650000000001 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.241583+0000", 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022013229999999998 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.711 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.241639+0000", 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6001000000000001e-05 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.242654+0000", 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010153930000000001 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.712 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182535 6.14 6:2e05be58:::benchmark_data_smithi135_344736_object79879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.733828+0000", 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.81819633100002, 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.497046002, 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.713 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182535 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733828+0000", 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733828+0000", 2023-09-23T20:08:07.714 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733827+0000", 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733833+0000", 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8189999999999997e-06 2023-09-23T20:08:07.715 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733840+0000", 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1180000000000003e-06 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733846+0000", 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4020000000000001e-06 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.716 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733919+0000", 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3867000000000003e-05 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.733934+0000", 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4487e-05 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209794+0000", 2023-09-23T20:08:07.717 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.475860226 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.209986+0000", 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000192134 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.210415+0000", 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00042892000000000001 2023-09-23T20:08:07.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216570+0000", 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061551009999999996 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221464+0000", 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048936450000000003 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.229936+0000", 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084721339999999992 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.229994+0000", 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7757000000000003e-05 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230874+0000", 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00088049799999999996 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182539 6.14 6:2bbcc305:::benchmark_data_smithi135_344736_object79883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.745414+0000", 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.806610288, 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4861406260000001, 2023-09-23T20:08:07.721 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182539 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745414+0000", 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745414+0000", 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745412+0000", 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745420+0000", 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8339999999999999e-06 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745432+0000", 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2692e-05 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.724 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745440+0000", 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5179999999999998e-06 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745501+0000", 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1546999999999999e-05 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.745512+0000", 2023-09-23T20:08:07.725 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0458999999999999e-05 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.212416+0000", 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.466904027 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.212572+0000", 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000155916 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.726 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.212947+0000", 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037548600000000002 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216578+0000", 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036308780000000001 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.727 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221544+0000", 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049656229999999997 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230942+0000", 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0093981930000000009 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.728 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.230998+0000", 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5649999999999997e-05 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231555+0000", 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00055735400000000003 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.729 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182551 6.14 6:293f61b4:::benchmark_data_smithi135_344736_object79895:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:34.810825+0000", 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 452.74120008699998, 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.421271562, 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182551 2023-09-23T20:08:07.730 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810825+0000", 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810825+0000", 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.731 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810823+0000", 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810830+0000", 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.55e-06 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.732 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810846+0000", 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5909e-05 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810854+0000", 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0730000000000005e-06 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810935+0000", 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1198000000000004e-05 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.810950+0000", 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.558e-05 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216606+0000", 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.405656003 2023-09-23T20:08:07.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.216785+0000", 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000178078 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.217257+0000", 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00047277099999999998 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.221473+0000", 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0042157610000000002 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.226942+0000", 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0054686429999999996 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:07.736 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231631+0000", 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046896519999999999 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.231675+0000", 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3658000000000001e-05 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:36.232096+0000", 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00042110599999999998 2023-09-23T20:08:07.737 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.738 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.738 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.738 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.738 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.738 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65645 8.3 8.74bd96f3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:08:07.738 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:17.963213+0000", 2023-09-23T20:08:07.738 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 49.588811376999999, 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96203893799999995, 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65645 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.739 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963213+0000", 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963213+0000", 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963213+0000", 2023-09-23T20:08:07.740 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963233+0000", 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9947999999999999e-05 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963239+0000", 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5390000000000004e-06 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963245+0000", 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6060000000000002e-06 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963283+0000", 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8461999999999999e-05 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:07.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963289+0000", 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7869999999999998e-06 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.925246+0000", 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96195676500000005 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.925252+0000", 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2899999999999999e-06 2023-09-23T20:08:07.743 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.744 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.744 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.744 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:07.744 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:07.744 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:08:07.826 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:07 smithi144 ceph-mon[194833]: pgmap v923: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 5.1 MiB/s wr, 469 op/s; 28452/245799 objects degraded (11.575%); 3756/245799 objects misplaced (1.528%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:08:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:07 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 28452/246474 objects degraded (11.544%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:08.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:07 smithi135 ceph-mon[213871]: pgmap v923: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 5.1 MiB/s wr, 469 op/s; 28452/245799 objects degraded (11.575%); 3756/245799 objects misplaced (1.528%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:08:08.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:08.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:07 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 28452/246474 objects degraded (11.544%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:08.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:07 smithi135 ceph-mon[221113]: pgmap v923: 99 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 5.1 MiB/s wr, 469 op/s; 28452/245799 objects degraded (11.575%); 3756/245799 objects misplaced (1.528%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:08:08.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:08.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:07 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 28452/246474 objects degraded (11.544%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:08.259 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:08.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:08.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:08.366 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:08.398 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:08.453 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:08.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:08.559 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:08.610 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:08.638 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:08:08.960 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:08 smithi135 ceph-mon[213871]: pgmap v924: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 1.3 MiB/s wr, 586 op/s; 27032/241347 objects degraded (11.200%); 4717/241347 objects misplaced (1.954%); 4.8 MiB/s, 75 objects/s recovering 2023-09-23T20:08:09.211 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:09.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:08 smithi135 ceph-mon[221113]: pgmap v924: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 1.3 MiB/s wr, 586 op/s; 27032/241347 objects degraded (11.200%); 4717/241347 objects misplaced (1.954%); 4.8 MiB/s, 75 objects/s recovering 2023-09-23T20:08:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:09.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:09.322 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:08:09.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:08 smithi144 ceph-mon[194833]: pgmap v924: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 1.3 MiB/s wr, 586 op/s; 27032/241347 objects degraded (11.200%); 4717/241347 objects misplaced (1.954%); 4.8 MiB/s, 75 objects/s recovering 2023-09-23T20:08:09.462 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:09.755 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:09.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:09.874 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:09.954 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:10.677 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:10.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:10.758 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:10.795 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:10.866 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:10.957 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:10 smithi135 ceph-mon[213871]: pgmap v925: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 27032/241347 objects degraded (11.200%); 4717/241347 objects misplaced (1.954%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:08:11.164 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:11.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:11.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:11.231 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:10 smithi135 ceph-mon[221113]: pgmap v925: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 27032/241347 objects degraded (11.200%); 4717/241347 objects misplaced (1.954%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:08:11.270 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:11.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:10 smithi144 ceph-mon[194833]: pgmap v925: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 27032/241347 objects degraded (11.200%); 4717/241347 objects misplaced (1.954%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:08:11.311 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:11.371 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:11.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:11.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:11.479 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:11.575 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:11.765 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:11.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:11.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:11.874 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:11.905 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:11.958 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:11.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:11.997 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:08:11.997 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:08:11.998 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-09-23T20:08:11.998 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:08:12.066 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:12.096 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:12.145 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:12.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:12.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:12.250 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:12.344 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:12.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 27032/241347 objects degraded (11.200%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:12.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 27032/241347 objects degraded (11.200%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:13.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 27032/241347 objects degraded (11.200%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:13.196 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:13.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:13.280 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:13.308 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:13.337 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:13.388 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:13.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:13.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:13.493 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:13.522 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:13.590 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:13.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:13.698 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:13.735 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:13.764 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:08:13.785 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:13.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:13.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3710221323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:08:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:13 smithi144 ceph-mon[194833]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 27032/239910 objects degraded (11.268%); 4568/239910 objects misplaced (1.904%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:08:13.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:13.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3710221323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:08:13.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:13 smithi135 ceph-mon[213871]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 27032/239910 objects degraded (11.268%); 4568/239910 objects misplaced (1.904%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:08:13.928 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:08:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:08:13] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-09-23T20:08:13.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3710221323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:08:13.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:13 smithi135 ceph-mon[221113]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 27032/239910 objects degraded (11.268%); 4568/239910 objects misplaced (1.904%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:08:13.986 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:14.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:14.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[213871]: failed to encode map e336 with expected crc 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3710221323' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[213871]: osdmap e336: 8 total, 8 up, 7 in 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[213871]: failed to encode map e336 with expected crc 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[213871]: failed to encode map e336 with expected crc 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[213871]: failed to encode map e336 with expected crc 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[221113]: failed to encode map e336 with expected crc 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3710221323' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[221113]: osdmap e336: 8 total, 8 up, 7 in 2023-09-23T20:08:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[221113]: failed to encode map e336 with expected crc 2023-09-23T20:08:14.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[221113]: failed to encode map e336 with expected crc 2023-09-23T20:08:14.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:14 smithi135 ceph-mon[221113]: failed to encode map e336 with expected crc 2023-09-23T20:08:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:14 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:14 smithi144 ceph-mon[194833]: failed to encode map e336 with expected crc 2023-09-23T20:08:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3710221323' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:08:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:14 smithi144 ceph-mon[194833]: osdmap e336: 8 total, 8 up, 7 in 2023-09-23T20:08:15.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:14 smithi144 ceph-mon[194833]: failed to encode map e336 with expected crc 2023-09-23T20:08:15.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:14 smithi144 ceph-mon[194833]: failed to encode map e336 with expected crc 2023-09-23T20:08:15.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:14 smithi144 ceph-mon[194833]: failed to encode map e336 with expected crc 2023-09-23T20:08:15.478 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:15.762 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:15.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:15.887 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:15.965 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:15 smithi135 ceph-mon[213871]: pgmap v928: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 26879/233460 objects degraded (11.513%); 4445/233460 objects misplaced (1.904%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T20:08:15.965 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:15 smithi135 ceph-mon[221113]: pgmap v928: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 26879/233460 objects degraded (11.513%); 4445/233460 objects misplaced (1.904%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T20:08:15.993 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:16.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:15 smithi144 ceph-mon[194833]: pgmap v928: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 26879/233460 objects degraded (11.513%); 4445/233460 objects misplaced (1.904%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T20:08:16.685 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:16.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:16.796 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:16.877 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:17.038 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:08:17.102 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:08:17.157 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:08:17.208 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:17.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:17.261 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:08:17.303 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:08:17.323 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:17.349 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:08:17.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:17 smithi144 ceph-mon[194833]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 26879/232650 objects degraded (11.553%); 4445/232650 objects misplaced (1.911%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:08:17.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/749939727' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:17.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:17 smithi135 ceph-mon[213871]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 26879/232650 objects degraded (11.553%); 4445/232650 objects misplaced (1.911%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:08:17.386 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/749939727' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:17.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:17 smithi135 ceph-mon[221113]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 26879/232650 objects degraded (11.553%); 4445/232650 objects misplaced (1.911%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:08:17.386 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/749939727' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:17.394 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:17.397 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:08:17.437 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:08:17.455 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:08:17.455 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:08:17.455 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:08:17.455 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:08:17.455 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:08:17.455 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:08:17.455 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:08:17.456 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:08:17.879 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:08:18.136 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:18.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:18.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:18.246 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:18.278 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:18.341 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:18.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:18.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:18.450 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:18.478 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:18.532 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:18.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:18.547 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:18.547 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:18.547 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:18 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 26879/232650 objects degraded (11.553%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:18.647 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:18.679 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:18 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 26879/232650 objects degraded (11.553%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:18 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 26879/232650 objects degraded (11.553%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:08:18.724 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:18.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:18.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:18.832 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:18.925 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:08:18.925 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:08:18.926 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:19.453 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:19.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:19.560 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:19.596 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:19.649 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:19.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:19.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:19.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:19 smithi135 ceph-mon[213871]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 26727/226398 objects degraded (11.805%); 4106/226398 objects misplaced (1.814%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:08:19.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:19.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:19 smithi135 ceph-mon[221113]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 26727/226398 objects degraded (11.805%); 4106/226398 objects misplaced (1.814%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:08:19.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:19.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:19.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:19.755 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:19.779 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:19 smithi144 ceph-mon[194833]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 26727/226398 objects degraded (11.805%); 4106/226398 objects misplaced (1.814%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:08:19.779 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:19.779 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:19.779 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:19.781 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:19.831 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:19.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:19.938 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:20.018 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:20.338 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:20.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:20.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:20.446 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:20.524 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:21.168 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:08:21.279 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:21.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:21.384 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:21.417 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:21.470 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:21.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:21.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:21.575 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:21.654 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:21.663 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:21.961 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:21.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:21.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:22.075 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:22.089 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:08:22.106 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:22.163 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:22.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:22.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:22.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:21 smithi135 ceph-mon[213871]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 26727/226398 objects degraded (11.805%); 4106/226398 objects misplaced (1.814%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:08:22.190 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:21 smithi135 ceph-mon[221113]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 26727/226398 objects degraded (11.805%); 4106/226398 objects misplaced (1.814%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:08:22.276 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:22.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:21 smithi144 ceph-mon[194833]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 26727/226398 objects degraded (11.805%); 4106/226398 objects misplaced (1.814%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:08:22.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:22.868 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:08:22.878 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:08:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:23.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 26727/226398 objects degraded (11.805%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:23.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:23.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 26727/226398 objects degraded (11.805%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 26727/226398 objects degraded (11.805%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:23.930 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:08:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:08:23] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-09-23T20:08:24.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:23 smithi135 ceph-mon[221113]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:08:24.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:23 smithi135 ceph-mon[221113]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 26727/225114 objects degraded (11.873%); 3935/225114 objects misplaced (1.748%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:08:24.236 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:23 smithi135 ceph-mon[213871]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:08:24.236 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:23 smithi135 ceph-mon[213871]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 26727/225114 objects degraded (11.873%); 3935/225114 objects misplaced (1.748%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:08:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:23 smithi144 ceph-mon[194833]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:08:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:23 smithi144 ceph-mon[194833]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 26727/225114 objects degraded (11.873%); 3935/225114 objects misplaced (1.748%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:08:25.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:24 smithi135 ceph-mon[213871]: pgmap v933: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 26552/218847 objects degraded (12.133%); 3729/218847 objects misplaced (1.704%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:08:25.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:24 smithi135 ceph-mon[221113]: pgmap v933: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 26552/218847 objects degraded (12.133%); 3729/218847 objects misplaced (1.704%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:08:25.234 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:25.234 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:25.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:25.267 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":931,"stamp":"2023-09-23T20:08:20.867572+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4948724924,"num_objects":75466,"num_object_clones":0,"num_object_copies":226398,"num_objects_missing_on_primary":988,"num_objects_missing":988,"num_objects_degraded":26727,"num_objects_misplaced":4106,"num_objects_unfound":0,"num_objects_dirty":75466,"num_whiteouts":0,"num_read":6114,"num_read_kb":13456,"num_write":93997,"num_write_kb":5404105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7162,"num_bytes_recovered":467508772,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87328,"ondisk_log_size":87328,"up":297,"acting":286,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":373,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18600868,"kb_used_data":15187116,"kb_used_omap":32,"kb_used_meta":3413663,"kb_avail":637615196,"statfs":{"total":671965249536,"available":652917960704,"internally_reserved":0,"allocated":15551606784,"data_stored":15542175609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3495591430},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,6,7,9,3,8,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-326565888,"num_objects":-4983,"num_object_clones":0,"num_object_copies":-14949,"num_objects_missing_on_primary":-286,"num_objects_missing":-286,"num_objects_degraded":-305,"num_objects_misplaced":-611,"num_objects_unfound":0,"num_objects_dirty":-4983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4983,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":916,"num_bytes_recovered":60030976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005469"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":334,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748312+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:08:05.748312+0000","last_peered":"2023-09-23T20:08:05.748312+0000","last_clean":"2023-09-23T20:08:05.748312+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:08:05.748312+0000","last_undegraded":"2023-09-23T20:08:05.748312+0000","last_fullsized":"2023-09-23T20:08:05.748312+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":163,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.935543+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:08:05.935543+0000","last_peered":"2023-09-23T20:08:05.935543+0000","last_clean":"2023-09-23T20:08:05.935543+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:08:05.935543+0000","last_undegraded":"2023-09-23T20:08:05.935543+0000","last_fullsized":"2023-09-23T20:08:05.935543+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.755588+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:08:05.755588+0000","last_peered":"2023-09-23T20:08:05.755588+0000","last_clean":"2023-09-23T20:08:05.755588+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:08:05.755588+0000","last_undegraded":"2023-09-23T20:08:05.755588+0000","last_fullsized":"2023-09-23T20:08:05.755588+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":24,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748790+0000","last_change":"2023-09-23T20:07:36.970799+0000","last_active":"2023-09-23T20:08:05.748790+0000","last_peered":"2023-09-23T20:08:05.748790+0000","last_clean":"2023-09-23T20:08:05.748790+0000","last_became_active":"2023-09-23T20:07:36.970478+0000","last_became_peered":"2023-09-23T20:07:36.970478+0000","last_unstale":"2023-09-23T20:08:05.748790+0000","last_undegraded":"2023-09-23T20:08:05.748790+0000","last_fullsized":"2023-09-23T20:08:05.748790+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:10.247703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":163,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.935384+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:08:05.935384+0000","last_peered":"2023-09-23T20:08:05.935384+0000","last_clean":"2023-09-23T20:08:05.935384+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:08:05.935384+0000","last_undegraded":"2023-09-23T20:08:05.935384+0000","last_fullsized":"2023-09-23T20:08:05.935384+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":281,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746632+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:08:05.746632+0000","last_peered":"2023-09-23T20:08:05.746632+0000","last_clean":"2023-09-23T20:08:05.746632+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:08:05.746632+0000","last_undegraded":"2023-09-23T20:08:05.746632+0000","last_fullsized":"2023-09-23T20:08:05.746632+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":262,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746533+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:08:05.746533+0000","last_peered":"2023-09-23T20:08:05.746533+0000","last_clean":"2023-09-23T20:08:05.746533+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:08:05.746533+0000","last_undegraded":"2023-09-23T20:08:05.746533+0000","last_fullsized":"2023-09-23T20:08:05.746533+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":177,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.756225+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:08:05.756225+0000","last_peered":"2023-09-23T20:08:05.756225+0000","last_clean":"2023-09-23T20:08:05.756225+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:08:05.756225+0000","last_undegraded":"2023-09-23T20:08:05.756225+0000","last_fullsized":"2023-09-23T20:08:05.756225+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":335,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746207+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:08:05.746207+0000","last_peered":"2023-09-23T20:08:05.746207+0000","last_clean":"2023-09-23T20:08:05.746207+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:08:05.746207+0000","last_undegraded":"2023-09-23T20:08:05.746207+0000","last_fullsized":"2023-09-23T20:08:05.746207+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":196,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.755493+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:08:05.755493+0000","last_peered":"2023-09-23T20:08:05.755493+0000","last_clean":"2023-09-23T20:08:05.755493+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:08:05.755493+0000","last_undegraded":"2023-09-23T20:08:05.755493+0000","last_fullsized":"2023-09-23T20:08:05.755493+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":29,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748377+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:08:05.748377+0000","last_peered":"2023-09-23T20:08:05.748377+0000","last_clean":"2023-09-23T20:08:05.748377+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:08:05.748377+0000","last_undegraded":"2023-09-23T20:08:05.748377+0000","last_fullsized":"2023-09-23T20:08:05.748377+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754399+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:08:05.754399+0000","last_peered":"2023-09-23T20:08:05.754399+0000","last_clean":"2023-09-23T20:08:05.754399+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:08:05.754399+0000","last_undegraded":"2023-09-23T20:08:05.754399+0000","last_fullsized":"2023-09-23T20:08:05.754399+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":390,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749141+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:08:05.749141+0000","last_peered":"2023-09-23T20:08:05.749141+0000","last_clean":"2023-09-23T20:08:05.749141+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:08:05.749141+0000","last_undegraded":"2023-09-23T20:08:05.749141+0000","last_fullsized":"2023-09-23T20:08:05.749141+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":332,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748825+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:08:05.748825+0000","last_peered":"2023-09-23T20:08:05.748825+0000","last_clean":"2023-09-23T20:08:05.748825+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:08:05.748825+0000","last_undegraded":"2023-09-23T20:08:05.748825+0000","last_fullsized":"2023-09-23T20:08:05.748825+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":29,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.750241+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:08:05.750241+0000","last_peered":"2023-09-23T20:08:05.750241+0000","last_clean":"2023-09-23T20:08:05.750241+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:08:05.750241+0000","last_undegraded":"2023-09-23T20:08:05.750241+0000","last_fullsized":"2023-09-23T20:08:05.750241+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":187,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.935492+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:08:05.935492+0000","last_peered":"2023-09-23T20:08:05.935492+0000","last_clean":"2023-09-23T20:08:05.935492+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:08:05.935492+0000","last_undegraded":"2023-09-23T20:08:05.935492+0000","last_fullsized":"2023-09-23T20:08:05.935492+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":24,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.750141+0000","last_change":"2023-09-23T20:07:36.972071+0000","last_active":"2023-09-23T20:08:05.750141+0000","last_peered":"2023-09-23T20:08:05.750141+0000","last_clean":"2023-09-23T20:08:05.750141+0000","last_became_active":"2023-09-23T20:07:36.971704+0000","last_became_peered":"2023-09-23T20:07:36.971704+0000","last_unstale":"2023-09-23T20:08:05.750141+0000","last_undegraded":"2023-09-23T20:08:05.750141+0000","last_fullsized":"2023-09-23T20:08:05.750141+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:44.287763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":334,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746943+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:08:05.746943+0000","last_peered":"2023-09-23T20:08:05.746943+0000","last_clean":"2023-09-23T20:08:05.746943+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:08:05.746943+0000","last_undegraded":"2023-09-23T20:08:05.746943+0000","last_fullsized":"2023-09-23T20:08:05.746943+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":23,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748022+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:08:05.748022+0000","last_peered":"2023-09-23T20:08:05.748022+0000","last_clean":"2023-09-23T20:08:05.748022+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:08:05.748022+0000","last_undegraded":"2023-09-23T20:08:05.748022+0000","last_fullsized":"2023-09-23T20:08:05.748022+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":320,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754163+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:08:05.754163+0000","last_peered":"2023-09-23T20:08:05.754163+0000","last_clean":"2023-09-23T20:08:05.754163+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:08:05.754163+0000","last_undegraded":"2023-09-23T20:08:05.754163+0000","last_fullsized":"2023-09-23T20:08:05.754163+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":129,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.745464+0000","last_change":"2023-09-23T20:01:04.467492+0000","last_active":"2023-09-23T20:08:05.745464+0000","last_peered":"2023-09-23T20:08:05.745464+0000","last_clean":"2023-09-23T20:08:05.745464+0000","last_became_active":"2023-09-23T20:01:04.467224+0000","last_became_peered":"2023-09-23T20:01:04.467224+0000","last_unstale":"2023-09-23T20:08:05.745464+0000","last_undegraded":"2023-09-23T20:08:05.745464+0000","last_fullsized":"2023-09-23T20:08:05.745464+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:26.855014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":110,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.755367+0000","last_change":"2023-09-23T20:07:36.968556+0000","last_active":"2023-09-23T20:08:05.755367+0000","last_peered":"2023-09-23T20:08:05.755367+0000","last_clean":"2023-09-23T20:08:05.755367+0000","last_became_active":"2023-09-23T20:07:36.968232+0000","last_became_peered":"2023-09-23T20:07:36.968232+0000","last_unstale":"2023-09-23T20:08:05.755367+0000","last_undegraded":"2023-09-23T20:08:05.755367+0000","last_fullsized":"2023-09-23T20:08:05.755367+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:08.890838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"336'5945","reported_seq":5990,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T20:08:18.144551+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:08:18.144551+0000","last_peered":"2023-09-23T20:08:18.144551+0000","last_clean":"2023-09-23T20:08:18.144551+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:08:18.144551+0000","last_undegraded":"2023-09-23T20:08:18.144551+0000","last_fullsized":"2023-09-23T20:08:18.144551+0000","mapping_epoch":299,"log_start":"306'1100","ondisk_log_start":"306'1100","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4845,"log_dups_size":0,"ondisk_log_size":4845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313589760,"num_objects":4785,"num_object_clones":0,"num_object_copies":14355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5945,"num_write_kb":343360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":27,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748618+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:08:05.748618+0000","last_peered":"2023-09-23T20:08:05.748618+0000","last_clean":"2023-09-23T20:08:05.748618+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:08:05.748618+0000","last_undegraded":"2023-09-23T20:08:05.748618+0000","last_fullsized":"2023-09-23T20:08:05.748618+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":344,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746598+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:08:05.746598+0000","last_peered":"2023-09-23T20:08:05.746598+0000","last_clean":"2023-09-23T20:08:05.746598+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:08:05.746598+0000","last_undegraded":"2023-09-23T20:08:05.746598+0000","last_fullsized":"2023-09-23T20:08:05.746598+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"336'5692","reported_seq":8331,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T20:08:18.539587+0000","last_change":"2023-09-23T20:07:45.697314+0000","last_active":"2023-09-23T20:08:18.539587+0000","last_peered":"2023-09-23T20:08:18.539587+0000","last_clean":"2023-09-23T20:08:18.539587+0000","last_became_active":"2023-09-23T20:07:45.690354+0000","last_became_peered":"2023-09-23T20:07:45.690354+0000","last_unstale":"2023-09-23T20:08:18.539587+0000","last_undegraded":"2023-09-23T20:08:18.539587+0000","last_fullsized":"2023-09-23T20:08:18.539587+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.926631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302907392,"num_objects":4622,"num_object_clones":0,"num_object_copies":13866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5692,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":28,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748888+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:08:05.748888+0000","last_peered":"2023-09-23T20:08:05.748888+0000","last_clean":"2023-09-23T20:08:05.748888+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:08:05.748888+0000","last_undegraded":"2023-09-23T20:08:05.748888+0000","last_fullsized":"2023-09-23T20:08:05.748888+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":372,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.747652+0000","last_change":"2023-09-23T20:07:18.955536+0000","last_active":"2023-09-23T20:08:05.747652+0000","last_peered":"2023-09-23T20:08:05.747652+0000","last_clean":"2023-09-23T20:08:05.747652+0000","last_became_active":"2023-09-23T20:07:18.954603+0000","last_became_peered":"2023-09-23T20:07:18.954603+0000","last_unstale":"2023-09-23T20:08:05.747652+0000","last_undegraded":"2023-09-23T20:08:05.747652+0000","last_fullsized":"2023-09-23T20:08:05.747652+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:03.907532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"336'5977","reported_seq":7545,"reported_epoch":336,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:18.641275+0000","last_change":"2023-09-23T20:07:37.064417+0000","last_active":"2023-09-23T20:08:18.641275+0000","last_peered":"2023-09-23T20:08:18.641275+0000","last_clean":"2023-09-23T20:06:59.831272+0000","last_became_active":"2023-09-23T20:07:37.059682+0000","last_became_peered":"2023-09-23T20:07:37.059682+0000","last_unstale":"2023-09-23T20:08:18.641275+0000","last_undegraded":"2023-09-23T20:07:36.983730+0000","last_fullsized":"2023-09-23T20:07:36.974121+0000","mapping_epoch":322,"log_start":"306'300","ondisk_log_start":"306'300","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:33.575626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316604416,"num_objects":4831,"num_object_clones":0,"num_object_copies":14493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5977,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":3440},{"shards":"0,2,3","objects":1391}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":27,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748234+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:08:05.748234+0000","last_peered":"2023-09-23T20:08:05.748234+0000","last_clean":"2023-09-23T20:08:05.748234+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:08:05.748234+0000","last_undegraded":"2023-09-23T20:08:05.748234+0000","last_fullsized":"2023-09-23T20:08:05.748234+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":39,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754439+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:08:05.754439+0000","last_peered":"2023-09-23T20:08:05.754439+0000","last_clean":"2023-09-23T20:08:05.754439+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:08:05.754439+0000","last_undegraded":"2023-09-23T20:08:05.754439+0000","last_fullsized":"2023-09-23T20:08:05.754439+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"336'5813","reported_seq":6022,"reported_epoch":336,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:18.621941+0000","last_change":"2023-09-23T20:07:37.066540+0000","last_active":"2023-09-23T20:08:18.621941+0000","last_peered":"2023-09-23T20:08:18.621941+0000","last_clean":"2023-09-23T20:07:17.891274+0000","last_became_active":"2023-09-23T20:07:20.676605+0000","last_became_peered":"2023-09-23T20:07:20.676605+0000","last_unstale":"2023-09-23T20:08:18.621941+0000","last_undegraded":"2023-09-23T20:07:37.063836+0000","last_fullsized":"2023-09-23T20:07:19.924413+0000","mapping_epoch":317,"log_start":"306'100","ondisk_log_start":"306'100","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5713,"log_dups_size":0,"ondisk_log_size":5713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:17.923115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309657600,"num_objects":4725,"num_object_clones":0,"num_object_copies":14175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5813,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":5111808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3","objects":715},{"shards":"2,3,4","objects":3544}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":374,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746078+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:08:05.746078+0000","last_peered":"2023-09-23T20:08:05.746078+0000","last_clean":"2023-09-23T20:08:05.746078+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:08:05.746078+0000","last_undegraded":"2023-09-23T20:08:05.746078+0000","last_fullsized":"2023-09-23T20:08:05.746078+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.756280+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:08:05.756280+0000","last_peered":"2023-09-23T20:08:05.756280+0000","last_clean":"2023-09-23T20:08:05.756280+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:08:05.756280+0000","last_undegraded":"2023-09-23T20:08:05.756280+0000","last_fullsized":"2023-09-23T20:08:05.756280+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"336'5632","reported_seq":5691,"reported_epoch":336,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:15.924165+0000","last_change":"2023-09-23T20:07:37.149002+0000","last_active":"2023-09-23T20:08:15.924165+0000","last_peered":"2023-09-23T20:08:15.924165+0000","last_clean":"2023-09-23T20:07:17.769108+0000","last_became_active":"2023-09-23T20:07:20.672993+0000","last_became_peered":"2023-09-23T20:07:20.672993+0000","last_unstale":"2023-09-23T20:08:15.924165+0000","last_undegraded":"2023-09-23T20:07:37.145948+0000","last_fullsized":"2023-09-23T20:07:19.923439+0000","mapping_epoch":317,"log_start":"306'1000","ondisk_log_start":"306'1000","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4632,"log_dups_size":0,"ondisk_log_size":4632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:56.185131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306446336,"num_objects":4676,"num_object_clones":0,"num_object_copies":14028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4002,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5632,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3525},{"shards":"3,5","objects":674}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":718,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.747993+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:08:05.747993+0000","last_peered":"2023-09-23T20:08:05.747993+0000","last_clean":"2023-09-23T20:08:05.747993+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:08:05.747993+0000","last_undegraded":"2023-09-23T20:08:05.747993+0000","last_fullsized":"2023-09-23T20:08:05.747993+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":26,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748738+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:08:05.748738+0000","last_peered":"2023-09-23T20:08:05.748738+0000","last_clean":"2023-09-23T20:08:05.748738+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:08:05.748738+0000","last_undegraded":"2023-09-23T20:08:05.748738+0000","last_fullsized":"2023-09-23T20:08:05.748738+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":362,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749044+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:08:05.749044+0000","last_peered":"2023-09-23T20:08:05.749044+0000","last_clean":"2023-09-23T20:08:05.749044+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:08:05.749044+0000","last_undegraded":"2023-09-23T20:08:05.749044+0000","last_fullsized":"2023-09-23T20:08:05.749044+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"336'5802","reported_seq":7899,"reported_epoch":336,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:08:17.751749+0000","last_change":"2023-09-23T20:07:50.805986+0000","last_active":"2023-09-23T20:08:17.751749+0000","last_peered":"2023-09-23T20:08:17.751749+0000","last_clean":"2023-09-23T20:07:17.847486+0000","last_became_active":"2023-09-23T20:07:20.666556+0000","last_became_peered":"2023-09-23T20:07:20.666556+0000","last_unstale":"2023-09-23T20:08:17.751749+0000","last_undegraded":"2023-09-23T20:08:17.751749+0000","last_fullsized":"2023-09-23T20:07:19.932561+0000","mapping_epoch":317,"log_start":"306'100","ondisk_log_start":"306'100","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:38.441814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307888128,"num_objects":4698,"num_object_clones":0,"num_object_copies":14094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3118,"num_objects_unfound":0,"num_objects_dirty":4698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5802,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1023,"num_bytes_recovered":67043328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":1580},{"shards":"0,2,5","objects":3118}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":695,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.745937+0000","last_change":"2023-09-23T19:59:44.949851+0000","last_active":"2023-09-23T20:08:05.745937+0000","last_peered":"2023-09-23T20:08:05.745937+0000","last_clean":"2023-09-23T20:08:05.745937+0000","last_became_active":"2023-09-23T19:59:44.949481+0000","last_became_peered":"2023-09-23T19:59:44.949481+0000","last_unstale":"2023-09-23T20:08:05.745937+0000","last_undegraded":"2023-09-23T20:08:05.745937+0000","last_fullsized":"2023-09-23T20:08:05.745937+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.983923+0000","scrub_duration":0.0087257840000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":302,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748218+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:08:05.748218+0000","last_peered":"2023-09-23T20:08:05.748218+0000","last_clean":"2023-09-23T20:08:05.748218+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:08:05.748218+0000","last_undegraded":"2023-09-23T20:08:05.748218+0000","last_fullsized":"2023-09-23T20:08:05.748218+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":332,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749406+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:08:05.749406+0000","last_peered":"2023-09-23T20:08:05.749406+0000","last_clean":"2023-09-23T20:08:05.749406+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:08:05.749406+0000","last_undegraded":"2023-09-23T20:08:05.749406+0000","last_fullsized":"2023-09-23T20:08:05.749406+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"336'5789","reported_seq":5834,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T20:08:18.550987+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:08:18.550987+0000","last_peered":"2023-09-23T20:08:18.550987+0000","last_clean":"2023-09-23T20:08:18.550987+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:08:18.550987+0000","last_undegraded":"2023-09-23T20:08:18.550987+0000","last_fullsized":"2023-09-23T20:08:18.550987+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5789,"log_dups_size":0,"ondisk_log_size":5789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306118656,"num_objects":4671,"num_object_clones":0,"num_object_copies":14013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5789,"num_write_kb":334720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":326,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.747424+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:08:05.747424+0000","last_peered":"2023-09-23T20:08:05.747424+0000","last_clean":"2023-09-23T20:08:05.747424+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:08:05.747424+0000","last_undegraded":"2023-09-23T20:08:05.747424+0000","last_fullsized":"2023-09-23T20:08:05.747424+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":362,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749163+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:08:05.749163+0000","last_peered":"2023-09-23T20:08:05.749163+0000","last_clean":"2023-09-23T20:08:05.749163+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:08:05.749163+0000","last_undegraded":"2023-09-23T20:08:05.749163+0000","last_fullsized":"2023-09-23T20:08:05.749163+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"336'5702","reported_seq":5750,"reported_epoch":336,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:18.529134+0000","last_change":"2023-09-23T20:07:20.703835+0000","last_active":"2023-09-23T20:08:18.529134+0000","last_peered":"2023-09-23T20:08:18.529134+0000","last_clean":"2023-09-23T20:07:17.901992+0000","last_became_active":"2023-09-23T20:07:20.692221+0000","last_became_peered":"2023-09-23T20:07:20.692221+0000","last_unstale":"2023-09-23T20:08:18.529134+0000","last_undegraded":"2023-09-23T20:07:19.969017+0000","last_fullsized":"2023-09-23T20:07:19.925015+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:57.811754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301334528,"num_objects":4598,"num_object_clones":0,"num_object_copies":13794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4021,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5702,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":3468},{"shards":"2,5","objects":577}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":674,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746871+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:08:05.746871+0000","last_peered":"2023-09-23T20:08:05.746871+0000","last_clean":"2023-09-23T20:08:05.746871+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:08:05.746871+0000","last_undegraded":"2023-09-23T20:08:05.746871+0000","last_fullsized":"2023-09-23T20:08:05.746871+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":110,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.755833+0000","last_change":"2023-09-23T20:07:36.965842+0000","last_active":"2023-09-23T20:08:05.755833+0000","last_peered":"2023-09-23T20:08:05.755833+0000","last_clean":"2023-09-23T20:08:05.755833+0000","last_became_active":"2023-09-23T20:07:36.965484+0000","last_became_peered":"2023-09-23T20:07:36.965484+0000","last_unstale":"2023-09-23T20:08:05.755833+0000","last_undegraded":"2023-09-23T20:08:05.755833+0000","last_fullsized":"2023-09-23T20:08:05.755833+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:55.912064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754075+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:08:05.754075+0000","last_peered":"2023-09-23T20:08:05.754075+0000","last_clean":"2023-09-23T20:08:05.754075+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:08:05.754075+0000","last_undegraded":"2023-09-23T20:08:05.754075+0000","last_fullsized":"2023-09-23T20:08:05.754075+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"336'5852","reported_seq":5898,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T20:08:18.666292+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:08:18.666292+0000","last_peered":"2023-09-23T20:08:18.666292+0000","last_clean":"2023-09-23T20:08:18.666292+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:08:18.666292+0000","last_undegraded":"2023-09-23T20:08:18.666292+0000","last_fullsized":"2023-09-23T20:08:18.666292+0000","mapping_epoch":299,"log_start":"306'100","ondisk_log_start":"306'100","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308674560,"num_objects":4710,"num_object_clones":0,"num_object_copies":14130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5852,"num_write_kb":337984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":694,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.755456+0000","last_change":"2023-09-23T20:07:40.033003+0000","last_active":"2023-09-23T20:08:05.755456+0000","last_peered":"2023-09-23T20:08:05.755456+0000","last_clean":"2023-09-23T20:08:05.755456+0000","last_became_active":"2023-09-23T20:07:40.032587+0000","last_became_peered":"2023-09-23T20:07:40.032587+0000","last_unstale":"2023-09-23T20:08:05.755456+0000","last_undegraded":"2023-09-23T20:08:05.755456+0000","last_fullsized":"2023-09-23T20:08:05.755456+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:24.945100+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":467,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754739+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:08:05.754739+0000","last_peered":"2023-09-23T20:08:05.754739+0000","last_clean":"2023-09-23T20:08:05.754739+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:08:05.754739+0000","last_undegraded":"2023-09-23T20:08:05.754739+0000","last_fullsized":"2023-09-23T20:08:05.754739+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":336,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746839+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:08:05.746839+0000","last_peered":"2023-09-23T20:08:05.746839+0000","last_clean":"2023-09-23T20:08:05.746839+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:08:05.746839+0000","last_undegraded":"2023-09-23T20:08:05.746839+0000","last_fullsized":"2023-09-23T20:08:05.746839+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754328+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:08:05.754328+0000","last_peered":"2023-09-23T20:08:05.754328+0000","last_clean":"2023-09-23T20:08:05.754328+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:08:05.754328+0000","last_undegraded":"2023-09-23T20:08:05.754328+0000","last_fullsized":"2023-09-23T20:08:05.754328+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"336'5814","reported_seq":10588,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T20:08:18.243472+0000","last_change":"2023-09-23T20:07:52.783215+0000","last_active":"2023-09-23T20:08:18.243472+0000","last_peered":"2023-09-23T20:08:18.243472+0000","last_clean":"2023-09-23T20:08:18.243472+0000","last_became_active":"2023-09-23T20:07:52.778874+0000","last_became_peered":"2023-09-23T20:07:52.778874+0000","last_unstale":"2023-09-23T20:08:18.243472+0000","last_undegraded":"2023-09-23T20:08:18.243472+0000","last_fullsized":"2023-09-23T20:08:18.243472+0000","mapping_epoch":330,"log_start":"306'1000","ondisk_log_start":"306'1000","created":299,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4814,"log_dups_size":0,"ondisk_log_size":4814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:31.036337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308674560,"num_objects":4710,"num_object_clones":0,"num_object_copies":14130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5814,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2349,"num_bytes_recovered":153944064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":647,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.745599+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:08:05.745599+0000","last_peered":"2023-09-23T20:08:05.745599+0000","last_clean":"2023-09-23T20:08:05.745599+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:08:05.745599+0000","last_undegraded":"2023-09-23T20:08:05.745599+0000","last_fullsized":"2023-09-23T20:08:05.745599+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":956,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746669+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:08:05.746669+0000","last_peered":"2023-09-23T20:08:05.746669+0000","last_clean":"2023-09-23T20:08:05.746669+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:08:05.746669+0000","last_undegraded":"2023-09-23T20:08:05.746669+0000","last_fullsized":"2023-09-23T20:08:05.746669+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":355,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749045+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:08:05.749045+0000","last_peered":"2023-09-23T20:08:05.749045+0000","last_clean":"2023-09-23T20:08:05.749045+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:08:05.749045+0000","last_undegraded":"2023-09-23T20:08:05.749045+0000","last_fullsized":"2023-09-23T20:08:05.749045+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754674+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:08:05.754674+0000","last_peered":"2023-09-23T20:08:05.754674+0000","last_clean":"2023-09-23T20:08:05.754674+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:08:05.754674+0000","last_undegraded":"2023-09-23T20:08:05.754674+0000","last_fullsized":"2023-09-23T20:08:05.754674+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"336'5662","reported_seq":5717,"reported_epoch":336,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:15.935632+0000","last_change":"2023-09-23T20:07:20.690249+0000","last_active":"2023-09-23T20:08:15.935632+0000","last_peered":"2023-09-23T20:08:15.935632+0000","last_clean":"2023-09-23T20:07:17.918417+0000","last_became_active":"2023-09-23T20:07:20.675349+0000","last_became_peered":"2023-09-23T20:07:20.675349+0000","last_unstale":"2023-09-23T20:08:15.935632+0000","last_undegraded":"2023-09-23T20:07:19.983837+0000","last_fullsized":"2023-09-23T20:07:19.924667+0000","mapping_epoch":317,"log_start":"306'1000","ondisk_log_start":"306'1000","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4662,"log_dups_size":0,"ondisk_log_size":4662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:52.893532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308150272,"num_objects":4702,"num_object_clones":0,"num_object_copies":14106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4017,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5662,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":3537},{"shards":"3,4","objects":685}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":515,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.750236+0000","last_change":"2023-09-23T20:07:40.030105+0000","last_active":"2023-09-23T20:08:05.750236+0000","last_peered":"2023-09-23T20:08:05.750236+0000","last_clean":"2023-09-23T20:08:05.750236+0000","last_became_active":"2023-09-23T20:07:40.029679+0000","last_became_peered":"2023-09-23T20:07:40.029679+0000","last_unstale":"2023-09-23T20:08:05.750236+0000","last_undegraded":"2023-09-23T20:08:05.750236+0000","last_fullsized":"2023-09-23T20:08:05.750236+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":970,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.740225+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:08:05.740225+0000","last_peered":"2023-09-23T20:08:05.740225+0000","last_clean":"2023-09-23T20:08:05.740225+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:08:05.740225+0000","last_undegraded":"2023-09-23T20:08:05.740225+0000","last_fullsized":"2023-09-23T20:08:05.740225+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754117+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:08:05.754117+0000","last_peered":"2023-09-23T20:08:05.754117+0000","last_clean":"2023-09-23T20:08:05.754117+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:08:05.754117+0000","last_undegraded":"2023-09-23T20:08:05.754117+0000","last_fullsized":"2023-09-23T20:08:05.754117+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754009+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:08:05.754009+0000","last_peered":"2023-09-23T20:08:05.754009+0000","last_clean":"2023-09-23T20:08:05.754009+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:08:05.754009+0000","last_undegraded":"2023-09-23T20:08:05.754009+0000","last_fullsized":"2023-09-23T20:08:05.754009+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"336'5826","reported_seq":5871,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T20:08:18.162531+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:08:18.162531+0000","last_peered":"2023-09-23T20:08:18.162531+0000","last_clean":"2023-09-23T20:08:18.162531+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:08:18.162531+0000","last_undegraded":"2023-09-23T20:08:18.162531+0000","last_fullsized":"2023-09-23T20:08:18.162531+0000","mapping_epoch":299,"log_start":"306'1000","ondisk_log_start":"306'1000","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4826,"log_dups_size":0,"ondisk_log_size":4826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310771712,"num_objects":4742,"num_object_clones":0,"num_object_copies":14226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5826,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1204,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:05.746071+0000","last_change":"2023-09-23T20:07:38.015014+0000","last_active":"2023-09-23T20:08:05.746071+0000","last_peered":"2023-09-23T20:08:05.746071+0000","last_clean":"2023-09-23T20:07:24.060460+0000","last_became_active":"2023-09-23T20:07:38.012311+0000","last_became_peered":"2023-09-23T20:07:38.012311+0000","last_unstale":"2023-09-23T20:08:05.746071+0000","last_undegraded":"2023-09-23T20:07:37.981480+0000","last_fullsized":"2023-09-23T20:07:37.980538+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:21.724545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":446,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748682+0000","last_change":"2023-09-23T20:07:40.036264+0000","last_active":"2023-09-23T20:08:05.748682+0000","last_peered":"2023-09-23T20:08:05.748682+0000","last_clean":"2023-09-23T20:08:05.748682+0000","last_became_active":"2023-09-23T20:07:40.035955+0000","last_became_peered":"2023-09-23T20:07:40.035955+0000","last_unstale":"2023-09-23T20:08:05.748682+0000","last_undegraded":"2023-09-23T20:08:05.748682+0000","last_fullsized":"2023-09-23T20:08:05.748682+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":201,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746772+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:08:05.746772+0000","last_peered":"2023-09-23T20:08:05.746772+0000","last_clean":"2023-09-23T20:08:05.746772+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:08:05.746772+0000","last_undegraded":"2023-09-23T20:08:05.746772+0000","last_fullsized":"2023-09-23T20:08:05.746772+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":231,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749757+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:08:05.749757+0000","last_peered":"2023-09-23T20:08:05.749757+0000","last_clean":"2023-09-23T20:08:05.749757+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:08:05.749757+0000","last_undegraded":"2023-09-23T20:08:05.749757+0000","last_fullsized":"2023-09-23T20:08:05.749757+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"336'5867","reported_seq":5917,"reported_epoch":336,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:18.654826+0000","last_change":"2023-09-23T20:07:20.707475+0000","last_active":"2023-09-23T20:08:18.654826+0000","last_peered":"2023-09-23T20:08:18.654826+0000","last_clean":"2023-09-23T20:07:17.907438+0000","last_became_active":"2023-09-23T20:07:20.693608+0000","last_became_peered":"2023-09-23T20:07:20.693608+0000","last_unstale":"2023-09-23T20:08:18.654826+0000","last_undegraded":"2023-09-23T20:07:19.959595+0000","last_fullsized":"2023-09-23T20:07:19.924202+0000","mapping_epoch":317,"log_start":"306'200","ondisk_log_start":"306'200","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5667,"log_dups_size":0,"ondisk_log_size":5667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:21.022797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312541184,"num_objects":4769,"num_object_clones":0,"num_object_copies":14307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4180,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5867,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":3631},{"shards":"2,5","objects":589}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":451,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.756355+0000","last_change":"2023-09-23T20:07:21.995787+0000","last_active":"2023-09-23T20:08:05.756355+0000","last_peered":"2023-09-23T20:08:05.756355+0000","last_clean":"2023-09-23T20:08:05.756355+0000","last_became_active":"2023-09-23T20:07:21.995407+0000","last_became_peered":"2023-09-23T20:07:21.995407+0000","last_unstale":"2023-09-23T20:08:05.756355+0000","last_undegraded":"2023-09-23T20:08:05.756355+0000","last_fullsized":"2023-09-23T20:08:05.756355+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":803,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:05.748080+0000","last_change":"2023-09-23T20:07:38.010675+0000","last_active":"2023-09-23T20:08:05.748080+0000","last_peered":"2023-09-23T20:08:05.748080+0000","last_clean":"2023-09-23T19:43:07.790771+0000","last_became_active":"2023-09-23T20:07:38.009799+0000","last_became_peered":"2023-09-23T20:07:38.009799+0000","last_unstale":"2023-09-23T20:08:05.748080+0000","last_undegraded":"2023-09-23T20:07:37.988964+0000","last_fullsized":"2023-09-23T20:07:37.988167+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:50.851157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":302,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.747731+0000","last_change":"2023-09-23T20:03:59.235180+0000","last_active":"2023-09-23T20:08:05.747731+0000","last_peered":"2023-09-23T20:08:05.747731+0000","last_clean":"2023-09-23T20:08:05.747731+0000","last_became_active":"2023-09-23T20:03:59.234877+0000","last_became_peered":"2023-09-23T20:03:59.234877+0000","last_unstale":"2023-09-23T20:08:05.747731+0000","last_undegraded":"2023-09-23T20:08:05.747731+0000","last_fullsized":"2023-09-23T20:08:05.747731+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:29.834972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.755408+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:08:05.755408+0000","last_peered":"2023-09-23T20:08:05.755408+0000","last_clean":"2023-09-23T20:08:05.755408+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:08:05.755408+0000","last_undegraded":"2023-09-23T20:08:05.755408+0000","last_fullsized":"2023-09-23T20:08:05.755408+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"336'5907","reported_seq":6778,"reported_epoch":336,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:08:18.720680+0000","last_change":"2023-09-23T20:08:05.796050+0000","last_active":"2023-09-23T20:08:18.720680+0000","last_peered":"2023-09-23T20:08:18.720680+0000","last_clean":"2023-09-23T20:07:17.917130+0000","last_became_active":"2023-09-23T20:08:05.761325+0000","last_became_peered":"2023-09-23T20:08:05.761325+0000","last_unstale":"2023-09-23T20:08:18.720680+0000","last_undegraded":"2023-09-23T20:08:05.751514+0000","last_fullsized":"2023-09-23T20:08:18.720680+0000","mapping_epoch":334,"log_start":"306'800","ondisk_log_start":"306'800","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5107,"log_dups_size":0,"ondisk_log_size":5107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:13.200860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312803328,"num_objects":4773,"num_object_clones":0,"num_object_copies":14319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3753,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5907,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":405,"num_bytes_recovered":26542080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2,3","objects":1182},{"shards":"2,3,4","objects":3591}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":513,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749324+0000","last_change":"2023-09-23T20:08:04.756122+0000","last_active":"2023-09-23T20:08:05.749324+0000","last_peered":"2023-09-23T20:08:05.749324+0000","last_clean":"2023-09-23T20:08:05.749324+0000","last_became_active":"2023-09-23T20:08:04.755767+0000","last_became_peered":"2023-09-23T20:08:04.755767+0000","last_unstale":"2023-09-23T20:08:05.749324+0000","last_undegraded":"2023-09-23T20:08:05.749324+0000","last_fullsized":"2023-09-23T20:08:05.749324+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:16:29.799074+0000","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":36,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748269+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:08:05.748269+0000","last_peered":"2023-09-23T20:08:05.748269+0000","last_clean":"2023-09-23T20:08:05.748269+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:08:05.748269+0000","last_undegraded":"2023-09-23T20:08:05.748269+0000","last_fullsized":"2023-09-23T20:08:05.748269+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":30,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.754637+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:08:05.754637+0000","last_peered":"2023-09-23T20:08:05.754637+0000","last_clean":"2023-09-23T20:08:05.754637+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:08:05.754637+0000","last_undegraded":"2023-09-23T20:08:05.754637+0000","last_fullsized":"2023-09-23T20:08:05.754637+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746367+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:08:05.746367+0000","last_peered":"2023-09-23T20:08:05.746367+0000","last_clean":"2023-09-23T20:08:05.746367+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:08:05.746367+0000","last_undegraded":"2023-09-23T20:08:05.746367+0000","last_fullsized":"2023-09-23T20:08:05.746367+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"336'5921","reported_seq":8184,"reported_epoch":336,"state":"active+recovering","last_fresh":"2023-09-23T20:08:18.555326+0000","last_change":"2023-09-23T20:08:06.590518+0000","last_active":"2023-09-23T20:08:18.555326+0000","last_peered":"2023-09-23T20:08:18.555326+0000","last_clean":"2023-09-23T20:06:59.894420+0000","last_became_active":"2023-09-23T20:08:05.774750+0000","last_became_peered":"2023-09-23T20:08:05.774750+0000","last_unstale":"2023-09-23T20:08:18.555326+0000","last_undegraded":"2023-09-23T20:08:18.555326+0000","last_fullsized":"2023-09-23T20:08:18.555326+0000","mapping_epoch":334,"log_start":"306'800","ondisk_log_start":"306'800","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5121,"log_dups_size":0,"ondisk_log_size":5121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.633506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309657600,"num_objects":4725,"num_object_clones":0,"num_object_copies":14175,"num_objects_missing_on_primary":988,"num_objects_missing":988,"num_objects_degraded":0,"num_objects_misplaced":988,"num_objects_unfound":0,"num_objects_dirty":4725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5921,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3,5","objects":988},{"shards":"0,3,7","objects":3737}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":714,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749251+0000","last_change":"2023-09-23T20:03:13.244953+0000","last_active":"2023-09-23T20:08:05.749251+0000","last_peered":"2023-09-23T20:08:05.749251+0000","last_clean":"2023-09-23T20:08:05.749251+0000","last_became_active":"2023-09-23T20:03:13.244569+0000","last_became_peered":"2023-09-23T20:03:13.244569+0000","last_unstale":"2023-09-23T20:08:05.749251+0000","last_undegraded":"2023-09-23T20:08:05.749251+0000","last_fullsized":"2023-09-23T20:08:05.749251+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.274222+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":178,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748348+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:08:05.748348+0000","last_peered":"2023-09-23T20:08:05.748348+0000","last_clean":"2023-09-23T20:08:05.748348+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:08:05.748348+0000","last_undegraded":"2023-09-23T20:08:05.748348+0000","last_fullsized":"2023-09-23T20:08:05.748348+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":372,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.755332+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:08:05.755332+0000","last_peered":"2023-09-23T20:08:05.755332+0000","last_clean":"2023-09-23T20:08:05.755332+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:08:05.755332+0000","last_undegraded":"2023-09-23T20:08:05.755332+0000","last_fullsized":"2023-09-23T20:08:05.755332+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":83,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.739994+0000","last_change":"2023-09-23T20:07:36.978344+0000","last_active":"2023-09-23T20:08:05.739994+0000","last_peered":"2023-09-23T20:08:05.739994+0000","last_clean":"2023-09-23T20:08:05.739994+0000","last_became_active":"2023-09-23T20:07:36.978048+0000","last_became_peered":"2023-09-23T20:07:36.978048+0000","last_unstale":"2023-09-23T20:08:05.739994+0000","last_undegraded":"2023-09-23T20:08:05.739994+0000","last_fullsized":"2023-09-23T20:08:05.739994+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:28.605336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.d","version":"336'5824","reported_seq":5894,"reported_epoch":336,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:18.171205+0000","last_change":"2023-09-23T20:07:37.106881+0000","last_active":"2023-09-23T20:08:18.171205+0000","last_peered":"2023-09-23T20:08:18.171205+0000","last_clean":"2023-09-23T20:06:59.856184+0000","last_became_active":"2023-09-23T20:07:37.089929+0000","last_became_peered":"2023-09-23T20:07:37.089929+0000","last_unstale":"2023-09-23T20:08:18.171205+0000","last_undegraded":"2023-09-23T20:07:36.973377+0000","last_fullsized":"2023-09-23T20:07:36.954757+0000","mapping_epoch":322,"log_start":"306'1000","ondisk_log_start":"306'1000","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4824,"log_dups_size":0,"ondisk_log_size":4824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:25.866127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307953688,"num_objects":4700,"num_object_clones":0,"num_object_copies":14100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1316,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4700,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5824,"num_write_kb":336705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"1,2,5","objects":459},{"shards":"2,4,5","objects":857},{"shards":"2,5","objects":3384}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":623,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:08:11.054673+0000","last_change":"2023-09-23T20:07:37.995077+0000","last_active":"2023-09-23T20:08:11.054673+0000","last_peered":"2023-09-23T20:08:11.054673+0000","last_clean":"2023-09-23T20:07:24.057948+0000","last_became_active":"2023-09-23T20:07:37.994679+0000","last_became_peered":"2023-09-23T20:07:37.994679+0000","last_unstale":"2023-09-23T20:08:11.054673+0000","last_undegraded":"2023-09-23T20:07:37.977845+0000","last_fullsized":"2023-09-23T20:07:37.977450+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:43.844346+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":37,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748605+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:08:05.748605+0000","last_peered":"2023-09-23T20:08:05.748605+0000","last_clean":"2023-09-23T20:08:05.748605+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:08:05.748605+0000","last_undegraded":"2023-09-23T20:08:05.748605+0000","last_fullsized":"2023-09-23T20:08:05.748605+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":34,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.755187+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:08:05.755187+0000","last_peered":"2023-09-23T20:08:05.755187+0000","last_clean":"2023-09-23T20:08:05.755187+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:08:05.755187+0000","last_undegraded":"2023-09-23T20:08:05.755187+0000","last_fullsized":"2023-09-23T20:08:05.755187+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":295,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.747517+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:08:05.747517+0000","last_peered":"2023-09-23T20:08:05.747517+0000","last_clean":"2023-09-23T20:08:05.747517+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:08:05.747517+0000","last_undegraded":"2023-09-23T20:08:05.747517+0000","last_fullsized":"2023-09-23T20:08:05.747517+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":109,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.744336+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:08:05.744336+0000","last_peered":"2023-09-23T20:08:05.744336+0000","last_clean":"2023-09-23T20:08:05.744336+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:08:05.744336+0000","last_undegraded":"2023-09-23T20:08:05.744336+0000","last_fullsized":"2023-09-23T20:08:05.744336+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:13.687532+0000","last_change":"2023-09-23T20:07:18.947757+0000","last_active":"2023-09-23T20:08:13.687532+0000","last_peered":"2023-09-23T20:08:13.687532+0000","last_clean":"2023-09-23T20:08:13.687532+0000","last_became_active":"2023-09-23T20:07:18.947439+0000","last_became_peered":"2023-09-23T20:07:18.947439+0000","last_unstale":"2023-09-23T20:08:13.687532+0000","last_undegraded":"2023-09-23T20:08:13.687532+0000","last_fullsized":"2023-09-23T20:08:13.687532+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:49.555624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":339,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.745980+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:08:05.745980+0000","last_peered":"2023-09-23T20:08:05.745980+0000","last_clean":"2023-09-23T20:08:05.745980+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:08:05.745980+0000","last_undegraded":"2023-09-23T20:08:05.745980+0000","last_fullsized":"2023-09-23T20:08:05.745980+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":352,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.749072+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:08:05.749072+0000","last_peered":"2023-09-23T20:08:05.749072+0000","last_clean":"2023-09-23T20:08:05.749072+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:08:05.749072+0000","last_undegraded":"2023-09-23T20:08:05.749072+0000","last_fullsized":"2023-09-23T20:08:05.749072+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":329,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746011+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:08:05.746011+0000","last_peered":"2023-09-23T20:08:05.746011+0000","last_clean":"2023-09-23T20:08:05.746011+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:08:05.746011+0000","last_undegraded":"2023-09-23T20:08:05.746011+0000","last_fullsized":"2023-09-23T20:08:05.746011+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":332,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.748130+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:08:05.748130+0000","last_peered":"2023-09-23T20:08:05.748130+0000","last_clean":"2023-09-23T20:08:05.748130+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:08:05.748130+0000","last_undegraded":"2023-09-23T20:08:05.748130+0000","last_fullsized":"2023-09-23T20:08:05.748130+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":35,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:08.682940+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:08:08.682940+0000","last_peered":"2023-09-23T20:08:08.682940+0000","last_clean":"2023-09-23T20:08:08.682940+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:08:08.682940+0000","last_undegraded":"2023-09-23T20:08:08.682940+0000","last_fullsized":"2023-09-23T20:08:08.682940+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:08.680806+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:08:08.680806+0000","last_peered":"2023-09-23T20:08:08.680806+0000","last_clean":"2023-09-23T20:08:08.680806+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:08:08.680806+0000","last_undegraded":"2023-09-23T20:08:08.680806+0000","last_fullsized":"2023-09-23T20:08:08.680806+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":155,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.746486+0000","last_change":"2023-09-23T20:03:10.178518+0000","last_active":"2023-09-23T20:08:05.746486+0000","last_peered":"2023-09-23T20:08:05.746486+0000","last_clean":"2023-09-23T20:08:05.746486+0000","last_became_active":"2023-09-23T20:03:10.174331+0000","last_became_peered":"2023-09-23T20:03:10.174331+0000","last_unstale":"2023-09-23T20:08:05.746486+0000","last_undegraded":"2023-09-23T20:08:05.746486+0000","last_fullsized":"2023-09-23T20:08:05.746486+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:44.577438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":382,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T20:08:05.747383+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:08:05.747383+0000","last_peered":"2023-09-23T20:08:05.747383+0000","last_clean":"2023-09-23T20:08:05.747383+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:08:05.747383+0000","last_undegraded":"2023-09-23T20:08:05.747383+0000","last_fullsized":"2023-09-23T20:08:05.747383+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":15062,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":139264,"data_stored":26196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4943773720,"num_objects":75437,"num_object_clones":0,"num_object_copies":226311,"num_objects_missing_on_primary":988,"num_objects_missing":988,"num_objects_degraded":26690,"num_objects_misplaced":4106,"num_objects_unfound":0,"num_objects_dirty":75437,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":93025,"num_write_kb":5390721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6985,"num_bytes_recovered":457768960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15454711808,"data_stored":15454699592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84525,"ondisk_log_size":84525,"up":48,"acting":40,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":751377,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":133,"num_read_kb":248,"num_write":65,"num_write_kb":659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1336085,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":36,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":140,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20992000,"data_stored":20971615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234326,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1603332,"kb_used_data":1282300,"kb_used_omap":0,"kb_used_meta":321024,"kb_avail":92141820,"statfs":{"total":95995035648,"available":94353223680,"internally_reserved":0,"allocated":1313075200,"data_stored":1312035674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":328728576},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.6160000000000001,"15min":2.2599999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.19700000000000001},"max":{"1min":6.4740000000000002,"5min":60.140000000000001,"15min":62.061999999999998},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.94899999999999995,"5min":2.875,"15min":2.5},"min":{"1min":0.27200000000000002,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":4.3230000000000004,"5min":62.389000000000003,"15min":62.389000000000003},"last":1.109}]},{"osd":1,"last update":"Sat Sep 23 20:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":4.9630000000000001,"15min":3.3029999999999999},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":4.0540000000000003,"5min":104.16800000000001,"15min":104.16800000000001},"last":4.7779999999999996},{"interface":"front","average":{"1min":1.2529999999999999,"5min":5.0979999999999999,"15min":3.4279999999999999},"min":{"1min":0.28299999999999997,"5min":0.223,"15min":0.21199999999999999},"max":{"1min":4.7220000000000004,"5min":104.003,"15min":104.003},"last":1.1439999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:07:42 2023","interfaces":[{"interface":"back","average":{"1min":3.343,"5min":4.5350000000000001,"15min":3.8069999999999999},"min":{"1min":0.27500000000000002,"5min":0.251,"15min":0.251},"max":{"1min":25.741,"5min":105.842,"15min":130.673},"last":1.5229999999999999},{"interface":"front","average":{"1min":3.7120000000000002,"5min":4.8419999999999996,"15min":4.1829999999999998},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":25.067,"5min":105.66,"15min":130.71100000000001},"last":1.0600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:07:35 2023","interfaces":[{"interface":"back","average":{"1min":4.3620000000000001,"5min":6.5839999999999996,"15min":4.1630000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":24.681999999999999,"5min":146.38800000000001,"15min":146.38800000000001},"last":0.754},{"interface":"front","average":{"1min":4.4130000000000003,"5min":6.4340000000000002,"15min":4.1660000000000004},"min":{"1min":0.44400000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":24.995000000000001,"5min":146.851,"15min":146.851},"last":2.5910000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:08:03 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.63500000000000001,"15min":1.903},"min":{"1min":0.28699999999999998,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":3.032,"5min":6.3109999999999999,"15min":143.255},"last":1.454},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.999,"15min":1.9370000000000001},"min":{"1min":0.33200000000000002,"5min":0.246,"15min":0.246},"max":{"1min":2.7949999999999999,"5min":41.822000000000003,"15min":143.41200000000001},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:08:03 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":1.024,"15min":1.5900000000000001},"min":{"1min":0.317,"5min":0.26500000000000001,"15min":0.25},"max":{"1min":4.4080000000000004,"5min":24.876000000000001,"15min":84.566999999999993},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.874,"15min":1.395},"min":{"1min":0.34999999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.2050000000000001,"5min":21.623000000000001,"15min":81.244},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:07:39 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1.091,"15min":0.83699999999999997},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.3710000000000004,"5min":41.902999999999999,"15min":41.902999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.74399999999999999,"15min":0.66100000000000003},"min":{"1min":0.36899999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.9889999999999999,"5min":6.3620000000000001,"15min":6.3620000000000001},"last":0.58799999999999997}]}]},{"osd":6,"up_from":0,"seq":1078036791367,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762527,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1073960,"kb_used_data":583968,"kb_used_omap":7,"kb_used_meta":489976,"kb_avail":92671192,"statfs":{"total":95995035648,"available":94895300608,"internally_reserved":0,"allocated":597983232,"data_stored":596447118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":501736250},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":4.3239999999999998,"15min":4.9089999999999998},"min":{"1min":0.254,"5min":0.17100000000000001,"15min":0.151},"max":{"1min":3.758,"5min":144.273,"15min":202.917},"last":4.992},{"interface":"front","average":{"1min":2.024,"5min":5.3289999999999997,"15min":5.2610000000000001},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.17899999999999999},"max":{"1min":10.73,"5min":144.321,"15min":203.34899999999999},"last":7.4169999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:08:02 2023","interfaces":[{"interface":"back","average":{"1min":4.8540000000000001,"5min":4.8760000000000003,"15min":6.54},"min":{"1min":0.222,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":21.241,"5min":98.531000000000006,"15min":231.57499999999999},"last":17.483000000000001},{"interface":"front","average":{"1min":4.3179999999999996,"5min":4.9109999999999996,"15min":6.5209999999999999},"min":{"1min":0.27500000000000002,"5min":0.20699999999999999,"15min":0.19600000000000001},"max":{"1min":12.922000000000001,"5min":98.346000000000004,"15min":236.077},"last":4.6500000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:07:19 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":3.71,"15min":3.7269999999999999},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.217},"max":{"1min":4.343,"5min":25.178999999999998,"15min":68.180999999999997},"last":7.2169999999999996},{"interface":"front","average":{"1min":1.284,"5min":3.8370000000000002,"15min":3.4870000000000001},"min":{"1min":0.28999999999999998,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":5.4109999999999996,"5min":28.437999999999999,"15min":66.509},"last":7.4530000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:07:19 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":2.3050000000000002,"15min":2.6619999999999999},"min":{"1min":0.443,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.3290000000000006,"5min":35.497999999999998,"15min":161.68299999999999},"last":3.7229999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.6920000000000002,"15min":2.6970000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.26400000000000001},"max":{"1min":5.2549999999999999,"5min":35.521000000000001,"15min":161.286},"last":3.895}]},{"osd":5,"last update":"Sat Sep 23 20:07:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":2.7530000000000001,"15min":3.6120000000000001},"min":{"1min":0.375,"5min":0.29899999999999999,"15min":0.25900000000000001},"max":{"1min":5.484,"5min":51.698,"15min":219.048},"last":3.9329999999999998},{"interface":"front","average":{"1min":0.93700000000000006,"5min":2.5169999999999999,"15min":3.7309999999999999},"min":{"1min":0.32800000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.6949999999999998,"5min":51.173000000000002,"15min":218.69499999999999},"last":5.9169999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:07:39 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.8620000000000001,"15min":2.1179999999999999},"min":{"1min":0.41899999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.3879999999999999,"5min":10.4,"15min":10.4},"last":3.7599999999999998},{"interface":"front","average":{"1min":2.468,"5min":2.5230000000000001,"15min":3.1379999999999999},"min":{"1min":0.40100000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":14.862,"5min":20.734999999999999,"15min":20.734999999999999},"last":3.9620000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":2.0750000000000002,"15min":2.3799999999999999},"min":{"1min":0.42399999999999999,"5min":0.30399999999999999,"15min":0.26800000000000002},"max":{"1min":2.6429999999999998,"5min":14.007,"15min":60.213000000000001},"last":5.8730000000000002},{"interface":"front","average":{"1min":1.3899999999999999,"5min":1.9770000000000001,"15min":2.0030000000000001},"min":{"1min":0.495,"5min":0.34100000000000003,"15min":0.30199999999999999},"max":{"1min":8.4280000000000008,"5min":14.188000000000001,"15min":61.298999999999999},"last":3.883}]}]},{"osd":0,"up_from":94,"seq":403726926062,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2885892,"kb_used_data":2281660,"kb_used_omap":8,"kb_used_meta":604215,"kb_avail":90859260,"statfs":{"total":95995035648,"available":93039882240,"internally_reserved":0,"allocated":2336419840,"data_stored":2334860716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":618716769},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,5,1,1,1,0,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:07:43 2023","interfaces":[{"interface":"back","average":{"1min":2.597,"5min":3.496,"15min":3.4740000000000002},"min":{"1min":0.23999999999999999,"5min":0.185,"15min":0.17799999999999999},"max":{"1min":14.813000000000001,"5min":48.087000000000003,"15min":94.055999999999997},"last":14.300000000000001},{"interface":"front","average":{"1min":2.2999999999999998,"5min":3.4279999999999999,"15min":3.7610000000000001},"min":{"1min":0.255,"5min":0.187,"15min":0.183},"max":{"1min":9.3059999999999992,"5min":48.119999999999997,"15min":91.632999999999996},"last":6.4790000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:08:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6200000000000001,"5min":4.8710000000000004,"15min":4.9480000000000004},"min":{"1min":0.26900000000000002,"5min":0.216,"15min":0.19800000000000001},"max":{"1min":11.518000000000001,"5min":162.62899999999999,"15min":270.51100000000002},"last":0.311},{"interface":"front","average":{"1min":3.4399999999999999,"5min":4.6479999999999997,"15min":4.9059999999999997},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.22600000000000001},"max":{"1min":9.9939999999999998,"5min":162.52500000000001,"15min":269.69299999999998},"last":6.1870000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:07:43 2023","interfaces":[{"interface":"back","average":{"1min":10.682,"5min":6.0019999999999998,"15min":5.8940000000000001},"min":{"1min":0.34699999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":126.95399999999999,"5min":126.95399999999999,"15min":145.72800000000001},"last":17.039999999999999},{"interface":"front","average":{"1min":10.698,"5min":5.9000000000000004,"15min":5.9909999999999997},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":127.294,"5min":127.294,"15min":145.50700000000001},"last":6.5369999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":3.125,"15min":3.085},"min":{"1min":0.40699999999999997,"5min":0.27600000000000002,"15min":0.27500000000000002},"max":{"1min":9.9209999999999994,"5min":148.20599999999999,"15min":148.20599999999999},"last":6.2329999999999997},{"interface":"front","average":{"1min":2.5209999999999999,"5min":3.3399999999999999,"15min":2.754},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":13.428000000000001,"5min":148.11000000000001,"15min":148.11000000000001},"last":6.4320000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":1.9330000000000001,"15min":2.8380000000000001},"min":{"1min":0.42499999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.5719999999999992,"5min":14.901999999999999,"15min":131.34399999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.7749999999999999,"5min":1.8089999999999999,"15min":2.9119999999999999},"min":{"1min":0.44900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.8789999999999996,"5min":12.307,"15min":131.386},"last":6.5039999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:07:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.732,"15min":1.6479999999999999},"min":{"1min":0.40899999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.9480000000000004,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":6.5529999999999999},{"interface":"front","average":{"1min":1.282,"5min":1.9390000000000001,"15min":1.917},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.309},"max":{"1min":5.0810000000000004,"5min":13.766999999999999,"15min":13.766999999999999},"last":6.7160000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7370000000000001,"5min":1.974,"15min":2.3639999999999999},"min":{"1min":0.42099999999999999,"5min":0.377,"15min":0.29699999999999999},"max":{"1min":9.6069999999999993,"5min":14.856,"15min":77.132999999999996},"last":0.51500000000000001},{"interface":"front","average":{"1min":3.1259999999999999,"5min":2.1000000000000001,"15min":2.4849999999999999},"min":{"1min":0.46000000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.9689999999999994,"5min":14.916,"15min":75.465999999999994},"last":6.4530000000000003}]}]},{"osd":2,"up_from":104,"seq":446676599001,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3662652,"kb_used_data":3001780,"kb_used_omap":12,"kb_used_meta":660851,"kb_avail":90082500,"statfs":{"total":95995035648,"available":92244480000,"internally_reserved":0,"allocated":3073822720,"data_stored":3072271076,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":676712446},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:08:04 2023","interfaces":[{"interface":"back","average":{"1min":5.1230000000000002,"5min":3.7229999999999999,"15min":3.496},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.154},"max":{"1min":17.876999999999999,"5min":76.251999999999995,"15min":76.251999999999995},"last":0.377},{"interface":"front","average":{"1min":4.5170000000000003,"5min":3.7440000000000002,"15min":3.4380000000000002},"min":{"1min":0.26500000000000001,"5min":0.20200000000000001,"15min":0.187},"max":{"1min":17.032,"5min":76.590000000000003,"15min":76.590000000000003},"last":15.170999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:08:04 2023","interfaces":[{"interface":"back","average":{"1min":4.6859999999999999,"5min":3.98,"15min":3.7490000000000001},"min":{"1min":0.33500000000000002,"5min":0.23200000000000001,"15min":0.184},"max":{"1min":19.562999999999999,"5min":50.192999999999998,"15min":94.768000000000001},"last":10.875},{"interface":"front","average":{"1min":3.9790000000000001,"5min":3.4830000000000001,"15min":3.7959999999999998},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":19.314,"5min":47.987000000000002,"15min":96.808000000000007},"last":0.72899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:07:28 2023","interfaces":[{"interface":"back","average":{"1min":3.181,"5min":6.8769999999999998,"15min":4.3049999999999997},"min":{"1min":0.34200000000000003,"5min":0.23899999999999999,"15min":0.20499999999999999},"max":{"1min":8.0120000000000005,"5min":107.381,"15min":107.381},"last":5.5449999999999999},{"interface":"front","average":{"1min":1.762,"5min":4.5750000000000002,"15min":3.7189999999999999},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":10.663,"5min":106.301,"15min":106.301},"last":0.624}]},{"osd":4,"last update":"Sat Sep 23 20:08:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8359999999999999,"5min":2.9180000000000001,"15min":2.415},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.26900000000000002},"max":{"1min":10.127000000000001,"5min":75.608000000000004,"15min":75.608000000000004},"last":0.441},{"interface":"front","average":{"1min":2.5670000000000002,"5min":3.3100000000000001,"15min":2.4820000000000002},"min":{"1min":0.499,"5min":0.28999999999999998,"15min":0.26700000000000002},"max":{"1min":8.125,"5min":76.161000000000001,"15min":76.161000000000001},"last":5.46}]},{"osd":5,"last update":"Sat Sep 23 20:08:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":4.1699999999999999,"15min":3.0329999999999999},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.28100000000000003},"max":{"1min":10.18,"5min":172.505,"15min":172.505},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.0819999999999999,"5min":4.4660000000000002,"15min":3.0019999999999998},"min":{"1min":0.435,"5min":0.35099999999999998,"15min":0.29099999999999998},"max":{"1min":6.9390000000000001,"5min":172.375,"15min":172.375},"last":10.936999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:07:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7749999999999999,"5min":1.617,"15min":1.623},"min":{"1min":0.40500000000000003,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.209,"5min":10.209,"15min":10.209},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.944,"15min":1.96},"min":{"1min":0.40500000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.7140000000000004,"5min":9.7210000000000001,"15min":9.7210000000000001},"last":10.977}]},{"osd":7,"last update":"Sat Sep 23 20:08:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.1840000000000002,"15min":2.165},"min":{"1min":0.68899999999999995,"5min":0.35099999999999998,"15min":0.27400000000000002},"max":{"1min":8.1519999999999992,"5min":9.7330000000000005,"15min":50.118000000000002},"last":5.5110000000000001},{"interface":"front","average":{"1min":2.8119999999999998,"5min":1.819,"15min":2.3839999999999999},"min":{"1min":0.38500000000000001,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":10.244,"5min":10.244,"15min":50.664999999999999},"last":0.47399999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435474,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3451904,"kb_used_data":2818296,"kb_used_omap":4,"kb_used_meta":633595,"kb_avail":90293248,"statfs":{"total":95995035648,"available":92460285952,"internally_reserved":0,"allocated":2885935104,"data_stored":2884256782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":648801389},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:07:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2309999999999999,"5min":3.3079999999999998,"15min":4.4930000000000003},"min":{"1min":0.20499999999999999,"5min":0.193,"15min":0.17399999999999999},"max":{"1min":11.34,"5min":21.100999999999999,"15min":240.40600000000001},"last":4.3310000000000004},{"interface":"front","average":{"1min":1.98,"5min":3.8050000000000002,"15min":5.0460000000000003},"min":{"1min":0.23699999999999999,"5min":0.19700000000000001,"15min":0.17199999999999999},"max":{"1min":16.917999999999999,"5min":53.201000000000001,"15min":240.352},"last":3.2999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:07:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0179999999999998,"5min":4.9640000000000004,"15min":3.0760000000000001},"min":{"1min":0.23599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":17.463000000000001,"5min":112.122,"15min":112.122},"last":4.1859999999999999},{"interface":"front","average":{"1min":2.5910000000000002,"5min":5.5369999999999999,"15min":3.2080000000000002},"min":{"1min":0.28899999999999998,"5min":0.249,"15min":0.19900000000000001},"max":{"1min":14.971,"5min":111.786,"15min":111.786},"last":9.2050000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":3.52,"15min":5.7160000000000002},"min":{"1min":0.252,"5min":0.23899999999999999,"15min":0.219},"max":{"1min":3.6280000000000001,"5min":82.111999999999995,"15min":264.97800000000001},"last":4.1630000000000003},{"interface":"front","average":{"1min":2.6579999999999999,"5min":3.8109999999999999,"15min":5.5800000000000001},"min":{"1min":0.32700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":14.927,"5min":30.768999999999998,"15min":257.40899999999999},"last":4.4580000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:07:25 2023","interfaces":[{"interface":"back","average":{"1min":2.1509999999999998,"5min":4.2560000000000002,"15min":3.097},"min":{"1min":0.38200000000000001,"5min":0.27200000000000002,"15min":0.25700000000000001},"max":{"1min":7.7679999999999998,"5min":109.66800000000001,"15min":109.66800000000001},"last":4.3920000000000003},{"interface":"front","average":{"1min":1.3380000000000001,"5min":4.3520000000000003,"15min":3.5099999999999998},"min":{"1min":0.37,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.8890000000000002,"5min":110.295,"15min":110.295},"last":3.2669999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:07:25 2023","interfaces":[{"interface":"back","average":{"1min":5.9489999999999998,"5min":3.907,"15min":3.1389999999999998},"min":{"1min":0.42899999999999999,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":77.206000000000003,"5min":82.138000000000005,"15min":113.881},"last":3.3279999999999998},{"interface":"front","average":{"1min":5.4870000000000001,"5min":3.4039999999999999,"15min":2.8959999999999999},"min":{"1min":0.35899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":77.968000000000004,"5min":82.25,"15min":114.598},"last":4.0650000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:07:39 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":2.4430000000000001,"15min":4.649},"min":{"1min":0.38600000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.2329999999999997,"5min":82.281999999999996,"15min":82.281999999999996},"last":4.1230000000000002},{"interface":"front","average":{"1min":3.915,"5min":3.2909999999999999,"15min":4.9299999999999997},"min":{"1min":0.41699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":27.113,"5min":82.233000000000004,"15min":82.233000000000004},"last":4.4889999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:07:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":3.8639999999999999,"15min":3.1320000000000001},"min":{"1min":0.33700000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.774,"5min":112.883,"15min":138.035},"last":4.0999999999999996},{"interface":"front","average":{"1min":2.3660000000000001,"5min":4.3070000000000004,"15min":3.1000000000000001},"min":{"1min":0.443,"5min":0.33100000000000002,"15min":0.28699999999999998},"max":{"1min":7.8609999999999998,"5min":111.85599999999999,"15min":138.054},"last":4.5220000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019236,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2665788,"kb_used_data":2343412,"kb_used_omap":0,"kb_used_meta":322368,"kb_avail":91079364,"statfs":{"total":95995035648,"available":93265268736,"internally_reserved":0,"allocated":2399653888,"data_stored":2398622518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":330104832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":2.4630000000000001,"15min":3.5369999999999999},"min":{"1min":0.248,"5min":0.22,"15min":0.189},"max":{"1min":2.9169999999999998,"5min":56.061999999999998,"15min":104.604},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.03,"5min":2.3239999999999998,"15min":3.4420000000000002},"min":{"1min":0.25900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":2.9340000000000002,"5min":56.075000000000003,"15min":104.687},"last":0.66400000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:07:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":9.8160000000000007,"15min":5.2400000000000002},"min":{"1min":0.28399999999999997,"5min":0.23200000000000001,"15min":0.216},"max":{"1min":4.1150000000000002,"5min":161.613,"15min":161.613},"last":2.4790000000000001},{"interface":"front","average":{"1min":1.119,"5min":9.5969999999999995,"15min":5.0659999999999998},"min":{"1min":0.26700000000000002,"5min":0.23400000000000001,"15min":0.22},"max":{"1min":6.8799999999999999,"5min":162.017,"15min":162.017},"last":1.9850000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:08:03 2023","interfaces":[{"interface":"back","average":{"1min":15.901999999999999,"5min":7.1849999999999996,"15min":6.3140000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":182.96700000000001,"5min":182.96700000000001,"15min":182.96700000000001},"last":3.9039999999999999},{"interface":"front","average":{"1min":15.569000000000001,"5min":7.569,"15min":6.1680000000000001},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":183.244,"5min":183.244,"15min":183.244},"last":1.4710000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:07:22 2023","interfaces":[{"interface":"back","average":{"1min":8.4309999999999992,"5min":7.3220000000000001,"15min":6.3129999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.255},"max":{"1min":78.108999999999995,"5min":118.31699999999999,"15min":266.54700000000003},"last":1.8839999999999999},{"interface":"front","average":{"1min":8.4450000000000003,"5min":7.2990000000000004,"15min":5.3310000000000004},"min":{"1min":0.42399999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":76.977000000000004,"5min":118.395,"15min":139.911},"last":2.2410000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:08:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":2.1899999999999999,"15min":2.3980000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":2.4300000000000002,"5min":176.81899999999999,"15min":266.59899999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.48599999999999999,"5min":2.2719999999999998,"15min":2.5329999999999999},"min":{"1min":0.27900000000000003,"5min":0.26600000000000001,"15min":0.24399999999999999},"max":{"1min":1.286,"5min":176.172,"15min":266.72899999999998},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:07:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53100000000000003,"15min":0.51200000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.5129999999999999,"5min":4.0410000000000004,"15min":4.0410000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.77400000000000002,"15min":0.57799999999999996},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.2889999999999997,"5min":4.2889999999999997,"15min":4.2889999999999997},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:07:57 2023","interfaces":[{"interface":"back","average":{"1min":4.25,"5min":2.8759999999999999,"15min":1.7370000000000001},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.26100000000000001},"max":{"1min":74.040000000000006,"5min":76.507000000000005,"15min":76.507000000000005},"last":0.81999999999999995},{"interface":"front","average":{"1min":4.0780000000000003,"5min":3.0640000000000001,"15min":1.8129999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":74.021000000000001,"5min":76.462000000000003,"15min":76.462000000000003},"last":0.80700000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757600,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3257340,"kb_used_data":2875700,"kb_used_omap":0,"kb_used_meta":381632,"kb_avail":90487812,"statfs":{"total":95995035648,"available":92659519488,"internally_reserved":0,"allocated":2944716800,"data_stored":2943681725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":390791168},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,0,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:08:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":3.1970000000000001,"15min":2.1459999999999999},"min":{"1min":0.224,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":4.4939999999999998,"5min":121.94799999999999,"15min":121.94799999999999},"last":1.2370000000000001},{"interface":"front","average":{"1min":0.89400000000000002,"5min":3.569,"15min":2.2730000000000001},"min":{"1min":0.23999999999999999,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":3.6160000000000001,"5min":121.964,"15min":121.964},"last":4.4269999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":7.1459999999999999,"15min":5.843},"min":{"1min":0.26900000000000002,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":5.0759999999999996,"5min":171.47300000000001,"15min":266.197},"last":3.2269999999999999},{"interface":"front","average":{"1min":1.776,"5min":7.71,"15min":5.9889999999999999},"min":{"1min":0.28299999999999997,"5min":0.27600000000000002,"15min":0.24099999999999999},"max":{"1min":9.8719999999999999,"5min":171.399,"15min":266.14999999999998},"last":6.1299999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:07:54 2023","interfaces":[{"interface":"back","average":{"1min":7.6120000000000001,"5min":4.5469999999999997,"15min":6.2380000000000004},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":96.733000000000004,"5min":177.816,"15min":238.96199999999999},"last":1.5049999999999999},{"interface":"front","average":{"1min":7.6360000000000001,"5min":4.617,"15min":6.2830000000000004},"min":{"1min":0.42399999999999999,"5min":0.32300000000000001,"15min":0.313},"max":{"1min":96.757999999999996,"5min":177.851,"15min":238.905},"last":1.5409999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:07:25 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":8.7799999999999994,"15min":5.0259999999999998},"min":{"1min":0.38100000000000001,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":14.808,"5min":201.50899999999999,"15min":201.50899999999999},"last":8.2560000000000002},{"interface":"front","average":{"1min":2.3069999999999999,"5min":8.3770000000000007,"15min":4.9020000000000001},"min":{"1min":0.34300000000000003,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":14.893000000000001,"5min":201.58099999999999,"15min":201.58099999999999},"last":1.4139999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:08:10 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.68400000000000005,"15min":1.661},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.21299999999999999},"max":{"1min":0.89400000000000002,"5min":9.7240000000000002,"15min":191.78899999999999},"last":0.746},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.628,"15min":1.615},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.54700000000000004,"5min":8.7360000000000007,"15min":191.821},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":0.77200000000000002,"15min":0.65700000000000003},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.9009999999999998,"5min":9.9009999999999998,"15min":9.9009999999999998},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.624,"15min":0.51600000000000001},"min":{"1min":0.33600000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0119999999999996,"5min":5.0119999999999996,"15min":5.0119999999999996},"last":0.82399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.65600000000000003,"15min":1.766},"min":{"1min":0.29799999999999999,"5min":0.28100000000000003,"15min":0.26200000000000001},"max":{"1min":4.2080000000000002,"5min":5.2670000000000003,"15min":191.84299999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.72399999999999998,"15min":1.972},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.26300000000000001},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":191.69},"last":0.83699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2326986752,"data_stored":2326986752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":588972032,"data_stored":588972032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3049000960,"data_stored":3048996888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2866216960,"data_stored":2866216960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2392195072,"data_stored":2392195072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2928414720,"data_stored":2928410648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1302925312,"data_stored":1302921240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:08:25.273 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 16 2023-09-23T20:08:25.344 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:25.373 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:24 smithi144 ceph-mon[194833]: pgmap v933: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 26552/218847 objects degraded (12.133%); 3729/218847 objects misplaced (1.704%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:08:25.376 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:25.436 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:25.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:25.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:25.479 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:25.541 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:25.648 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:26.061 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:26.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:26.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:26.172 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:26.267 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:26.854 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:08:26.892 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1207781015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:26.892 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:26.897 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1207781015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:26.897 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:27.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1207781015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:27.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:27 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:08:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:27 smithi144 ceph-mon[194833]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:27 smithi144 ceph-mon[194833]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:27 smithi144 ceph-mon[194833]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:27 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:08:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:27 smithi144 ceph-mon[194833]: osdmap e337: 8 total, 8 up, 7 in 2023-09-23T20:08:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:27 smithi144 ceph-mon[194833]: pgmap v935: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 26552/217827 objects degraded (12.189%); 3729/217827 objects misplaced (1.712%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:08:28.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 26552/218847 objects degraded (12.133%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:28.228 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:08:28.228 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[213871]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.228 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[213871]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.228 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[213871]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.229 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:08:28.229 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[213871]: osdmap e337: 8 total, 8 up, 7 in 2023-09-23T20:08:28.229 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[213871]: pgmap v935: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 26552/217827 objects degraded (12.189%); 3729/217827 objects misplaced (1.712%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:08:28.229 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 26552/218847 objects degraded (12.133%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:28.229 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:08:28.229 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[221113]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.229 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[221113]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.229 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[221113]: failed to encode map e337 with expected crc 2023-09-23T20:08:28.230 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:08:28.230 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[221113]: osdmap e337: 8 total, 8 up, 7 in 2023-09-23T20:08:28.230 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[221113]: pgmap v935: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 26552/217827 objects degraded (12.189%); 3729/217827 objects misplaced (1.712%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:08:28.230 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 26552/218847 objects degraded (12.133%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:28.230 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:28.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:28.338 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:28.422 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:29.387 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:29.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:29.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:29.504 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:29.554 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:29.943 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:29 smithi135 ceph-mon[213871]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 26552/213969 objects degraded (12.409%); 3279/213969 objects misplaced (1.532%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:08:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:29 smithi135 ceph-mon[221113]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 26552/213969 objects degraded (12.409%); 3279/213969 objects misplaced (1.532%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:08:30.250 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:30.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:30.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:30.363 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:29 smithi144 ceph-mon[194833]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 26552/213969 objects degraded (12.409%); 3279/213969 objects misplaced (1.532%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:08:30.402 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:30.500 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:30.752 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:30.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:30.869 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:30.900 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:08:30.904 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:30.954 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:08:30.963 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2885041110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:30.963 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:30 smithi135 ceph-mon[213871]: pgmap v937: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 26401/211092 objects degraded (12.507%); 3279/211092 objects misplaced (1.553%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T20:08:30.965 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:30.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:30.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:30.992 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:08:31.038 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:08:31.070 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:31.081 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:08:31.125 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:08:31.150 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:31.170 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:08:31.212 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:08:31.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2885041110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:31.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:30 smithi135 ceph-mon[221113]: pgmap v937: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 26401/211092 objects degraded (12.507%); 3279/211092 objects misplaced (1.553%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T20:08:31.243 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:08:31.243 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:08:31.243 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:08:31.243 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:08:31.243 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:08:31.244 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:08:31.244 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:08:31.244 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:08:31.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2885041110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:31.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:30 smithi144 ceph-mon[194833]: pgmap v937: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 26401/211092 objects degraded (12.507%); 3279/211092 objects misplaced (1.553%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T20:08:31.388 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:08:31.388 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:08:31.388 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:08:31.388 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:08:31.388 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:08:31.437 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:31.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:31.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:31.548 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:31.656 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:32.050 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:08:32.891 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:32.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:32.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 26401/211092 objects degraded (12.507%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:32.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 26401/211092 objects degraded (12.507%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:32.998 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:33.026 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:33.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 26401/211092 objects degraded (12.507%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:33.568 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:33.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:33.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:33.678 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:33.707 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:33.757 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:33.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:33.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:33.861 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:33.953 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:33 smithi135 ceph-mon[213871]: pgmap v938: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 26401/209616 objects degraded (12.595%); 3095/209616 objects misplaced (1.477%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:08:33.954 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:08:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:08:33] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-09-23T20:08:33.955 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:33 smithi135 ceph-mon[221113]: pgmap v938: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 26401/209616 objects degraded (12.595%); 3095/209616 objects misplaced (1.477%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:08:33.965 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:34.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:33 smithi144 ceph-mon[194833]: pgmap v938: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 26401/209616 objects degraded (12.595%); 3095/209616 objects misplaced (1.477%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:08:34.724 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:08:34.746 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:34.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:34.864 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:34.904 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:34.959 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:35.183 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:35.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:35.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:34 smithi135 ceph-mon[213871]: pgmap v939: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 26238/203358 objects degraded (12.902%); 2598/203358 objects misplaced (1.278%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:08:35.231 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:34 smithi135 ceph-mon[221113]: pgmap v939: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 26238/203358 objects degraded (12.902%); 2598/203358 objects misplaced (1.278%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:08:35.294 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:34 smithi144 ceph-mon[194833]: pgmap v939: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 26238/203358 objects degraded (12.902%); 2598/203358 objects misplaced (1.278%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:08:35.386 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:35.561 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:35.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:35.683 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:35.763 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:35.918 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:35.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:35.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:36.035 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:36.051 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 8 pgp_num to 16 2023-09-23T20:08:36.066 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:36.072 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:35 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2073610485' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:08:36.072 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:35 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:08:36.074 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:35 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2073610485' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:08:36.074 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:35 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:08:36.124 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:36.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:36.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:36.229 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:36.305 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:35 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2073610485' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:08:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:35 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:08:36.583 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:08:37.288 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:37.292 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[221113]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.292 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[221113]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.293 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:08:37.293 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[221113]: osdmap e338: 8 total, 8 up, 7 in 2023-09-23T20:08:37.293 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[221113]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.293 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[221113]: pgmap v941: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 26238/202497 objects degraded (12.957%); 2598/202497 objects misplaced (1.283%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:08:37.293 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[213871]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.293 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[213871]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.293 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:08:37.293 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[213871]: osdmap e338: 8 total, 8 up, 7 in 2023-09-23T20:08:37.294 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[213871]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.294 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:37 smithi135 ceph-mon[213871]: pgmap v941: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 26238/202497 objects degraded (12.957%); 2598/202497 objects misplaced (1.283%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:08:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:37.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:37 smithi144 ceph-mon[194833]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:37 smithi144 ceph-mon[194833]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:37 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:08:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:37 smithi144 ceph-mon[194833]: osdmap e338: 8 total, 8 up, 7 in 2023-09-23T20:08:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:37 smithi144 ceph-mon[194833]: failed to encode map e338 with expected crc 2023-09-23T20:08:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:37 smithi144 ceph-mon[194833]: pgmap v941: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 26238/202497 objects degraded (12.957%); 2598/202497 objects misplaced (1.283%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:08:37.395 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:37.458 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:38.269 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:38.347 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:38.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:38.465 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:38 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:08:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 26238/202497 objects degraded (12.957%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:38 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:08:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:38.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 26238/202497 objects degraded (12.957%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:38.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:38 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:08:38.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:38.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:38.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 26238/202497 objects degraded (12.957%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:38.493 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:38.545 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:38.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:38.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:38.651 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:38.677 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:38.728 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:38.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:38.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:38.835 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:38.962 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:39.336 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:39.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:39.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:39.449 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:39 smithi135 ceph-mon[213871]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26238/198915 objects degraded (13.191%); 2379/198915 objects misplaced (1.196%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:08:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:39 smithi135 ceph-mon[221113]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26238/198915 objects degraded (13.191%); 2379/198915 objects misplaced (1.196%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:08:39.568 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:39 smithi144 ceph-mon[194833]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26238/198915 objects degraded (13.191%); 2379/198915 objects misplaced (1.196%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:08:39.658 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:39.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:39.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:39.761 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:39.787 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:39.829 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:39.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:39.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:39.932 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:39.960 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:40.006 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:40.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:40.112 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:40.140 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:40.193 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:40 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/291992064' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:08:40.193 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:40 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:08:40.195 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:40.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:40.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:40.208 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:08:40.303 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:40 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/291992064' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:08:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:40 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:08:40.404 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:40.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:40 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/291992064' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:08:40.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:40 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:08:40.637 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:40.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:40.744 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:40.772 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:40.826 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:40.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:40.932 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:40.960 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:41.010 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:41.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:41.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:41.115 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:41.143 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:41.192 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:41.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:41.297 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:41.328 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:41.378 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:41.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:41.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:08:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[221113]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[221113]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:08:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[221113]: osdmap e339: 8 total, 8 up, 7 in 2023-09-23T20:08:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[221113]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[221113]: pgmap v944: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26082/196182 objects degraded (13.295%); 2379/196182 objects misplaced (1.213%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:08:41.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:08:41.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:41.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[213871]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[213871]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:08:41.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[213871]: osdmap e339: 8 total, 8 up, 7 in 2023-09-23T20:08:41.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[213871]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:41 smithi135 ceph-mon[213871]: pgmap v944: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26082/196182 objects degraded (13.295%); 2379/196182 objects misplaced (1.213%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:08:41.488 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:41.546 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:41.566 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:41 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:08:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:41 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:41.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:41 smithi144 ceph-mon[194833]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:41 smithi144 ceph-mon[194833]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:41 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:08:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:41 smithi144 ceph-mon[194833]: osdmap e339: 8 total, 8 up, 7 in 2023-09-23T20:08:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:41 smithi144 ceph-mon[194833]: failed to encode map e339 with expected crc 2023-09-23T20:08:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:41 smithi144 ceph-mon[194833]: pgmap v944: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26082/196182 objects degraded (13.295%); 2379/196182 objects misplaced (1.213%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:08:41.723 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:41.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:41.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:41.833 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:41.863 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:41.917 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:41.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:41.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:42.022 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:42.048 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:42.096 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:42.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:42.121 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:08:42.121 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:08:42.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:08:42.121 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:08:42.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.121 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52207 8.6 8:6fd8f8f3:::benchmark_data_smithi135_460343_object52206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 102.20410853200001, 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1234665740000001, 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52207 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.122 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909076+0000", 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909102+0000", 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.605e-05 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.124 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909495+0000", 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00039314200000000002 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909506+0000", 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0699999999999999e-05 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.125 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910549+0000", 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001043034 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910557+0000", 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.816e-06 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914406+0000", 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0038485909999999 2023-09-23T20:08:42.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914418+0000", 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2016e-05 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.031329+0000", 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1169108190000001 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.032544+0000", 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001214705 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52216 8.6 8:60b8198d:::benchmark_data_smithi135_460343_object52215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 102.203667479, 2023-09-23T20:08:42.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1446887769999998, 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52216 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:08:42.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909517+0000", 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909548+0000", 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0108999999999999e-05 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909557+0000", 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.842e-06 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909563+0000", 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0759999999999999e-06 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910589+0000", 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010254730000000001 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910596+0000", 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4970000000000004e-06 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914445+0000", 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.003848268 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914452+0000", 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7100000000000007e-06 2023-09-23T20:08:42.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.033864+0000", 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1194116679999999 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.034445+0000", 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00058121900000000001 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.034641+0000", 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019598099999999999 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050209+0000", 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015567925 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050294+0000", 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5099000000000006e-05 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.052593+0000", 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022989210000000002 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.052660+0000", 2023-09-23T20:08:42.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6950000000000001e-05 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054207+0000", 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001546658 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.137 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52226 8.6 8:64c4735f:::benchmark_data_smithi135_460343_object52225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 102.19808939399999, 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1418229260000001, 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52226 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.138 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915095+0000", 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915130+0000", 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.536e-05 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.918885+0000", 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0037552050000000002 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.918902+0000", 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6691e-05 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.919727+0000", 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00082474700000000002 2023-09-23T20:08:42.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.919738+0000", 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1235e-05 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914477+0000", 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.99473878900000001 2023-09-23T20:08:42.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914486+0000", 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0350000000000007e-06 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036650+0000", 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.122164451 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036775+0000", 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012457700000000001 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036925+0000", 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000150464 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050301+0000", 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013375705999999999 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054287+0000", 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0039856179999999998 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055326+0000", 2023-09-23T20:08:42.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001039744 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055373+0000", 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6264999999999998e-05 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.056919+0000", 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001546551 2023-09-23T20:08:42.146 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52231 8.6 8:64b25628:::benchmark_data_smithi135_460343_object52230:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 102.184931519, 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.130363241, 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.147 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52231 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928273+0000", 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9911e-05 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928281+0000", 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2910000000000002e-06 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928286+0000", 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3159999999999996e-06 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928328+0000", 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1780000000000003e-05 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928337+0000", 2023-09-23T20:08:42.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5899999999999997e-06 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914511+0000", 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98617375299999999 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914519+0000", 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5680000000000002e-06 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.038973+0000", 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.12445419 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039130+0000", 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000156861 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:08:42.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039293+0000", 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000163535 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050307+0000", 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011013761 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054370+0000", 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0040626919999999997 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.057016+0000", 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026464700000000002 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.057055+0000", 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8605000000000001e-05 2023-09-23T20:08:42.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058617+0000", 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015624199999999999 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52236 8.6 8:693d9ca7:::benchmark_data_smithi135_460343_object52235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:08:42.156 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 102.174965095, 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1218329040000001, 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52236 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938241+0000", 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1342999999999999e-05 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938249+0000", 2023-09-23T20:08:42.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7800000000000001e-06 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938253+0000", 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.777e-06 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.939183+0000", 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00093006800000000004 2023-09-23T20:08:42.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.939192+0000", 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2129999999999997e-06 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914544+0000", 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97535231300000003 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914552+0000", 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5190000000000003e-06 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041404+0000", 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.126851673 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041553+0000", 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000148877 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041722+0000", 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016971799999999999 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050312+0000", 2023-09-23T20:08:42.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085901160000000001 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054415+0000", 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041022569999999998 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058671+0000", 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0042560890000000002 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058717+0000", 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6471000000000003e-05 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060053+0000", 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001336222 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52242 8.6 8:663215dc:::benchmark_data_smithi135_460343_object52241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 102.168162298, 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.116376196, 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.166 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52242 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945022+0000", 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945047+0000", 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5188e-05 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945066+0000", 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8719000000000002e-05 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945074+0000", 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0460000000000006e-06 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945123+0000", 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9542000000000001e-05 2023-09-23T20:08:42.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945133+0000", 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6490000000000003e-06 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914577+0000", 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96944391900000004 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914584+0000", 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4880000000000001e-06 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043641+0000", 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.129056426 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043776+0000", 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0001348 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043935+0000", 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015974000000000001 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050322+0000", 2023-09-23T20:08:42.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0063863770000000004 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054682+0000", 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00436075 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060087+0000", 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005404252 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060126+0000", 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9467e-05 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061399+0000", 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001273281 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.174 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52249 8.6 8:6b568471:::benchmark_data_smithi135_460343_object52248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 102.163613058, 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1131886460000002, 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.175 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52249 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949571+0000", 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949593+0000", 2023-09-23T20:08:42.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1430999999999999e-05 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949607+0000", 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4381e-05 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949614+0000", 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0330000000000003e-06 2023-09-23T20:08:42.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949662+0000", 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7685000000000002e-05 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949675+0000", 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2575000000000001e-05 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.179 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.915769+0000", 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96609494600000001 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.915781+0000", 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1364e-05 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.047967+0000", 2023-09-23T20:08:42.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1321858119999999 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.048151+0000", 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000184138 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.048355+0000", 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020468600000000001 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050329+0000", 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019734399999999999 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055290+0000", 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049610740000000002 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.182 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061446+0000", 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061564150000000002 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061507+0000", 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0739999999999998e-05 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.062761+0000", 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012539999999999999 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65642 8.7 8:e801740d:::benchmark_data_smithi135_460343_object65641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 83.187079988999997, 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.760197558, 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.184 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65642 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.185 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926104+0000", 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.186 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926155+0000", 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1158e-05 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926172+0000", 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.664e-05 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926183+0000", 2023-09-23T20:08:42.187 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0896999999999999e-05 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926311+0000", 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012874499999999999 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926319+0000", 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.294e-06 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927738+0000", 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001418891 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927746+0000", 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5960000000000004e-06 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.684466+0000", 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.756720061 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.686303+0000", 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001836935 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.190 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65643 8.3 8:c7d581fe:::benchmark_data_smithi135_460343_object65642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 83.186983119999994, 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7775143610000002, 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65643 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.191 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.192 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926223+0000", 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0891999999999999e-05 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.193 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926230+0000", 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3799999999999996e-06 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926236+0000", 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2179999999999996e-06 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926302+0000", 2023-09-23T20:08:42.194 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5423e-05 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926309+0000", 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3889999999999999e-06 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927982+0000", 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016733589999999999 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927990+0000", 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8369999999999997e-06 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.701891+0000", 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.773900823 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.703717+0000", 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018257550000000001 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65645 8.3 8:cf69bd2e:::benchmark_data_smithi135_460343_object65644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 83.186939244000001, 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7804623400000001, 2023-09-23T20:08:42.197 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65645 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:08:42.198 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926264+0000", 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8465e-05 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926271+0000", 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.849e-06 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926334+0000", 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2448999999999998e-05 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926375+0000", 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1554999999999998e-05 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926381+0000", 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.083e-06 2023-09-23T20:08:42.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.928014+0000", 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001632362 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.928020+0000", 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4629999999999998e-06 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.706613+0000", 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7785930159999999 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.706709+0000", 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5585e-05 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.203 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65667 8.a 8:5e718162:::benchmark_data_smithi135_460343_object65666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 83.186844101999995, 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.820842632, 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.204 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65667 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:08:42.205 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926366+0000", 2023-09-23T20:08:42.206 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4946000000000001e-05 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926374+0000", 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0150000000000003e-06 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926380+0000", 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.981e-06 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.933090+0000", 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067101180000000002 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.933100+0000", 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0087e-05 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704577+0000", 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7714773450000001 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704719+0000", 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000141182 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704870+0000", 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015157699999999999 2023-09-23T20:08:42.209 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713627+0000", 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0087563789999999999 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713675+0000", 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7994000000000002e-05 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.210 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.743266+0000", 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029591681000000002 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.743323+0000", 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7122e-05 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.211 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747184+0000", 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038607680000000001 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65682 8.a 8:51fe7b50:::benchmark_data_smithi135_460343_object65681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 82.979402132000004, 2023-09-23T20:08:42.212 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.61738943, 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65682 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.213 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133782+0000", 2023-09-23T20:08:42.214 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133803+0000", 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0800000000000001e-05 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133815+0000", 2023-09-23T20:08:42.215 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2177999999999999e-05 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133821+0000", 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0170000000000002e-06 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133873+0000", 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2039000000000001e-05 2023-09-23T20:08:42.216 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133885+0000", 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1158000000000001e-05 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710505+0000", 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.576620546 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.217 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710683+0000", 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017801700000000001 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710887+0000", 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020342900000000001 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.218 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713666+0000", 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027793280000000002 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713740+0000", 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4468000000000006e-05 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747234+0000", 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.033493756 2023-09-23T20:08:42.219 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747286+0000", 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.185e-05 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.751173+0000", 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038869239999999999 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.220 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86869 8.f 8:fd41b8fd:::benchmark_data_smithi135_460343_object2636:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.371055585999997, 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0711724490000001, 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.221 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86869 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.222 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742129+0000", 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742133+0000", 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5920000000000002e-06 2023-09-23T20:08:42.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742144+0000", 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0358e-05 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742153+0000", 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1579999999999994e-06 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.224 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742207+0000", 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4622e-05 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742216+0000", 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3820000000000004e-06 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.225 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744303+0000", 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020874449999999998 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744310+0000", 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9330000000000004e-06 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.226 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.767416+0000", 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.023105787 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.768079+0000", 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00066287699999999998 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.799565+0000", 2023-09-23T20:08:42.227 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.031486449999999999 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.800979+0000", 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001414092 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.801070+0000", 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0818999999999999e-05 2023-09-23T20:08:42.228 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803811+0000", 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002740592 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804527+0000", 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00071653999999999999 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.229 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.811597+0000", 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0070696429999999996 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.811634+0000", 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7245e-05 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.813302+0000", 2023-09-23T20:08:42.230 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016681619999999999 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86884 8.f 8:f6bbdc58:::benchmark_data_smithi135_460343_object2651:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.371018055, 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1026705059999999, 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.231 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86884 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742167+0000", 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742170+0000", 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5610000000000001e-06 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742176+0000", 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0070000000000003e-06 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742180+0000", 2023-09-23T20:08:42.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.651e-06 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742295+0000", 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011517499999999999 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742308+0000", 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2274e-05 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744328+0000", 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002019982 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744334+0000", 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2779999999999999e-06 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.236 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803248+0000", 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0589140269999999 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803789+0000", 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00054099499999999995 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817686+0000", 2023-09-23T20:08:42.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013896531 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817787+0000", 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101207 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817881+0000", 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4238000000000001e-05 2023-09-23T20:08:42.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.832086+0000", 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014205493 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843262+0000", 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011175433 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843318+0000", 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6292000000000002e-05 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843373+0000", 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4737000000000001e-05 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844838+0000", 2023-09-23T20:08:42.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001465084 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86888 8.f 8:ff8ee601:::benchmark_data_smithi135_460343_object2655:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.370976034999998, 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1038876470000001, 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.241 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86888 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.242 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742209+0000", 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742214+0000", 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8230000000000002e-06 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742260+0000", 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5964000000000002e-05 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742266+0000", 2023-09-23T20:08:42.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6209999999999998e-06 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742385+0000", 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011896900000000001 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742393+0000", 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4590000000000001e-06 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744351+0000", 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019579710000000002 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744358+0000", 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803866+0000", 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0595081719999999 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804470+0000", 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060409500000000002 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.247 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823610+0000", 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019139976999999999 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823691+0000", 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0525999999999994e-05 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:08:42.248 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823816+0000", 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000125029 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.837821+0000", 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014005521999999999 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844894+0000", 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007073368 2023-09-23T20:08:42.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844944+0000", 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9799999999999998e-05 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.845000+0000", 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5933999999999997e-05 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846097+0000", 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001096999 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86947 8.f 8:fc570a5a:::benchmark_data_smithi135_460343_object2714:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.370897065999998, 2023-09-23T20:08:42.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.10541584, 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86947 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.253 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.253 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:08:42.253 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.253 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.253 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.253 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:42.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742288+0000", 2023-09-23T20:08:42.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742292+0000", 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4179999999999999e-06 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742299+0000", 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0110000000000003e-06 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742304+0000", 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4710000000000001e-06 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742429+0000", 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012482199999999999 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742435+0000", 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5150000000000003e-06 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744369+0000", 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001934203 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744375+0000", 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9190000000000004e-06 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804569+0000", 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0601939 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805072+0000", 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050289400000000004 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829232+0000", 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024160198000000001 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829317+0000", 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4447e-05 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829401+0000", 2023-09-23T20:08:42.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4468999999999994e-05 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843205+0000", 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013804293 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846168+0000", 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029630279999999999 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846218+0000", 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9574999999999999e-05 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846273+0000", 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5058e-05 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847704+0000", 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014312109999999999 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86957 8.f 8:fdc750af:::benchmark_data_smithi135_460343_object2724:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:08:42.265 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.370877065000002, 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.106922379, 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86957 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.266 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742308+0000", 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742311+0000", 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5840000000000002e-06 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742316+0000", 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.746e-06 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742320+0000", 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1649999999999999e-06 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742500+0000", 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000180003 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:08:42.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742506+0000", 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.263e-06 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744392+0000", 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001886284 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744398+0000", 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1149999999999999e-06 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805115+0000", 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0607168629999999 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805787+0000", 2023-09-23T20:08:42.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00067139499999999998 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.834880+0000", 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029093853999999999 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.835004+0000", 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123545 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.835169+0000", 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000164506 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844858+0000", 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0096890859999999995 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847759+0000", 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029018820000000002 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847800+0000", 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0590999999999998e-05 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847847+0000", 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6615e-05 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849231+0000", 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0013841960000000001 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.275 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:87007 8.f 8:ff8f150f:::benchmark_data_smithi135_460343_object2774:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.370862535000001, 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.108736019, 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 87007 2023-09-23T20:08:42.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742325+0000", 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.824e-06 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742329+0000", 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7099999999999998e-06 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742333+0000", 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8589999999999998e-06 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742551+0000", 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00021761999999999999 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742559+0000", 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2020000000000004e-06 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744418+0000", 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001859468 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744426+0000", 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0770000000000002e-06 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805831+0000", 2023-09-23T20:08:42.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0614059330000001 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.806364+0000", 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00053292300000000003 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840788+0000", 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.034423435000000002 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840892+0000", 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010409299999999999 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840986+0000", 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3930000000000004e-05 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846116+0000", 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005129761 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849292+0000", 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031762359999999998 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849356+0000", 2023-09-23T20:08:42.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4146000000000006e-05 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849403+0000", 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7151999999999998e-05 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.851059+0000", 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001655903 2023-09-23T20:08:42.285 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:91634 8.f 8:f6420982:::benchmark_data_smithi135_460343_object7401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:14.919420+0000", 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 27.193765852999999, 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.40329260099999997, 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.286 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 91634 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.919420+0000", 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.919420+0000", 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.919418+0000", 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.919428+0000", 2023-09-23T20:08:42.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0536e-05 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.919452+0000", 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3285000000000002e-05 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.919462+0000", 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.047e-05 2023-09-23T20:08:42.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.935286+0000", 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015823845 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.935936+0000", 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00065008999999999998 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.955673+0000", 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019737329000000001 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.955783+0000", 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011003899999999999 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:08:42.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.955920+0000", 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013621099999999999 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.984193+0000", 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.028273377999999998 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.985996+0000", 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018029439999999999 2023-09-23T20:08:42.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.987366+0000", 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0013698510000000001 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.987412+0000", 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.587e-05 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.322712+0000", 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33530069600000001 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:91662 8.3 8:c680a306:::benchmark_data_smithi135_460343_object7429:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:14.984470+0000", 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 27.128715266, 2023-09-23T20:08:42.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.38284268100000002, 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 91662 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.984470+0000", 2023-09-23T20:08:42.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.984470+0000", 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.984470+0000", 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:08:42.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.984473+0000", 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7189999999999999e-06 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.984482+0000", 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7010000000000002e-06 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.984488+0000", 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2859999999999996e-06 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.324692+0000", 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.340203798 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.324856+0000", 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000163392 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.325027+0000", 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017105199999999999 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.330918+0000", 2023-09-23T20:08:42.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005891594 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.330956+0000", 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7363000000000001e-05 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.364427+0000", 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.033470888999999997 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.364490+0000", 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3032000000000002e-05 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.367313+0000", 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028234050000000002 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.301 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:08:42.302 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:08:42.302 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:08:42.303 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:42.331 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:42.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:42.437 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:42.543 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:42.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 26082/196182 objects degraded (13.295%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:42.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 26082/196182 objects degraded (13.295%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:43.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 26082/196182 objects degraded (13.295%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:43.365 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:08:43.366 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:08:43.366 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:08:43.366 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T20:08:43.507 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:08:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:08:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:08:43] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-09-23T20:08:43.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:43 smithi135 ceph-mon[221113]: pgmap v945: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 26082/194757 objects degraded (13.392%); 2184/194757 objects misplaced (1.121%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:08:43.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:43 smithi135 ceph-mon[213871]: pgmap v945: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 26082/194757 objects degraded (13.392%); 2184/194757 objects misplaced (1.121%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:08:44.022 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:44.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:44.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:44.126 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:44.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:43 smithi144 ceph-mon[194833]: pgmap v945: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 26082/194757 objects degraded (13.392%); 2184/194757 objects misplaced (1.121%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:08:44.154 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:44.458 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:08:44.707 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:44.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:44.828 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:44.894 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:44.971 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:45.011 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:45.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:45.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:45.121 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:45.126 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:45.154 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:45.203 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:45.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:45.308 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:45.378 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:45.676 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:45.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:45.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:45.742 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:08:45.782 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:08:45.786 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:45.812 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:45.829 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:08:45.858 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:45.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:45.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:45.879 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:08:45.914 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:08:45.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:45 smithi135 ceph-mon[213871]: pgmap v946: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 25896/188304 objects degraded (13.752%); 2184/188304 objects misplaced (1.160%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:08:45.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2290067620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:45.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:45 smithi135 ceph-mon[221113]: pgmap v946: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 25896/188304 objects degraded (13.752%); 2184/188304 objects misplaced (1.160%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:08:45.949 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2290067620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:45.960 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:08:45.966 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:45.993 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:46.003 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:08:46.037 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:08:46.051 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:46.052 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:08:46.052 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:08:46.052 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:08:46.052 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:08:46.052 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:08:46.052 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:08:46.053 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:08:46.053 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:08:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:46.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:45 smithi144 ceph-mon[194833]: pgmap v946: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 25896/188304 objects degraded (13.752%); 2184/188304 objects misplaced (1.160%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:08:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2290067620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:46.158 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:08:46.185 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:46.246 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:08:46.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:46.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:08:46.360 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:46.386 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:46.430 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:46.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:46.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:46.534 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:46.549 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:46.620 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:47.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:47 smithi135 ceph-mon[213871]: pgmap v947: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 25896/187461 objects degraded (13.814%); 2184/187461 objects misplaced (1.165%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:08:47.951 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 25896/187461 objects degraded (13.814%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:48.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:47 smithi135 ceph-mon[221113]: pgmap v947: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 25896/187461 objects degraded (13.814%); 2184/187461 objects misplaced (1.165%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:08:48.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 25896/187461 objects degraded (13.814%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:47 smithi144 ceph-mon[194833]: pgmap v947: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 25896/187461 objects degraded (13.814%); 2184/187461 objects misplaced (1.165%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:08:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 25896/187461 objects degraded (13.814%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:48.637 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:48.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:48.749 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:48.778 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:48.829 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:48.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:48.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:48.935 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:49.039 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:49.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:49 smithi144 ceph-mon[194833]: pgmap v948: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 25896/183753 objects degraded (14.093%); 1956/183753 objects misplaced (1.064%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:08:49.154 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:49.161 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:49 smithi135 ceph-mon[213871]: pgmap v948: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 25896/183753 objects degraded (14.093%); 1956/183753 objects misplaced (1.064%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:08:49.161 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:49 smithi135 ceph-mon[221113]: pgmap v948: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 25896/183753 objects degraded (14.093%); 1956/183753 objects misplaced (1.064%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:08:49.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:49.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:49.266 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:49.347 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:49.630 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:49.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:49.743 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:49.768 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:49.815 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:49.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:49.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:49.924 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:49.950 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:49.997 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:50.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:50.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:50.055 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4172553270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:50.056 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:50 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:50.056 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4172553270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:50.056 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:50 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:50.068 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:08:50.102 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:50.196 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4172553270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:50 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:08:50.398 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T20:08:50.398 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T20:08:50.398 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 5.01019 -> 5.01019 (expected baseline 6.03053) 2023-09-23T20:08:50.398 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 34 -> 34 pgs (0.801347 -> 0.801347 * mean) 2023-09-23T20:08:50.398 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 49 -> 49 pgs (1.15488 -> 1.15488 * mean) 2023-09-23T20:08:50.398 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:08:50.398 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T20:08:50.399 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 3 2023-09-23T20:08:50.399 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:08:50.399 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0248 2023-09-23T20:08:50.399 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.2482 2023-09-23T20:08:50.433 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:08:50.899 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:08:51.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:51 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:51.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:51 smithi144 ceph-mon[194833]: failed to encode map e340 with expected crc 2023-09-23T20:08:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:51 smithi144 ceph-mon[194833]: failed to encode map e340 with expected crc 2023-09-23T20:08:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:08:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:51 smithi144 ceph-mon[194833]: osdmap e340: 8 total, 8 up, 7 in 2023-09-23T20:08:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:51 smithi144 ceph-mon[194833]: from='client.45662 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:51.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:51 smithi144 ceph-mon[194833]: pgmap v950: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 25734/180888 objects degraded (14.226%); 1956/180888 objects misplaced (1.081%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:08:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[213871]: failed to encode map e340 with expected crc 2023-09-23T20:08:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[213871]: failed to encode map e340 with expected crc 2023-09-23T20:08:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:08:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[213871]: osdmap e340: 8 total, 8 up, 7 in 2023-09-23T20:08:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[213871]: from='client.45662 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:51.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[213871]: pgmap v950: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 25734/180888 objects degraded (14.226%); 1956/180888 objects misplaced (1.081%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:08:51.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:08:51.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[221113]: failed to encode map e340 with expected crc 2023-09-23T20:08:51.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[221113]: failed to encode map e340 with expected crc 2023-09-23T20:08:51.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:08:51.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[221113]: osdmap e340: 8 total, 8 up, 7 in 2023-09-23T20:08:51.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[221113]: from='client.45662 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:51.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:51 smithi135 ceph-mon[221113]: pgmap v950: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 25734/180888 objects degraded (14.226%); 1956/180888 objects misplaced (1.081%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:08:53.305 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:53.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:53.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:53.318 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T20:08:53.318 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T20:08:53.414 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:08:53.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:53.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:53.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 25734/180888 objects degraded (14.226%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:53.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:53 smithi135 ceph-mon[221113]: pgmap v951: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 25734/179616 objects degraded (14.327%); 1736/179616 objects misplaced (0.967%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:08:53.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:53.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:53.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 25734/180888 objects degraded (14.226%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:53.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:53 smithi135 ceph-mon[213871]: pgmap v951: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 25734/179616 objects degraded (14.327%); 1736/179616 objects misplaced (0.967%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:08:53.559 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:53.625 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:08:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:08:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 25734/180888 objects degraded (14.226%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:53 smithi144 ceph-mon[194833]: pgmap v951: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 25734/179616 objects degraded (14.327%); 1736/179616 objects misplaced (0.967%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:08:53.734 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:08:53.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:53.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:08:53.856 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:08:53.884 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:53.933 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:08:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:53.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:08:53.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:08:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:08:53] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T20:08:54.038 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:08:54.162 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:54.188 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:08:54.188 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:08:54.903 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:08:55.006 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:08:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:55.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:08:55.112 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:55.246 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:55.955 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:55 smithi135 ceph-mon[213871]: pgmap v952: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 25539/173001 objects degraded (14.762%); 1736/173001 objects misplaced (1.003%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:08:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:55 smithi135 ceph-mon[221113]: pgmap v952: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 25539/173001 objects degraded (14.762%); 1736/173001 objects misplaced (1.003%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:08:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:55 smithi144 ceph-mon[194833]: pgmap v952: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 25539/173001 objects degraded (14.762%); 1736/173001 objects misplaced (1.003%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:08:56.436 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:56.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:56.548 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:08:56.550 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:56.556 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:56.577 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:56.622 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:56.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:56.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:56.728 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:56.857 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:57.087 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:57.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:57.197 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:57.311 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:57.422 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:57.505 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:57.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:57.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:57.621 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:08:57.720 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:57.782 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:08:57.834 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:08:57.853 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:08:57.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:57.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:08:57.872 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:08:57.927 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:08:57.954 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:08:57.967 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:08:57.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:57 smithi135 ceph-mon[213871]: pgmap v953: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 25539/172128 objects degraded (14.837%); 1736/172128 objects misplaced (1.009%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:08:57.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 25539/172128 objects degraded (14.837%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:57.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2330433768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:57.977 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:57 smithi135 ceph-mon[221113]: pgmap v953: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 25539/172128 objects degraded (14.837%); 1736/172128 objects misplaced (1.009%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:08:57.977 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 25539/172128 objects degraded (14.837%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:57.977 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2330433768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:57.990 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:08:58.002 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:58.023 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:08:58.054 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:08:58.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:58.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:08:58.063 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:08:58.160 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:58.187 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:58.229 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:58.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:58.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:58.330 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:08:58.330 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:08:58.330 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:08:58.331 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:08:58.331 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:08:58.331 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:08:58.331 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:08:58.331 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:08:58.332 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:08:58.376 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:57 smithi144 ceph-mon[194833]: pgmap v953: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 25539/172128 objects degraded (14.837%); 1736/172128 objects misplaced (1.009%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:08:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 25539/172128 objects degraded (14.837%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:08:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2330433768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:08:59.754 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:08:59.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:59.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:08:59.859 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:08:59.888 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:08:59.935 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:08:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:08:59.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:08:59.943 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:00.043 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:00.138 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:08:59 smithi135 ceph-mon[213871]: pgmap v954: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 25539/168294 objects degraded (15.175%); 1499/168294 objects misplaced (0.891%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:09:00.139 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:08:59 smithi135 ceph-mon[221113]: pgmap v954: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 25539/168294 objects degraded (15.175%); 1499/168294 objects misplaced (0.891%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:09:00.155 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:08:59 smithi144 ceph-mon[194833]: pgmap v954: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 25539/168294 objects degraded (15.175%); 1499/168294 objects misplaced (0.891%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:09:00.685 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:00.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:00.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:00.791 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:00.892 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:01.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:00 smithi135 ceph-mon[221113]: pgmap v955: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 25386/165363 objects degraded (15.352%); 1499/165363 objects misplaced (0.906%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:09:01.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:00 smithi135 ceph-mon[213871]: pgmap v955: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 25386/165363 objects degraded (15.352%); 1499/165363 objects misplaced (0.906%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:09:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:00 smithi144 ceph-mon[194833]: pgmap v955: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 25386/165363 objects degraded (15.352%); 1499/165363 objects misplaced (0.906%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:09:01.873 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:01.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:01.982 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:09:01.989 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:02.043 INFO:teuthology.orchestra.run.smithi135.stdout:moved 16 / 297 (5.38721%) 2023-09-23T20:09:02.043 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T20:09:02.043 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 5.01019 -> 7.06818 (expected baseline 6.03053) 2023-09-23T20:09:02.043 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 34 -> 31 pgs (0.801347 -> 0.73064 * mean) 2023-09-23T20:09:02.043 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 49 -> 53 pgs (1.15488 -> 1.24916 * mean) 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 1 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0235 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0259 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 weight 1.0000 -> 0.7459 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 weight 1.0000 -> 0.7588 2023-09-23T20:09:02.044 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 weight 1.0000 -> 0.8455 2023-09-23T20:09:02.122 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:02.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:02 smithi135 ceph-mon[221113]: from='client.35772 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:02.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'48884','3':'49730','5':'55409'}"}]: dispatch 2023-09-23T20:09:02.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:02 smithi135 ceph-mon[213871]: from='client.35772 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:02.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'48884','3':'49730','5':'55409'}"}]: dispatch 2023-09-23T20:09:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:02 smithi144 ceph-mon[194833]: from='client.35772 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:02.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'48884','3':'49730','5':'55409'}"}]: dispatch 2023-09-23T20:09:02.384 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:02.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:02.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:02.488 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:02.515 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:02.557 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:02.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:02.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:02.662 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:02.689 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:03.365 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:03.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'48884','3':'49730','5':'55409'}"}]': finished 2023-09-23T20:09:03.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: osdmap e341: 8 total, 8 up, 7 in 2023-09-23T20:09:03.367 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 25386/165363 objects degraded (15.352%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: failed to encode map e342 with expected crc 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: osdmap e342: 8 total, 8 up, 7 in 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[221113]: pgmap v958: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 25386/164091 objects degraded (15.471%); 1178/164091 objects misplaced (0.718%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'48884','3':'49730','5':'55409'}"}]': finished 2023-09-23T20:09:03.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: osdmap e341: 8 total, 8 up, 7 in 2023-09-23T20:09:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 25386/165363 objects degraded (15.352%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: failed to encode map e342 with expected crc 2023-09-23T20:09:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: osdmap e342: 8 total, 8 up, 7 in 2023-09-23T20:09:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:03 smithi135 ceph-mon[213871]: pgmap v958: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 25386/164091 objects degraded (15.471%); 1178/164091 objects misplaced (0.718%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:09:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:03.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:03.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'48884','3':'49730','5':'55409'}"}]': finished 2023-09-23T20:09:03.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: osdmap e341: 8 total, 8 up, 7 in 2023-09-23T20:09:03.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: failed to encode map e341 with expected crc 2023-09-23T20:09:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 25386/165363 objects degraded (15.352%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: failed to encode map e342 with expected crc 2023-09-23T20:09:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: osdmap e342: 8 total, 8 up, 7 in 2023-09-23T20:09:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:03 smithi144 ceph-mon[194833]: pgmap v958: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 25386/164091 objects degraded (15.471%); 1178/164091 objects misplaced (0.718%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:09:03.470 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:03.498 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:03.599 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:09:03.599 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T20:09:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:09:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:09:03] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T20:09:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: osdmap e343: 8 total, 8 up, 7 in 2023-09-23T20:09:04.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:04 smithi144 ceph-mon[194833]: failed to encode map e343 with expected crc 2023-09-23T20:09:04.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: osdmap e343: 8 total, 8 up, 7 in 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[221113]: failed to encode map e343 with expected crc 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: failed to encode map e342 with expected crc 2023-09-23T20:09:04.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: osdmap e343: 8 total, 8 up, 7 in 2023-09-23T20:09:04.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:04 smithi135 ceph-mon[213871]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.135 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:05.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:05.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:05.241 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:05.268 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:05.271 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:05.314 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:05.317 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.317 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.317 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.318 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.318 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.318 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.318 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.318 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: failed to encode map e344 with expected crc 2023-09-23T20:09:05.318 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: osdmap e344: 8 total, 8 up, 7 in 2023-09-23T20:09:05.318 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:05 smithi144 ceph-mon[194833]: pgmap v961: 99 pgs: 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 426 op/s; 17954/160443 objects degraded (11.190%); 4424/160443 objects misplaced (2.757%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T20:09:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:05.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:05.323 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:05.420 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:05.454 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.454 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: failed to encode map e344 with expected crc 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: osdmap e344: 8 total, 8 up, 7 in 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[213871]: pgmap v961: 99 pgs: 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 426 op/s; 17954/160443 objects degraded (11.190%); 4424/160443 objects misplaced (2.757%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.455 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: failed to encode map e343 with expected crc 2023-09-23T20:09:05.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: failed to encode map e344 with expected crc 2023-09-23T20:09:05.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: osdmap e344: 8 total, 8 up, 7 in 2023-09-23T20:09:05.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:05 smithi135 ceph-mon[221113]: pgmap v961: 99 pgs: 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 426 op/s; 17954/160443 objects degraded (11.190%); 4424/160443 objects misplaced (2.757%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T20:09:05.460 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:05.639 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:05.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:05.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:05.666 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:09:05.738 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:09:05.738 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:09:05.738 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:09:05.738 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:09:05.739 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:09:05.746 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:05.772 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:05.816 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:05.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:05.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:05.922 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:05.948 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:05.992 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:06.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:06.100 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:06.126 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:06.172 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:06.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:06.278 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:06.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.310 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.311 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.311 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3272368272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:06.311 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:09:06.311 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:06.311 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: failed to encode map e345 with expected crc 2023-09-23T20:09:06.311 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3272368272' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:09:06.312 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[221113]: osdmap e345: 8 total, 8 up, 7 in 2023-09-23T20:09:06.312 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.312 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.312 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3272368272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:06.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:09:06.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:06.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: failed to encode map e345 with expected crc 2023-09-23T20:09:06.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3272368272' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:09:06.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:06 smithi135 ceph-mon[213871]: osdmap e345: 8 total, 8 up, 7 in 2023-09-23T20:09:06.314 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: failed to encode map e344 with expected crc 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3272368272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: failed to encode map e345 with expected crc 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3272368272' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:09:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:06 smithi144 ceph-mon[194833]: osdmap e345: 8 total, 8 up, 7 in 2023-09-23T20:09:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:07 smithi144 ceph-mon[194833]: failed to encode map e345 with expected crc 2023-09-23T20:09:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:07 smithi144 ceph-mon[194833]: failed to encode map e345 with expected crc 2023-09-23T20:09:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:07 smithi144 ceph-mon[194833]: pgmap v963: 99 pgs: 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 341 op/s; 17954/160350 objects degraded (11.197%); 4424/160350 objects misplaced (2.759%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T20:09:07.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:07 smithi135 ceph-mon[221113]: failed to encode map e345 with expected crc 2023-09-23T20:09:07.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:07 smithi135 ceph-mon[221113]: failed to encode map e345 with expected crc 2023-09-23T20:09:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:07 smithi135 ceph-mon[221113]: pgmap v963: 99 pgs: 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 341 op/s; 17954/160350 objects degraded (11.197%); 4424/160350 objects misplaced (2.759%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T20:09:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:07 smithi135 ceph-mon[213871]: failed to encode map e345 with expected crc 2023-09-23T20:09:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:07 smithi135 ceph-mon[213871]: failed to encode map e345 with expected crc 2023-09-23T20:09:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:07 smithi135 ceph-mon[213871]: pgmap v963: 99 pgs: 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 341 op/s; 17954/160350 objects degraded (11.197%); 4424/160350 objects misplaced (2.759%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T20:09:07.898 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:07.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:07.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:08.003 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:08.036 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 17954/160350 objects degraded (11.197%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[213871]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[213871]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.142 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[213871]: osdmap e346: 8 total, 8 up, 7 in 2023-09-23T20:09:08.142 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[213871]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 17954/160350 objects degraded (11.197%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[221113]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[221113]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[221113]: osdmap e346: 8 total, 8 up, 7 in 2023-09-23T20:09:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:08 smithi135 ceph-mon[221113]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:08 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 17954/160350 objects degraded (11.197%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:08 smithi144 ceph-mon[194833]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:08 smithi144 ceph-mon[194833]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:08 smithi144 ceph-mon[194833]: osdmap e346: 8 total, 8 up, 7 in 2023-09-23T20:09:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:08 smithi144 ceph-mon[194833]: failed to encode map e346 with expected crc 2023-09-23T20:09:08.648 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:08.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:08.753 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:08.785 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:09.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:09:09.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[221113]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[221113]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[221113]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[221113]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[221113]: pgmap v966: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 26144/160305 objects degraded (16.309%); 1998/160305 objects misplaced (1.246%); 19 MiB/s, 304 objects/s recovering 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[213871]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[213871]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[213871]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[213871]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:09 smithi135 ceph-mon[213871]: pgmap v966: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 26144/160305 objects degraded (16.309%); 1998/160305 objects misplaced (1.246%); 19 MiB/s, 304 objects/s recovering 2023-09-23T20:09:09.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:09 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:09:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:09 smithi144 ceph-mon[194833]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:09 smithi144 ceph-mon[194833]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T20:09:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:09 smithi144 ceph-mon[194833]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:09 smithi144 ceph-mon[194833]: failed to encode map e347 with expected crc 2023-09-23T20:09:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:09 smithi144 ceph-mon[194833]: pgmap v966: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 26144/160305 objects degraded (16.309%); 1998/160305 objects misplaced (1.246%); 19 MiB/s, 304 objects/s recovering 2023-09-23T20:09:10.291 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:09:10.295 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:10.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:10.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:10.401 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:10.429 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:10 smithi144 ceph-mon[194833]: failed to encode map e347 with expected crc 2023-09-23T20:09:10.430 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:10 smithi144 ceph-mon[194833]: failed to encode map e347 with expected crc 2023-09-23T20:09:10.430 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:10 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:09:10.432 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:10 smithi135 ceph-mon[221113]: failed to encode map e347 with expected crc 2023-09-23T20:09:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:10 smithi135 ceph-mon[221113]: failed to encode map e347 with expected crc 2023-09-23T20:09:10.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:10 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:09:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:10 smithi135 ceph-mon[213871]: failed to encode map e347 with expected crc 2023-09-23T20:09:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:10 smithi135 ceph-mon[213871]: failed to encode map e347 with expected crc 2023-09-23T20:09:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:10 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:09:10.481 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:10.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:10.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:10.587 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:10.619 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:11.135 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:11.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:11.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:11.211 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:09:11.252 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:11.289 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:11.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:11 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:09:11.427 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:11 smithi135 ceph-mon[213871]: pgmap v967: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 26144/160305 objects degraded (16.309%); 1606/160305 objects misplaced (1.002%); 17 MiB/s, 280 objects/s recovering 2023-09-23T20:09:11.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:11 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:09:11.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:11 smithi135 ceph-mon[221113]: pgmap v967: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 26144/160305 objects degraded (16.309%); 1606/160305 objects misplaced (1.002%); 17 MiB/s, 280 objects/s recovering 2023-09-23T20:09:11.590 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:11.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:11 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:09:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:11 smithi144 ceph-mon[194833]: pgmap v967: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 26144/160305 objects degraded (16.309%); 1606/160305 objects misplaced (1.002%); 17 MiB/s, 280 objects/s recovering 2023-09-23T20:09:11.635 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:11.694 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:11.726 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:11.774 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:11.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:11.881 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:11.922 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:12.018 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:09:12.067 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:12.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:12.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:12.086 INFO:teuthology.orchestra.run.smithi135.stdout:moved 27 / 297 (9.09091%) 2023-09-23T20:09:12.086 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T20:09:12.086 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.06818 -> 12.3272 (expected baseline 6.03053) 2023-09-23T20:09:12.086 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 31 -> 23 pgs (0.73064 -> 0.542088 * mean) 2023-09-23T20:09:12.086 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 53 -> 56 pgs (1.24916 -> 1.31987 * mean) 2023-09-23T20:09:12.086 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:09:12.086 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T20:09:12.087 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 1 2023-09-23T20:09:12.087 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:09:12.087 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0231 2023-09-23T20:09:12.087 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0254 2023-09-23T20:09:12.087 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 weight 0.7459 -> 0.5591 2023-09-23T20:09:12.087 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 weight 0.7588 -> 0.5819 2023-09-23T20:09:12.087 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 weight 0.8455 -> 0.7199 2023-09-23T20:09:12.090 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:09:12.172 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:12.212 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:12.325 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:12.329 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/222180985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:12.329 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:09:12 smithi135 conmon[201366]: 2023-09-23T20:09:12.084+0000 7fcf49f9e700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T20:09:12.329 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/222180985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:12.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:12.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:12.431 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:12.464 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:12.568 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:12.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:12.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/222180985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:12.674 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:12.706 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[213871]: from='client.35781 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:09:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[213871]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:09:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 26144/160305 objects degraded (16.309%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[213871]: pgmap v968: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 26144/160305 objects degraded (16.309%); 1606/160305 objects misplaced (1.002%); 14 MiB/s, 232 objects/s recovering 2023-09-23T20:09:13.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[221113]: from='client.35781 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:13.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:09:13.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[221113]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:09:13.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 26144/160305 objects degraded (16.309%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:13.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:13 smithi135 ceph-mon[221113]: pgmap v968: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 26144/160305 objects degraded (16.309%); 1606/160305 objects misplaced (1.002%); 14 MiB/s, 232 objects/s recovering 2023-09-23T20:09:13.614 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:13.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:13.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:13 smithi144 ceph-mon[194833]: from='client.35781 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:13 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:09:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:13 smithi144 ceph-mon[194833]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:09:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:13 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 26144/160305 objects degraded (16.309%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:13.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:13 smithi144 ceph-mon[194833]: pgmap v968: 99 pgs: 2 peering, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 26144/160305 objects degraded (16.309%); 1606/160305 objects misplaced (1.002%); 14 MiB/s, 232 objects/s recovering 2023-09-23T20:09:13.718 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:13.849 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:09:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:09:13] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T20:09:14.607 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:14.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:14.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:14.711 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:14.743 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:14.792 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:14.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:14.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:14.899 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:14.926 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:14.970 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:14.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:15.076 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:15.119 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:15.179 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:15.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:15.191 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:09:15.191 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T20:09:15.229 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:15.283 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:15.325 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:15.403 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:15.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:15.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:15.508 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:15.547 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:15.662 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:15.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:15.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:15.769 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:15.808 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:15.962 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:09:15.962 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:09:15.962 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:09:15.963 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:09:15.963 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.963 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148788 6.f 6:f326eaaa:::benchmark_data_smithi135_344736_object46132:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:09:15.963 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:09:15.963 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 573.02874044199996, 2023-09-23T20:09:15.963 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5928274390000001, 2023-09-23T20:09:15.963 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:15.963 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:09:15.964 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:15.964 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:09:15.964 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:15.964 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148788 2023-09-23T20:09:15.964 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.964 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:15.964 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.964 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:15.965 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:09:15.965 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:15.965 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.965 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.965 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:15.965 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928733+0000", 2023-09-23T20:09:15.965 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:15.965 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.966 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.966 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:15.966 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928731+0000", 2023-09-23T20:09:15.966 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:09:15.966 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.966 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.966 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:15.966 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928740+0000", 2023-09-23T20:09:15.967 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4009999999999997e-06 2023-09-23T20:09:15.967 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.967 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.967 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:15.967 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928755+0000", 2023-09-23T20:09:15.967 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5481999999999998e-05 2023-09-23T20:09:15.967 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.967 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.968 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:15.968 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928765+0000", 2023-09-23T20:09:15.968 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0239e-05 2023-09-23T20:09:15.968 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.968 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.968 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.968 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928822+0000", 2023-09-23T20:09:15.968 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6518000000000003e-05 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928829+0000", 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3819999999999998e-06 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.969 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944149+0000", 2023-09-23T20:09:15.970 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015319442000000001 2023-09-23T20:09:15.970 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.970 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.970 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:15.970 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944158+0000", 2023-09-23T20:09:15.970 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3000000000000007e-06 2023-09-23T20:09:15.970 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.970 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049126+0000", 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.104968422 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049138+0000", 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1644999999999999e-05 2023-09-23T20:09:15.971 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.972 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.972 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.972 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.265679+0000", 2023-09-23T20:09:15.972 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2165408280000001 2023-09-23T20:09:15.972 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.972 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.972 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:09:15.973 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.265834+0000", 2023-09-23T20:09:15.973 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015490799999999999 2023-09-23T20:09:15.973 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.973 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.973 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.973 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.521437+0000", 2023-09-23T20:09:15.973 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25560343699999999 2023-09-23T20:09:15.973 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.974 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.974 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:15.974 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.521561+0000", 2023-09-23T20:09:15.974 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123342 2023-09-23T20:09:15.974 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:15.974 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:15.974 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:15.974 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.975 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.975 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148791 6.f 6:fce301f6:::benchmark_data_smithi135_344736_object46135:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:09:15.975 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:09:15.975 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 573.02868257700004, 2023-09-23T20:09:15.975 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5984101370000001, 2023-09-23T20:09:15.975 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:15.975 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:09:15.975 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:15.976 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:09:15.976 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:15.976 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148791 2023-09-23T20:09:15.976 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.976 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:15.976 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.976 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:15.976 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:09:15.977 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:15.977 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.977 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.977 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:15.977 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928791+0000", 2023-09-23T20:09:15.977 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:15.977 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.977 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.978 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:15.978 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928790+0000", 2023-09-23T20:09:15.978 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:15.978 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.978 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.978 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:15.978 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928794+0000", 2023-09-23T20:09:15.978 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8800000000000001e-06 2023-09-23T20:09:15.979 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.979 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:15.979 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928802+0000", 2023-09-23T20:09:15.979 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0499999999999992e-06 2023-09-23T20:09:15.979 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.979 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.979 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:15.980 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928809+0000", 2023-09-23T20:09:15.980 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1800000000000001e-06 2023-09-23T20:09:15.980 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.980 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.980 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.980 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928882+0000", 2023-09-23T20:09:15.980 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3524999999999998e-05 2023-09-23T20:09:15.980 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.981 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.981 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:15.981 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.928887+0000", 2023-09-23T20:09:15.981 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0010000000000001e-06 2023-09-23T20:09:15.981 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.981 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.981 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.981 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944176+0000", 2023-09-23T20:09:15.982 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015289166 2023-09-23T20:09:15.982 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.982 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.982 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:15.982 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.944182+0000", 2023-09-23T20:09:15.982 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.105e-06 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049162+0000", 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1049793050000001 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049170+0000", 2023-09-23T20:09:15.983 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3869999999999995e-06 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268117+0000", 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2189466419999999 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:09:15.984 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268204+0000", 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7319999999999997e-05 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527103+0000", 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.25889852899999999 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.985 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.527201+0000", 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8562999999999998e-05 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148804 6.b 6:d3899bd3:::benchmark_data_smithi135_344736_object46148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:09:15.986 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 573.02185674199995, 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1034934710000002, 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148804 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.987 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935617+0000", 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:15.988 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935615+0000", 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935620+0000", 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.775e-06 2023-09-23T20:09:15.989 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935630+0000", 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9310000000000003e-06 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.935637+0000", 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3440000000000003e-06 2023-09-23T20:09:15.990 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.936071+0000", 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00043330099999999998 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.936077+0000", 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6900000000000003e-06 2023-09-23T20:09:15.991 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.941346+0000", 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052686859999999999 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.941360+0000", 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3464e-05 2023-09-23T20:09:15.992 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050328+0000", 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.108968468 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.050338+0000", 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0088999999999999e-05 2023-09-23T20:09:15.993 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.003742+0000", 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.95340369300000005 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.003918+0000", 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017656799999999999 2023-09-23T20:09:15.994 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.004058+0000", 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000139682 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.007338+0000", 2023-09-23T20:09:15.995 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003280303 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.021476+0000", 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014137442 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.038947+0000", 2023-09-23T20:09:15.996 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017471477999999999 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.039000+0000", 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3022000000000002e-05 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.039110+0000", 2023-09-23T20:09:15.997 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000109966 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148813 6.f 6:f77e0ce5:::benchmark_data_smithi135_344736_object46157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 572.99653061000004, 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.545090514, 2023-09-23T20:09:15.998 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148813 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:15.999 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960943+0000", 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960941+0000", 2023-09-23T20:09:16.000 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960950+0000", 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6949999999999995e-06 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960969+0000", 2023-09-23T20:09:16.001 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8785999999999999e-05 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.960980+0000", 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0703000000000001e-05 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.002 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.961043+0000", 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2600000000000004e-05 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:42.961057+0000", 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4535999999999999e-05 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.003 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049192+0000", 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0881350000000001 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.049200+0000", 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9049999999999997e-06 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.004 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268242+0000", 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.219042129 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268348+0000", 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010592 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.005 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.268839+0000", 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00049129899999999995 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279896+0000", 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011056725 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.006 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279913+0000", 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6673000000000001e-05 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.502819+0000", 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22290622900000001 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.007 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.502884+0000", 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4616000000000006e-05 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506033+0000", 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031498979999999999 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.008 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:148815 6.f 6:f7552838:::benchmark_data_smithi135_344736_object46159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 571.83447949799995, 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.385714227, 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.009 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 148815 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.010 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122994+0000", 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.122993+0000", 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:09:16.011 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123000+0000", 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9290000000000001e-06 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123009+0000", 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5689999999999994e-06 2023-09-23T20:09:16.012 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.123015+0000", 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8189999999999997e-06 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.152030+0000", 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029014671999999998 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.013 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:44.152042+0000", 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1756000000000001e-05 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276241+0000", 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.124199189 2023-09-23T20:09:16.014 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276452+0000", 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000210934 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.276679+0000", 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00022696700000000001 2023-09-23T20:09:16.015 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279921+0000", 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032419129999999999 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.279990+0000", 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9807000000000004e-05 2023-09-23T20:09:16.016 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506095+0000", 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22610454399999999 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.506139+0000", 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3659000000000003e-05 2023-09-23T20:09:16.017 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T19:59:45.508708+0000", 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002569661 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.018 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:182372 6.b 6:d373db44:::benchmark_data_smithi135_344736_object79716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e222)", 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 522.50768650500004, 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1349242150000001, 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 182372 2023-09-23T20:09:16.019 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.020 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449787+0000", 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449786+0000", 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.021 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449790+0000", 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4469999999999999e-06 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449800+0000", 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6909999999999992e-06 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.022 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449809+0000", 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3009999999999995e-06 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.449948+0000", 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013898400000000001 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.023 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.450078+0000", 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012999600000000001 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.450185+0000", 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010655299999999999 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.024 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.457033+0000", 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0068485000000000004 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:33.457047+0000", 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.376e-05 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.025 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.567645+0000", 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1105975640000001 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.567815+0000", 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000170104 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.026 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584606+0000", 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016791006000000001 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:00:34.584711+0000", 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000105554 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.027 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201858 6.10 6:0e829f63:::benchmark_data_smithi135_344736_object99202:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 491.49087758600001, 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.12779443, 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.028 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201858 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.029 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466596+0000", 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:16.030 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466598+0000", 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8920000000000001e-06 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466601+0000", 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0869999999999998e-06 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.031 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466604+0000", 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.596e-06 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468607+0000", 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020026620000000001 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.032 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468616+0000", 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7739999999999993e-06 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560293+0000", 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0916775329999999 2023-09-23T20:09:16.033 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.560929+0000", 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00063605599999999997 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576296+0000", 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01536671 2023-09-23T20:09:16.034 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576359+0000", 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3100000000000002e-05 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.576772+0000", 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00041272599999999998 2023-09-23T20:09:16.035 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582458+0000", 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056866119999999997 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585662+0000", 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032034730000000001 2023-09-23T20:09:16.036 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594087+0000", 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084249849999999994 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594137+0000", 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9614000000000003e-05 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.037 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.594390+0000", 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00025390100000000001 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.038 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.44942.0:201912 6.10 6:0a9a0b94:::benchmark_data_smithi135_344736_object99256:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 491.49084841199999, 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1498471859999999, 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.44942", 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 201912 2023-09-23T20:09:16.039 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.040 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466625+0000", 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466628+0000", 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6249999999999999e-06 2023-09-23T20:09:16.041 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466633+0000", 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6210000000000001e-06 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.466638+0000", 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6809999999999997e-06 2023-09-23T20:09:16.042 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468671+0000", 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020332229999999998 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:04.468677+0000", 2023-09-23T20:09:16.043 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8409999999999996e-06 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561014+0000", 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0923374370000001 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.561497+0000", 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00048270000000000002 2023-09-23T20:09:16.044 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579372+0000", 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017874660000000001 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579464+0000", 2023-09-23T20:09:16.045 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2050000000000001e-05 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.579912+0000", 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00044868700000000002 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.046 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.582554+0000", 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00264197 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.585730+0000", 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031761210000000001 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.047 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.615988+0000", 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030257373000000001 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.616043+0000", 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4648000000000003e-05 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.048 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:01:05.616472+0000", 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00042985000000000001 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52229 8.c 8:3f49a7df:::benchmark_data_smithi135_460343_object52228:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.928128+0000", 2023-09-23T20:09:16.049 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 136.02934525699999, 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1212938079999999, 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52229 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.050 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928128+0000", 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928128+0000", 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.051 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928127+0000", 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928153+0000", 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5808000000000001e-05 2023-09-23T20:09:16.052 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928166+0000", 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2986999999999999e-05 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928174+0000", 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5099999999999998e-06 2023-09-23T20:09:16.053 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.930595+0000", 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024209869999999999 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.930607+0000", 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1447000000000001e-05 2023-09-23T20:09:16.054 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.917025+0000", 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98641873899999999 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.917042+0000", 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6416999999999999e-05 2023-09-23T20:09:16.055 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039344+0000", 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.122302524 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039506+0000", 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016151699999999999 2023-09-23T20:09:16.056 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039685+0000", 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000179338 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.046933+0000", 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0072474749999999998 2023-09-23T20:09:16.057 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.046990+0000", 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7259999999999997e-05 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.047001+0000", 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1182999999999999e-05 2023-09-23T20:09:16.058 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.047052+0000", 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1165999999999998e-05 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.049422+0000", 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023699659999999998 2023-09-23T20:09:16.059 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52240 8.c 8:3f854097:::benchmark_data_smithi135_460343_object52239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.938268+0000", 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 136.01920584300001, 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1138651510000002, 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.060 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52240 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938268+0000", 2023-09-23T20:09:16.061 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938268+0000", 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.062 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938267+0000", 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938289+0000", 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1744000000000001e-05 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938296+0000", 2023-09-23T20:09:16.063 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5880000000000003e-06 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938301+0000", 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8100000000000003e-06 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.064 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938361+0000", 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.961e-05 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938371+0000", 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0276e-05 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.065 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.917072+0000", 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97870070099999995 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.917081+0000", 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8950000000000004e-06 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.066 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.044654+0000", 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.127573537 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.044749+0000", 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4976999999999995e-05 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:09:16.067 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.044850+0000", 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100753 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.049492+0000", 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046419199999999999 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.068 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.049542+0000", 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0111999999999997e-05 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.049554+0000", 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1758e-05 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.069 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.049617+0000", 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2615999999999998e-05 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.052133+0000", 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025163360000000001 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.070 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65644 8.0 8:0083046d:::benchmark_data_smithi135_460343_object65643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:17.908214+0000", 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 118.04925918000001, 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7742204049999999, 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.071 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65644 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908214+0000", 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.072 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908214+0000", 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908213+0000", 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.073 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908238+0000", 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4936e-05 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908248+0000", 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0114e-05 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.074 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908254+0000", 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2809999999999997e-06 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908304+0000", 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9453000000000002e-05 2023-09-23T20:09:16.075 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908434+0000", 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000130674 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.908583+0000", 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014822199999999999 2023-09-23T20:09:16.076 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.913537+0000", 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049542329999999997 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.915945+0000", 2023-09-23T20:09:16.077 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024087280000000002 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.943388+0000", 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.027442350000000001 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.078 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.943405+0000", 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7008999999999999e-05 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.936360+0000", 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.99295548300000003 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.079 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.936388+0000", 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7259000000000001e-05 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.682319+0000", 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7459312380000001 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.080 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.682435+0000", 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011600200000000001 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65651 8.f 8:f50c31e1:::benchmark_data_smithi135_460343_object65650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:17.919873+0000", 2023-09-23T20:09:16.081 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 118.037600735, 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8252364349999999, 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65651 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.082 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.919873+0000", 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.919873+0000", 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.083 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.919871+0000", 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.919902+0000", 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0358e-05 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.084 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.923296+0000", 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033941729999999999 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.923310+0000", 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4164e-05 2023-09-23T20:09:16.085 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.949622+0000", 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.026312076 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.949630+0000", 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6410000000000002e-06 2023-09-23T20:09:16.086 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.944143+0000", 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.99451354800000002 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.944156+0000", 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3115000000000001e-05 2023-09-23T20:09:16.087 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.714451+0000", 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7702946160000002 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.714555+0000", 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000104182 2023-09-23T20:09:16.088 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.714687+0000", 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013173300000000001 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.722362+0000", 2023-09-23T20:09:16.089 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0076756319999999999 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.725441+0000", 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030786429999999998 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.737883+0000", 2023-09-23T20:09:16.090 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012441847000000001 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.737951+0000", 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8446000000000002e-05 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.745109+0000", 2023-09-23T20:09:16.091 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0071579529999999999 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65658 8.f 8:fa091232:::benchmark_data_smithi135_460343_object65657:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:17.963277+0000", 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 117.99419625100001, 2023-09-23T20:09:16.092 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7854484319999999, 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65658 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.093 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963277+0000", 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963277+0000", 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.094 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963276+0000", 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963298+0000", 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2238e-05 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.095 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963311+0000", 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2799999999999999e-05 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963320+0000", 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9060000000000008e-06 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.096 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963375+0000", 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5581000000000002e-05 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963390+0000", 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4487999999999999e-05 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.097 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.944193+0000", 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98080345700000005 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.944202+0000", 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5210000000000002e-06 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.098 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.722406+0000", 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.77820445 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.722595+0000", 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018889800000000001 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.099 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.722773+0000", 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000177417 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.734745+0000", 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011972271 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.100 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.745165+0000", 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010419793 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.745196+0000", 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1337999999999997e-05 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.101 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.745258+0000", 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1915999999999995e-05 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.748726+0000", 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034678769999999999 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.102 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65659 8.f 8:fc975883:::benchmark_data_smithi135_460343_object65658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:17.963338+0000", 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 117.99413595599999, 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7904322829999999, 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.103 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65659 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963338+0000", 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.104 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963338+0000", 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963337+0000", 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963360+0000", 2023-09-23T20:09:16.105 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.27e-05 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963368+0000", 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3720000000000005e-06 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.106 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963374+0000", 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9599999999999997e-06 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963411+0000", 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7011999999999997e-05 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963417+0000", 2023-09-23T20:09:16.107 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.609e-06 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.944250+0000", 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98083321499999998 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.944260+0000", 2023-09-23T20:09:16.108 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0346999999999999e-05 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.725465+0000", 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.78120426 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.109 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.725623+0000", 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000158128 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.725797+0000", 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000174208 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.110 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.734751+0000", 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0089538250000000003 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.748781+0000", 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014030248 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.111 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.750607+0000", 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018264069999999999 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.750671+0000", 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3538000000000001e-05 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.112 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.753770+0000", 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030989640000000001 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65665 8.b 8:d11fc3d2:::benchmark_data_smithi135_460343_object65664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:17.963808+0000", 2023-09-23T20:09:16.113 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 117.99366593000001, 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7405857720000002, 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65665 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.114 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963808+0000", 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963808+0000", 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.115 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963807+0000", 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963828+0000", 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1180999999999999e-05 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.116 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963837+0000", 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4609999999999997e-06 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963844+0000", 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5209999999999999e-06 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.117 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963889+0000", 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4582000000000001e-05 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:17.963899+0000", 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0739e-05 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.118 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.943019+0000", 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97911959699999995 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.943033+0000", 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3876999999999999e-05 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.119 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.687789+0000", 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.744755847 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.687950+0000", 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016151199999999999 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.120 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.688078+0000", 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012796199999999999 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.693239+0000", 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051605180000000002 2023-09-23T20:09:16.121 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.694685+0000", 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001446751 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.701262+0000", 2023-09-23T20:09:16.122 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065768709999999998 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.701318+0000", 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5738000000000001e-05 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704393+0000", 2023-09-23T20:09:16.123 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030754409999999999 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65669 8.f 8:fe1ca6f2:::benchmark_data_smithi135_460343_object65668:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.131610+0000", 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 117.825864018, 2023-09-23T20:09:16.124 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6329595069999998, 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65669 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.125 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.131610+0000", 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.131610+0000", 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.126 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.131608+0000", 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.131634+0000", 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5919999999999999e-05 2023-09-23T20:09:16.127 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.131645+0000", 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.129e-05 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.131655+0000", 2023-09-23T20:09:16.128 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0067e-05 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.131800+0000", 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000145189 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.129 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.131813+0000", 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2646999999999999e-05 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.944323+0000", 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.81250993000000005 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.130 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.944335+0000", 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1851999999999999e-05 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.728878+0000", 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7845433530000001 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.131 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.729024+0000", 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014601300000000001 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.729175+0000", 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000150715 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.748835+0000", 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019660499000000001 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.753833+0000", 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049974879999999996 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.761190+0000", 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073575079999999996 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.761237+0000", 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6715000000000003e-05 2023-09-23T20:09:16.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.764569+0000", 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033321990000000001 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.135 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65670 8.0 8:07c470c9:::benchmark_data_smithi135_460343_object65669:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.133919+0000", 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 117.82355462300001, 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5860679549999999, 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.136 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65670 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.133919+0000", 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.133919+0000", 2023-09-23T20:09:16.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.133918+0000", 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.133941+0000", 2023-09-23T20:09:16.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2997999999999999e-05 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.133951+0000", 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0579e-05 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.133959+0000", 2023-09-23T20:09:16.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4329999999999998e-06 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.134022+0000", 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3471000000000003e-05 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.134035+0000", 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2435999999999999e-05 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.936414+0000", 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.80237972000000002 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.936422+0000", 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1670000000000002e-06 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.685601+0000", 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.749179099 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.685774+0000", 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017345899999999999 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.685963+0000", 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018891099999999999 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.691562+0000", 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005598553 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.695027+0000", 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034656330000000001 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.716810+0000", 2023-09-23T20:09:16.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021782497000000001 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.716869+0000", 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9046999999999999e-05 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.719987+0000", 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031181619999999998 2023-09-23T20:09:16.145 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65671 8.0 8:0faa2209:::benchmark_data_smithi135_460343_object65670:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.137287+0000", 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 117.820186886, 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5858094249999999, 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.146 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65671 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.137287+0000", 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.137287+0000", 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.137285+0000", 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.137308+0000", 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2192999999999999e-05 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.137323+0000", 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5676e-05 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.137333+0000", 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2599999999999994e-06 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.137380+0000", 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7023e-05 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.137387+0000", 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5800000000000003e-06 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.936445+0000", 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.79905781899999995 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.936453+0000", 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3089999999999996e-06 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.688446+0000", 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7519931039999999 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.688585+0000", 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013858100000000001 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.688728+0000", 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014281499999999999 2023-09-23T20:09:16.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.691597+0000", 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028694979999999998 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.695034+0000", 2023-09-23T20:09:16.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034367289999999999 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.720059+0000", 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.025025209999999999 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.720122+0000", 2023-09-23T20:09:16.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2448999999999998e-05 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.723096+0000", 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029744350000000001 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65672 8.f 8:f447461e:::benchmark_data_smithi135_460343_object65671:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:19.107522+0000", 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.849951629, 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6606425840000001, 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.157 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65672 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.107522+0000", 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.107522+0000", 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.107520+0000", 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.107546+0000", 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5578000000000001e-05 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.107560+0000", 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3621000000000001e-05 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.107567+0000", 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1280000000000002e-06 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.107614+0000", 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6962000000000002e-05 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.107622+0000", 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9189999999999999e-06 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.731859+0000", 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6242370859999999 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.732010+0000", 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015131899999999999 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.732155+0000", 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014474600000000001 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.748842+0000", 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016687794999999998 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.753865+0000", 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050222269999999998 2023-09-23T20:09:16.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.764646+0000", 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010780979 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.764706+0000", 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0024e-05 2023-09-23T20:09:16.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.768165+0000", 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034588259999999999 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65683 8.0 8:077905cb:::benchmark_data_smithi135_460343_object65682:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:19.141218+0000", 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 116.816255504, 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5850940100000002, 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65683 2023-09-23T20:09:16.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.141218+0000", 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.141218+0000", 2023-09-23T20:09:16.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.141217+0000", 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.141256+0000", 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9269e-05 2023-09-23T20:09:16.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.141270+0000", 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4036000000000001e-05 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.141279+0000", 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4200000000000007e-06 2023-09-23T20:09:16.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.141330+0000", 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0986999999999998e-05 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.141338+0000", 2023-09-23T20:09:16.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2449999999999998e-06 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.691622+0000", 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5502846460000002 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.691805+0000", 2023-09-23T20:09:16.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018276699999999999 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.692006+0000", 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000200736 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.695040+0000", 2023-09-23T20:09:16.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030336349999999998 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.698209+0000", 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003169748 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.723168+0000", 2023-09-23T20:09:16.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024958693000000001 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.723204+0000", 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5924999999999999e-05 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.726312+0000", 2023-09-23T20:09:16.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031081820000000001 2023-09-23T20:09:16.176 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.176 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.176 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.176 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:09:16.176 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:09:16.176 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:09:16.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:16.177 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:16.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:16.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:15 smithi135 ceph-mon[221113]: pgmap v969: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 26144/160305 objects degraded (16.309%); 1191/160305 objects misplaced (0.743%); 16 MiB/s, 261 objects/s recovering 2023-09-23T20:09:16.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:15 smithi135 ceph-mon[213871]: pgmap v969: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 26144/160305 objects degraded (16.309%); 1191/160305 objects misplaced (0.743%); 16 MiB/s, 261 objects/s recovering 2023-09-23T20:09:16.279 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:16.308 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:15 smithi144 ceph-mon[194833]: pgmap v969: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 26144/160305 objects degraded (16.309%); 1191/160305 objects misplaced (0.743%); 16 MiB/s, 261 objects/s recovering 2023-09-23T20:09:16.490 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:09:16.519 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:09:16.519 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:16.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:16.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:16.547 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:09:16.577 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:09:16.602 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:09:16.625 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:16.628 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:09:16.654 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:09:16.655 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:16.655 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:09:16.655 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:09:16.656 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:09:16.656 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:09:16.656 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:09:16.656 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:09:16.656 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:09:16.656 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:09:16.702 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:16.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:16.807 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:16.834 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:16.877 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:16.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:16.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:16.984 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:17.010 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:17.050 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:17.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:17.156 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:17.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:16 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:09:17.184 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:16 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:09:17.187 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:17.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:16 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:09:18.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[213871]: pgmap v970: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 32339/159696 objects degraded (20.250%); 1191/159696 objects misplaced (0.746%); 15 MiB/s, 237 objects/s recovering 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[213871]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[213871]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[213871]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[213871]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32339/159696 objects degraded (20.250%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[221113]: pgmap v970: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 32339/159696 objects degraded (20.250%); 1191/159696 objects misplaced (0.746%); 15 MiB/s, 237 objects/s recovering 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[221113]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[221113]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[221113]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[221113]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32339/159696 objects degraded (20.250%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:17 smithi144 ceph-mon[194833]: pgmap v970: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 32339/159696 objects degraded (20.250%); 1191/159696 objects misplaced (0.746%); 15 MiB/s, 237 objects/s recovering 2023-09-23T20:09:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:17 smithi144 ceph-mon[194833]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:17 smithi144 ceph-mon[194833]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T20:09:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:17 smithi144 ceph-mon[194833]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:17 smithi144 ceph-mon[194833]: failed to encode map e348 with expected crc 2023-09-23T20:09:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32339/159696 objects degraded (20.250%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:19.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[221113]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[221113]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[221113]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[221113]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T20:09:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[221113]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:09:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[213871]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[213871]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[213871]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[213871]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T20:09:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[213871]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:18 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:09:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:18 smithi144 ceph-mon[194833]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:18 smithi144 ceph-mon[194833]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:18 smithi144 ceph-mon[194833]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:18 smithi144 ceph-mon[194833]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T20:09:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:18 smithi144 ceph-mon[194833]: failed to encode map e349 with expected crc 2023-09-23T20:09:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:18 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:09:19.717 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:09:19.807 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:19.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:19.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:19.825 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:19.919 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:19.950 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:20.000 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:20.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:20.006 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:19 smithi144 ceph-mon[194833]: pgmap v973: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 32335/159624 objects degraded (20.257%); 1191/159624 objects misplaced (0.746%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T20:09:20.006 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:19 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:09:20.006 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:19 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:20.101 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:20.106 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:20.159 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:19 smithi135 ceph-mon[221113]: pgmap v973: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 32335/159624 objects degraded (20.257%); 1191/159624 objects misplaced (0.746%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T20:09:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:19 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:09:20.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:19 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:19 smithi135 ceph-mon[213871]: pgmap v973: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 32335/159624 objects degraded (20.257%); 1191/159624 objects misplaced (0.746%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T20:09:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:19 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:09:20.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:19 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:20.423 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:20.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:20.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:20.535 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:20.567 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:20.613 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:20.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:20.716 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:20.744 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:20.990 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:09:20.991 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:20.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:21.095 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:21.122 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:20 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:09:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2723229411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:20 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:09:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2723229411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:21.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:20 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:09:21.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2723229411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:21.849 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:09:22.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[213871]: pgmap v974: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 32335/159552 objects degraded (20.266%); 682/159552 objects misplaced (0.427%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T20:09:22.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:09:22.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[213871]: failed to encode map e350 with expected crc 2023-09-23T20:09:22.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2723229411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[213871]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[213871]: failed to encode map e350 with expected crc 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[213871]: failed to encode map e350 with expected crc 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[221113]: pgmap v974: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 32335/159552 objects degraded (20.266%); 682/159552 objects misplaced (0.427%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[221113]: failed to encode map e350 with expected crc 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2723229411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[221113]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[221113]: failed to encode map e350 with expected crc 2023-09-23T20:09:22.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:21 smithi135 ceph-mon[221113]: failed to encode map e350 with expected crc 2023-09-23T20:09:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:21 smithi144 ceph-mon[194833]: pgmap v974: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 32335/159552 objects degraded (20.266%); 682/159552 objects misplaced (0.427%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T20:09:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:21 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:09:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:21 smithi144 ceph-mon[194833]: failed to encode map e350 with expected crc 2023-09-23T20:09:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2723229411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:21 smithi144 ceph-mon[194833]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T20:09:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:21 smithi144 ceph-mon[194833]: failed to encode map e350 with expected crc 2023-09-23T20:09:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:21 smithi144 ceph-mon[194833]: failed to encode map e350 with expected crc 2023-09-23T20:09:23.287 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:23.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:23.391 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:23.393 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:23.393 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:23.393 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32335/159552 objects degraded (20.266%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:23.393 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:23 smithi144 ceph-mon[194833]: pgmap v976: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 20 op/s; 25192/159324 objects degraded (15.812%); 4003/159324 objects misplaced (2.512%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:09:23.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:23.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:23.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32335/159552 objects degraded (20.266%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:23.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:23 smithi135 ceph-mon[221113]: pgmap v976: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 20 op/s; 25192/159324 objects degraded (15.812%); 4003/159324 objects misplaced (2.512%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:09:23.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:23.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:23.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32335/159552 objects degraded (20.266%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:23.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:23 smithi135 ceph-mon[213871]: pgmap v976: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 20 op/s; 25192/159324 objects degraded (15.812%); 4003/159324 objects misplaced (2.512%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:09:23.430 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:23.530 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:23.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:23.634 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:23.664 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:23.841 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:23.863 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:23.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:23.939 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 297 (0%) 2023-09-23T20:09:23.939 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T20:09:23.939 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.06818 -> 7.06818 (expected baseline 6.03053) 2023-09-23T20:09:23.939 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 31 -> 31 pgs (0.73064 -> 0.73064 * mean) 2023-09-23T20:09:23.939 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 53 -> 53 pgs (1.24916 -> 1.24916 * mean) 2023-09-23T20:09:23.939 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:09:23.939 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T20:09:23.939 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 3 2023-09-23T20:09:23.940 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:09:23.940 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0232 2023-09-23T20:09:23.940 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.2315 2023-09-23T20:09:23.946 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:09:23.967 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:09:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:09:23] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:09:23.996 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:24.048 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:24.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:24.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:24.093 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:09:24.093 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:09:24.152 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:24.202 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:24.934 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:24 smithi144 ceph-mon[194833]: from='client.35799 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:24.935 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:24 smithi144 ceph-mon[194833]: 1.0 deep-scrub starts 2023-09-23T20:09:24.952 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:09:24.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:24 smithi135 ceph-mon[213871]: from='client.35799 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:24.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:24 smithi135 ceph-mon[213871]: 1.0 deep-scrub starts 2023-09-23T20:09:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:24 smithi135 ceph-mon[221113]: from='client.35799 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:24.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:24 smithi135 ceph-mon[221113]: 1.0 deep-scrub starts 2023-09-23T20:09:25.210 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:09:25.453 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:25.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:25 smithi135 ceph-mon[221113]: pgmap v977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 45 op/s; 25164/158613 objects degraded (15.865%); 3458/158613 objects misplaced (2.180%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T20:09:25.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:25 smithi135 ceph-mon[213871]: pgmap v977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 45 op/s; 25164/158613 objects degraded (15.865%); 3458/158613 objects misplaced (2.180%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T20:09:26.067 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:25 smithi144 ceph-mon[194833]: pgmap v977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 45 op/s; 25164/158613 objects degraded (15.865%); 3458/158613 objects misplaced (2.180%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T20:09:26.436 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:26.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:26.541 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:26.569 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:26.615 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:26.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:26.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:26.719 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:26.732 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:09:26.732 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T20:09:26.746 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:27.005 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:27.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:27.091 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:27.108 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:27.136 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:27.213 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:27.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:27.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:27.316 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:27.347 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:27.448 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:27.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:27.509 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:09:27.510 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:09:27.553 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:27.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:27.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:27.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:27 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:09:27.581 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:27 smithi144 ceph-mon[194833]: pgmap v978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 49 op/s; 25138/158373 objects degraded (15.873%); 3426/158373 objects misplaced (2.163%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:09:27.584 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:27.594 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:09:27.614 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:27.614 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:27.614 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:27 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:09:27.615 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:27 smithi135 ceph-mon[213871]: pgmap v978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 49 op/s; 25138/158373 objects degraded (15.873%); 3426/158373 objects misplaced (2.163%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:09:27.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:27.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:27.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:27 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:09:27.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:27 smithi135 ceph-mon[221113]: pgmap v978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 49 op/s; 25138/158373 objects degraded (15.873%); 3426/158373 objects misplaced (2.163%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:09:27.622 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:09:27.656 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:09:27.679 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:27.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:27.688 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:09:27.713 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:09:27.738 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:09:27.764 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:09:27.782 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:27.787 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:09:27.790 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:09:27.790 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:09:27.790 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:09:27.790 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:09:27.790 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:09:27.790 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:09:27.790 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:09:27.790 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:09:27.809 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:27.821 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:27.854 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:27.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:27.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:27.958 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:27.984 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:28.234 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:09:28.235 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:28.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:28.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:28.343 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:28.382 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/636242360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 25138/158373 objects degraded (15.873%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:28.640 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:28.648 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:28.652 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/636242360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:28.653 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 25138/158373 objects degraded (15.873%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:28.653 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/636242360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:28.653 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 25138/158373 objects degraded (15.873%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:28.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:28.755 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:28.780 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:28.978 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:28.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:28.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:29.082 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:29.110 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:29.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:29 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:09:29.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:29 smithi135 ceph-mon[213871]: pgmap v979: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 61 op/s; 25115/157776 objects degraded (15.918%); 3426/157776 objects misplaced (2.171%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:09:29.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:29 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:09:29.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:29 smithi135 ceph-mon[221113]: pgmap v979: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 61 op/s; 25115/157776 objects degraded (15.918%); 3426/157776 objects misplaced (2.171%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:09:29.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:29 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:09:29.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:29 smithi144 ceph-mon[194833]: pgmap v979: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 61 op/s; 25115/157776 objects degraded (15.918%); 3426/157776 objects misplaced (2.171%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:09:30.098 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:30.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:30.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:30.206 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:30.247 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:30.362 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:30.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:30.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:30.382 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148794 6.e 6:78f29f8c:::benchmark_data_smithi135_344736_object46138:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.915025+0000", 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 587.46428706999995, 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1236764180000001, 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.383 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148794 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915025+0000", 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.384 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915025+0000", 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915024+0000", 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.385 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915032+0000", 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8589999999999994e-06 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915046+0000", 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3339000000000001e-05 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915058+0000", 2023-09-23T20:09:30.386 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2602999999999999e-05 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915112+0000", 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3155000000000002e-05 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915232+0000", 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000120548 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.387 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915352+0000", 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011983 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.918233+0000", 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028810120000000001 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.388 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.935108+0000", 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016874914000000001 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.935118+0000", 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0329000000000001e-05 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.930839+0000", 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.995720665 2023-09-23T20:09:30.389 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.930850+0000", 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0882e-05 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.038649+0000", 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1077991949999999 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.390 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.038701+0000", 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2340999999999999e-05 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148799 6.2 6:4491f51b:::benchmark_data_smithi135_344736_object46143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.915417+0000", 2023-09-23T20:09:30.391 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 587.46389457199996, 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.088989105, 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148799 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.392 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915417+0000", 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915417+0000", 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915417+0000", 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915421+0000", 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7010000000000004e-06 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915432+0000", 2023-09-23T20:09:30.394 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1032e-05 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915438+0000", 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6840000000000001e-06 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915487+0000", 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9128000000000001e-05 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.395 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915579+0000", 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1897000000000005e-05 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.915936+0000", 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00035673200000000001 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.396 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.917937+0000", 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020014439999999998 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.926398+0000", 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0084603830000000001 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.931596+0000", 2023-09-23T20:09:30.397 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0051984850000000001 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.931610+0000", 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3713999999999999e-05 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.941803+0000", 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.010192763 2023-09-23T20:09:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:43.941818+0000", 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5097e-05 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.001097+0000", 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0592793089999999 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.399 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for missing object", 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.001256+0000", 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015914100000000001 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.004349+0000", 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0030928499999999999 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.400 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:45.004406+0000", 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7466000000000002e-05 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148801 6.12 6:4c566438:::benchmark_data_smithi135_344736_object46145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.924033+0000", 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 587.45527890899996, 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.19486709, 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.401 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148801 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924033+0000", 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.402 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924033+0000", 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924032+0000", 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.403 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924039+0000", 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4200000000000001e-06 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924052+0000", 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2663000000000001e-05 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924059+0000", 2023-09-23T20:09:30.404 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7769999999999997e-06 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924281+0000", 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000222189 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924434+0000", 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000153628 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.405 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.924824+0000", 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00038961200000000001 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.926508+0000", 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0016836069999999999 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.931639+0000", 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0051310870000000003 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.931646+0000", 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4540000000000001e-06 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.073693+0000", 2023-09-23T20:09:30.407 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.142047239 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for missing object", 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.073896+0000", 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000202649 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.118824+0000", 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.044928382000000003 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.408 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.118900+0000", 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5635000000000003e-05 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:148811 6.12 6:4f11fd05:::benchmark_data_smithi135_344736_object46155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:42.955172+0000", 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 587.42413984200005, 2023-09-23T20:09:30.409 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1686808200000001, 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 148811 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.410 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.955172+0000", 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.955172+0000", 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.955171+0000", 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:09:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.955175+0000", 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1400000000000002e-06 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.955183+0000", 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7190000000000001e-06 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.412 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.955189+0000", 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3690000000000003e-06 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.972515+0000", 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.017325885999999999 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:42.972526+0000", 2023-09-23T20:09:30.413 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1282e-05 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.118931+0000", 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1464046589999999 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.119042+0000", 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000110574 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.119361+0000", 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00031942400000000001 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.121144+0000", 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001782898 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.121574+0000", 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043046000000000002 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.122696+0000", 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0011217379999999999 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.416 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.122737+0000", 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0655999999999998e-05 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:44.123853+0000", 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0011160619999999999 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.417 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52248 8.5 8:aed026a3:::benchmark_data_smithi135_460343_object52247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.949589+0000", 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 150.42972313300001, 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1372650540000002, 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52248 2023-09-23T20:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949589+0000", 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949589+0000", 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.419 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949588+0000", 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949658+0000", 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0339999999999994e-05 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.420 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949669+0000", 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1156e-05 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949675+0000", 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3979999999999998e-06 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949734+0000", 2023-09-23T20:09:30.421 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9015000000000003e-05 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949744+0000", 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0358e-05 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.926958+0000", 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97721380000000002 2023-09-23T20:09:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.926969+0000", 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1173e-05 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.070949+0000", 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.143979493 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.423 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.071095+0000", 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000146501 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.071258+0000", 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016240699999999999 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.424 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.075199+0000", 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0039416440000000002 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.075207+0000", 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0730000000000005e-06 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.084221+0000", 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090141570000000001 2023-09-23T20:09:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.084282+0000", 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0473000000000002e-05 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.086854+0000", 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025719039999999999 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.426 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:65654 8.5 8:abe5195a:::benchmark_data_smithi135_460343_object65653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:17.919947+0000", 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 132.45936506699999, 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.190072045, 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.427 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 65654 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919947+0000", 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.428 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919947+0000", 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919946+0000", 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920012+0000", 2023-09-23T20:09:30.429 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6530000000000002e-05 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920030+0000", 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7927999999999998e-05 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920037+0000", 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8449999999999997e-06 2023-09-23T20:09:30.430 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920085+0000", 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7766000000000003e-05 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920214+0000", 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000129589 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.431 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920312+0000", 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7917000000000005e-05 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.922814+0000", 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025019669999999999 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.432 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.924821+0000", 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020062880000000002 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.945068+0000", 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.020247023999999999 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.945081+0000", 2023-09-23T20:09:30.433 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3132999999999999e-05 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.109930+0000", 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.164849646 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.110019+0000", 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8443999999999995e-05 2023-09-23T20:09:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:65661 8.5 8:a319973d:::benchmark_data_smithi135_460343_object65660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:17.963073+0000", 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 132.416239319, 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1591320270000001, 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.435 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 65661 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963073+0000", 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963073+0000", 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963071+0000", 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.437 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963115+0000", 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4014999999999998e-05 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963131+0000", 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5480000000000001e-05 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.438 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963137+0000", 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0739999999999998e-06 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963201+0000", 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4103999999999995e-05 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963212+0000", 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0712e-05 2023-09-23T20:09:30.439 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.110045+0000", 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.146833177 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.110156+0000", 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000110993 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.110267+0000", 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111184 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.112139+0000", 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018723139999999999 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.441 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.115899+0000", 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003759877 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.119057+0000", 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031581249999999999 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.119101+0000", 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3328000000000003e-05 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.122205+0000", 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003103916 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.443 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77498 8.5 8:a5b293b3:::benchmark_data_smithi135_460343_object77497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.694200+0000", 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 105.68511223599999, 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.006215106, 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77498 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.444 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694200+0000", 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694200+0000", 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.445 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694199+0000", 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694254+0000", 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5170000000000002e-05 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.446 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694271+0000", 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7184e-05 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694278+0000", 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7870000000000004e-06 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698780+0000", 2023-09-23T20:09:30.447 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045027890000000001 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698792+0000", 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1778e-05 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.694233+0000", 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99544110799999996 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.448 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.694325+0000", 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1958000000000003e-05 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.694433+0000", 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107877 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.449 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700191+0000", 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0057577390000000004 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700198+0000", 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0609999999999998e-06 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.450 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700324+0000", 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000125535 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700376+0000", 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2336000000000002e-05 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.451 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700415+0000", 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8923e-05 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77501 8.5 8:ad6d7fa4:::benchmark_data_smithi135_460343_object77500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.698362+0000", 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 105.680950045, 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0068878059999999, 2023-09-23T20:09:30.452 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77501 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.453 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698362+0000", 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698362+0000", 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698361+0000", 2023-09-23T20:09:30.454 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698392+0000", 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1779000000000002e-05 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698404+0000", 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1348999999999999e-05 2023-09-23T20:09:30.455 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698409+0000", 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0799999999999996e-06 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698881+0000", 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000472029 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.456 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698888+0000", 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1720000000000001e-06 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.697427+0000", 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99853948599999998 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.697530+0000", 2023-09-23T20:09:30.457 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010283100000000001 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.697659+0000", 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000129064 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.702943+0000", 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052838679999999997 2023-09-23T20:09:30.458 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705153+0000", 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022095389999999999 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705191+0000", 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7898999999999997e-05 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.459 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705225+0000", 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4557999999999997e-05 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705250+0000", 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4436000000000001e-05 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.460 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77504 8.5 8:aa509932:::benchmark_data_smithi135_460343_object77503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.711477+0000", 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 105.66783487799999, 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.998780051, 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77504 2023-09-23T20:09:30.461 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711477+0000", 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711477+0000", 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.462 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711475+0000", 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711503+0000", 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8280999999999999e-05 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.463 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711523+0000", 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9758999999999998e-05 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711530+0000", 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3559999999999998e-06 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711578+0000", 2023-09-23T20:09:30.464 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8469e-05 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711586+0000", 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2150000000000001e-06 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700445+0000", 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98885881099999995 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.465 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700548+0000", 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102729 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700726+0000", 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017792599999999999 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.466 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705297+0000", 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045714950000000001 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.707825+0000", 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025279289999999999 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.467 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710195+0000", 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002369894 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710231+0000", 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6044999999999998e-05 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710257+0000", 2023-09-23T20:09:30.468 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6038999999999999e-05 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77527 8.5 8:a80e791e:::benchmark_data_smithi135_460343_object77526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.758268+0000", 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 105.62104406500001, 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.95206617400000004, 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.469 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77527 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758268+0000", 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.470 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758268+0000", 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758266+0000", 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.471 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758296+0000", 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9828e-05 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758315+0000", 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8927999999999999e-05 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758321+0000", 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9900000000000002e-06 2023-09-23T20:09:30.472 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758388+0000", 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6811999999999997e-05 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758401+0000", 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3227999999999999e-05 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.473 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.702999+0000", 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.94459813199999998 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.703108+0000", 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000108748 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:09:30.474 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.703205+0000", 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7096999999999995e-05 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710088+0000", 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.006883134 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710132+0000", 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4391000000000002e-05 2023-09-23T20:09:30.475 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710292+0000", 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015950099999999999 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710315+0000", 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2713000000000001e-05 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.476 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710334+0000", 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9429e-05 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77549 8.5 8:a49adce5:::benchmark_data_smithi135_460343_object77548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.798013+0000", 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 105.581299211, 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.924997548, 2023-09-23T20:09:30.477 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77549 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798013+0000", 2023-09-23T20:09:30.478 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798013+0000", 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798011+0000", 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.479 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798051+0000", 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9983e-05 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798069+0000", 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7640000000000001e-05 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.480 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798076+0000", 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.413e-06 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798143+0000", 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6481000000000003e-05 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798154+0000", 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0956e-05 2023-09-23T20:09:30.481 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705321+0000", 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.90716749699999999 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705404+0000", 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3079999999999997e-05 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.482 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705494+0000", 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9672000000000005e-05 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710157+0000", 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046633819999999998 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.483 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710166+0000", 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7220000000000005e-06 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.722950+0000", 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012784617999999999 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.722985+0000", 2023-09-23T20:09:30.484 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4872000000000001e-05 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.723010+0000", 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4885999999999999e-05 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77558 8.5 8:a6a5e11a:::benchmark_data_smithi135_460343_object77557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:09:30.485 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.814035+0000", 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 105.56527730800001, 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.90909048400000003, 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77558 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.486 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814035+0000", 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814035+0000", 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.487 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814033+0000", 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814079+0000", 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5802e-05 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814099+0000", 2023-09-23T20:09:30.488 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0177999999999999e-05 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814107+0000", 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1820000000000006e-06 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814202+0000", 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5297999999999995e-05 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.489 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814214+0000", 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1311999999999999e-05 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.707855+0000", 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.89364169800000004 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.490 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.707982+0000", 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012681899999999999 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.708127+0000", 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000145325 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.491 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.712213+0000", 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040858919999999998 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.712247+0000", 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3890999999999998e-05 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.723050+0000", 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010802959000000001 2023-09-23T20:09:30.492 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.723097+0000", 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.672e-05 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.723125+0000", 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8221999999999998e-05 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.493 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:113253 8.5 8:a0597b52:::benchmark_data_smithi135_460343_object29020:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:57.993093+0000", 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 32.386218650000004, 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.028671068000000001, 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.494 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 113253 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.993093+0000", 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.495 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.993093+0000", 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.993093+0000", 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.993096+0000", 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7129999999999999e-06 2023-09-23T20:09:30.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.993104+0000", 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.357e-06 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.993109+0000", 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6970000000000001e-06 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.497 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.996725+0000", 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0036166430000000001 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.996830+0000", 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000104484 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.498 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:57.996944+0000", 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114462 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:58.001105+0000", 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0041605790000000002 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:58.002053+0000", 2023-09-23T20:09:30.499 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00094796400000000003 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:58.021674+0000", 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.01962092 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:58.021721+0000", 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7150000000000001e-05 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.500 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:58.021764+0000", 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3575000000000002e-05 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:114091 8.2 8:431bbd5a:::benchmark_data_smithi135_460343_object29858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:59.947598+0000", 2023-09-23T20:09:30.501 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 30.431713757000001, 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.034013479999999999, 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 114091 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.502 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.947598+0000", 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.947598+0000", 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.947598+0000", 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:30.503 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.947600+0000", 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.72e-06 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.947608+0000", 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8019999999999998e-06 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.504 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.947611+0000", 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4080000000000001e-06 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.950858+0000", 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032471930000000002 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.950991+0000", 2023-09-23T20:09:30.505 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013305200000000001 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.951118+0000", 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012730099999999999 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.953466+0000", 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023477519999999998 2023-09-23T20:09:30.506 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.953621+0000", 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015500600000000001 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.981527+0000", 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.027906014 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.507 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.981575+0000", 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7809000000000003e-05 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:59.981612+0000", 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6696000000000002e-05 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.508 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:114630 8.2 8:4c95d0a8:::benchmark_data_smithi135_460343_object30397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:00.907046+0000", 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 29.472266257000001, 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.028864834999999998, 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 114630 2023-09-23T20:09:30.509 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907046+0000", 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907046+0000", 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.510 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907044+0000", 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907051+0000", 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7909999999999999e-06 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.511 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907065+0000", 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4171e-05 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907071+0000", 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2339999999999999e-06 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.512 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907110+0000", 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8581999999999998e-05 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907200+0000", 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0173000000000006e-05 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.907273+0000", 2023-09-23T20:09:30.513 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2545000000000004e-05 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.909823+0000", 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025504120000000002 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.910062+0000", 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00023838599999999999 2023-09-23T20:09:30.514 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.935808+0000", 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.02574688 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.935866+0000", 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7553000000000003e-05 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.515 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:00.935910+0000", 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4455e-05 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:115187 8.b 8:ddd6b759:::benchmark_data_smithi135_460343_object30954:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e342)", 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:02.643566+0000", 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 27.735746120000002, 2023-09-23T20:09:30.516 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2096566420000001, 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 115187 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643566+0000", 2023-09-23T20:09:30.517 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643566+0000", 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643564+0000", 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:30.518 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643571+0000", 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.114e-06 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643589+0000", 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7634000000000001e-05 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.519 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643597+0000", 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.303e-06 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643662+0000", 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4257999999999994e-05 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643673+0000", 2023-09-23T20:09:30.520 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1575e-05 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.650917+0000", 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0072438399999999997 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.650925+0000", 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2230000000000007e-06 2023-09-23T20:09:30.521 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.827062+0000", 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1761365800000001 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.827706+0000", 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00064398799999999996 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.522 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.837267+0000", 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0095610520000000004 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.838831+0000", 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001564466 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.838932+0000", 2023-09-23T20:09:30.523 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101042 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851044+0000", 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012111709999999999 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851081+0000", 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6539999999999999e-05 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.524 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851972+0000", 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00089110199999999995 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.852008+0000", 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6337999999999997e-05 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.525 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.853222+0000", 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001214302 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:115197 8.b 8:d990aab7:::benchmark_data_smithi135_460343_object30964:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e342)", 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:02.643610+0000", 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 27.735701975000001, 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2161859939999999, 2023-09-23T20:09:30.526 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 115197 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643610+0000", 2023-09-23T20:09:30.527 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643610+0000", 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643609+0000", 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:30.528 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643613+0000", 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3869999999999999e-06 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643620+0000", 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7240000000000003e-06 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.529 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643624+0000", 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2980000000000002e-06 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643758+0000", 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000133769 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:30.530 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643768+0000", 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0671e-05 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.651097+0000", 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0073282039999999996 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.651106+0000", 2023-09-23T20:09:30.531 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6639999999999993e-06 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.829201+0000", 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1780947209999999 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.829783+0000", 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000581997 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.532 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.844348+0000", 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014565058000000001 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.844431+0000", 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3151000000000004e-05 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.533 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.844522+0000", 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1051000000000006e-05 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851914+0000", 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0073918050000000004 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.853255+0000", 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001341096 2023-09-23T20:09:30.534 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.857930+0000", 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046753999999999997 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.857971+0000", 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.049e-05 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.859796+0000", 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018249539999999999 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:115254 8.b 8:d147ebd0:::benchmark_data_smithi135_460343_object31021:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e342)", 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:02.643628+0000", 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 27.735683681000001, 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2309227030000001, 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.536 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 115254 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643628+0000", 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.537 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643628+0000", 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643628+0000", 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.538 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643630+0000", 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1050000000000002e-06 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643636+0000", 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.105e-06 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643640+0000", 2023-09-23T20:09:30.539 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2939999999999999e-06 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643799+0000", 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000158932 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643810+0000", 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0139999999999999e-05 2023-09-23T20:09:30.540 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.651130+0000", 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0073204899999999998 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.651137+0000", 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2150000000000004e-06 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.541 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.829847+0000", 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.178710275 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.830512+0000", 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00066481899999999998 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.848989+0000", 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.018476328 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.849062+0000", 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3139000000000006e-05 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.849154+0000", 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2279000000000002e-05 2023-09-23T20:09:30.543 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851919+0000", 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002764533 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.859839+0000", 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0079203439999999993 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.871956+0000", 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012117146000000001 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.872034+0000", 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7768999999999994e-05 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.874551+0000", 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025170689999999998 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:116078 8.5 8:ad389036:::benchmark_data_smithi135_460343_object31845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:28.841168+0000", 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 1.5381439690000001, 2023-09-23T20:09:30.546 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.055073830999999997, 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116078 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:30.547 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841168+0000", 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841168+0000", 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841167+0000", 2023-09-23T20:09:30.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841173+0000", 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5030000000000002e-06 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841187+0000", 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3896999999999999e-05 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841192+0000", 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2839999999999997e-06 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841237+0000", 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4274999999999999e-05 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.550 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841341+0000", 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103953 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841422+0000", 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1631999999999994e-05 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.844545+0000", 2023-09-23T20:09:30.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031223710000000001 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.844907+0000", 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036229099999999999 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.896146+0000", 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.051239543999999998 2023-09-23T20:09:30.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.896203+0000", 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6385000000000003e-05 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.896242+0000", 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8949999999999998e-05 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.553 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.554 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:30.554 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:30.554 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:09:30.554 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:30.586 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:30.630 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:30.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:30.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:30.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:30 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:09:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:30 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:09:30.733 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:30.757 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:30 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:09:30.759 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:30.802 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:30.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:30.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:30.906 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:30.932 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:31.068 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:31.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:31.125 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:09:31.172 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:31.203 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:31.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:31 smithi135 ceph-mon[213871]: pgmap v980: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 68 op/s; 24549/157494 objects degraded (15.587%); 3289/157494 objects misplaced (2.088%); 5.2 MiB/s, 139 objects/s recovering 2023-09-23T20:09:31.481 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:31.559 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:31.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:31.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:31.662 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:31.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:31 smithi144 ceph-mon[194833]: pgmap v980: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 68 op/s; 24549/157494 objects degraded (15.587%); 3289/157494 objects misplaced (2.088%); 5.2 MiB/s, 139 objects/s recovering 2023-09-23T20:09:31.691 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:31.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:31 smithi135 ceph-mon[221113]: pgmap v980: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 68 op/s; 24549/157494 objects degraded (15.587%); 3289/157494 objects misplaced (2.088%); 5.2 MiB/s, 139 objects/s recovering 2023-09-23T20:09:31.735 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:31.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:31.839 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:31.862 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:09:31.900 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:32.000 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:32.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:32.104 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:32.152 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:32.332 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:32.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:32.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:32 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:09:32.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/416160453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:09:32.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:32 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:09:32.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:32 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:09:32.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/416160453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:09:32.420 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:32 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:09:32.439 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:32.439 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:09:32.467 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:32.632 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:32.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:32.737 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:32.762 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:32 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:09:32.762 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/416160453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:09:32.763 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:32 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:09:32.765 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:32.811 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:32.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:32.914 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:32.949 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:33.156 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:09:33.157 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:09:33.157 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:09:33.157 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:09:33.427 INFO:teuthology.orchestra.run.smithi135.stdout:moved 12 / 297 (4.0404%) 2023-09-23T20:09:33.427 INFO:teuthology.orchestra.run.smithi135.stdout:avg 42.4286 2023-09-23T20:09:33.427 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.06818 -> 9.30218 (expected baseline 6.03053) 2023-09-23T20:09:33.427 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 31 -> 26 pgs (0.73064 -> 0.612795 * mean) 2023-09-23T20:09:33.427 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 53 -> 53 pgs (1.24916 -> 1.24916 * mean) 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0230 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0253 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 weight 0.7459 -> 0.6959 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 weight 0.7588 -> 0.7088 2023-09-23T20:09:33.428 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5 weight 0.8455 -> 0.7955 2023-09-23T20:09:33.430 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:09:33.544 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:33.544 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.544 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:09:33.545 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T20:09:33.545 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.545 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.545 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:33.545 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:33.546 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 24549/157494 objects degraded (15.587%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:33.546 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:33 smithi144 ceph-mon[194833]: pgmap v982: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 24521/157170 objects degraded (15.602%); 3257/157170 objects misplaced (2.072%); 5.3 MiB/s, 141 objects/s recovering 2023-09-23T20:09:33.683 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:09:33 smithi135 conmon[201366]: 2023-09-23T20:09:33.426+0000 7fcf49f9e700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T20:09:33.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:33.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:09:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T20:09:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 24549/157494 objects degraded (15.587%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[213871]: pgmap v982: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 24521/157170 objects degraded (15.602%); 3257/157170 objects misplaced (2.072%); 5.3 MiB/s, 141 objects/s recovering 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: failed to encode map e351 with expected crc 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 24549/157494 objects degraded (15.587%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:33.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:33 smithi135 ceph-mon[221113]: pgmap v982: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 24521/157170 objects degraded (15.602%); 3257/157170 objects misplaced (2.072%); 5.3 MiB/s, 141 objects/s recovering 2023-09-23T20:09:33.926 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:33.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:33.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:09:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:09:33] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:09:34.031 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:34.067 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:34.092 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:09:34.460 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:34.632 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:34.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:34.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:34.692 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:34.741 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:34.777 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:34.831 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:34.833 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:34 smithi135 ceph-mon[213871]: from='client.55585 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:34.833 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:34 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:34.833 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:34 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:34.833 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:34 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:34.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:34.834 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:34 smithi135 ceph-mon[221113]: from='client.55585 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:34.835 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:34 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:34.835 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:34 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:34.835 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:34 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:34.935 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:34.963 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:34 smithi144 ceph-mon[194833]: from='client.55585 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:34.963 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:34 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:34.964 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:34 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:34.964 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:34 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:34.968 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:35.119 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:35.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:35.224 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:35.260 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:35.415 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:35.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:35.519 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:35.545 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:35.590 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:35.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:35.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:35.695 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:35.725 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:35.783 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:35.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:35.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:35.887 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:35.931 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:35 smithi135 ceph-mon[221113]: pgmap v983: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 23545/156042 objects degraded (15.089%); 3257/156042 objects misplaced (2.087%); 1.9 MiB/s, 181 objects/s recovering 2023-09-23T20:09:35.931 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:35 smithi135 ceph-mon[213871]: pgmap v983: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 23545/156042 objects degraded (15.089%); 3257/156042 objects misplaced (2.087%); 1.9 MiB/s, 181 objects/s recovering 2023-09-23T20:09:35.935 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:35.979 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:35 smithi144 ceph-mon[194833]: pgmap v983: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 23545/156042 objects degraded (15.089%); 3257/156042 objects misplaced (2.087%); 1.9 MiB/s, 181 objects/s recovering 2023-09-23T20:09:35.995 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:35.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:36.100 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:36.143 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:36.201 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:09:36.201 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:09:36.201 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:09:36.201 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:09:36.201 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:09:36.204 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:36.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:36.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:36.315 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:36.347 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:36.418 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:36.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:36.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:36.522 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:36.558 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:36.603 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:09:36.659 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:36.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:36.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:36.709 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:09:36.757 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:09:36.767 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:36.779 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:09:36.797 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:36.815 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:09:36.839 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:09:36.845 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:36.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:36.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:36.861 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:09:36.893 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:09:36.901 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:09:36.922 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:09:36.927 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:09:36.927 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:09:36.927 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:09:36.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:09:36.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:09:36.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:09:36.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:09:36.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:09:36.949 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:36.985 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:37.200 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:37.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:37.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:37.259 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:37.304 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:37.344 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:37.405 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:37.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:37.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:37.508 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:37.537 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:37.584 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:37.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:37.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:37.668 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.668 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.688 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:37.723 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:37.835 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.835 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T20:09:37.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/284851745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:37.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: pgmap v985: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 21029/155712 objects degraded (13.505%); 5290/155712 objects misplaced (3.397%); 2.4 MiB/s, 277 objects/s recovering 2023-09-23T20:09:37.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:37.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:37.836 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:37 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 23545/156042 objects degraded (15.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:37.875 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:37.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:37.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:37.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T20:09:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/284851745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: pgmap v985: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 21029/155712 objects degraded (13.505%); 5290/155712 objects misplaced (3.397%); 2.4 MiB/s, 277 objects/s recovering 2023-09-23T20:09:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:37.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 23545/156042 objects degraded (15.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:37.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T20:09:37.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/284851745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:37.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: failed to encode map e352 with expected crc 2023-09-23T20:09:37.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: pgmap v985: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 21029/155712 objects degraded (13.505%); 5290/155712 objects misplaced (3.397%); 2.4 MiB/s, 277 objects/s recovering 2023-09-23T20:09:37.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:09:37.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:37.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:37 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 23545/156042 objects degraded (15.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:37.979 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:38.013 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:38.052 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:38.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:38.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:38.156 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:38.188 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:38.248 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:38.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:38.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:38.353 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:38.368 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:38.382 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:38.433 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:38.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:38.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:38.538 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:38.569 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:38.631 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:38.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:38.736 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:38.768 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:38.872 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:38.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[221113]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[221113]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[221113]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.874 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[221113]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.875 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[221113]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T20:09:38.875 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[213871]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.875 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[213871]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.875 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[213871]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.875 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[213871]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.875 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:38 smithi135 ceph-mon[213871]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T20:09:38.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:38.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:38.914 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:09:38.914 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:09:38.935 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:38 smithi144 ceph-mon[194833]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.935 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:38 smithi144 ceph-mon[194833]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.935 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:38 smithi144 ceph-mon[194833]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.935 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:38 smithi144 ceph-mon[194833]: failed to encode map e353 with expected crc 2023-09-23T20:09:38.935 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:38 smithi144 ceph-mon[194833]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T20:09:38.977 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:39.007 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:39.158 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:39.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:39.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:39.262 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:39.289 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:39.537 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:39.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:39.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:39.591 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:09:39.618 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:09:39.618 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:09:39.618 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:09:39.619 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:09:39.641 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:39.669 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:39.714 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:39.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:39.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:39.819 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:39.844 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:39 smithi144 ceph-mon[194833]: pgmap v987: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 21004/154851 objects degraded (13.564%); 5290/154851 objects misplaced (3.416%); 1.3 MiB/s, 229 objects/s recovering 2023-09-23T20:09:39.847 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:39.893 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:39.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:39.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:39 smithi135 ceph-mon[221113]: pgmap v987: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 21004/154851 objects degraded (13.564%); 5290/154851 objects misplaced (3.416%); 1.3 MiB/s, 229 objects/s recovering 2023-09-23T20:09:39.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:39 smithi135 ceph-mon[213871]: pgmap v987: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 21004/154851 objects degraded (13.564%); 5290/154851 objects misplaced (3.416%); 1.3 MiB/s, 229 objects/s recovering 2023-09-23T20:09:39.996 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:40.008 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:40.024 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:40.071 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:40.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:40.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:40.175 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:40.216 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:40.259 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:40.280 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:40.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:40.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:40.323 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:09:40.385 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:40.414 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:40.458 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:40.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:40.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:40.562 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:40.603 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:40.663 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:40.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:40.767 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:40.825 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:40.914 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:40.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:41.021 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:41.048 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:41.095 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:41.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:41.200 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:41.233 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:41.328 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:41.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:41.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:41.434 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:41.462 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:41.515 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:41.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:41.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:41.574 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:09:41.574 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:09:41.619 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:41.648 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:41.697 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:41.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:41.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:41.800 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:41.831 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:41.875 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:41.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:41.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:41.966 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:09:41.979 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:42.020 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:41 smithi144 ceph-mon[194833]: pgmap v988: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 21004/154602 objects degraded (13.586%); 5290/154602 objects misplaced (3.422%); 1.1 MiB/s, 185 objects/s recovering 2023-09-23T20:09:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1063567985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:42.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:41 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:42.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:41 smithi135 ceph-mon[221113]: pgmap v988: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 21004/154602 objects degraded (13.586%); 5290/154602 objects misplaced (3.422%); 1.1 MiB/s, 185 objects/s recovering 2023-09-23T20:09:42.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1063567985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:42.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:41 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:41 smithi135 ceph-mon[213871]: pgmap v988: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 21004/154602 objects degraded (13.586%); 5290/154602 objects misplaced (3.422%); 1.1 MiB/s, 185 objects/s recovering 2023-09-23T20:09:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1063567985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:41 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:42.350 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:09:42.424 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:42.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:42.530 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:42.559 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:42.605 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:42.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:42.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:42.704 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:42.715 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:42.743 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:42.934 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:43.150 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:43.152 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:43.152 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[221113]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.153 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[221113]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.153 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:43.153 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[221113]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T20:09:43.153 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[221113]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.153 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 21004/154602 objects degraded (13.586%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:43.153 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:43.153 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[213871]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.153 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[213871]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.154 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:43.154 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[213871]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T20:09:43.154 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[213871]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.154 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 21004/154602 objects degraded (13.586%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:43.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:43.255 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:43.275 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:42 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:43.276 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:42 smithi144 ceph-mon[194833]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.276 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:42 smithi144 ceph-mon[194833]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.276 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:42 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:43.276 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:42 smithi144 ceph-mon[194833]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T20:09:43.276 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:42 smithi144 ceph-mon[194833]: failed to encode map e354 with expected crc 2023-09-23T20:09:43.276 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 21004/154602 objects degraded (13.586%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:43.286 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:43.385 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:43.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:43.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:43.488 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:43.541 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:43.921 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:43.924 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:09:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:09:43] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T20:09:43.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:43.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:44.026 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:44.044 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:09:44.045 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:09:44.046 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:43 smithi144 ceph-mon[194833]: pgmap v990: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 20970/154221 objects degraded (13.597%); 4897/154221 objects misplaced (3.175%); 5.1 MiB/s, 147 objects/s recovering 2023-09-23T20:09:44.063 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:44.210 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:44.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:44.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:44.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:43 smithi135 ceph-mon[221113]: pgmap v990: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 20970/154221 objects degraded (13.597%); 4897/154221 objects misplaced (3.175%); 5.1 MiB/s, 147 objects/s recovering 2023-09-23T20:09:44.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:43 smithi135 ceph-mon[213871]: pgmap v990: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 20970/154221 objects degraded (13.597%); 4897/154221 objects misplaced (3.175%); 5.1 MiB/s, 147 objects/s recovering 2023-09-23T20:09:44.287 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:09:44.287 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:09:44.313 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:44.348 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:45.023 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:09:45.260 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:09:45.388 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:45.870 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:45.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:45.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:45.973 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":990,"stamp":"2023-09-23T20:09:42.903898+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3371850818,"num_objects":51407,"num_object_clones":0,"num_object_copies":154221,"num_objects_missing_on_primary":6037,"num_objects_missing":6037,"num_objects_degraded":20970,"num_objects_misplaced":4897,"num_objects_unfound":0,"num_objects_dirty":51407,"num_whiteouts":0,"num_read":6158,"num_read_kb":13539,"num_write":118077,"num_write_kb":5404324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17678,"num_bytes_recovered":1029443839,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88734,"ondisk_log_size":88734,"up":297,"acting":288,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":342,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14909068,"kb_used_data":11367444,"kb_used_omap":32,"kb_used_meta":3541535,"kb_avail":641306996,"statfs":{"total":671965249536,"available":656698363904,"internally_reserved":0,"allocated":11640262656,"data_stored":11630234057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3626532358},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-37486592,"num_objects":-572,"num_object_clones":0,"num_object_copies":-1716,"num_objects_missing_on_primary":-115,"num_objects_missing":-115,"num_objects_degraded":-498,"num_objects_misplaced":-420,"num_objects_unfound":0,"num_objects_dirty":-572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":572,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":918,"num_bytes_recovered":33226752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.225800"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":351,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.681015+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:09:36.681015+0000","last_peered":"2023-09-23T20:09:36.681015+0000","last_clean":"2023-09-23T20:09:36.681015+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:09:36.681015+0000","last_undegraded":"2023-09-23T20:09:36.681015+0000","last_fullsized":"2023-09-23T20:09:36.681015+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":180,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674491+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:09:36.674491+0000","last_peered":"2023-09-23T20:09:36.674491+0000","last_clean":"2023-09-23T20:09:36.674491+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:09:36.674491+0000","last_undegraded":"2023-09-23T20:09:36.674491+0000","last_fullsized":"2023-09-23T20:09:36.674491+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674866+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:09:36.674866+0000","last_peered":"2023-09-23T20:09:36.674866+0000","last_clean":"2023-09-23T20:09:36.674866+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:09:36.674866+0000","last_undegraded":"2023-09-23T20:09:36.674866+0000","last_fullsized":"2023-09-23T20:09:36.674866+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":51,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676484+0000","last_change":"2023-09-23T20:09:02.653309+0000","last_active":"2023-09-23T20:09:36.676484+0000","last_peered":"2023-09-23T20:09:36.676484+0000","last_clean":"2023-09-23T20:09:36.676484+0000","last_became_active":"2023-09-23T20:09:02.652950+0000","last_became_peered":"2023-09-23T20:09:02.652950+0000","last_unstale":"2023-09-23T20:09:36.676484+0000","last_undegraded":"2023-09-23T20:09:36.676484+0000","last_fullsized":"2023-09-23T20:09:36.676484+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:19.137730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":180,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676385+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:09:36.676385+0000","last_peered":"2023-09-23T20:09:36.676385+0000","last_clean":"2023-09-23T20:09:36.676385+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:09:36.676385+0000","last_undegraded":"2023-09-23T20:09:36.676385+0000","last_fullsized":"2023-09-23T20:09:36.676385+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":298,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.671091+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:09:36.671091+0000","last_peered":"2023-09-23T20:09:36.671091+0000","last_clean":"2023-09-23T20:09:36.671091+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:09:36.671091+0000","last_undegraded":"2023-09-23T20:09:36.671091+0000","last_fullsized":"2023-09-23T20:09:36.671091+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":279,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.671029+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:09:36.671029+0000","last_peered":"2023-09-23T20:09:36.671029+0000","last_clean":"2023-09-23T20:09:36.671029+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:09:36.671029+0000","last_undegraded":"2023-09-23T20:09:36.671029+0000","last_fullsized":"2023-09-23T20:09:36.671029+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":195,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.675476+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:09:36.675476+0000","last_peered":"2023-09-23T20:09:36.675476+0000","last_clean":"2023-09-23T20:09:36.675476+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:09:36.675476+0000","last_undegraded":"2023-09-23T20:09:36.675476+0000","last_fullsized":"2023-09-23T20:09:36.675476+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":352,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.671705+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:09:36.671705+0000","last_peered":"2023-09-23T20:09:36.671705+0000","last_clean":"2023-09-23T20:09:36.671705+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:09:36.671705+0000","last_undegraded":"2023-09-23T20:09:36.671705+0000","last_fullsized":"2023-09-23T20:09:36.671705+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":214,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670924+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:09:36.670924+0000","last_peered":"2023-09-23T20:09:36.670924+0000","last_clean":"2023-09-23T20:09:36.670924+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:09:36.670924+0000","last_undegraded":"2023-09-23T20:09:36.670924+0000","last_fullsized":"2023-09-23T20:09:36.670924+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":46,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676756+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:09:36.676756+0000","last_peered":"2023-09-23T20:09:36.676756+0000","last_clean":"2023-09-23T20:09:36.676756+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:09:36.676756+0000","last_undegraded":"2023-09-23T20:09:36.676756+0000","last_fullsized":"2023-09-23T20:09:36.676756+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:37.726970+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:09:37.726970+0000","last_peered":"2023-09-23T20:09:37.726970+0000","last_clean":"2023-09-23T20:09:37.726970+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:09:37.726970+0000","last_undegraded":"2023-09-23T20:09:37.726970+0000","last_fullsized":"2023-09-23T20:09:37.726970+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":407,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.683836+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:09:36.683836+0000","last_peered":"2023-09-23T20:09:36.683836+0000","last_clean":"2023-09-23T20:09:36.683836+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:09:36.683836+0000","last_undegraded":"2023-09-23T20:09:36.683836+0000","last_fullsized":"2023-09-23T20:09:36.683836+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":349,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.681163+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:09:36.681163+0000","last_peered":"2023-09-23T20:09:36.681163+0000","last_clean":"2023-09-23T20:09:36.681163+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:09:36.681163+0000","last_undegraded":"2023-09-23T20:09:36.681163+0000","last_fullsized":"2023-09-23T20:09:36.681163+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":46,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676439+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:09:36.676439+0000","last_peered":"2023-09-23T20:09:36.676439+0000","last_clean":"2023-09-23T20:09:36.676439+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:09:36.676439+0000","last_undegraded":"2023-09-23T20:09:36.676439+0000","last_fullsized":"2023-09-23T20:09:36.676439+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":204,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.673884+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:09:36.673884+0000","last_peered":"2023-09-23T20:09:36.673884+0000","last_clean":"2023-09-23T20:09:36.673884+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:09:36.673884+0000","last_undegraded":"2023-09-23T20:09:36.673884+0000","last_fullsized":"2023-09-23T20:09:36.673884+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676316+0000","last_change":"2023-09-23T20:09:02.653351+0000","last_active":"2023-09-23T20:09:36.676316+0000","last_peered":"2023-09-23T20:09:36.676316+0000","last_clean":"2023-09-23T20:09:36.676316+0000","last_became_active":"2023-09-23T20:09:02.653005+0000","last_became_peered":"2023-09-23T20:09:02.653005+0000","last_unstale":"2023-09-23T20:09:36.676316+0000","last_undegraded":"2023-09-23T20:09:36.676316+0000","last_fullsized":"2023-09-23T20:09:36.676316+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:39.222451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":351,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670737+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:09:36.670737+0000","last_peered":"2023-09-23T20:09:36.670737+0000","last_clean":"2023-09-23T20:09:36.670737+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:09:36.670737+0000","last_undegraded":"2023-09-23T20:09:36.670737+0000","last_fullsized":"2023-09-23T20:09:36.670737+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":40,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676261+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:09:36.676261+0000","last_peered":"2023-09-23T20:09:36.676261+0000","last_clean":"2023-09-23T20:09:36.676261+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:09:36.676261+0000","last_undegraded":"2023-09-23T20:09:36.676261+0000","last_fullsized":"2023-09-23T20:09:36.676261+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":338,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.675138+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:09:36.675138+0000","last_peered":"2023-09-23T20:09:36.675138+0000","last_clean":"2023-09-23T20:09:36.675138+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:09:36.675138+0000","last_undegraded":"2023-09-23T20:09:36.675138+0000","last_fullsized":"2023-09-23T20:09:36.675138+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":134,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.675277+0000","last_change":"2023-09-23T20:09:02.939926+0000","last_active":"2023-09-23T20:09:36.675277+0000","last_peered":"2023-09-23T20:09:36.675277+0000","last_clean":"2023-09-23T20:09:36.675277+0000","last_became_active":"2023-09-23T20:09:02.654234+0000","last_became_peered":"2023-09-23T20:09:02.654234+0000","last_unstale":"2023-09-23T20:09:36.675277+0000","last_undegraded":"2023-09-23T20:09:36.675277+0000","last_fullsized":"2023-09-23T20:09:36.675277+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:20.747680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":112,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676590+0000","last_change":"2023-09-23T20:09:02.935358+0000","last_active":"2023-09-23T20:09:36.676590+0000","last_peered":"2023-09-23T20:09:36.676590+0000","last_clean":"2023-09-23T20:09:36.676590+0000","last_became_active":"2023-09-23T20:09:02.935154+0000","last_became_peered":"2023-09-23T20:09:02.935154+0000","last_unstale":"2023-09-23T20:09:36.676590+0000","last_undegraded":"2023-09-23T20:09:36.676590+0000","last_fullsized":"2023-09-23T20:09:36.676590+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:27.125673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"353'7456","reported_seq":7518,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:37.913859+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:09:37.913859+0000","last_peered":"2023-09-23T20:09:37.913859+0000","last_clean":"2023-09-23T20:09:37.913859+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:09:37.913859+0000","last_undegraded":"2023-09-23T20:09:37.913859+0000","last_fullsized":"2023-09-23T20:09:37.913859+0000","mapping_epoch":299,"log_start":"309'2665","ondisk_log_start":"309'2665","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4791,"log_dups_size":0,"ondisk_log_size":4791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214564864,"num_objects":3274,"num_object_clones":0,"num_object_copies":9822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7456,"num_write_kb":343360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":44,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676664+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:09:36.676664+0000","last_peered":"2023-09-23T20:09:36.676664+0000","last_clean":"2023-09-23T20:09:36.676664+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:09:36.676664+0000","last_undegraded":"2023-09-23T20:09:36.676664+0000","last_fullsized":"2023-09-23T20:09:36.676664+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":361,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670341+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:09:36.670341+0000","last_peered":"2023-09-23T20:09:36.670341+0000","last_clean":"2023-09-23T20:09:36.670341+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:09:36.670341+0000","last_undegraded":"2023-09-23T20:09:36.670341+0000","last_fullsized":"2023-09-23T20:09:36.670341+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"353'7131","reported_seq":9787,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:38.082584+0000","last_change":"2023-09-23T20:07:45.697314+0000","last_active":"2023-09-23T20:09:38.082584+0000","last_peered":"2023-09-23T20:09:38.082584+0000","last_clean":"2023-09-23T20:09:38.082584+0000","last_became_active":"2023-09-23T20:07:45.690354+0000","last_became_peered":"2023-09-23T20:07:45.690354+0000","last_unstale":"2023-09-23T20:09:38.082584+0000","last_undegraded":"2023-09-23T20:09:38.082584+0000","last_fullsized":"2023-09-23T20:09:38.082584+0000","mapping_epoch":327,"log_start":"306'1459","ondisk_log_start":"306'1459","created":299,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5672,"log_dups_size":0,"ondisk_log_size":5672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.926631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208601088,"num_objects":3183,"num_object_clones":0,"num_object_copies":9549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7131,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":45,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676546+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:09:36.676546+0000","last_peered":"2023-09-23T20:09:36.676546+0000","last_clean":"2023-09-23T20:09:36.676546+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:09:36.676546+0000","last_undegraded":"2023-09-23T20:09:36.676546+0000","last_fullsized":"2023-09-23T20:09:36.676546+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":399,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.680698+0000","last_change":"2023-09-23T20:09:02.653936+0000","last_active":"2023-09-23T20:09:36.680698+0000","last_peered":"2023-09-23T20:09:36.680698+0000","last_clean":"2023-09-23T20:09:36.680698+0000","last_became_active":"2023-09-23T20:09:02.653232+0000","last_became_peered":"2023-09-23T20:09:02.653232+0000","last_unstale":"2023-09-23T20:09:36.680698+0000","last_undegraded":"2023-09-23T20:09:36.680698+0000","last_fullsized":"2023-09-23T20:09:36.680698+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:23.060335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"353'7480","reported_seq":9066,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:09:37.910641+0000","last_change":"2023-09-23T20:07:37.064417+0000","last_active":"2023-09-23T20:09:37.910641+0000","last_peered":"2023-09-23T20:09:37.910641+0000","last_clean":"2023-09-23T20:06:59.831272+0000","last_became_active":"2023-09-23T20:07:37.059682+0000","last_became_peered":"2023-09-23T20:07:37.059682+0000","last_unstale":"2023-09-23T20:09:37.910641+0000","last_undegraded":"2023-09-23T20:07:36.983730+0000","last_fullsized":"2023-09-23T20:07:36.974121+0000","mapping_epoch":322,"log_start":"307'1909","ondisk_log_start":"307'1909","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5571,"log_dups_size":0,"ondisk_log_size":5571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:33.575626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218103808,"num_objects":3328,"num_object_clones":0,"num_object_copies":9984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7480,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":1937},{"shards":"0,2,3","objects":1391}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":44,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676649+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:09:36.676649+0000","last_peered":"2023-09-23T20:09:36.676649+0000","last_clean":"2023-09-23T20:09:36.676649+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:09:36.676649+0000","last_undegraded":"2023-09-23T20:09:36.676649+0000","last_fullsized":"2023-09-23T20:09:36.676649+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":57,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676272+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:09:36.676272+0000","last_peered":"2023-09-23T20:09:36.676272+0000","last_clean":"2023-09-23T20:09:36.676272+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:09:36.676272+0000","last_undegraded":"2023-09-23T20:09:36.676272+0000","last_fullsized":"2023-09-23T20:09:36.676272+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"353'7328","reported_seq":7832,"reported_epoch":353,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:09:41.776649+0000","last_change":"2023-09-23T20:09:18.013029+0000","last_active":"2023-09-23T20:09:41.776649+0000","last_peered":"2023-09-23T20:09:41.776649+0000","last_clean":"2023-09-23T20:07:17.891274+0000","last_became_active":"2023-09-23T20:09:16.552874+0000","last_became_peered":"2023-09-23T20:09:16.552874+0000","last_unstale":"2023-09-23T20:09:41.776649+0000","last_undegraded":"2023-09-23T20:09:41.776649+0000","last_fullsized":"2023-09-23T20:09:03.653890+0000","mapping_epoch":342,"log_start":"307'1637","ondisk_log_start":"307'1637","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5691,"log_dups_size":0,"ondisk_log_size":5691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:56.551539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210370560,"num_objects":3210,"num_object_clones":0,"num_object_copies":9630,"num_objects_missing_on_primary":3868,"num_objects_missing":3868,"num_objects_degraded":0,"num_objects_misplaced":3204,"num_objects_unfound":0,"num_objects_dirty":3210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7328,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":214,"num_bytes_recovered":14024704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"1,2,3","objects":1181},{"shards":"1,2,3,4","objects":6},{"shards":"2,3,4","objects":2023}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":391,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670733+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:09:36.670733+0000","last_peered":"2023-09-23T20:09:36.670733+0000","last_clean":"2023-09-23T20:09:36.670733+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:09:36.670733+0000","last_undegraded":"2023-09-23T20:09:36.670733+0000","last_fullsized":"2023-09-23T20:09:36.670733+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":48,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.712418+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:09:36.712418+0000","last_peered":"2023-09-23T20:09:36.712418+0000","last_clean":"2023-09-23T20:09:36.712418+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:09:36.712418+0000","last_undegraded":"2023-09-23T20:09:36.712418+0000","last_fullsized":"2023-09-23T20:09:36.712418+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"353'7106","reported_seq":7950,"reported_epoch":353,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:09:41.087938+0000","last_change":"2023-09-23T20:09:35.845295+0000","last_active":"2023-09-23T20:09:41.087938+0000","last_peered":"2023-09-23T20:09:41.087938+0000","last_clean":"2023-09-23T20:07:17.769108+0000","last_became_active":"2023-09-23T20:07:20.672993+0000","last_became_peered":"2023-09-23T20:07:20.672993+0000","last_unstale":"2023-09-23T20:09:41.087938+0000","last_undegraded":"2023-09-23T20:07:37.145948+0000","last_fullsized":"2023-09-23T20:07:19.923439+0000","mapping_epoch":317,"log_start":"308'2400","ondisk_log_start":"308'2400","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4706,"log_dups_size":0,"ondisk_log_size":4706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:56.185131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209846272,"num_objects":3202,"num_object_clones":0,"num_object_copies":9606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1926,"num_objects_misplaced":1693,"num_objects_unfound":0,"num_objects_dirty":3202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7106,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":384,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1693},{"shards":"3,5","objects":-417}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":735,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676262+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:09:36.676262+0000","last_peered":"2023-09-23T20:09:36.676262+0000","last_clean":"2023-09-23T20:09:36.676262+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:09:36.676262+0000","last_undegraded":"2023-09-23T20:09:36.676262+0000","last_fullsized":"2023-09-23T20:09:36.676262+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":43,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.681115+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:09:36.681115+0000","last_peered":"2023-09-23T20:09:36.681115+0000","last_clean":"2023-09-23T20:09:36.681115+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:09:36.681115+0000","last_undegraded":"2023-09-23T20:09:36.681115+0000","last_fullsized":"2023-09-23T20:09:36.681115+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":379,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.683784+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:09:36.683784+0000","last_peered":"2023-09-23T20:09:36.683784+0000","last_clean":"2023-09-23T20:09:36.683784+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:09:36.683784+0000","last_undegraded":"2023-09-23T20:09:36.683784+0000","last_fullsized":"2023-09-23T20:09:36.683784+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"354'7342","reported_seq":13378,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:09:42.391140+0000","last_change":"2023-09-23T20:09:02.928277+0000","last_active":"2023-09-23T20:09:42.391140+0000","last_peered":"2023-09-23T20:09:42.391140+0000","last_clean":"2023-09-23T20:07:17.847486+0000","last_became_active":"2023-09-23T20:07:20.666556+0000","last_became_peered":"2023-09-23T20:07:20.666556+0000","last_unstale":"2023-09-23T20:09:42.391140+0000","last_undegraded":"2023-09-23T20:09:02.927042+0000","last_fullsized":"2023-09-23T20:07:19.932561+0000","mapping_epoch":317,"log_start":"307'1972","ondisk_log_start":"307'1972","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5370,"log_dups_size":0,"ondisk_log_size":5370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:38.441814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206962688,"num_objects":3158,"num_object_clones":0,"num_object_copies":9474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1159,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7342,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2982,"num_bytes_recovered":195428352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":1999},{"shards":"0,2,5","objects":1159}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":435,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:09:36.671471+0000","last_change":"2023-09-23T20:09:03.664597+0000","last_active":"2023-09-23T20:09:36.671471+0000","last_peered":"2023-09-23T20:09:36.671471+0000","last_clean":"2023-09-23T19:45:37.119152+0000","last_became_active":"2023-09-23T20:09:03.664321+0000","last_became_peered":"2023-09-23T20:09:03.664321+0000","last_unstale":"2023-09-23T20:09:36.671471+0000","last_undegraded":"2023-09-23T20:09:03.654896+0000","last_fullsized":"2023-09-23T20:09:03.654523+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3","objects":-1},{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":319,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.680982+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:09:36.680982+0000","last_peered":"2023-09-23T20:09:36.680982+0000","last_clean":"2023-09-23T20:09:36.680982+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:09:36.680982+0000","last_undegraded":"2023-09-23T20:09:36.680982+0000","last_fullsized":"2023-09-23T20:09:36.680982+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":349,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.683879+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:09:36.683879+0000","last_peered":"2023-09-23T20:09:36.683879+0000","last_clean":"2023-09-23T20:09:36.683879+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:09:36.683879+0000","last_undegraded":"2023-09-23T20:09:36.683879+0000","last_fullsized":"2023-09-23T20:09:36.683879+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"353'7296","reported_seq":7358,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:37.906686+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:09:37.906686+0000","last_peered":"2023-09-23T20:09:37.906686+0000","last_clean":"2023-09-23T20:09:37.906686+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:09:37.906686+0000","last_undegraded":"2023-09-23T20:09:37.906686+0000","last_fullsized":"2023-09-23T20:09:37.906686+0000","mapping_epoch":299,"log_start":"306'1615","ondisk_log_start":"306'1615","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5681,"log_dups_size":0,"ondisk_log_size":5681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207355904,"num_objects":3164,"num_object_clones":0,"num_object_copies":9492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7296,"num_write_kb":334720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":343,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.680593+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:09:36.680593+0000","last_peered":"2023-09-23T20:09:36.680593+0000","last_clean":"2023-09-23T20:09:36.680593+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:09:36.680593+0000","last_undegraded":"2023-09-23T20:09:36.680593+0000","last_fullsized":"2023-09-23T20:09:36.680593+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":379,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.683950+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:09:36.683950+0000","last_peered":"2023-09-23T20:09:36.683950+0000","last_clean":"2023-09-23T20:09:36.683950+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:09:36.683950+0000","last_undegraded":"2023-09-23T20:09:36.683950+0000","last_fullsized":"2023-09-23T20:09:36.683950+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"353'7177","reported_seq":15298,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:40.703009+0000","last_change":"2023-09-23T20:09:37.763558+0000","last_active":"2023-09-23T20:09:40.703009+0000","last_peered":"2023-09-23T20:09:40.703009+0000","last_clean":"2023-09-23T20:09:40.703009+0000","last_became_active":"2023-09-23T20:09:37.732414+0000","last_became_peered":"2023-09-23T20:09:37.732414+0000","last_unstale":"2023-09-23T20:09:40.703009+0000","last_undegraded":"2023-09-23T20:09:40.703009+0000","last_fullsized":"2023-09-23T20:09:40.703009+0000","mapping_epoch":352,"log_start":"306'1515","ondisk_log_start":"306'1515","created":299,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:39.792571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204668928,"num_objects":3123,"num_object_clones":0,"num_object_copies":9369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7177,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4021,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":691,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670945+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:09:36.670945+0000","last_peered":"2023-09-23T20:09:36.670945+0000","last_clean":"2023-09-23T20:09:36.670945+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:09:36.670945+0000","last_undegraded":"2023-09-23T20:09:36.670945+0000","last_fullsized":"2023-09-23T20:09:36.670945+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":111,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676450+0000","last_change":"2023-09-23T20:09:02.652469+0000","last_active":"2023-09-23T20:09:36.676450+0000","last_peered":"2023-09-23T20:09:36.676450+0000","last_clean":"2023-09-23T20:09:36.676450+0000","last_became_active":"2023-09-23T20:09:02.652158+0000","last_became_peered":"2023-09-23T20:09:02.652158+0000","last_unstale":"2023-09-23T20:09:36.676450+0000","last_undegraded":"2023-09-23T20:09:36.676450+0000","last_fullsized":"2023-09-23T20:09:36.676450+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:13.467939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":44,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.737459+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:09:36.737459+0000","last_peered":"2023-09-23T20:09:36.737459+0000","last_clean":"2023-09-23T20:09:36.737459+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:09:36.737459+0000","last_undegraded":"2023-09-23T20:09:36.737459+0000","last_fullsized":"2023-09-23T20:09:36.737459+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"353'7370","reported_seq":7434,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:38.974466+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:09:38.974466+0000","last_peered":"2023-09-23T20:09:38.974466+0000","last_clean":"2023-09-23T20:09:38.974466+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:09:38.974466+0000","last_undegraded":"2023-09-23T20:09:38.974466+0000","last_fullsized":"2023-09-23T20:09:38.974466+0000","mapping_epoch":299,"log_start":"307'1796","ondisk_log_start":"307'1796","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5574,"log_dups_size":0,"ondisk_log_size":5574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209190912,"num_objects":3192,"num_object_clones":0,"num_object_copies":9576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7370,"num_write_kb":337984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":700,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676545+0000","last_change":"2023-09-23T20:09:18.024224+0000","last_active":"2023-09-23T20:09:36.676545+0000","last_peered":"2023-09-23T20:09:36.676545+0000","last_clean":"2023-09-23T20:09:36.676545+0000","last_became_active":"2023-09-23T20:09:18.023811+0000","last_became_peered":"2023-09-23T20:09:18.023811+0000","last_unstale":"2023-09-23T20:09:36.676545+0000","last_undegraded":"2023-09-23T20:09:36.676545+0000","last_fullsized":"2023-09-23T20:09:36.676545+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:20.046957+0000","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":485,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674408+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:09:36.674408+0000","last_peered":"2023-09-23T20:09:36.674408+0000","last_clean":"2023-09-23T20:09:36.674408+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:09:36.674408+0000","last_undegraded":"2023-09-23T20:09:36.674408+0000","last_fullsized":"2023-09-23T20:09:36.674408+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":353,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670708+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:09:36.670708+0000","last_peered":"2023-09-23T20:09:36.670708+0000","last_clean":"2023-09-23T20:09:36.670708+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:09:36.670708+0000","last_undegraded":"2023-09-23T20:09:36.670708+0000","last_fullsized":"2023-09-23T20:09:36.670708+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676078+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:09:36.676078+0000","last_peered":"2023-09-23T20:09:36.676078+0000","last_clean":"2023-09-23T20:09:36.676078+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:09:36.676078+0000","last_undegraded":"2023-09-23T20:09:36.676078+0000","last_fullsized":"2023-09-23T20:09:36.676078+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"353'7303","reported_seq":12094,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:38.069631+0000","last_change":"2023-09-23T20:07:52.783215+0000","last_active":"2023-09-23T20:09:38.069631+0000","last_peered":"2023-09-23T20:09:38.069631+0000","last_clean":"2023-09-23T20:09:38.069631+0000","last_became_active":"2023-09-23T20:07:52.778874+0000","last_became_peered":"2023-09-23T20:07:52.778874+0000","last_unstale":"2023-09-23T20:09:38.069631+0000","last_undegraded":"2023-09-23T20:09:38.069631+0000","last_fullsized":"2023-09-23T20:09:38.069631+0000","mapping_epoch":330,"log_start":"308'2400","ondisk_log_start":"308'2400","created":299,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4903,"log_dups_size":0,"ondisk_log_size":4903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:31.036337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211091456,"num_objects":3221,"num_object_clones":0,"num_object_copies":9663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7303,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2349,"num_bytes_recovered":153944064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":664,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670434+0000","last_change":"2023-09-23T19:52:57.794800+0000","last_active":"2023-09-23T20:09:36.670434+0000","last_peered":"2023-09-23T20:09:36.670434+0000","last_clean":"2023-09-23T20:09:36.670434+0000","last_became_active":"2023-09-23T19:52:57.351426+0000","last_became_peered":"2023-09-23T19:52:57.351426+0000","last_unstale":"2023-09-23T20:09:36.670434+0000","last_undegraded":"2023-09-23T20:09:36.670434+0000","last_fullsized":"2023-09-23T20:09:36.670434+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:40.042226+0000","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":973,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670813+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:09:36.670813+0000","last_peered":"2023-09-23T20:09:36.670813+0000","last_clean":"2023-09-23T20:09:36.670813+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:09:36.670813+0000","last_undegraded":"2023-09-23T20:09:36.670813+0000","last_fullsized":"2023-09-23T20:09:36.670813+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":372,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.681239+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:09:36.681239+0000","last_peered":"2023-09-23T20:09:36.681239+0000","last_clean":"2023-09-23T20:09:36.681239+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:09:36.681239+0000","last_undegraded":"2023-09-23T20:09:36.681239+0000","last_fullsized":"2023-09-23T20:09:36.681239+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":48,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674327+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:09:36.674327+0000","last_peered":"2023-09-23T20:09:36.674327+0000","last_clean":"2023-09-23T20:09:36.674327+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:09:36.674327+0000","last_undegraded":"2023-09-23T20:09:36.674327+0000","last_fullsized":"2023-09-23T20:09:36.674327+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"353'7165","reported_seq":7477,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:09:38.376701+0000","last_change":"2023-09-23T20:09:03.827876+0000","last_active":"2023-09-23T20:09:38.376701+0000","last_peered":"2023-09-23T20:09:38.376701+0000","last_clean":"2023-09-23T20:07:17.918417+0000","last_became_active":"2023-09-23T20:09:03.809396+0000","last_became_peered":"2023-09-23T20:09:03.809396+0000","last_unstale":"2023-09-23T20:09:38.376701+0000","last_undegraded":"2023-09-23T20:09:03.761556+0000","last_fullsized":"2023-09-23T20:09:03.662140+0000","mapping_epoch":342,"log_start":"308'2400","ondisk_log_start":"308'2400","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4765,"log_dups_size":0,"ondisk_log_size":4765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:30.207351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209649664,"num_objects":3199,"num_object_clones":0,"num_object_copies":9597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6933,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7165,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":7602176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"1,3,4","objects":1165},{"shards":"2,3,4","objects":2034}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":553,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676421+0000","last_change":"2023-09-23T20:09:29.459954+0000","last_active":"2023-09-23T20:09:36.676421+0000","last_peered":"2023-09-23T20:09:36.676421+0000","last_clean":"2023-09-23T20:09:36.676421+0000","last_became_active":"2023-09-23T20:07:40.029679+0000","last_became_peered":"2023-09-23T20:07:40.029679+0000","last_unstale":"2023-09-23T20:09:36.676421+0000","last_undegraded":"2023-09-23T20:09:36.676421+0000","last_fullsized":"2023-09-23T20:09:36.676421+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":988,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.673015+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:09:36.673015+0000","last_peered":"2023-09-23T20:09:36.673015+0000","last_clean":"2023-09-23T20:09:36.673015+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:09:36.673015+0000","last_undegraded":"2023-09-23T20:09:36.673015+0000","last_fullsized":"2023-09-23T20:09:36.673015+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676536+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:09:36.676536+0000","last_peered":"2023-09-23T20:09:36.676536+0000","last_clean":"2023-09-23T20:09:36.676536+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:09:36.676536+0000","last_undegraded":"2023-09-23T20:09:36.676536+0000","last_fullsized":"2023-09-23T20:09:36.676536+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":44,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.731589+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:09:36.731589+0000","last_peered":"2023-09-23T20:09:36.731589+0000","last_clean":"2023-09-23T20:09:36.731589+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:09:36.731589+0000","last_undegraded":"2023-09-23T20:09:36.731589+0000","last_fullsized":"2023-09-23T20:09:36.731589+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"353'7354","reported_seq":7416,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:38.069436+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:09:38.069436+0000","last_peered":"2023-09-23T20:09:38.069436+0000","last_clean":"2023-09-23T20:09:38.069436+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:09:38.069436+0000","last_undegraded":"2023-09-23T20:09:38.069436+0000","last_fullsized":"2023-09-23T20:09:38.069436+0000","mapping_epoch":299,"log_start":"308'2565","ondisk_log_start":"308'2565","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4789,"log_dups_size":0,"ondisk_log_size":4789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210632704,"num_objects":3214,"num_object_clones":0,"num_object_copies":9642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7354,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1261,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670957+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:09:36.670957+0000","last_peered":"2023-09-23T20:09:36.670957+0000","last_clean":"2023-09-23T20:09:36.670957+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:09:36.670957+0000","last_undegraded":"2023-09-23T20:09:36.670957+0000","last_fullsized":"2023-09-23T20:09:36.670957+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":484,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676273+0000","last_change":"2023-09-23T20:09:31.457986+0000","last_active":"2023-09-23T20:09:36.676273+0000","last_peered":"2023-09-23T20:09:36.676273+0000","last_clean":"2023-09-23T20:09:36.676273+0000","last_became_active":"2023-09-23T20:07:40.035955+0000","last_became_peered":"2023-09-23T20:07:40.035955+0000","last_unstale":"2023-09-23T20:09:36.676273+0000","last_undegraded":"2023-09-23T20:09:36.676273+0000","last_fullsized":"2023-09-23T20:09:36.676273+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":218,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670756+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:09:36.670756+0000","last_peered":"2023-09-23T20:09:36.670756+0000","last_clean":"2023-09-23T20:09:36.670756+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:09:36.670756+0000","last_undegraded":"2023-09-23T20:09:36.670756+0000","last_fullsized":"2023-09-23T20:09:36.670756+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":248,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.684036+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:09:36.684036+0000","last_peered":"2023-09-23T20:09:36.684036+0000","last_clean":"2023-09-23T20:09:36.684036+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:09:36.684036+0000","last_undegraded":"2023-09-23T20:09:36.684036+0000","last_fullsized":"2023-09-23T20:09:36.684036+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"353'7386","reported_seq":7454,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:09:38.908198+0000","last_change":"2023-09-23T20:07:20.707475+0000","last_active":"2023-09-23T20:09:38.908198+0000","last_peered":"2023-09-23T20:09:38.908198+0000","last_clean":"2023-09-23T20:07:17.907438+0000","last_became_active":"2023-09-23T20:07:20.693608+0000","last_became_peered":"2023-09-23T20:07:20.693608+0000","last_unstale":"2023-09-23T20:09:38.908198+0000","last_undegraded":"2023-09-23T20:07:19.959595+0000","last_fullsized":"2023-09-23T20:07:19.924202+0000","mapping_epoch":317,"log_start":"307'1825","ondisk_log_start":"307'1825","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5561,"log_dups_size":0,"ondisk_log_size":5561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:21.022797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212992000,"num_objects":3250,"num_object_clones":0,"num_object_copies":9750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4180,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7386,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":2112},{"shards":"2,5","objects":-930}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":469,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674730+0000","last_change":"2023-09-23T20:07:21.995787+0000","last_active":"2023-09-23T20:09:36.674730+0000","last_peered":"2023-09-23T20:09:36.674730+0000","last_clean":"2023-09-23T20:09:36.674730+0000","last_became_active":"2023-09-23T20:07:21.995407+0000","last_became_peered":"2023-09-23T20:07:21.995407+0000","last_unstale":"2023-09-23T20:09:36.674730+0000","last_undegraded":"2023-09-23T20:09:36.674730+0000","last_fullsized":"2023-09-23T20:09:36.674730+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":822,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:09:37.671745+0000","last_change":"2023-09-23T20:07:38.010675+0000","last_active":"2023-09-23T20:09:37.671745+0000","last_peered":"2023-09-23T20:09:37.671745+0000","last_clean":"2023-09-23T19:43:07.790771+0000","last_became_active":"2023-09-23T20:07:38.009799+0000","last_became_peered":"2023-09-23T20:07:38.009799+0000","last_unstale":"2023-09-23T20:09:37.671745+0000","last_undegraded":"2023-09-23T20:07:37.988964+0000","last_fullsized":"2023-09-23T20:07:37.988167+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:50.851157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":329,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.680781+0000","last_change":"2023-09-23T20:09:02.654113+0000","last_active":"2023-09-23T20:09:36.680781+0000","last_peered":"2023-09-23T20:09:36.680781+0000","last_clean":"2023-09-23T20:09:36.680781+0000","last_became_active":"2023-09-23T20:09:02.653626+0000","last_became_peered":"2023-09-23T20:09:02.653626+0000","last_unstale":"2023-09-23T20:09:36.680781+0000","last_undegraded":"2023-09-23T20:09:36.680781+0000","last_fullsized":"2023-09-23T20:09:36.680781+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:42.754098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":48,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.675551+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:09:36.675551+0000","last_peered":"2023-09-23T20:09:36.675551+0000","last_clean":"2023-09-23T20:09:36.675551+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:09:36.675551+0000","last_undegraded":"2023-09-23T20:09:36.675551+0000","last_fullsized":"2023-09-23T20:09:36.675551+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"353'7483","reported_seq":11541,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:09:41.637188+0000","last_change":"2023-09-23T20:09:17.760517+0000","last_active":"2023-09-23T20:09:41.637188+0000","last_peered":"2023-09-23T20:09:41.637188+0000","last_clean":"2023-09-23T20:07:17.917130+0000","last_became_active":"2023-09-23T20:09:16.561110+0000","last_became_peered":"2023-09-23T20:09:16.561110+0000","last_unstale":"2023-09-23T20:09:41.637188+0000","last_undegraded":"2023-09-23T20:09:16.413378+0000","last_fullsized":"2023-09-23T20:09:03.673314+0000","mapping_epoch":342,"log_start":"307'1794","ondisk_log_start":"307'1794","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:13.844790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209518592,"num_objects":3197,"num_object_clones":0,"num_object_copies":9591,"num_objects_missing_on_primary":2169,"num_objects_missing":2169,"num_objects_degraded":5358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7483,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":129826816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"1,2,3","objects":1182},{"shards":"1,2,3,4","objects":8},{"shards":"2,3,4","objects":2007}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":530,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.684040+0000","last_change":"2023-09-23T20:08:04.756122+0000","last_active":"2023-09-23T20:09:36.684040+0000","last_peered":"2023-09-23T20:09:36.684040+0000","last_clean":"2023-09-23T20:09:36.684040+0000","last_became_active":"2023-09-23T20:08:04.755767+0000","last_became_peered":"2023-09-23T20:08:04.755767+0000","last_unstale":"2023-09-23T20:09:36.684040+0000","last_undegraded":"2023-09-23T20:09:36.684040+0000","last_fullsized":"2023-09-23T20:09:36.684040+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:16:29.799074+0000","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":53,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676659+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:09:36.676659+0000","last_peered":"2023-09-23T20:09:36.676659+0000","last_clean":"2023-09-23T20:09:36.676659+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:09:36.676659+0000","last_undegraded":"2023-09-23T20:09:36.676659+0000","last_fullsized":"2023-09-23T20:09:36.676659+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674256+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:09:36.674256+0000","last_peered":"2023-09-23T20:09:36.674256+0000","last_clean":"2023-09-23T20:09:36.674256+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:09:36.674256+0000","last_undegraded":"2023-09-23T20:09:36.674256+0000","last_fullsized":"2023-09-23T20:09:36.674256+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":179,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.671036+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:09:36.671036+0000","last_peered":"2023-09-23T20:09:36.671036+0000","last_clean":"2023-09-23T20:09:36.671036+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:09:36.671036+0000","last_undegraded":"2023-09-23T20:09:36.671036+0000","last_fullsized":"2023-09-23T20:09:36.671036+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"353'7424","reported_seq":11684,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:38.090024+0000","last_change":"2023-09-23T20:08:33.396812+0000","last_active":"2023-09-23T20:09:38.090024+0000","last_peered":"2023-09-23T20:09:38.090024+0000","last_clean":"2023-09-23T20:09:38.090024+0000","last_became_active":"2023-09-23T20:08:05.774750+0000","last_became_peered":"2023-09-23T20:08:05.774750+0000","last_unstale":"2023-09-23T20:09:38.090024+0000","last_undegraded":"2023-09-23T20:09:38.090024+0000","last_fullsized":"2023-09-23T20:09:38.090024+0000","mapping_epoch":334,"log_start":"306'800","ondisk_log_start":"306'800","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6624,"log_dups_size":0,"ondisk_log_size":6624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.633506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211156992,"num_objects":3222,"num_object_clones":0,"num_object_copies":9666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7424,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2086,"num_bytes_recovered":136708096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":751,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.683790+0000","last_change":"2023-09-23T20:09:05.666351+0000","last_active":"2023-09-23T20:09:36.683790+0000","last_peered":"2023-09-23T20:09:36.683790+0000","last_clean":"2023-09-23T20:09:36.683790+0000","last_became_active":"2023-09-23T20:09:05.666014+0000","last_became_peered":"2023-09-23T20:09:05.666014+0000","last_unstale":"2023-09-23T20:09:36.683790+0000","last_undegraded":"2023-09-23T20:09:36.683790+0000","last_fullsized":"2023-09-23T20:09:36.683790+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:47.481729+0000","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":195,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676503+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:09:36.676503+0000","last_peered":"2023-09-23T20:09:36.676503+0000","last_clean":"2023-09-23T20:09:36.676503+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:09:36.676503+0000","last_undegraded":"2023-09-23T20:09:36.676503+0000","last_fullsized":"2023-09-23T20:09:36.676503+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":389,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.675507+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:09:36.675507+0000","last_peered":"2023-09-23T20:09:36.675507+0000","last_clean":"2023-09-23T20:09:36.675507+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:09:36.675507+0000","last_undegraded":"2023-09-23T20:09:36.675507+0000","last_fullsized":"2023-09-23T20:09:36.675507+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676123+0000","last_change":"2023-09-23T20:09:02.654900+0000","last_active":"2023-09-23T20:09:36.676123+0000","last_peered":"2023-09-23T20:09:36.676123+0000","last_clean":"2023-09-23T20:09:36.676123+0000","last_became_active":"2023-09-23T20:09:02.654672+0000","last_became_peered":"2023-09-23T20:09:02.654672+0000","last_unstale":"2023-09-23T20:09:36.676123+0000","last_undegraded":"2023-09-23T20:09:36.676123+0000","last_fullsized":"2023-09-23T20:09:36.676123+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:38.124348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"353'7282","reported_seq":10013,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:38.066566+0000","last_change":"2023-09-23T20:09:08.683160+0000","last_active":"2023-09-23T20:09:38.066566+0000","last_peered":"2023-09-23T20:09:38.066566+0000","last_clean":"2023-09-23T20:09:38.066566+0000","last_became_active":"2023-09-23T20:09:08.670527+0000","last_became_peered":"2023-09-23T20:09:08.670527+0000","last_unstale":"2023-09-23T20:09:38.066566+0000","last_undegraded":"2023-09-23T20:09:38.066566+0000","last_fullsized":"2023-09-23T20:09:38.066566+0000","mapping_epoch":346,"log_start":"308'2400","ondisk_log_start":"308'2400","created":299,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:47.531381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212402200,"num_objects":3242,"num_object_clones":0,"num_object_copies":9726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3242,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7282,"num_write_kb":336705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":653,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:09:37.678312+0000","last_change":"2023-09-23T20:09:37.678312+0000","last_active":"2023-09-23T20:09:37.678312+0000","last_peered":"2023-09-23T20:09:37.678312+0000","last_clean":"2023-09-23T20:09:37.678312+0000","last_became_active":"2023-09-23T20:09:37.677534+0000","last_became_peered":"2023-09-23T20:09:37.677534+0000","last_unstale":"2023-09-23T20:09:37.678312+0000","last_undegraded":"2023-09-23T20:09:37.678312+0000","last_fullsized":"2023-09-23T20:09:37.678312+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:33.062873+0000","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":54,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.676633+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:09:36.676633+0000","last_peered":"2023-09-23T20:09:36.676633+0000","last_clean":"2023-09-23T20:09:36.676633+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:09:36.676633+0000","last_undegraded":"2023-09-23T20:09:36.676633+0000","last_fullsized":"2023-09-23T20:09:36.676633+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":52,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674901+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:09:36.674901+0000","last_peered":"2023-09-23T20:09:36.674901+0000","last_clean":"2023-09-23T20:09:36.674901+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:09:36.674901+0000","last_undegraded":"2023-09-23T20:09:36.674901+0000","last_fullsized":"2023-09-23T20:09:36.674901+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":312,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.680616+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:09:36.680616+0000","last_peered":"2023-09-23T20:09:36.680616+0000","last_clean":"2023-09-23T20:09:36.680616+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:09:36.680616+0000","last_undegraded":"2023-09-23T20:09:36.680616+0000","last_fullsized":"2023-09-23T20:09:36.680616+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":127,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674853+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:09:36.674853+0000","last_peered":"2023-09-23T20:09:36.674853+0000","last_clean":"2023-09-23T20:09:36.674853+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:09:36.674853+0000","last_undegraded":"2023-09-23T20:09:36.674853+0000","last_fullsized":"2023-09-23T20:09:36.674853+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":112,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.674178+0000","last_change":"2023-09-23T20:09:02.654095+0000","last_active":"2023-09-23T20:09:36.674178+0000","last_peered":"2023-09-23T20:09:36.674178+0000","last_clean":"2023-09-23T20:09:36.674178+0000","last_became_active":"2023-09-23T20:09:02.653713+0000","last_became_peered":"2023-09-23T20:09:02.653713+0000","last_unstale":"2023-09-23T20:09:36.674178+0000","last_undegraded":"2023-09-23T20:09:36.674178+0000","last_fullsized":"2023-09-23T20:09:36.674178+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:04.759813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":356,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670645+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:09:36.670645+0000","last_peered":"2023-09-23T20:09:36.670645+0000","last_clean":"2023-09-23T20:09:36.670645+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:09:36.670645+0000","last_undegraded":"2023-09-23T20:09:36.670645+0000","last_fullsized":"2023-09-23T20:09:36.670645+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":369,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.683730+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:09:36.683730+0000","last_peered":"2023-09-23T20:09:36.683730+0000","last_clean":"2023-09-23T20:09:36.683730+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:09:36.683730+0000","last_undegraded":"2023-09-23T20:09:36.683730+0000","last_fullsized":"2023-09-23T20:09:36.683730+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":346,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670655+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:09:36.670655+0000","last_peered":"2023-09-23T20:09:36.670655+0000","last_clean":"2023-09-23T20:09:36.670655+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:09:36.670655+0000","last_undegraded":"2023-09-23T20:09:36.670655+0000","last_fullsized":"2023-09-23T20:09:36.670655+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":349,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.680935+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:09:36.680935+0000","last_peered":"2023-09-23T20:09:36.680935+0000","last_clean":"2023-09-23T20:09:36.680935+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:09:36.680935+0000","last_undegraded":"2023-09-23T20:09:36.680935+0000","last_fullsized":"2023-09-23T20:09:36.680935+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":52,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.673755+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:09:36.673755+0000","last_peered":"2023-09-23T20:09:36.673755+0000","last_clean":"2023-09-23T20:09:36.673755+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:09:36.673755+0000","last_undegraded":"2023-09-23T20:09:36.673755+0000","last_fullsized":"2023-09-23T20:09:36.673755+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":209,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.673093+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:09:36.673093+0000","last_peered":"2023-09-23T20:09:36.673093+0000","last_clean":"2023-09-23T20:09:36.673093+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:09:36.673093+0000","last_undegraded":"2023-09-23T20:09:36.673093+0000","last_fullsized":"2023-09-23T20:09:36.673093+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":188,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.670982+0000","last_change":"2023-09-23T20:09:02.657265+0000","last_active":"2023-09-23T20:09:36.670982+0000","last_peered":"2023-09-23T20:09:36.670982+0000","last_clean":"2023-09-23T20:09:36.670982+0000","last_became_active":"2023-09-23T20:09:02.653043+0000","last_became_peered":"2023-09-23T20:09:02.653043+0000","last_unstale":"2023-09-23T20:09:36.670982+0000","last_undegraded":"2023-09-23T20:09:36.670982+0000","last_fullsized":"2023-09-23T20:09:36.670982+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:23.580711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":399,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:09:36.680664+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:09:36.680664+0000","last_peered":"2023-09-23T20:09:36.680664+0000","last_clean":"2023-09-23T20:09:36.680664+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:09:36.680664+0000","last_undegraded":"2023-09-23T20:09:36.680664+0000","last_fullsized":"2023-09-23T20:09:36.680664+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":139264,"data_stored":26728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3367108632,"num_objects":51379,"num_object_clones":0,"num_object_copies":154137,"num_objects_missing_on_primary":6037,"num_objects_missing":6037,"num_objects_degraded":20947,"num_objects_misplaced":4897,"num_objects_unfound":0,"num_objects_dirty":51379,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":117083,"num_write_kb":5390721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17482,"num_bytes_recovered":1019740160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11582713856,"data_stored":11582701640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85931,"ondisk_log_size":85931,"up":48,"acting":41,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":542359,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":177,"num_read_kb":331,"num_write":87,"num_write_kb":878,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1299398,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":154,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234342,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1279248,"kb_used_data":955272,"kb_used_omap":0,"kb_used_meta":323968,"kb_avail":92465904,"statfs":{"total":95995035648,"available":94685085696,"internally_reserved":0,"allocated":978198528,"data_stored":977093234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331743232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:09:33 2023","interfaces":[{"interface":"back","average":{"1min":3.9950000000000001,"5min":2.0230000000000001,"15min":2.456},"min":{"1min":0.24299999999999999,"5min":0.20799999999999999,"15min":0.19700000000000001},"max":{"1min":43.597999999999999,"5min":43.597999999999999,"15min":62.061999999999998},"last":0.33200000000000002},{"interface":"front","average":{"1min":3.5219999999999998,"5min":2.0470000000000002,"15min":2.673},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.19900000000000001},"max":{"1min":43.579999999999998,"5min":43.579999999999998,"15min":62.389000000000003},"last":0.23799999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:09:33 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":2.0169999999999999,"15min":3.4500000000000002},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":4.0590000000000002,"5min":26.309999999999999,"15min":104.16800000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.996,"15min":3.5910000000000002},"min":{"1min":0.28399999999999997,"5min":0.223,"15min":0.21199999999999999},"max":{"1min":6.1029999999999998,"5min":26.545000000000002,"15min":104.003},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:08:44 2023","interfaces":[{"interface":"back","average":{"1min":7.601,"5min":4.5460000000000003,"15min":4.1159999999999997},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":92.319000000000003,"5min":92.319000000000003,"15min":130.673},"last":0.39300000000000002},{"interface":"front","average":{"1min":8.5449999999999999,"5min":5.1479999999999997,"15min":4.5759999999999996},"min":{"1min":0.44400000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":92.049000000000007,"5min":92.049000000000007,"15min":130.71100000000001},"last":0.27600000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:08:40 2023","interfaces":[{"interface":"back","average":{"1min":7.8849999999999998,"5min":6.2480000000000002,"15min":4.6269999999999998},"min":{"1min":0.38200000000000001,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":94.953000000000003,"5min":94.953000000000003,"15min":146.38800000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":7.266,"5min":6.0529999999999999,"15min":4.5830000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":93.423000000000002,"5min":93.423000000000002,"15min":146.851},"last":0.442}]},{"osd":4,"last update":"Sat Sep 23 20:09:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.58899999999999997,"15min":1.855},"min":{"1min":0.26200000000000001,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":1.9670000000000001,"5min":5.431,"15min":143.255},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.022,"15min":1.8779999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":2.6360000000000001,"5min":41.822000000000003,"15min":143.41200000000001},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:09:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.0129999999999999,"15min":1.534},"min":{"1min":0.27200000000000002,"5min":0.26500000000000001,"15min":0.25},"max":{"1min":3.2400000000000002,"5min":24.876000000000001,"15min":84.566999999999993},"last":4.3109999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.85099999999999998,"15min":1.337},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.27300000000000002},"max":{"1min":3.3599999999999999,"5min":21.623000000000001,"15min":81.244},"last":4.2030000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:08:40 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.0780000000000001,"15min":0.83199999999999996},"min":{"1min":0.318,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.383,"5min":41.902999999999999,"15min":41.902999999999999},"last":0.376},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.73299999999999998,"15min":0.65700000000000003},"min":{"1min":0.36499999999999999,"5min":0.317,"15min":0.30399999999999999},"max":{"1min":2.149,"5min":6.3620000000000001,"15min":6.3620000000000001},"last":0.40300000000000002}]}]},{"osd":6,"up_from":0,"seq":1078036791383,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762542,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1249052,"kb_used_data":735252,"kb_used_omap":7,"kb_used_meta":513784,"kb_avail":92496100,"statfs":{"total":95995035648,"available":94716006400,"internally_reserved":0,"allocated":752898048,"data_stored":751258955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":526115642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:09:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":4.6239999999999997,"15min":4.1890000000000001},"min":{"1min":0.27700000000000002,"5min":0.17100000000000001,"15min":0.151},"max":{"1min":8.9339999999999993,"5min":144.273,"15min":186.84},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.3010000000000002,"5min":5.4020000000000001,"15min":4.5259999999999998},"min":{"1min":0.24199999999999999,"5min":0.19400000000000001,"15min":0.17899999999999999},"max":{"1min":9.0039999999999996,"5min":144.321,"15min":191.322},"last":0.23400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:09:03 2023","interfaces":[{"interface":"back","average":{"1min":6.6769999999999996,"5min":5.5449999999999999,"15min":5.9589999999999996},"min":{"1min":0.27300000000000002,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":19.465,"5min":98.531000000000006,"15min":180.25200000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":4.4130000000000003,"5min":5.0259999999999998,"15min":5.8040000000000003},"min":{"1min":0.28999999999999998,"5min":0.20699999999999999,"15min":0.19600000000000001},"max":{"1min":9.5800000000000001,"5min":98.346000000000004,"15min":180.05000000000001},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:09:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":3.9750000000000001,"15min":4.0149999999999997},"min":{"1min":0.32700000000000001,"5min":0.22,"15min":0.217},"max":{"1min":14.4,"5min":25.178999999999998,"15min":68.180999999999997},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.9399999999999999,"5min":3.871,"15min":3.8799999999999999},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":9.077,"5min":40.716000000000001,"15min":66.509},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:09:23 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.909,"15min":2.7109999999999999},"min":{"1min":0.36199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.0030000000000001,"5min":10.368,"15min":161.68299999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.617,"5min":2.2040000000000002,"15min":2.8980000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":8.9649999999999999,"5min":14.890000000000001,"15min":161.286},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:09:23 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":2.21,"15min":3.7850000000000001},"min":{"1min":0.34499999999999997,"5min":0.34399999999999997,"15min":0.25900000000000001},"max":{"1min":8.8989999999999991,"5min":14.83,"15min":219.048},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.8740000000000001,"5min":2.3140000000000001,"15min":3.9580000000000002},"min":{"1min":0.29299999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.5129999999999999,"5min":14.547000000000001,"15min":218.69499999999999},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 20:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":1.9039999999999999,"15min":2.105},"min":{"1min":0.40000000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":7.3540000000000001,"5min":10.4,"15min":10.4},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.3859999999999999,"5min":2.012,"15min":2.9550000000000001},"min":{"1min":0.37,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":9.0410000000000004,"5min":14.862,"15min":20.734999999999999},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:09:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":2.3820000000000001,"15min":2.5750000000000002},"min":{"1min":0.33300000000000002,"5min":0.30399999999999999,"15min":0.26800000000000002},"max":{"1min":7.4859999999999998,"5min":14.499000000000001,"15min":60.213000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.8169999999999999,"5min":2.0329999999999999,"15min":2.0499999999999998},"min":{"1min":0.41199999999999998,"5min":0.34100000000000003,"15min":0.30199999999999999},"max":{"1min":9.0800000000000001,"5min":10.323,"15min":61.298999999999999},"last":0.41399999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926079,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2199872,"kb_used_data":1591544,"kb_used_omap":8,"kb_used_meta":608311,"kb_avail":91545280,"statfs":{"total":95995035648,"available":93742366720,"internally_reserved":0,"allocated":1629741056,"data_stored":1628078400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":622911073},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:08:43 2023","interfaces":[{"interface":"back","average":{"1min":4.1059999999999999,"5min":3.581,"15min":3.7040000000000002},"min":{"1min":0.214,"5min":0.19700000000000001,"15min":0.17799999999999999},"max":{"1min":14.300000000000001,"5min":48.087000000000003,"15min":94.055999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":4.3289999999999997,"5min":3.5009999999999999,"15min":4.0010000000000003},"min":{"1min":0.23400000000000001,"5min":0.187,"15min":0.183},"max":{"1min":13.569000000000001,"5min":48.119999999999997,"15min":91.632999999999996},"last":2.391}]},{"osd":2,"last update":"Sat Sep 23 20:09:00 2023","interfaces":[{"interface":"back","average":{"1min":11.720000000000001,"5min":6.7380000000000004,"15min":5.4429999999999996},"min":{"1min":0.309,"5min":0.216,"15min":0.19800000000000001},"max":{"1min":102.65300000000001,"5min":162.62899999999999,"15min":270.51100000000002},"last":0.374},{"interface":"front","average":{"1min":10.291,"5min":6.1440000000000001,"15min":5.2530000000000001},"min":{"1min":0.29499999999999998,"5min":0.23200000000000001,"15min":0.22600000000000001},"max":{"1min":96.578999999999994,"5min":162.52500000000001,"15min":269.69299999999998},"last":0.36299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:08:43 2023","interfaces":[{"interface":"back","average":{"1min":8.3279999999999994,"5min":6.4119999999999999,"15min":6.4130000000000003},"min":{"1min":0.33300000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":32.204999999999998,"5min":126.95399999999999,"15min":145.72800000000001},"last":2.4369999999999998},{"interface":"front","average":{"1min":8.6760000000000002,"5min":6.3390000000000004,"15min":6.5369999999999999},"min":{"1min":0.30399999999999999,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":33.457000000000001,"5min":127.294,"15min":145.50700000000001},"last":2.6269999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":3.0880000000000001,"15min":3.0019999999999998},"min":{"1min":0.379,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":5.7359999999999998,"5min":148.20599999999999,"15min":148.20599999999999},"last":0.498},{"interface":"front","average":{"1min":1.7689999999999999,"5min":3.2389999999999999,"15min":2.7719999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":8.8490000000000002,"5min":148.11000000000001,"15min":148.11000000000001},"last":2.4670000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":1.917,"15min":2.8500000000000001},"min":{"1min":0.39300000000000002,"5min":0.34100000000000003,"15min":0.26900000000000002},"max":{"1min":4.1719999999999997,"5min":14.901999999999999,"15min":131.34399999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.28,"5min":1.8240000000000001,"15min":2.8540000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":5.7969999999999997,"5min":12.307,"15min":131.386},"last":0.60399999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":1.9279999999999999,"15min":1.714},"min":{"1min":0.43099999999999999,"5min":0.38,"15min":0.33800000000000002},"max":{"1min":9.9169999999999998,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.57199999999999995},{"interface":"front","average":{"1min":3.1800000000000002,"5min":2.194,"15min":2.0019999999999998},"min":{"1min":0.33900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":13.470000000000001,"5min":13.766999999999999,"15min":13.766999999999999},"last":2.4929999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.724,"5min":1.956,"15min":2.3460000000000001},"min":{"1min":0.432,"5min":0.38100000000000001,"15min":0.29699999999999999},"max":{"1min":4.2030000000000003,"5min":14.856,"15min":77.132999999999996},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.298,"5min":1.9910000000000001,"15min":2.4039999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":5.8700000000000001,"5min":14.916,"15min":75.465999999999994},"last":0.52500000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599017,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2837292,"kb_used_data":2132132,"kb_used_omap":12,"kb_used_meta":705139,"kb_avail":90907860,"statfs":{"total":95995035648,"available":93089648640,"internally_reserved":0,"allocated":2183303168,"data_stored":2181652331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":722063358},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:09:05 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":3.726,"15min":3.4689999999999999},"min":{"1min":0.29199999999999998,"5min":0.19900000000000001,"15min":0.154},"max":{"1min":6.915,"5min":76.251999999999995,"15min":76.251999999999995},"last":0.64600000000000002},{"interface":"front","average":{"1min":4.3700000000000001,"5min":4.3250000000000002,"15min":3.532},"min":{"1min":0.316,"5min":0.215,"15min":0.187},"max":{"1min":18.561,"5min":76.590000000000003,"15min":76.590000000000003},"last":0.98399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:09:05 2023","interfaces":[{"interface":"back","average":{"1min":4.2460000000000004,"5min":3.5409999999999999,"15min":3.9060000000000001},"min":{"1min":0.26500000000000001,"5min":0.23200000000000001,"15min":0.184},"max":{"1min":10.875,"5min":25.102,"15min":94.768000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":3.2330000000000001,"5min":2.7749999999999999,"15min":3.9390000000000001},"min":{"1min":0.27100000000000002,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":10.086,"5min":19.314,"15min":96.808000000000007},"last":0.61199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:09:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":6.2039999999999997,"15min":4.5709999999999997},"min":{"1min":0.30199999999999999,"5min":0.28100000000000003,"15min":0.214},"max":{"1min":7.9320000000000004,"5min":107.381,"15min":107.381},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.653,"5min":4.6079999999999997,"15min":3.9700000000000002},"min":{"1min":0.29899999999999999,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":20.308,"5min":106.301,"15min":106.301},"last":0.67200000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:09:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":3.0699999999999998,"15min":2.4729999999999999},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.26900000000000002},"max":{"1min":6.8339999999999996,"5min":75.608000000000004,"15min":75.608000000000004},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.4060000000000001,"5min":3.3300000000000001,"15min":2.5350000000000001},"min":{"1min":0.34999999999999998,"5min":0.34100000000000003,"15min":0.26700000000000002},"max":{"1min":5.4720000000000004,"5min":76.161000000000001,"15min":76.161000000000001},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:09:05 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":2.5009999999999999,"15min":3.0699999999999998},"min":{"1min":0.375,"5min":0.33500000000000002,"15min":0.28100000000000003},"max":{"1min":6.8600000000000003,"5min":65.921999999999997,"15min":172.505},"last":0.72199999999999998},{"interface":"front","average":{"1min":3.194,"5min":3.1030000000000002,"15min":3.0990000000000002},"min":{"1min":0.38900000000000001,"5min":0.38800000000000001,"15min":0.29099999999999998},"max":{"1min":14.176,"5min":63.792000000000002,"15min":172.375},"last":1.0549999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:08:38 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.8320000000000001,"15min":1.694},"min":{"1min":0.44900000000000001,"5min":0.371,"15min":0.35199999999999998},"max":{"1min":8.3309999999999995,"5min":10.209,"15min":10.209},"last":0.74199999999999999},{"interface":"front","average":{"1min":3.2330000000000001,"5min":2.1970000000000001,"15min":2.0449999999999999},"min":{"1min":0.49099999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":14.279999999999999,"5min":14.279999999999999,"15min":14.279999999999999},"last":1.028}]},{"osd":7,"last update":"Sat Sep 23 20:09:05 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":2.2109999999999999,"15min":2.2519999999999998},"min":{"1min":0.40699999999999997,"5min":0.35099999999999998,"15min":0.27400000000000002},"max":{"1min":8.0920000000000005,"5min":9.1170000000000009,"15min":50.118000000000002},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.593,"5min":1.9910000000000001,"15min":2.4529999999999998},"min":{"1min":0.47399999999999998,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":6.806,"5min":10.244,"15min":50.664999999999999},"last":0.70699999999999996}]}]},{"osd":3,"up_from":109,"seq":468151435491,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2770032,"kb_used_data":2105640,"kb_used_omap":4,"kb_used_meta":664379,"kb_avail":90975120,"statfs":{"total":95995035648,"available":93158522880,"internally_reserved":0,"allocated":2156175360,"data_stored":2154398136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":680324205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:09:30 2023","interfaces":[{"interface":"back","average":{"1min":3.8759999999999999,"5min":5.4180000000000001,"15min":4.7169999999999996},"min":{"1min":0.191,"5min":0.191,"15min":0.17399999999999999},"max":{"1min":53.593000000000004,"5min":74.323999999999998,"15min":240.40600000000001},"last":0.20399999999999999},{"interface":"front","average":{"1min":4.0720000000000001,"5min":5.1529999999999996,"15min":5.2030000000000003},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":53.478000000000002,"5min":74.447000000000003,"15min":240.352},"last":0.215}]},{"osd":1,"last update":"Sat Sep 23 20:09:30 2023","interfaces":[{"interface":"back","average":{"1min":3.9700000000000002,"5min":4.2439999999999998,"15min":3.4529999999999998},"min":{"1min":0.216,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":53.497999999999998,"5min":112.122,"15min":112.122},"last":0.48899999999999999},{"interface":"front","average":{"1min":6.8630000000000004,"5min":5.2519999999999998,"15min":3.7730000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.19900000000000001},"max":{"1min":58.378,"5min":111.786,"15min":111.786},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:09:30 2023","interfaces":[{"interface":"back","average":{"1min":12.622,"5min":4.4980000000000002,"15min":5.3259999999999996},"min":{"1min":0.26400000000000001,"5min":0.23899999999999999,"15min":0.219},"max":{"1min":88.486999999999995,"5min":88.486999999999995,"15min":264.97800000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":12.926,"5min":5.4130000000000003,"15min":5.2229999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":88.387,"5min":88.387,"15min":257.40899999999999},"last":0.373}]},{"osd":4,"last update":"Sat Sep 23 20:09:30 2023","interfaces":[{"interface":"back","average":{"1min":6.6500000000000004,"5min":3.004,"15min":3.508},"min":{"1min":0.35699999999999998,"5min":0.29899999999999999,"15min":0.25700000000000001},"max":{"1min":58.472999999999999,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":7.0350000000000001,"5min":3.3780000000000001,"15min":4.024},"min":{"1min":0.312,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":58.408000000000001,"5min":58.408000000000001,"15min":110.295},"last":0.28299999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:09:30 2023","interfaces":[{"interface":"back","average":{"1min":7.048,"5min":4.0499999999999998,"15min":3.5880000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":58.506,"5min":77.206000000000003,"15min":113.881},"last":0.31},{"interface":"front","average":{"1min":6.851,"5min":4.0099999999999998,"15min":3.448},"min":{"1min":0.36699999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":58.643000000000001,"5min":77.968000000000004,"15min":114.598},"last":0.57399999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:09:40 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":2.8380000000000001,"15min":4.5359999999999996},"min":{"1min":0.35599999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.4329999999999998,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.93899999999999995,"5min":3.427,"15min":4.7149999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.33400000000000002},"max":{"1min":5.1970000000000001,"5min":58.701000000000001,"15min":82.233000000000004},"last":0.41799999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:09:30 2023","interfaces":[{"interface":"back","average":{"1min":6.8559999999999999,"5min":3.105,"15min":3.6680000000000001},"min":{"1min":0.39100000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":58.686,"5min":58.686,"15min":138.035},"last":0.55900000000000005},{"interface":"front","average":{"1min":6.7229999999999999,"5min":3.4550000000000001,"15min":3.577},"min":{"1min":0.41099999999999998,"5min":0.40799999999999997,"15min":0.28699999999999998},"max":{"1min":58.667999999999999,"5min":58.667999999999999,"15min":138.054},"last":0.441}]}]},{"osd":4,"up_from":30,"seq":128849019252,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2185740,"kb_used_data":1853764,"kb_used_omap":0,"kb_used_meta":331968,"kb_avail":91559412,"statfs":{"total":95995035648,"available":93756837888,"internally_reserved":0,"allocated":1898254336,"data_stored":1897153656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":339935232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":2.6760000000000002,"15min":2.9350000000000001},"min":{"1min":0.26000000000000001,"5min":0.22,"15min":0.189},"max":{"1min":11.743,"5min":56.061999999999998,"15min":104.604},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.998,"5min":2.5489999999999999,"15min":2.8500000000000001},"min":{"1min":0.313,"5min":0.218,"15min":0.218},"max":{"1min":11.776,"5min":56.075000000000003,"15min":104.687},"last":0.45500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":3.8700000000000001,"15min":5.7450000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.224},"max":{"1min":4.9100000000000001,"5min":119.892,"15min":161.613},"last":0.26800000000000002},{"interface":"front","average":{"1min":1.3180000000000001,"5min":3.8260000000000001,"15min":5.6779999999999999},"min":{"1min":0.27700000000000002,"5min":0.254,"15min":0.22},"max":{"1min":7.0209999999999999,"5min":119.351,"15min":162.017},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:09:03 2023","interfaces":[{"interface":"back","average":{"1min":13.195,"5min":6.9740000000000002,"15min":6.8899999999999997},"min":{"1min":0.378,"5min":0.245,"15min":0.245},"max":{"1min":154.94999999999999,"5min":182.96700000000001,"15min":182.96700000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":11.275,"5min":6.8949999999999996,"15min":6.6660000000000004},"min":{"1min":0.42699999999999999,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":155.51300000000001,"5min":183.244,"15min":183.244},"last":1.54}]},{"osd":3,"last update":"Sat Sep 23 20:09:25 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":6.3179999999999996,"15min":7.0780000000000003},"min":{"1min":0.28499999999999998,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":5.899,"5min":184.37100000000001,"15min":266.54700000000003},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.5760000000000001,"5min":6.4939999999999998,"15min":6.0350000000000001},"min":{"1min":0.432,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.1920000000000002,"5min":184.22300000000001,"15min":184.22300000000001},"last":0.57299999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:09:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":2.1709999999999998,"15min":2.3149999999999999},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":3.3290000000000002,"5min":176.81899999999999,"15min":266.59899999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.2719999999999998,"15min":2.468},"min":{"1min":0.28899999999999998,"5min":0.26600000000000001,"15min":0.24399999999999999},"max":{"1min":1.722,"5min":176.172,"15min":266.72899999999998},"last":0.41999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:08:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.51400000000000001},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.5489999999999999,"5min":4.0410000000000004,"15min":4.0410000000000004},"last":0.502},{"interface":"front","average":{"1min":0.497,"5min":0.77700000000000002,"15min":0.57899999999999996},"min":{"1min":0.29499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":1.6240000000000001,"5min":4.2889999999999997,"15min":4.2889999999999997},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:09:00 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.879,"15min":1.7230000000000001},"min":{"1min":0.33400000000000002,"5min":0.314,"15min":0.26100000000000001},"max":{"1min":1.5960000000000001,"5min":76.507000000000005,"15min":76.507000000000005},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.61799999999999999,"5min":3.0870000000000002,"15min":1.792},"min":{"1min":0.27200000000000002,"5min":0.222,"15min":0.222},"max":{"1min":1.4870000000000001,"5min":76.462000000000003,"15min":76.462000000000003},"last":0.48799999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757616,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2387832,"kb_used_data":1993840,"kb_used_omap":0,"kb_used_meta":393984,"kb_avail":91357320,"statfs":{"total":95995035648,"available":93549895680,"internally_reserved":0,"allocated":2041692160,"data_stored":2040599345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":403439616},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5269999999999999,"5min":3.3900000000000001,"15min":2.1429999999999998},"min":{"1min":0.224,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":8.6739999999999995,"5min":121.94799999999999,"15min":121.94799999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":2.0920000000000001,"5min":3.6309999999999998,"15min":2.3170000000000002},"min":{"1min":0.26600000000000001,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":10.327,"5min":121.964,"15min":121.964},"last":0.34799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:09:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":3.8690000000000002,"15min":5.883},"min":{"1min":0.26900000000000002,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":3.5640000000000001,"5min":127.41800000000001,"15min":266.197},"last":75.204999999999998},{"interface":"front","average":{"1min":1.1950000000000001,"5min":4.1619999999999999,"15min":6.0510000000000002},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.6070000000000002,"5min":127.37,"15min":266.14999999999998},"last":75.231999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:08:55 2023","interfaces":[{"interface":"back","average":{"1min":15.69,"5min":5.532,"15min":6.7450000000000001},"min":{"1min":0.44500000000000001,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":240.57300000000001,"5min":240.57300000000001,"15min":240.57300000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":15.714,"5min":5.6040000000000001,"15min":6.79},"min":{"1min":0.47099999999999997,"5min":0.32300000000000001,"15min":0.313},"max":{"1min":240.464,"5min":240.464,"15min":240.464},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:09:33 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":5.7400000000000002,"15min":5.117},"min":{"1min":0.32400000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.4859999999999998,"5min":201.50899999999999,"15min":201.50899999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.139,"5min":5.5579999999999998,"15min":4.9029999999999996},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":4.0579999999999998,"5min":201.58099999999999,"15min":201.58099999999999},"last":0.41999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:09:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.70999999999999996,"15min":1.6299999999999999},"min":{"1min":0.24199999999999999,"5min":0.24099999999999999,"15min":0.21299999999999999},"max":{"1min":4.0510000000000002,"5min":9.7240000000000002,"15min":191.78899999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.59999999999999998,"15min":1.5409999999999999},"min":{"1min":0.26900000000000002,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":1.4359999999999999,"5min":8.7360000000000007,"15min":191.821},"last":0.57899999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:08:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.76800000000000002,"15min":0.65600000000000003},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.7330000000000001,"5min":9.9009999999999998,"15min":9.9009999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.628,"15min":0.51700000000000002},"min":{"1min":0.33600000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.82399999999999995,"5min":5.0119999999999996,"15min":5.0119999999999996},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.65100000000000002,"15min":1.72},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":0.66700000000000004,"5min":5.2670000000000003,"15min":191.84299999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.72699999999999998,"15min":1.9179999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":2.819,"5min":9.6920000000000002,"15min":191.69},"last":0.46300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1620180992,"data_stored":1620180992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":744947712,"data_stored":744947712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2172850176,"data_stored":2172846104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2148728832,"data_stored":2148728832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1893203968,"data_stored":1893203968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2032865280,"data_stored":2032861208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":969936896,"data_stored":969932824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:09:45.980 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:09:45.981 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-09-23T20:09:45.981 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:46.009 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:46.055 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:46.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:46.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:46.108 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:46.159 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:46.194 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:45 smithi135 ceph-mon[221113]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:09:46.194 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:45 smithi135 ceph-mon[221113]: pgmap v991: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 20952/153534 objects degraded (13.646%); 4897/153534 objects misplaced (3.190%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:09:46.194 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:45 smithi135 ceph-mon[213871]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:09:46.194 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:45 smithi135 ceph-mon[213871]: pgmap v991: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 20952/153534 objects degraded (13.646%); 4897/153534 objects misplaced (3.190%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:09:46.200 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:45 smithi144 ceph-mon[194833]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:09:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:45 smithi144 ceph-mon[194833]: pgmap v991: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 20952/153534 objects degraded (13.646%); 4897/153534 objects misplaced (3.190%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:09:46.387 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:46.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:46.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:46.491 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:46.519 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:46.548 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:09:46.570 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:46.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:46.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:46.580 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:09:46.611 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:09:46.633 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:09:46.655 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:09:46.675 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:09:46.677 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:46.707 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:09:46.716 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:46.752 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:09:46.755 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:09:46.755 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:09:46.755 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:09:46.755 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:09:46.756 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:09:46.756 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:09:46.756 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:09:46.756 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:09:47.026 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:09:47.026 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:09:47.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:09:47.026 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:149989 6.15 6:afeafb5c:::benchmark_data_smithi135_344736_object47333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 599.99206035500004, 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0666702159999999, 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.027 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149989 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030923+0000", 2023-09-23T20:09:47.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030922+0000", 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030930+0000", 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9789999999999993e-06 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030945+0000", 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5231999999999999e-05 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.030952+0000", 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3880000000000002e-06 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.031049+0000", 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6500000000000001e-05 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.031062+0000", 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2901000000000001e-05 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090321+0000", 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.059259199 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090433+0000", 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000112318 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.090708+0000", 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00027474699999999999 2023-09-23T20:09:47.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094761+0000", 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040525600000000002 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.094797+0000", 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6569000000000002e-05 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097524+0000", 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002726657 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097554+0000", 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0074e-05 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097594+0000", 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9827999999999999e-05 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:149996 6.15 6:ac0d5d46:::benchmark_data_smithi135_344736_object47340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 599.98256633200003, 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.057323408, 2023-09-23T20:09:47.035 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 149996 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:09:47.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040418+0000", 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040417+0000", 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040422+0000", 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5979999999999996e-06 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040433+0000", 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0444e-05 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040445+0000", 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2872e-05 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040503+0000", 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7289e-05 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:47.040512+0000", 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6269999999999995e-06 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092269+0000", 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.05175639 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092337+0000", 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8631000000000001e-05 2023-09-23T20:09:47.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.092523+0000", 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018535200000000001 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.095904+0000", 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033811940000000001 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097643+0000", 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001738697 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097688+0000", 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5427999999999999e-05 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097718+0000", 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0437999999999999e-05 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T19:59:48.097741+0000", 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2449000000000001e-05 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.043 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201802 6.9 6:91b29cdb:::benchmark_data_smithi135_344736_object99146:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 522.556646301, 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1705403599999999, 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.044 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201802 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466338+0000", 2023-09-23T20:09:47.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466336+0000", 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466346+0000", 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0098000000000001e-05 2023-09-23T20:09:47.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466364+0000", 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7572999999999999e-05 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466371+0000", 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.238e-06 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466440+0000", 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8442000000000006e-05 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466449+0000", 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9490000000000008e-06 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469797+0000", 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033470090000000002 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469805+0000", 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2930000000000001e-06 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593229+0000", 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.123424108 2023-09-23T20:09:47.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593367+0000", 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000138338 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.593937+0000", 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00056937199999999996 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608226+0000", 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014289792000000001 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614886+0000", 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0066598359999999997 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.051 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614925+0000", 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8294999999999998e-05 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636724+0000", 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021799457000000001 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636765+0000", 2023-09-23T20:09:47.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1247e-05 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636878+0000", 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000112652 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.053 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201804 6.19 6:9cfa48b5:::benchmark_data_smithi135_344736_object99148:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 522.55659636300004, 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1704891850000001, 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201804 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.054 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466387+0000", 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466390+0000", 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.599e-06 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466405+0000", 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5133000000000001e-05 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466410+0000", 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8169999999999998e-06 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5672000000000002e-05 2023-09-23T20:09:47.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466510+0000", 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6859999999999997e-06 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.471432+0000", 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0049220280000000002 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.471445+0000", 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2568000000000001e-05 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594208+0000", 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1227630580000001 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594319+0000", 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111501 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.594813+0000", 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000494181 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612638+0000", 2023-09-23T20:09:47.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.017824584000000001 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614937+0000", 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022991069999999999 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614972+0000", 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4678999999999998e-05 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.061 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636699+0000", 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021726942999999999 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636748+0000", 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9106e-05 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636877+0000", 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012898200000000001 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201813 6.9 6:96974822:::benchmark_data_smithi135_344736_object99157:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 522.55656062900005, 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1705933580000001, 2023-09-23T20:09:47.063 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201813 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466423+0000", 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.065 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466425+0000", 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5450000000000002e-06 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466432+0000", 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6059999999999999e-06 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466436+0000", 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9979999999999997e-06 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466527+0000", 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0829999999999996e-05 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466531+0000", 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.765e-06 2023-09-23T20:09:47.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469820+0000", 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032892910000000002 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469824+0000", 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5639999999999998e-06 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.598827+0000", 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1290025159999999 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.598953+0000", 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012642799999999999 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:09:47.069 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.599456+0000", 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000502974 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614525+0000", 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015068381 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614956+0000", 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043108699999999997 2023-09-23T20:09:47.070 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614987+0000", 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1103999999999999e-05 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636915+0000", 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021927720000000001 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.636954+0000", 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8930000000000002e-05 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637017+0000", 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2964000000000002e-05 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.072 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201824 6.9 6:912a71a1:::benchmark_data_smithi135_344736_object99168:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 522.55654217400001, 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1708038080000001, 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201824 2023-09-23T20:09:47.073 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466442+0000", 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.074 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466441+0000", 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466444+0000", 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2010000000000002e-06 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466450+0000", 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9680000000000003e-06 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466454+0000", 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4009999999999999e-06 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466551+0000", 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6948999999999996e-05 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466557+0000", 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4999999999999996e-06 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469839+0000", 2023-09-23T20:09:47.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032815219999999998 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469843+0000", 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.527e-06 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602356+0000", 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1325121519999999 2023-09-23T20:09:47.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602467+0000", 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111412 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.602926+0000", 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000458691 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614536+0000", 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011610646000000001 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615029+0000", 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00049241599999999997 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615073+0000", 2023-09-23T20:09:47.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4329999999999997e-05 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637077+0000", 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022004391000000002 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637136+0000", 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8501000000000001e-05 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637245+0000", 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010948800000000001 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201827 6.9 6:937e540a:::benchmark_data_smithi135_344736_object99171:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:09:47.082 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 522.55652577900003, 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1709668390000001, 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201827 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466458+0000", 2023-09-23T20:09:47.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466460+0000", 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1710000000000001e-06 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466466+0000", 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8119999999999996e-06 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466470+0000", 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8290000000000001e-06 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466732+0000", 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000262759 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466747+0000", 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4204e-05 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469858+0000", 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003111426 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469875+0000", 2023-09-23T20:09:47.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6639000000000001e-05 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605181+0000", 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.135306167 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605324+0000", 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014373500000000001 2023-09-23T20:09:47.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.605687+0000", 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036256599999999998 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614615+0000", 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0089275680000000003 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615114+0000", 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00049912899999999996 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615172+0000", 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8260000000000001e-05 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637300+0000", 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022127563999999999 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637335+0000", 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5825999999999999e-05 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.637425+0000", 2023-09-23T20:09:47.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9488999999999996e-05 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201882 6.9 6:972c2747:::benchmark_data_smithi135_344736_object99226:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 522.55651031000002, 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.173680238, 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.092 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201882 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466474+0000", 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466473+0000", 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466475+0000", 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8220000000000001e-06 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466481+0000", 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7080000000000002e-06 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466485+0000", 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.749e-06 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466901+0000", 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000416542 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466909+0000", 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469891+0000", 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029817060000000002 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469895+0000", 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4510000000000002e-06 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608273+0000", 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.138377135 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608376+0000", 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010315 2023-09-23T20:09:47.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.608741+0000", 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036480300000000002 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614621+0000", 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0058808610000000002 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615135+0000", 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00051361899999999999 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615214+0000", 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8611999999999999e-05 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640006+0000", 2023-09-23T20:09:47.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.024792756999999999 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640051+0000", 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4193999999999997e-05 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640154+0000", 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010318 2023-09-23T20:09:47.102 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201893 6.9 6:9279cfff:::benchmark_data_smithi135_344736_object99237:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 522.55647875499994, 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.173837531, 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.103 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201893 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466505+0000", 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466500+0000", 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999952 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466507+0000", 2023-09-23T20:09:47.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.905e-06 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466514+0000", 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6449999999999999e-06 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466518+0000", 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.985e-06 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466950+0000", 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043239999999999999 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466957+0000", 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1590000000000004e-06 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469913+0000", 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029559510000000001 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469918+0000", 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4739999999999999e-06 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.610696+0000", 2023-09-23T20:09:47.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1407785420000001 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.610812+0000", 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115654 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.611100+0000", 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00028765000000000003 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614629+0000", 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035288590000000001 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615255+0000", 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00062691999999999999 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615334+0000", 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8597000000000007e-05 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640192+0000", 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.024858410000000001 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640237+0000", 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4200999999999999e-05 2023-09-23T20:09:47.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640343+0000", 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010594000000000001 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.44942.0:201908 6.9 6:9284cb28:::benchmark_data_smithi135_344736_object99252:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:09:47.112 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 522.55645564999998, 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.174007815, 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.44942", 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/2866858603", 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 201908 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466528+0000", 2023-09-23T20:09:47.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466530+0000", 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0990000000000001e-06 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466541+0000", 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0922e-05 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.115 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466545+0000", 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.788e-06 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466984+0000", 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000439586 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.466991+0000", 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9190000000000002e-06 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469933+0000", 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002941534 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:04.469937+0000", 2023-09-23T20:09:47.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2810000000000002e-06 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612655+0000", 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1427177049999999 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612740+0000", 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5024000000000005e-05 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2,3", 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.612994+0000", 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00025448499999999999 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.614834+0000", 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018393560000000001 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615298+0000", 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000464303 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.615356+0000", 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7827e-05 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640379+0000", 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.025023627999999999 2023-09-23T20:09:47.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640429+0000", 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.922e-05 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:01:05.640536+0000", 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107451 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.121 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86850 8.c 8:349d498f:::benchmark_data_smithi135_460343_object2617:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739145+0000", 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.283838678, 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.085194797, 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.122 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86850 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739145+0000", 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739145+0000", 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739139+0000", 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999938 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739153+0000", 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3427e-05 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.124 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739174+0000", 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.109e-05 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739182+0000", 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4689999999999997e-06 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739246+0000", 2023-09-23T20:09:47.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3938999999999999e-05 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739258+0000", 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2038999999999999e-05 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.744912+0000", 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005654236 2023-09-23T20:09:47.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.744923+0000", 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0957e-05 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:05.822899+0000", 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.077975949 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:05.824340+0000", 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00144074 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86863 8.c 8:380a5fea:::benchmark_data_smithi135_460343_object2630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739195+0000", 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.28378835700001, 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8269497810000002, 2023-09-23T20:09:47.128 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86863 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739195+0000", 2023-09-23T20:09:47.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739195+0000", 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739195+0000", 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.130 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739199+0000", 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7620000000000001e-06 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739206+0000", 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0180000000000004e-06 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739210+0000", 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2899999999999996e-06 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739360+0000", 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014967299999999999 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739366+0000", 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9220000000000002e-06 2023-09-23T20:09:47.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.744973+0000", 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0056069969999999998 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.744980+0000", 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1969999999999999e-06 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.133 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.549574+0000", 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8045945909999999 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.551797+0000", 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022222050000000001 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.554752+0000", 2023-09-23T20:09:47.134 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029555440000000001 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.559261+0000", 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045087060000000003 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.559914+0000", 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000652935 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.135 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.560940+0000", 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001026662 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.561820+0000", 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00087918800000000004 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.136 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.566145+0000", 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043255960000000001 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86871 8.c 8:31b0809c:::benchmark_data_smithi135_460343_object2638:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739234+0000", 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.28374961900001, 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8548918140000001, 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.137 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86871 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739234+0000", 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.138 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739234+0000", 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739234+0000", 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.139 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739237+0000", 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4000000000000001e-06 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739244+0000", 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9090000000000003e-06 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739250+0000", 2023-09-23T20:09:47.140 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4940000000000003e-06 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739440+0000", 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019021199999999999 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739449+0000", 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4739999999999994e-06 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.141 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745023+0000", 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055736689999999998 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745033+0000", 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7259999999999997e-06 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.142 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.566364+0000", 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.82133119 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.568556+0000", 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021923670000000002 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.571455+0000", 2023-09-23T20:09:47.143 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028990069999999999 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.578549+0000", 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070936990000000002 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.579111+0000", 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00056198900000000004 2023-09-23T20:09:47.144 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.589552+0000", 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010441071 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.590517+0000", 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00096461099999999998 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.145 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.594126+0000", 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003609456 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86885 8.c 8:32317725:::benchmark_data_smithi135_460343_object2652:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739267+0000", 2023-09-23T20:09:47.146 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.28371713, 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8611840710000001, 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86885 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739267+0000", 2023-09-23T20:09:47.147 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739267+0000", 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739266+0000", 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.148 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739270+0000", 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.63e-06 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739276+0000", 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4579999999999998e-06 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.149 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739281+0000", 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8369999999999996e-06 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739527+0000", 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00024542900000000002 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739532+0000", 2023-09-23T20:09:47.150 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1830000000000002e-06 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745069+0000", 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055371109999999999 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745076+0000", 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9589999999999998e-06 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.151 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.579317+0000", 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.834241362 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.581453+0000", 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002135924 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.152 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.584274+0000", 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028212060000000001 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.594764+0000", 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010489986 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.595380+0000", 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00061564400000000004 2023-09-23T20:09:47.153 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.595556+0000", 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017588099999999999 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.596442+0000", 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00088592499999999997 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.154 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.600451+0000", 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004009047 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86890 8.c 8:307d7fb6:::benchmark_data_smithi135_460343_object2657:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739286+0000", 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.283698196, 2023-09-23T20:09:47.155 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.893261627, 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86890 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739286+0000", 2023-09-23T20:09:47.156 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739286+0000", 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739285+0000", 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.157 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739288+0000", 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.807e-06 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739294+0000", 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1850000000000001e-06 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.158 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739298+0000", 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9940000000000002e-06 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739580+0000", 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000282226 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739592+0000", 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1204e-05 2023-09-23T20:09:47.159 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745098+0000", 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055061060000000002 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745104+0000", 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5470000000000002e-06 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.160 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.600774+0000", 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.855669483 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.602963+0000", 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021891580000000001 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.605747+0000", 2023-09-23T20:09:47.161 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027836179999999999 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.626390+0000", 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.020643702999999999 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.627218+0000", 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000827927 2023-09-23T20:09:47.162 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.627853+0000", 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00063512200000000001 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.628838+0000", 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00098511600000000007 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.163 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.632547+0000", 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037087980000000001 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86908 8.c 8:32cc60b2:::benchmark_data_smithi135_460343_object2675:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739349+0000", 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.28363512999999, 2023-09-23T20:09:47.164 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.898834269, 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86908 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739349+0000", 2023-09-23T20:09:47.165 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739349+0000", 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739348+0000", 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.166 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739351+0000", 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.345e-06 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739356+0000", 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8080000000000004e-06 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.167 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739359+0000", 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9569999999999998e-06 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739628+0000", 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00026991800000000001 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739632+0000", 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9620000000000004e-06 2023-09-23T20:09:47.168 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745126+0000", 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054939289999999998 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745134+0000", 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2590000000000003e-06 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.169 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.609209+0000", 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8640758420000001 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.611665+0000", 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0024555200000000001 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.170 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.614562+0000", 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002896844 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.626565+0000", 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012003363 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.633352+0000", 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0067871950000000002 2023-09-23T20:09:47.171 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.633894+0000", 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00054150400000000005 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.634773+0000", 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00087868799999999997 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.172 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.638183+0000", 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003410419 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86920 8.c 8:3ed02899:::benchmark_data_smithi135_460343_object2687:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739361+0000", 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.28362269100001, 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9050515560000001, 2023-09-23T20:09:47.173 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86920 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739361+0000", 2023-09-23T20:09:47.174 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739361+0000", 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739361+0000", 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967296 2023-09-23T20:09:47.175 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739363+0000", 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6789999999999999e-06 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739366+0000", 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2160000000000002e-06 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.176 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739368+0000", 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.356e-06 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739674+0000", 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00030608300000000002 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739684+0000", 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0084999999999999e-05 2023-09-23T20:09:47.177 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745166+0000", 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054815239999999998 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745173+0000", 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5170000000000002e-06 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.178 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.617808+0000", 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.872634696 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.620036+0000", 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022280609999999999 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.623068+0000", 2023-09-23T20:09:47.179 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003031607 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.632728+0000", 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0096604940000000004 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.638860+0000", 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0061315099999999997 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.180 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.639660+0000", 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00080000299999999998 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.640816+0000", 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0011565 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.181 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.644413+0000", 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035964320000000001 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86956 8.c 8:3f54cf6a:::benchmark_data_smithi135_460343_object2723:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739380+0000", 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.283603934, 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9306421629999999, 2023-09-23T20:09:47.182 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86956 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739380+0000", 2023-09-23T20:09:47.183 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739380+0000", 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739380+0000", 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.184 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739383+0000", 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6720000000000002e-06 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739399+0000", 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5656999999999999e-05 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.185 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739404+0000", 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2429999999999996e-06 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739718+0000", 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00031407100000000002 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739722+0000", 2023-09-23T20:09:47.186 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8639999999999998e-06 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745199+0000", 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054767610000000001 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745206+0000", 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7070000000000003e-06 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.187 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.644782+0000", 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8995764450000001 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.647638+0000", 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028560899999999999 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.650609+0000", 2023-09-23T20:09:47.188 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002970767 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.664765+0000", 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014155753 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.664972+0000", 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020733399999999999 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.189 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.665590+0000", 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00061814900000000002 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.666503+0000", 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00091274599999999998 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.190 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.670022+0000", 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035192499999999998 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86986 8.c 8:34297654:::benchmark_data_smithi135_460343_object2753:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739422+0000", 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.283561445, 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.936387968, 2023-09-23T20:09:47.191 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86986 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739422+0000", 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.192 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739422+0000", 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739422+0000", 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.193 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739426+0000", 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9639999999999997e-06 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739443+0000", 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7269999999999999e-05 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.194 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739450+0000", 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4130000000000003e-06 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739764+0000", 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00031484200000000002 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739775+0000", 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0380000000000001e-05 2023-09-23T20:09:47.195 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745230+0000", 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054552330000000003 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745237+0000", 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4900000000000003e-06 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.196 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.654209+0000", 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.908971516 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.656817+0000", 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002608239 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.197 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.660560+0000", 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037429049999999999 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.670625+0000", 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010065075999999999 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.670808+0000", 2023-09-23T20:09:47.198 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018266199999999999 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.671366+0000", 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00055820200000000005 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.672259+0000", 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00089319400000000004 2023-09-23T20:09:47.199 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.675810+0000", 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035510759999999998 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:87008 8.c 8:3e426314:::benchmark_data_smithi135_460343_object2775:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739455+0000", 2023-09-23T20:09:47.200 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 102.28352878, 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9619393139999999, 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 87008 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.201 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739455+0000", 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739455+0000", 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739455+0000", 2023-09-23T20:09:47.202 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739459+0000", 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1200000000000004e-06 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739466+0000", 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5449999999999998e-06 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.203 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739471+0000", 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4229999999999998e-06 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739791+0000", 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00032025199999999998 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739795+0000", 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8290000000000001e-06 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745262+0000", 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005467253 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.205 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745269+0000", 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0690000000000004e-06 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.676029+0000", 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9307603389999999 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.678302+0000", 2023-09-23T20:09:47.206 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022723969999999998 2023-09-23T20:09:47.207 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.207 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.207 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:09:47.207 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.681140+0000", 2023-09-23T20:09:47.207 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028378449999999999 2023-09-23T20:09:47.207 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.207 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.207 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.695255+0000", 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014115384 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.695838+0000", 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00058260400000000002 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.696048+0000", 2023-09-23T20:09:47.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00021043400000000001 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.697041+0000", 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00099287799999999999 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.701394+0000", 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043534610000000003 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.209 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.210 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.210 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:09:47.210 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:09:47.210 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:09:47.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/632143209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:47.211 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/632143209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:47.211 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/632143209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:47.750 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:09:48.112 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:47 smithi144 ceph-mon[194833]: pgmap v992: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 20919/153198 objects degraded (13.655%); 4465/153198 objects misplaced (2.915%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T20:09:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 20919/153198 objects degraded (13.655%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:48.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:47 smithi135 ceph-mon[221113]: pgmap v992: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 20919/153198 objects degraded (13.655%); 4465/153198 objects misplaced (2.915%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T20:09:48.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 20919/153198 objects degraded (13.655%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:48.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:47 smithi135 ceph-mon[213871]: pgmap v992: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 20919/153198 objects degraded (13.655%); 4465/153198 objects misplaced (2.915%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T20:09:48.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 20919/153198 objects degraded (13.655%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:48.721 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:48.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:48.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:48.829 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:48.855 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:48.859 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:48.903 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:48.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:49.006 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:49.044 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:49.101 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:49.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:49.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:49.208 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:49.252 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:49.371 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:49.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:49.476 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:49.508 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:49.640 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:09:49.640 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:09:49.640 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:09:49.640 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:09:49.669 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:49.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:49.775 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:49.813 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:49.864 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:49.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:49.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:49.968 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:50.002 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:09:50.012 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:50.060 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:50.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:50.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:50.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:49 smithi144 ceph-mon[194833]: pgmap v993: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 20887/152379 objects degraded (13.707%); 4465/152379 objects misplaced (2.930%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:09:50.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:49 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2233465335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:50.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:49 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:50.167 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:50.194 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:50.247 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:50.249 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:49 smithi135 ceph-mon[221113]: pgmap v993: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 20887/152379 objects degraded (13.707%); 4465/152379 objects misplaced (2.930%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:09:50.250 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:49 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2233465335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:50.250 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:49 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:50.250 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:49 smithi135 ceph-mon[213871]: pgmap v993: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 20887/152379 objects degraded (13.707%); 4465/152379 objects misplaced (2.930%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:09:50.250 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:49 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2233465335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:50.250 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:49 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:09:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:50.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:50.352 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:50.366 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:09:50.379 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:50.423 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:50.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:50.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:50.529 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:50.555 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:50.600 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:50.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:50.706 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:50.733 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:50.736 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:50.754 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:09:51.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:51.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[213871]: failed to encode map e355 with expected crc 2023-09-23T20:09:51.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:09:51.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[213871]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T20:09:51.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[213871]: failed to encode map e355 with expected crc 2023-09-23T20:09:51.257 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[213871]: pgmap v995: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 20887/152070 objects degraded (13.735%); 4465/152070 objects misplaced (2.936%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:09:51.257 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:51.257 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[221113]: failed to encode map e355 with expected crc 2023-09-23T20:09:51.257 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:09:51.257 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[221113]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T20:09:51.258 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[221113]: failed to encode map e355 with expected crc 2023-09-23T20:09:51.258 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:50 smithi135 ceph-mon[221113]: pgmap v995: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 20887/152070 objects degraded (13.735%); 4465/152070 objects misplaced (2.936%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:09:51.308 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:51.316 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:50 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:09:51.316 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:50 smithi144 ceph-mon[194833]: failed to encode map e355 with expected crc 2023-09-23T20:09:51.316 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:50 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:09:51.316 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:50 smithi144 ceph-mon[194833]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T20:09:51.316 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:50 smithi144 ceph-mon[194833]: failed to encode map e355 with expected crc 2023-09-23T20:09:51.316 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:50 smithi144 ceph-mon[194833]: pgmap v995: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 20887/152070 objects degraded (13.735%); 4465/152070 objects misplaced (2.936%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:09:51.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:51.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:51.419 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:51.448 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:51.489 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:51.492 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:51.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:51.596 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:51.636 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:51.754 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:51.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:51.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:51.858 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:51.907 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:52.071 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:52.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:52.118 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:52 smithi144 ceph-mon[194833]: failed to encode map e355 with expected crc 2023-09-23T20:09:52.175 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:52.205 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:52.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:52 smithi135 ceph-mon[213871]: failed to encode map e355 with expected crc 2023-09-23T20:09:52.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:52 smithi135 ceph-mon[221113]: failed to encode map e355 with expected crc 2023-09-23T20:09:52.301 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:52.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:52.346 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:09:52.347 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:09:52.347 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:09:52.347 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:09:52.347 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:09:52.404 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:52.432 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:52.588 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:52.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:52.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:52.691 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:52.717 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:52.760 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:52.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:52.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:52.865 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:52.891 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:53.028 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 1 pg_num to 21 2023-09-23T20:09:53.096 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:09:53.109 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:53.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:53.214 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:53.240 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:53.285 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:53.288 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/964696481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:09:53.288 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:09:53.288 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:53.288 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 20887/152070 objects degraded (13.735%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:53.288 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:53 smithi144 ceph-mon[194833]: pgmap v996: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 84 op/s; 20849/151671 objects degraded (13.746%); 3997/151671 objects misplaced (2.635%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:09:53.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:53.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:53.326 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:09:53.332 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/964696481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:09:53.332 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:09:53.332 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:53.332 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 20887/152070 objects degraded (13.735%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:53.332 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[221113]: pgmap v996: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 84 op/s; 20849/151671 objects degraded (13.746%); 3997/151671 objects misplaced (2.635%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:09:53.332 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/964696481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:09:53.333 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:09:53.333 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:09:53.333 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 20887/152070 objects degraded (13.735%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:53.333 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:53 smithi135 ceph-mon[213871]: pgmap v996: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 84 op/s; 20849/151671 objects degraded (13.746%); 3997/151671 objects misplaced (2.635%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:09:53.390 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:53.418 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:53.475 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:09:53.515 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:53.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:53.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:53.623 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:53.664 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:09:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:09:53] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:09:53.983 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:53.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:54.087 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:54.117 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:54.161 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:54.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:54.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:54.212 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:54.264 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:09:54.288 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[221113]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.288 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:09:54.288 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[221113]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T20:09:54.288 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[221113]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.288 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[221113]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[213871]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:09:54.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[213871]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T20:09:54.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[213871]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:54 smithi135 ceph-mon[213871]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.307 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:54 smithi144 ceph-mon[194833]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:54 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:09:54.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:54 smithi144 ceph-mon[194833]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T20:09:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:54 smithi144 ceph-mon[194833]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:54 smithi144 ceph-mon[194833]: failed to encode map e356 with expected crc 2023-09-23T20:09:54.568 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:09:54.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:54.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:09:54.615 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:09:54.648 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:09:54.673 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:54.676 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:09:54.699 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:09:54.700 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:54.720 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:09:54.742 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:09:54.772 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:09:54.793 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:09:54.798 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:09:54.798 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:09:54.798 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:09:54.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:09:54.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:09:54.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:09:54.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:09:54.799 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:09:54.847 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:54.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:54.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:09:54.951 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:54.984 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:55.085 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:55.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:55.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:55.169 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:09:55.169 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:09:55.188 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:55.189 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:55 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:09:55.189 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:55 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/429536488' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:55.189 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:55 smithi144 ceph-mon[194833]: pgmap v998: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 20824/150966 objects degraded (13.794%); 3997/150966 objects misplaced (2.648%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:09:55.189 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:55 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:09:55.216 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:55.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:55 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:09:55.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:55 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/429536488' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:55.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:55 smithi135 ceph-mon[221113]: pgmap v998: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 20824/150966 objects degraded (13.794%); 3997/150966 objects misplaced (2.648%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:09:55.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:55 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:09:55.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:55 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:09:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:55 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/429536488' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:09:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:55 smithi135 ceph-mon[213871]: pgmap v998: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 20824/150966 objects degraded (13.794%); 3997/150966 objects misplaced (2.648%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:09:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:55 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:09:55.412 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:55.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:55.518 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:55.545 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:55.592 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:55.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:55.696 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:55.722 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:55.931 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:09:55.934 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:55.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:55.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:56.041 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:56.069 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:56.113 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:56.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:56.218 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:56.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:09:56.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T20:09:56.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[221113]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T20:09:56.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:09:56.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T20:09:56.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:56 smithi135 ceph-mon[213871]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T20:09:56.266 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:09:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T20:09:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: failed to encode map e357 with expected crc 2023-09-23T20:09:56.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:56 smithi144 ceph-mon[194833]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T20:09:56.903 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:56.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:56.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:57.011 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:57.057 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:57.125 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:57.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:57.138 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:57.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[213871]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[213871]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[213871]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[213871]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.209 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[213871]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.210 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[213871]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.210 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[213871]: pgmap v1001: 109 pgs: 8 unknown, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 66 op/s; 20824/150873 objects degraded (13.802%); 3481/150873 objects misplaced (2.307%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:09:57.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[221113]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[221113]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[221113]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[221113]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.210 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[221113]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[221113]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:57 smithi135 ceph-mon[221113]: pgmap v1001: 109 pgs: 8 unknown, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 66 op/s; 20824/150873 objects degraded (13.802%); 3481/150873 objects misplaced (2.307%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:09:57.231 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:09:57.257 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:57.299 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:09:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:57.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:09:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:57 smithi144 ceph-mon[194833]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:57 smithi144 ceph-mon[194833]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:57 smithi144 ceph-mon[194833]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:57 smithi144 ceph-mon[194833]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:57 smithi144 ceph-mon[194833]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:57 smithi144 ceph-mon[194833]: failed to encode map e358 with expected crc 2023-09-23T20:09:57.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:57 smithi144 ceph-mon[194833]: pgmap v1001: 109 pgs: 8 unknown, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 66 op/s; 20824/150873 objects degraded (13.802%); 3481/150873 objects misplaced (2.307%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:09:57.410 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:09:57.448 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:57.517 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:09:57.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:57.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:09:57.621 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:57.647 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:57.692 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:57.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:57.800 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:09:57.826 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:57.867 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:09:57.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:57.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:09:57.973 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:58.006 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:58.058 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:58.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:58.162 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:09:58.190 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:58 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 20824/150873 objects degraded (13.802%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2182147397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:58.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:58 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:58 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 20824/150873 objects degraded (13.802%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2182147397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:58.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:58 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:58.237 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:09:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:58.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:09:58.343 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:09:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:58 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 20824/150873 objects degraded (13.802%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:09:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2182147397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:58 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:09:58.444 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:58.672 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:09:58.955 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:09:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:58.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:09:59.043 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:09:59.059 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:09:59.114 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:09:59.114 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:09:59.114 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:09:59.114 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T20:09:59.115 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:59.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:09:59.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:09:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[221113]: failed to encode map e359 with expected crc 2023-09-23T20:09:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[221113]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T20:09:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[221113]: pgmap v1003: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 20763/149796 objects degraded (13.861%); 3456/149796 objects misplaced (2.307%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:09:59.230 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:09:59.230 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:09:59.230 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[213871]: failed to encode map e359 with expected crc 2023-09-23T20:09:59.230 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:59.230 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[213871]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T20:09:59.230 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:09:59 smithi135 ceph-mon[213871]: pgmap v1003: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 20763/149796 objects degraded (13.861%); 3456/149796 objects misplaced (2.307%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:09:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:59 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:09:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:59 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:09:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:59 smithi144 ceph-mon[194833]: failed to encode map e359 with expected crc 2023-09-23T20:09:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:59 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:09:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:59 smithi144 ceph-mon[194833]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T20:09:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:09:59 smithi144 ceph-mon[194833]: pgmap v1003: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 20763/149796 objects degraded (13.861%); 3456/149796 objects misplaced (2.307%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:09:59.776 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:09:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:59.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:09:59.879 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:09:59.905 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:09:59.949 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:09:59.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:09:59.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:00.055 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:00.097 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:00.239 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:00.262 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:00.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:00.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:00.301 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:00.338 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:10:00.338 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:10:00.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: failed to encode map e359 with expected crc 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: failed to encode map e359 with expected crc 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: Health detail: HEALTH_WARN Degraded data redundancy: 20763/149796 objects degraded (13.861%), 8 pgs degraded, 5 pgs undersized 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: [WRN] PG_DEGRADED: Degraded data redundancy: 20763/149796 objects degraded (13.861%), 8 pgs degraded, 5 pgs undersized 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,3] 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: pg 2.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,1] 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: pg 8.0 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,5] 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: pg 8.1 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: pg 8.6 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: pg 8.a is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-23T20:10:00.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[221113]: pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-09-23T20:10:00.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: failed to encode map e359 with expected crc 2023-09-23T20:10:00.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: failed to encode map e359 with expected crc 2023-09-23T20:10:00.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: Health detail: HEALTH_WARN Degraded data redundancy: 20763/149796 objects degraded (13.861%), 8 pgs degraded, 5 pgs undersized 2023-09-23T20:10:00.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: [WRN] PG_DEGRADED: Degraded data redundancy: 20763/149796 objects degraded (13.861%), 8 pgs degraded, 5 pgs undersized 2023-09-23T20:10:00.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,3] 2023-09-23T20:10:00.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: pg 2.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,1] 2023-09-23T20:10:00.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: pg 8.0 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,5] 2023-09-23T20:10:00.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: pg 8.1 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2023-09-23T20:10:00.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: pg 8.6 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2023-09-23T20:10:00.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: pg 8.a is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-09-23T20:10:00.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-23T20:10:00.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:00 smithi135 ceph-mon[213871]: pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-09-23T20:10:00.373 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: failed to encode map e359 with expected crc 2023-09-23T20:10:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: failed to encode map e359 with expected crc 2023-09-23T20:10:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: Health detail: HEALTH_WARN Degraded data redundancy: 20763/149796 objects degraded (13.861%), 8 pgs degraded, 5 pgs undersized 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: [WRN] PG_DEGRADED: Degraded data redundancy: 20763/149796 objects degraded (13.861%), 8 pgs degraded, 5 pgs undersized 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,3] 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: pg 2.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,1] 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: pg 8.0 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,5] 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: pg 8.1 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: pg 8.6 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: pg 8.a is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-23T20:10:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:00 smithi144 ceph-mon[194833]: pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-09-23T20:10:00.406 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:00.775 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:00.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:00.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:00.879 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:00.916 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:01 smithi144 ceph-mon[194833]: pgmap v1004: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 20763/149562 objects degraded (13.883%); 3456/149562 objects misplaced (2.311%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:10:01.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:01 smithi135 ceph-mon[221113]: pgmap v1004: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 20763/149562 objects degraded (13.883%); 3456/149562 objects misplaced (2.311%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:10:01.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:01 smithi135 ceph-mon[213871]: pgmap v1004: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 20763/149562 objects degraded (13.883%); 3456/149562 objects misplaced (2.311%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:10:01.862 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:10:01.957 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:10:01.964 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:01.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:01.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:02.091 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:02.118 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:02.160 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:02.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:02.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:02.267 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:02.305 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:02 smithi135 ceph-mon[221113]: 1.7 scrub starts 2023-09-23T20:10:02.305 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:02 smithi135 ceph-mon[221113]: 1.12 scrub starts 2023-09-23T20:10:02.306 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:02 smithi135 ceph-mon[213871]: 1.7 scrub starts 2023-09-23T20:10:02.306 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:02 smithi135 ceph-mon[213871]: 1.12 scrub starts 2023-09-23T20:10:02.313 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:02 smithi144 ceph-mon[194833]: 1.7 scrub starts 2023-09-23T20:10:02.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:02 smithi144 ceph-mon[194833]: 1.12 scrub starts 2023-09-23T20:10:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:03 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 20763/149562 objects degraded (13.883%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:03.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:03 smithi144 ceph-mon[194833]: pgmap v1005: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 20421/149139 objects degraded (13.693%); 3100/149139 objects misplaced (2.079%); 7.8 MiB/s, 164 objects/s recovering 2023-09-23T20:10:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:03 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 20763/149562 objects degraded (13.883%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:03 smithi135 ceph-mon[221113]: pgmap v1005: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 20421/149139 objects degraded (13.693%); 3100/149139 objects misplaced (2.079%); 7.8 MiB/s, 164 objects/s recovering 2023-09-23T20:10:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:03 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 20763/149562 objects degraded (13.883%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:03 smithi135 ceph-mon[213871]: pgmap v1005: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 20421/149139 objects degraded (13.693%); 3100/149139 objects misplaced (2.079%); 7.8 MiB/s, 164 objects/s recovering 2023-09-23T20:10:03.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:10:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:10:03] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:10:03.987 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:03.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:04.093 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:04.122 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:04.149 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:04.166 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:04.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:04.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:04.182 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:04.271 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:04.302 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:04.345 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:04.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:04.451 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:04.478 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:04.525 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:04.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:04.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:04.635 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:04.648 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:10:04.660 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:04.679 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:10:04.704 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:04.707 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:10:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:04.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:04.732 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:10:04.761 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:10:04.786 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:10:04.807 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:10:04.809 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:04.829 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:10:04.833 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3975615948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:04.836 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:10:04.836 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:10:04.836 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:10:04.836 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:10:04.836 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:10:04.836 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:10:04.836 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:10:04.837 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:10:04.837 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:04.866 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3975615948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:04.866 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3975615948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:04.882 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:04.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:04.990 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:05.048 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:05.173 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 327 (0%) 2023-09-23T20:10:05.173 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T20:10:05.173 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.86882 -> 7.86882 (expected baseline 6.32778) 2023-09-23T20:10:05.173 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 32 -> 32 pgs (0.685015 -> 0.685015 * mean) 2023-09-23T20:10:05.173 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 57 -> 57 pgs (1.22018 -> 1.22018 * mean) 2023-09-23T20:10:05.173 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:10:05.174 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T20:10:05.174 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T20:10:05.174 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:10:05.174 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0222 2023-09-23T20:10:05.174 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.2222 2023-09-23T20:10:05.176 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:10:05.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:05 smithi135 ceph-mon[221113]: pgmap v1006: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 20395/148356 objects degraded (13.747%); 3100/148356 objects misplaced (2.090%); 7.1 MiB/s, 149 objects/s recovering 2023-09-23T20:10:05.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:05 smithi135 ceph-mon[221113]: 1.10 scrub starts 2023-09-23T20:10:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:05 smithi135 ceph-mon[213871]: pgmap v1006: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 20395/148356 objects degraded (13.747%); 3100/148356 objects misplaced (2.090%); 7.1 MiB/s, 149 objects/s recovering 2023-09-23T20:10:05.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:05 smithi135 ceph-mon[213871]: 1.10 scrub starts 2023-09-23T20:10:06.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:05 smithi144 ceph-mon[194833]: pgmap v1006: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 20395/148356 objects degraded (13.747%); 3100/148356 objects misplaced (2.090%); 7.1 MiB/s, 149 objects/s recovering 2023-09-23T20:10:06.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:05 smithi144 ceph-mon[194833]: 1.10 scrub starts 2023-09-23T20:10:06.375 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:06.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:06.483 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:06.509 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:06.549 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:06.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:06.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:06.655 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:06.662 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:06 smithi135 ceph-mon[213871]: from='client.45776 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:06.662 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:06 smithi135 ceph-mon[213871]: 1.f scrub starts 2023-09-23T20:10:06.681 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:06.723 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:06.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:06.828 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:06.860 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:06 smithi144 ceph-mon[194833]: from='client.45776 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:06 smithi144 ceph-mon[194833]: 1.f scrub starts 2023-09-23T20:10:06.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:06 smithi135 ceph-mon[221113]: from='client.45776 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:06.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:06 smithi135 ceph-mon[221113]: 1.f scrub starts 2023-09-23T20:10:07.217 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:07.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:07.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:07.326 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:07.367 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:07.469 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T20:10:07.470 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T20:10:07.691 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:07.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:07.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:07.763 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:07 smithi135 ceph-mon[213871]: pgmap v1007: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 19532/148227 objects degraded (13.177%); 3100/148227 objects misplaced (2.091%); 3.1 MiB/s, 167 objects/s recovering 2023-09-23T20:10:07.764 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:07.764 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:07.764 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:07 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 20395/148356 objects degraded (13.747%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:07.764 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:07 smithi135 ceph-mon[221113]: pgmap v1007: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 19532/148227 objects degraded (13.177%); 3100/148227 objects misplaced (2.091%); 3.1 MiB/s, 167 objects/s recovering 2023-09-23T20:10:07.764 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:07.765 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:07.765 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:07 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 20395/148356 objects degraded (13.747%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:07.796 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:07.831 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:07 smithi144 ceph-mon[194833]: pgmap v1007: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 19532/148227 objects degraded (13.177%); 3100/148227 objects misplaced (2.091%); 3.1 MiB/s, 167 objects/s recovering 2023-09-23T20:10:08.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:08.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:08.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:07 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 20395/148356 objects degraded (13.747%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:08.491 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:08.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:08.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:08.595 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:08.621 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:08.663 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:08.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:08.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:08.694 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:10:08.705 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:08.770 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:08.799 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:08.843 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:08.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:08 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1605264002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:08.948 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:08.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:08 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1605264002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:08.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:08 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1605264002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:08.973 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:09.016 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:09.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:09.121 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:09.146 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:09.190 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:09.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:09.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:09.296 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:09.330 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:09.421 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:10:09.469 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:10:09.469 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:10:09.469 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:10:09.469 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:10:09.469 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:10:09.484 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:09.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:09.590 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:09.617 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:09.662 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:09.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:09.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:09.670 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:09.769 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:09.808 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:09.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:09.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1605264002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:10:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T20:10:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:09 smithi144 ceph-mon[194833]: pgmap v1009: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 19473/147216 objects degraded (13.228%); 3066/147216 objects misplaced (2.083%); 3.2 MiB/s, 168 objects/s recovering 2023-09-23T20:10:09.924 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:09.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:09.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1605264002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[221113]: pgmap v1009: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 19473/147216 objects degraded (13.228%); 3066/147216 objects misplaced (2.083%); 3.2 MiB/s, 168 objects/s recovering 2023-09-23T20:10:09.927 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1605264002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:10:09.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T20:10:09.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: failed to encode map e360 with expected crc 2023-09-23T20:10:09.928 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:09 smithi135 ceph-mon[213871]: pgmap v1009: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 19473/147216 objects degraded (13.228%); 3066/147216 objects misplaced (2.083%); 3.2 MiB/s, 168 objects/s recovering 2023-09-23T20:10:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:09.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:10.030 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:10.058 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:10.463 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:10.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:10.511 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:10:10.568 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:10.603 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:10.671 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:10.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:10.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:10.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[213871]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[213871]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[213871]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T20:10:10.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[213871]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[213871]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[213871]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[221113]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[221113]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[221113]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T20:10:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[221113]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[221113]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:10 smithi135 ceph-mon[221113]: failed to encode map e361 with expected crc 2023-09-23T20:10:10.776 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:10.835 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:10.841 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:10.918 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:10.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:10.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:11.032 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:11.055 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:10 smithi144 ceph-mon[194833]: failed to encode map e361 with expected crc 2023-09-23T20:10:11.056 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:10 smithi144 ceph-mon[194833]: failed to encode map e361 with expected crc 2023-09-23T20:10:11.056 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:10 smithi144 ceph-mon[194833]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T20:10:11.056 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:10 smithi144 ceph-mon[194833]: failed to encode map e361 with expected crc 2023-09-23T20:10:11.056 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:10 smithi144 ceph-mon[194833]: failed to encode map e361 with expected crc 2023-09-23T20:10:11.056 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:10 smithi144 ceph-mon[194833]: failed to encode map e361 with expected crc 2023-09-23T20:10:11.058 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:11.101 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:11.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:11.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:11.206 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:11.233 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:11.276 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:11.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:11.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:11.382 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:11.415 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:11.638 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:11.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:11.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:11.741 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:11.773 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:11.854 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:11.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:11.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:11.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[213871]: failed to encode map e362 with expected crc 2023-09-23T20:10:11.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[213871]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T20:10:11.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[213871]: failed to encode map e362 with expected crc 2023-09-23T20:10:11.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[213871]: failed to encode map e362 with expected crc 2023-09-23T20:10:11.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[213871]: failed to encode map e362 with expected crc 2023-09-23T20:10:11.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[213871]: pgmap v1012: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 19473/146991 objects degraded (13.248%); 3066/146991 objects misplaced (2.086%); 981 KiB/s, 159 objects/s recovering 2023-09-23T20:10:11.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[221113]: failed to encode map e362 with expected crc 2023-09-23T20:10:11.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[221113]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T20:10:11.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[221113]: failed to encode map e362 with expected crc 2023-09-23T20:10:11.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[221113]: failed to encode map e362 with expected crc 2023-09-23T20:10:11.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[221113]: failed to encode map e362 with expected crc 2023-09-23T20:10:11.953 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:11 smithi135 ceph-mon[221113]: pgmap v1012: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 19473/146991 objects degraded (13.248%); 3066/146991 objects misplaced (2.086%); 981 KiB/s, 159 objects/s recovering 2023-09-23T20:10:11.959 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:11.990 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:12.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:11 smithi144 ceph-mon[194833]: failed to encode map e362 with expected crc 2023-09-23T20:10:12.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:11 smithi144 ceph-mon[194833]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T20:10:12.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:11 smithi144 ceph-mon[194833]: failed to encode map e362 with expected crc 2023-09-23T20:10:12.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:11 smithi144 ceph-mon[194833]: failed to encode map e362 with expected crc 2023-09-23T20:10:12.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:11 smithi144 ceph-mon[194833]: failed to encode map e362 with expected crc 2023-09-23T20:10:12.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:11 smithi144 ceph-mon[194833]: pgmap v1012: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 19473/146991 objects degraded (13.248%); 3066/146991 objects misplaced (2.086%); 981 KiB/s, 159 objects/s recovering 2023-09-23T20:10:12.152 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:12.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:12.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:12.210 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:10:12.256 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:12.283 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:12.329 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:12.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:12.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:12.434 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:12.466 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:12.699 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 19473/146991 objects degraded (13.248%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:12.809 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:12.938 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:12.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:12.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:12.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 19473/146991 objects degraded (13.248%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:13.042 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:13.066 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 19473/146991 objects degraded (13.248%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:13.067 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:13.069 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:13.117 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:13.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:13.223 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:13.268 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:13.357 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 327 (0%) 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.86882 -> 7.86882 (expected baseline 6.32778) 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 32 -> 32 pgs (0.685015 -> 0.685015 * mean) 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 57 -> 57 pgs (1.22018 -> 1.22018 * mean) 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0221 2023-09-23T20:10:13.358 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.2205 2023-09-23T20:10:13.361 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:10:13.528 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:13.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:13.633 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:13.661 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:13.708 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:13.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:13.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:13.805 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:13 smithi135 ceph-mon[213871]: pgmap v1013: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 18697/146490 objects degraded (12.763%); 3028/146490 objects misplaced (2.067%); 1.6 MiB/s, 150 objects/s recovering 2023-09-23T20:10:13.805 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2533064800' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:13.805 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:13 smithi135 ceph-mon[221113]: pgmap v1013: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 18697/146490 objects degraded (12.763%); 3028/146490 objects misplaced (2.067%); 1.6 MiB/s, 150 objects/s recovering 2023-09-23T20:10:13.806 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2533064800' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:13.806 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:10:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:10:13] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:10:13.813 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:13.839 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:13.884 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:13.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:13.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:10:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:10:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:10:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:10:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:10:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:10:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:10:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:10:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:10:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:10:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:10:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:10:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:10:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:10:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:10:13.968 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:10:13.990 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:14.014 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:13 smithi144 ceph-mon[194833]: pgmap v1013: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 18697/146490 objects degraded (12.763%); 3028/146490 objects misplaced (2.067%); 1.6 MiB/s, 150 objects/s recovering 2023-09-23T20:10:14.014 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2533064800' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:14.017 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:14.060 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:14.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:14.166 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:14.193 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:14.238 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:14.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:14.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:14.343 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:14.384 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:14.711 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:14 smithi135 ceph-mon[221113]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:14.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:14 smithi135 ceph-mon[213871]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:14.853 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:10:14.854 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T20:10:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:14 smithi144 ceph-mon[194833]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:15.800 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:15.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:15.903 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:15.938 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:15.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:15 smithi135 ceph-mon[221113]: pgmap v1014: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 11895/145365 objects degraded (8.183%); 5786/145365 objects misplaced (3.980%); 3.5 MiB/s, 178 objects/s recovering 2023-09-23T20:10:15.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:15 smithi135 ceph-mon[213871]: pgmap v1014: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 11895/145365 objects degraded (8.183%); 5786/145365 objects misplaced (3.980%); 3.5 MiB/s, 178 objects/s recovering 2023-09-23T20:10:16.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:15 smithi144 ceph-mon[194833]: pgmap v1014: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 11895/145365 objects degraded (8.183%); 5786/145365 objects misplaced (3.980%); 3.5 MiB/s, 178 objects/s recovering 2023-09-23T20:10:16.288 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:16.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:16.403 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:16.412 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:16.451 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:16.622 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:16.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:16.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:16.726 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:16.754 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:16.796 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:16.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:16.901 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:16.949 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:17.216 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:17.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:17.247 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:10:17.247 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:10:17.254 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:10:17.323 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:17.349 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:17.393 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:17.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:17.499 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:17.525 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:17.570 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:17.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:17.676 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:17.703 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:17.748 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:17.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:17.856 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:17.904 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:17.974 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:10:18.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:17 smithi135 ceph-mon[213871]: pgmap v1015: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 11895/145260 objects degraded (8.189%); 5786/145260 objects misplaced (3.983%); 2.7 MiB/s, 139 objects/s recovering 2023-09-23T20:10:18.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/473271681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:18.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11895/145260 objects degraded (8.189%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:18.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:17 smithi135 ceph-mon[221113]: pgmap v1015: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 11895/145260 objects degraded (8.189%); 5786/145260 objects misplaced (3.983%); 2.7 MiB/s, 139 objects/s recovering 2023-09-23T20:10:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/473271681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:18.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11895/145260 objects degraded (8.189%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:17 smithi144 ceph-mon[194833]: pgmap v1015: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 11895/145260 objects degraded (8.189%); 5786/145260 objects misplaced (3.983%); 2.7 MiB/s, 139 objects/s recovering 2023-09-23T20:10:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/473271681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11895/145260 objects degraded (8.189%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:18.556 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:18.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:18.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:18.659 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:18.688 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:18.842 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:10:19.233 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:19.233 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[213871]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.233 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/473271681' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:19.233 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[213871]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T20:10:19.233 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[213871]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.233 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[213871]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.234 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:19.234 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[221113]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.234 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/473271681' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:19.234 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[221113]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T20:10:19.234 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[221113]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.234 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:18 smithi135 ceph-mon[221113]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:18 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:18 smithi144 ceph-mon[194833]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/473271681' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:18 smithi144 ceph-mon[194833]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T20:10:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:18 smithi144 ceph-mon[194833]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:18 smithi144 ceph-mon[194833]: failed to encode map e363 with expected crc 2023-09-23T20:10:19.384 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:19.409 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:19.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:19.515 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:19.579 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:19.581 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:19.681 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:19.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:19.757 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 327 (0%) 2023-09-23T20:10:19.757 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T20:10:19.757 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.86882 -> 7.86882 (expected baseline 6.32778) 2023-09-23T20:10:19.758 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 32 -> 32 pgs (0.685015 -> 0.685015 * mean) 2023-09-23T20:10:19.758 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 57 -> 57 pgs (1.22018 -> 1.22018 * mean) 2023-09-23T20:10:19.758 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:10:19.758 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T20:10:19.758 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T20:10:19.758 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:10:19.758 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0218 2023-09-23T20:10:19.758 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.2183 2023-09-23T20:10:19.760 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:10:19.787 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:20.188 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:19 smithi135 ceph-mon[221113]: pgmap v1017: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 11867/144063 objects degraded (8.237%); 5405/144063 objects misplaced (3.752%); 5.8 MiB/s, 185 objects/s recovering 2023-09-23T20:10:20.188 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:19 smithi135 ceph-mon[213871]: pgmap v1017: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 11867/144063 objects degraded (8.237%); 5405/144063 objects misplaced (3.752%); 5.8 MiB/s, 185 objects/s recovering 2023-09-23T20:10:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:19 smithi144 ceph-mon[194833]: pgmap v1017: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 11867/144063 objects degraded (8.237%); 5405/144063 objects misplaced (3.752%); 5.8 MiB/s, 185 objects/s recovering 2023-09-23T20:10:20.742 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:10:20.742 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T20:10:20.952 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:21.066 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:21.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:21.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:21.173 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:10:21.174 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:21.209 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:21.344 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:20 smithi135 ceph-mon[213871]: from='client.45803 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:21.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:20 smithi135 ceph-mon[221113]: from='client.45803 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:21.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:20 smithi144 ceph-mon[194833]: from='client.45803 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:21.867 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:21.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:21.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:21.975 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:22.122 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:22.138 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:22.168 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:22.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:22.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:21 smithi135 ceph-mon[221113]: pgmap v1018: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 11867/143787 objects degraded (8.253%); 5405/143787 objects misplaced (3.759%); 4.8 MiB/s, 152 objects/s recovering 2023-09-23T20:10:22.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:21 smithi135 ceph-mon[213871]: pgmap v1018: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 11867/143787 objects degraded (8.253%); 5405/143787 objects misplaced (3.759%); 4.8 MiB/s, 152 objects/s recovering 2023-09-23T20:10:22.277 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:22.300 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:21 smithi144 ceph-mon[194833]: pgmap v1018: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 11867/143787 objects degraded (8.253%); 5405/143787 objects misplaced (3.759%); 4.8 MiB/s, 152 objects/s recovering 2023-09-23T20:10:22.303 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:22.349 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:22.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:22.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:22.431 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:22.454 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:22.480 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:22.525 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:22.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:22.537 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:10:22.537 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:10:22.631 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:22.658 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:22.702 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:22.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:22.808 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:22.835 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:23.078 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:23.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:23.186 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:23.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:23.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:23.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11867/143787 objects degraded (8.253%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:23.213 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:23 smithi135 ceph-mon[221113]: pgmap v1019: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 11828/143220 objects degraded (8.259%); 5374/143220 objects misplaced (3.752%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:10:23.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:23.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:23.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11867/143787 objects degraded (8.253%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:23.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:23 smithi135 ceph-mon[213871]: pgmap v1019: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 11828/143220 objects degraded (8.259%); 5374/143220 objects misplaced (3.752%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:10:23.216 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11867/143787 objects degraded (8.253%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:23 smithi144 ceph-mon[194833]: pgmap v1019: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 11828/143220 objects degraded (8.259%); 5374/143220 objects misplaced (3.752%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:10:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:10:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:10:23] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T20:10:24.260 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:24.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:24.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:10:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:10:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:10:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:10:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:10:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:10:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:10:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:10:24.309 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:10:24.309 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:10:24.309 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:10:24.309 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:10:24.309 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:10:24.309 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:10:24.309 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:10:24.309 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:10:24.366 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:24.378 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:10:24.400 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4164111131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:24.400 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4164111131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4164111131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:24.910 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:25.171 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:25.174 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:25 smithi144 ceph-mon[194833]: pgmap v1020: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 11828/142164 objects degraded (8.320%); 5031/142164 objects misplaced (3.539%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T20:10:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:25.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:25.276 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:25.303 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:25 smithi135 ceph-mon[221113]: pgmap v1020: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 11828/142164 objects degraded (8.320%); 5031/142164 objects misplaced (3.539%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T20:10:25.303 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:25 smithi135 ceph-mon[213871]: pgmap v1020: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 11828/142164 objects degraded (8.320%); 5031/142164 objects misplaced (3.539%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T20:10:25.307 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:25.562 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:25.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:25.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:25.667 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:25.696 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:25.854 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:25.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:25.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:25.959 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:25.990 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:26.104 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:26.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:26.163 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:26.212 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:26.263 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:10:26.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1476727890' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:10:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:10:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:26 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1476727890' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:10:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:10:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:26 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:26.621 INFO:teuthology.orchestra.run.smithi135.stdout:moved 7 / 327 (2.14067%) 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.86882 -> 9.37648 (expected baseline 6.32778) 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 32 -> 27 pgs (0.685015 -> 0.577982 * mean) 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 57 -> 57 pgs (1.22018 -> 1.22018 * mean) 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:oload 110 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 0.05 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0216 2023-09-23T20:10:26.622 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.0238 2023-09-23T20:10:26.623 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2 weight 0.7459 -> 0.6959 2023-09-23T20:10:26.623 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3 weight 0.7588 -> 0.7088 2023-09-23T20:10:26.623 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:10:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1476727890' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:10:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:10:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:26 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:26.943 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:26.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:10:26 smithi135 conmon[201366]: 2023-09-23T20:10:26.538+0000 7fcf49f9e700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T20:10:27.130 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:27.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:27.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:27.239 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:27.274 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[221113]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[221113]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[221113]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[221113]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[221113]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[221113]: pgmap v1022: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 11828/142047 objects degraded (8.327%); 5031/142047 objects misplaced (3.542%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[213871]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[213871]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[213871]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:10:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[213871]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T20:10:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[213871]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:27.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:27 smithi135 ceph-mon[213871]: pgmap v1022: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 11828/142047 objects degraded (8.327%); 5031/142047 objects misplaced (3.542%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:10:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:27 smithi144 ceph-mon[194833]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:27 smithi144 ceph-mon[194833]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:27 smithi144 ceph-mon[194833]: failed to encode map e364 with expected crc 2023-09-23T20:10:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:27 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:10:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:27 smithi144 ceph-mon[194833]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T20:10:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:27 smithi144 ceph-mon[194833]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:27 smithi144 ceph-mon[194833]: pgmap v1022: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 11828/142047 objects degraded (8.327%); 5031/142047 objects misplaced (3.542%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:10:27.773 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:10:27.850 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T20:10:27.851 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T20:10:28.041 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:28.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:28.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:28.146 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:28.189 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11828/142047 objects degraded (8.327%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:28 smithi135 ceph-mon[221113]: 1.1 scrub starts 2023-09-23T20:10:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11828/142047 objects degraded (8.327%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:28 smithi135 ceph-mon[213871]: 1.1 scrub starts 2023-09-23T20:10:28.488 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:28.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:28.579 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:28.594 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:28.617 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11828/142047 objects degraded (8.327%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:28.617 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:28 smithi144 ceph-mon[194833]: 1.1 scrub starts 2023-09-23T20:10:28.620 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:28.665 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:28.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:28.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:28.686 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:28.769 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:28.795 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:28.844 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:28.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:28.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:28.950 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:28.978 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:29.024 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:29.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:29.065 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:10:29.065 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:10:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:10:29.066 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:10:29.066 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:10:29.130 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:29.157 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:29.168 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:10:29.203 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:29.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:29.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:29.309 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:29.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:29 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:10:29.349 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:29 smithi135 ceph-mon[213871]: pgmap v1023: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 11800/140928 objects degraded (8.373%); 4574/140928 objects misplaced (3.246%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:29.349 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:29 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:10:29.349 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:29 smithi135 ceph-mon[221113]: pgmap v1023: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 11800/140928 objects degraded (8.373%); 4574/140928 objects misplaced (3.246%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:29.350 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:29.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:29 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:10:29.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:29 smithi144 ceph-mon[194833]: pgmap v1023: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 11800/140928 objects degraded (8.373%); 4574/140928 objects misplaced (3.246%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:30.463 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:10:31.074 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:31.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:31.178 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:31.221 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:31.267 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:31.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:31.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:31.373 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:31.400 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:31.450 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:31.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:31.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:31.522 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:31.558 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:31.568 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:31.585 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:31.594 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:10:31.623 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:10:31.634 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:31.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:31.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:31.660 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:10:31.691 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:10:31.712 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:10:31.734 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:10:31.739 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:31.755 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:10:31.765 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:31.781 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:10:31.791 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:10:31.791 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:10:31.791 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:10:31.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:10:31.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:10:31.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:10:31.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:10:31.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:10:31.815 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:31.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:31.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:31.921 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:31.948 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:31.993 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:31.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:31.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:32.099 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:32.126 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:32.182 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:32.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:32.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:32.206 INFO:teuthology.orchestra.run.smithi135.stdout:moved 0 / 327 (0%) 2023-09-23T20:10:32.206 INFO:teuthology.orchestra.run.smithi135.stdout:avg 46.7143 2023-09-23T20:10:32.206 INFO:teuthology.orchestra.run.smithi135.stdout:stddev 7.86882 -> 7.86882 (expected baseline 6.32778) 2023-09-23T20:10:32.206 INFO:teuthology.orchestra.run.smithi135.stdout:min osd.2 with 32 -> 32 pgs (0.685015 -> 0.685015 * mean) 2023-09-23T20:10:32.206 INFO:teuthology.orchestra.run.smithi135.stdout:max osd.1 with 57 -> 57 pgs (1.22018 -> 1.22018 * mean) 2023-09-23T20:10:32.206 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:10:32.206 INFO:teuthology.orchestra.run.smithi135.stdout:oload 1000 2023-09-23T20:10:32.207 INFO:teuthology.orchestra.run.smithi135.stdout:max_change 1 2023-09-23T20:10:32.207 INFO:teuthology.orchestra.run.smithi135.stdout:max_change_osds 4 2023-09-23T20:10:32.207 INFO:teuthology.orchestra.run.smithi135.stdout:average_utilization 0.0215 2023-09-23T20:10:32.207 INFO:teuthology.orchestra.run.smithi135.stdout:overload_utilization 0.2153 2023-09-23T20:10:32.207 INFO:teuthology.orchestra.run.smithi135.stderr:no change 2023-09-23T20:10:32.242 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:31 smithi135 ceph-mon[221113]: pgmap v1024: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 11800/140607 objects degraded (8.392%); 4574/140607 objects misplaced (3.253%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:32.243 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:31 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3106105221' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:32.243 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:31 smithi135 ceph-mon[213871]: pgmap v1024: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 11800/140607 objects degraded (8.392%); 4574/140607 objects misplaced (3.253%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:32.243 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:31 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3106105221' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:32.289 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:32.324 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:32.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:31 smithi144 ceph-mon[194833]: pgmap v1024: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 11800/140607 objects degraded (8.392%); 4574/140607 objects misplaced (3.253%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:32.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:31 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3106105221' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:32.767 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:32.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:32.872 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:32.904 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:32 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:10:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11800/140607 objects degraded (8.392%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:33.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:32 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:10:33.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11800/140607 objects degraded (8.392%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:33.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:32 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:10:33.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11800/140607 objects degraded (8.392%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:33.910 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:33.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:33.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:10:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:10:33] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T20:10:34.015 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:34.041 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:34.085 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:34.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:34.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:34.191 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:34.218 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:34.250 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:33 smithi135 ceph-mon[213871]: from='client.35931 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:34.250 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:33 smithi135 ceph-mon[213871]: pgmap v1025: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 11771/140076 objects degraded (8.403%); 4535/140076 objects misplaced (3.238%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:34.251 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:33 smithi135 ceph-mon[221113]: from='client.35931 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:34.251 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:33 smithi135 ceph-mon[221113]: pgmap v1025: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 11771/140076 objects degraded (8.403%); 4535/140076 objects misplaced (3.238%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:34.269 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:34.279 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:33 smithi144 ceph-mon[194833]: from='client.35931 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:34.279 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:33 smithi144 ceph-mon[194833]: pgmap v1025: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 11771/140076 objects degraded (8.403%); 4535/140076 objects misplaced (3.238%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:34.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:34.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:34.381 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:34.456 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:34.561 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:34.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:34.666 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:34.699 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:34.828 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:10:35.025 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:10:35.026 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:10:35.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:34 smithi135 ceph-mon[221113]: 1.7 scrub starts 2023-09-23T20:10:35.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:34 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:35.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:34 smithi135 ceph-mon[213871]: 1.7 scrub starts 2023-09-23T20:10:35.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:34 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:34 smithi144 ceph-mon[194833]: 1.7 scrub starts 2023-09-23T20:10:35.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:34 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:35.716 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:35.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:35.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:35.820 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:35.823 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:10:35.854 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:35.988 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:36.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:36.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:36.105 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:36.133 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:36.134 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:36.183 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:36.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:36.287 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:36.313 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:35 smithi144 ceph-mon[194833]: pgmap v1026: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 11771/138666 objects degraded (8.489%); 4138/138666 objects misplaced (2.984%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:10:36.316 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:36.365 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:36.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:36.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:36.470 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:35 smithi135 ceph-mon[221113]: pgmap v1026: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 11771/138666 objects degraded (8.489%); 4138/138666 objects misplaced (2.984%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:10:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:35 smithi135 ceph-mon[213871]: pgmap v1026: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 11771/138666 objects degraded (8.489%); 4138/138666 objects misplaced (2.984%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:10:36.519 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:36.689 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:36.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:36.793 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:36.836 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:37.007 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:10:37.034 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/313641439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:37.034 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:37.034 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/313641439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:37.034 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/313641439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:38.009 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:38.016 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:38.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:38.128 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:38.160 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:38.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: pgmap v1027: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 11771/138534 objects degraded (8.497%); 4138/138534 objects misplaced (2.987%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11771/138534 objects degraded (8.497%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: pgmap v1027: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 11771/138534 objects degraded (8.497%); 4138/138534 objects misplaced (2.987%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:38.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:37 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11771/138534 objects degraded (8.497%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:38.318 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:38.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: pgmap v1027: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 11771/138534 objects degraded (8.497%); 4138/138534 objects misplaced (2.987%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:10:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:10:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T20:10:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: failed to encode map e365 with expected crc 2023-09-23T20:10:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:37 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11771/138534 objects degraded (8.497%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:38.422 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:38.577 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:10:38.605 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:38.606 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:10:38.642 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:10:38.666 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:10:38.688 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:10:38.709 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:10:38.730 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:10:38.760 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:10:38.762 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:10:38.762 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:10:38.762 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:10:38.763 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:10:38.763 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:10:38.763 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:10:38.763 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:10:38.763 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:10:39.012 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:10:39.012 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:10:39.210 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:39.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:39.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:39.320 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:39.348 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2973690042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:39.348 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:39 smithi135 ceph-mon[221113]: pgmap v1029: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 11735/137193 objects degraded (8.554%); 4009/137193 objects misplaced (2.922%); 4.0 MiB/s, 94 objects/s recovering 2023-09-23T20:10:39.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2973690042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:39.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:39 smithi135 ceph-mon[213871]: pgmap v1029: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 11735/137193 objects degraded (8.554%); 4009/137193 objects misplaced (2.922%); 4.0 MiB/s, 94 objects/s recovering 2023-09-23T20:10:39.351 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2973690042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:39 smithi144 ceph-mon[194833]: pgmap v1029: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 11735/137193 objects degraded (8.554%); 4009/137193 objects misplaced (2.922%); 4.0 MiB/s, 94 objects/s recovering 2023-09-23T20:10:39.810 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:39.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:39.917 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:39.947 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:40.029 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:10:40.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:40 smithi135 ceph-mon[221113]: 1.12 scrub starts 2023-09-23T20:10:40.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:40 smithi135 ceph-mon[221113]: 1.11 scrub starts 2023-09-23T20:10:40.067 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:40 smithi135 ceph-mon[213871]: 1.12 scrub starts 2023-09-23T20:10:40.067 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:40 smithi135 ceph-mon[213871]: 1.11 scrub starts 2023-09-23T20:10:40.117 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:40.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:40.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:40.225 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:40.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:40 smithi144 ceph-mon[194833]: 1.12 scrub starts 2023-09-23T20:10:40.252 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:40 smithi144 ceph-mon[194833]: 1.11 scrub starts 2023-09-23T20:10:40.254 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:40.351 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:40.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:40.454 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:40.481 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:40.536 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:40.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:40.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:40.642 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:40.681 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:40.761 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:40.869 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:10:40.869 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:10:40.869 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T20:10:40.869 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:10:40.893 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:40.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:41.010 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:41.037 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:41.089 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:41.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:41.193 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:41.195 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:41 smithi135 ceph-mon[213871]: 1.1 scrub starts 2023-09-23T20:10:41.195 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:41 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:41.195 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:41 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:41.195 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:41 smithi135 ceph-mon[213871]: pgmap v1030: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 11735/136818 objects degraded (8.577%); 4009/136818 objects misplaced (2.930%); 4.0 MiB/s, 94 objects/s recovering 2023-09-23T20:10:41.195 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:41 smithi135 ceph-mon[221113]: 1.1 scrub starts 2023-09-23T20:10:41.195 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:41 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:41.196 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:41 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:41.196 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:41 smithi135 ceph-mon[221113]: pgmap v1030: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 11735/136818 objects degraded (8.577%); 4009/136818 objects misplaced (2.930%); 4.0 MiB/s, 94 objects/s recovering 2023-09-23T20:10:41.236 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:41.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:41 smithi144 ceph-mon[194833]: 1.1 scrub starts 2023-09-23T20:10:41.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:41 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:41.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:41 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:41.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:41 smithi144 ceph-mon[194833]: pgmap v1030: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 11735/136818 objects degraded (8.577%); 4009/136818 objects misplaced (2.930%); 4.0 MiB/s, 94 objects/s recovering 2023-09-23T20:10:41.460 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:41.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:41.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:41.566 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:41.593 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:41.638 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:41.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:41.749 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:41.776 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:41.822 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:41.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:41.931 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:41.957 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:42.000 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:42.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:42.108 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:42.139 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:42.220 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:42.242 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:10:42.262 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:10:42.262 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:42 smithi135 ceph-mon[221113]: 1.10 scrub starts 2023-09-23T20:10:42.262 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:42 smithi135 ceph-mon[221113]: 1.11 scrub starts 2023-09-23T20:10:42.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:42 smithi135 ceph-mon[213871]: 1.10 scrub starts 2023-09-23T20:10:42.263 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:42 smithi135 ceph-mon[213871]: 1.11 scrub starts 2023-09-23T20:10:42.349 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:42.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:42 smithi144 ceph-mon[194833]: 1.10 scrub starts 2023-09-23T20:10:42.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:42 smithi144 ceph-mon[194833]: 1.11 scrub starts 2023-09-23T20:10:42.453 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:42.486 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:42.840 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:42.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:42.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:42.947 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:42.984 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:43.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:43 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11735/136818 objects degraded (8.577%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:43.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:43 smithi135 ceph-mon[213871]: pgmap v1031: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 11702/136305 objects degraded (8.585%); 3974/136305 objects misplaced (2.916%); 3.9 MiB/s, 93 objects/s recovering 2023-09-23T20:10:43.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:43 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11735/136818 objects degraded (8.577%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:43.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:43 smithi135 ceph-mon[221113]: pgmap v1031: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 11702/136305 objects degraded (8.585%); 3974/136305 objects misplaced (2.916%); 3.9 MiB/s, 93 objects/s recovering 2023-09-23T20:10:43.254 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:43.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:43 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11735/136818 objects degraded (8.577%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:43.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:43 smithi144 ceph-mon[194833]: pgmap v1031: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 11702/136305 objects degraded (8.585%); 3974/136305 objects misplaced (2.916%); 3.9 MiB/s, 93 objects/s recovering 2023-09-23T20:10:43.478 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:43.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:43.583 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:43.583 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:43.610 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:43.662 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:43.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:43.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:43.768 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:43.807 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:10:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:10:43] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T20:10:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:44 smithi135 ceph-mon[213871]: 1.1 scrub starts 2023-09-23T20:10:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:44 smithi135 ceph-mon[221113]: 1.1 scrub starts 2023-09-23T20:10:44.489 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:44.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:44.594 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:44.620 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:44.620 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:44.620 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:44 smithi144 ceph-mon[194833]: 1.1 scrub starts 2023-09-23T20:10:44.623 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:44.672 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:44.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:44.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:44.780 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:44.813 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:45.342 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:10:45.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:45.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:45.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:45.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/520014338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:10:45.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[213871]: pgmap v1032: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 11702/135024 objects degraded (8.667%); 3974/135024 objects misplaced (2.943%); 1.5 MiB/s, 99 objects/s recovering 2023-09-23T20:10:45.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:45.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:45.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:45.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/520014338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:10:45.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:45 smithi135 ceph-mon[221113]: pgmap v1032: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 11702/135024 objects degraded (8.667%); 3974/135024 objects misplaced (2.943%); 1.5 MiB/s, 99 objects/s recovering 2023-09-23T20:10:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:45 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:45 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:45 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/520014338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:10:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:45 smithi144 ceph-mon[194833]: pgmap v1032: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 11702/135024 objects degraded (8.667%); 3974/135024 objects misplaced (2.943%); 1.5 MiB/s, 99 objects/s recovering 2023-09-23T20:10:45.788 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:45.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:45.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:45.893 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:45.921 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:45.970 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:45.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:45.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:46.078 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:46.104 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:46.156 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:46.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:46.265 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:46.297 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:46.353 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:10:46.557 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[221113]: 1.f scrub starts 2023-09-23T20:10:46.557 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[221113]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.557 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[221113]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.557 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/520014338' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:10:46.557 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[221113]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T20:10:46.557 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[221113]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3662829437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[213871]: 1.f scrub starts 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[213871]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[213871]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/520014338' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[213871]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[213871]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3662829437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:46.558 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:46 smithi144 ceph-mon[194833]: 1.f scrub starts 2023-09-23T20:10:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:46 smithi144 ceph-mon[194833]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:46 smithi144 ceph-mon[194833]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/520014338' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:10:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:46 smithi144 ceph-mon[194833]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T20:10:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:46 smithi144 ceph-mon[194833]: failed to encode map e366 with expected crc 2023-09-23T20:10:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3662829437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:10:47.614 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:10:47.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:47.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[221113]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[221113]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:10:47.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[221113]: osdmap e367: 8 total, 8 up, 7 in 2023-09-23T20:10:47.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[221113]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[221113]: pgmap v1035: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 11702/134910 objects degraded (8.674%); 3974/134910 objects misplaced (2.946%); 536 KiB/s, 64 objects/s recovering 2023-09-23T20:10:47.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:10:47.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:47.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[213871]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[213871]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:10:47.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[213871]: osdmap e367: 8 total, 8 up, 7 in 2023-09-23T20:10:47.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[213871]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.616 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:47 smithi135 ceph-mon[213871]: pgmap v1035: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 11702/134910 objects degraded (8.674%); 3974/134910 objects misplaced (2.946%); 536 KiB/s, 64 objects/s recovering 2023-09-23T20:10:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:47 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:10:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:47 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:47 smithi144 ceph-mon[194833]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:47 smithi144 ceph-mon[194833]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:47 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:10:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:47 smithi144 ceph-mon[194833]: osdmap e367: 8 total, 8 up, 7 in 2023-09-23T20:10:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:47 smithi144 ceph-mon[194833]: failed to encode map e367 with expected crc 2023-09-23T20:10:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:47 smithi144 ceph-mon[194833]: pgmap v1035: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 11702/134910 objects degraded (8.674%); 3974/134910 objects misplaced (2.946%); 536 KiB/s, 64 objects/s recovering 2023-09-23T20:10:47.709 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:10:47.765 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:47.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:47.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:47.872 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:47.944 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:48.107 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:48.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:48.211 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:48.237 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:48.253 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:10:48.253 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:10:48.253 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:10:48.253 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:10:48.254 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:10:48.284 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:48.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:48.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:48.355 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:10:48.387 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:10:48.392 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:48.411 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:10:48.432 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:48.438 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:10:48.470 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:10:48.499 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:10:48.515 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:48.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:48.531 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:10:48.556 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:10:48.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:10:48.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:10:48.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:10:48.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:10:48.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:10:48.558 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:10:48.558 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:10:48.558 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:10:48.582 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:48 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11702/134910 objects degraded (8.674%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:48.582 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:48 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/216354908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:48.582 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:48 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11702/134910 objects degraded (8.674%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:48.583 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:48 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/216354908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:48.621 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:48 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11702/134910 objects degraded (8.674%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:48 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/216354908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:48.647 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:48.689 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:48.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:48.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:48.794 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:48.821 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:48.871 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:48.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:48.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:48.978 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:49.005 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:49.052 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:49.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:49.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:49.157 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:10:49.189 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:49.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:49 smithi135 ceph-mon[213871]: pgmap v1036: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 11671/133815 objects degraded (8.722%); 3941/133815 objects misplaced (2.945%); 1.0 MiB/s, 153 objects/s recovering 2023-09-23T20:10:49.534 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:10:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:49.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:10:49.539 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:49.567 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:10:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:49 smithi144 ceph-mon[194833]: pgmap v1036: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 11671/133815 objects degraded (8.722%); 3941/133815 objects misplaced (2.945%); 1.0 MiB/s, 153 objects/s recovering 2023-09-23T20:10:49.640 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:49.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:49 smithi135 ceph-mon[221113]: pgmap v1036: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 11671/133815 objects degraded (8.722%); 3941/133815 objects misplaced (2.945%); 1.0 MiB/s, 153 objects/s recovering 2023-09-23T20:10:49.680 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:49.795 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:10:49.816 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:49.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:49.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:49.924 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:49.950 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:49.996 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:50.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:50.101 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:50.148 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:50.220 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:50.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:50.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:50.327 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:50.359 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:50.688 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:50.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:50.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:50.793 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:50.820 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:50.867 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:50.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:50.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:50.890 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:50.918 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:50.974 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:51.007 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:51.175 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:51.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:51.282 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:51.308 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:51.354 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:51.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:51.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:51.460 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:51.486 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:51.532 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:51.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:51.604 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:10:51.636 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:51.662 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:51.705 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:51.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:51.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:51.810 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:51.840 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:51.886 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:51.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:51.984 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:51 smithi144 ceph-mon[194833]: pgmap v1037: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 11671/133509 objects degraded (8.742%); 3941/133509 objects misplaced (2.952%); 504 KiB/s, 145 objects/s recovering 2023-09-23T20:10:51.984 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1388058777' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:10:51.984 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:10:51.991 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:51.998 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:10:52.017 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:52.065 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:52.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:52.099 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:51 smithi135 ceph-mon[221113]: pgmap v1037: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 11671/133509 objects degraded (8.742%); 3941/133509 objects misplaced (2.952%); 504 KiB/s, 145 objects/s recovering 2023-09-23T20:10:52.099 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1388058777' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:10:52.099 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:10:52.100 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:51 smithi135 ceph-mon[213871]: pgmap v1037: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 11671/133509 objects degraded (8.742%); 3941/133509 objects misplaced (2.952%); 504 KiB/s, 145 objects/s recovering 2023-09-23T20:10:52.100 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1388058777' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:10:52.100 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:10:52.171 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:52.198 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:52.242 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:52.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:52.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:52.348 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:52.374 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:52.421 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:52.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:52.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:52.527 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:52.555 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:52.954 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:52.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:52.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:53.059 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:53.088 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:53.156 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: osdmap e368: 8 total, 8 up, 7 in 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11671/133509 objects degraded (8.742%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: failed to encode map e369 with expected crc 2023-09-23T20:10:53.157 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[213871]: osdmap e369: 8 total, 8 up, 7 in 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: osdmap e368: 8 total, 8 up, 7 in 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11671/133509 objects degraded (8.742%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: failed to encode map e369 with expected crc 2023-09-23T20:10:53.158 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:52 smithi135 ceph-mon[221113]: osdmap e369: 8 total, 8 up, 7 in 2023-09-23T20:10:53.174 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:10:53.336 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:53.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:53.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: failed to encode map e368 with expected crc 2023-09-23T20:10:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:10:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: osdmap e368: 8 total, 8 up, 7 in 2023-09-23T20:10:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:10:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:10:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11671/133509 objects degraded (8.742%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: failed to encode map e369 with expected crc 2023-09-23T20:10:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:52 smithi144 ceph-mon[194833]: osdmap e369: 8 total, 8 up, 7 in 2023-09-23T20:10:53.440 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:53.467 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:53.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:10:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:10:53] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-09-23T20:10:53.954 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:53.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:53.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:54.038 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:54.068 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:54.070 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:54.109 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:54.197 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:54.199 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.199 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: pgmap v1040: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 6731/133053 objects degraded (5.059%); 1165/133053 objects misplaced (0.876%); 1.1 MiB/s, 116 objects/s recovering 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: failed to encode map e370 with expected crc 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: osdmap e370: 8 total, 8 up, 7 in 2023-09-23T20:10:54.200 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1095067800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:54.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:54.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:54.201 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: pgmap v1040: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 6731/133053 objects degraded (5.059%); 1165/133053 objects misplaced (0.876%); 1.1 MiB/s, 116 objects/s recovering 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: failed to encode map e370 with expected crc 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: osdmap e370: 8 total, 8 up, 7 in 2023-09-23T20:10:54.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1095067800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:54.302 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:54.325 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: failed to encode map e369 with expected crc 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: pgmap v1040: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 6731/133053 objects degraded (5.059%); 1165/133053 objects misplaced (0.876%); 1.1 MiB/s, 116 objects/s recovering 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: failed to encode map e370 with expected crc 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: osdmap e370: 8 total, 8 up, 7 in 2023-09-23T20:10:54.326 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1095067800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:10:54.328 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:54.372 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:54.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:54.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:54.478 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:10:54.504 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:54.549 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:10:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:54.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:10:54.655 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:54.701 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:10:54.704 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:54.987 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[213871]: failed to encode map e370 with expected crc 2023-09-23T20:10:54.987 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[213871]: failed to encode map e370 with expected crc 2023-09-23T20:10:54.987 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:54.987 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[213871]: failed to encode map e371 with expected crc 2023-09-23T20:10:54.988 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1095067800' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:55.178 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:55.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:55.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:55.258 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[221113]: failed to encode map e370 with expected crc 2023-09-23T20:10:55.258 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[221113]: failed to encode map e370 with expected crc 2023-09-23T20:10:55.259 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:55.259 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[221113]: failed to encode map e371 with expected crc 2023-09-23T20:10:55.259 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1095067800' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:55.259 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[221113]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T20:10:55.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:54 smithi135 ceph-mon[213871]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T20:10:55.284 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:55.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:54 smithi144 ceph-mon[194833]: failed to encode map e370 with expected crc 2023-09-23T20:10:55.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:54 smithi144 ceph-mon[194833]: failed to encode map e370 with expected crc 2023-09-23T20:10:55.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:54 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:10:55.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:54 smithi144 ceph-mon[194833]: failed to encode map e371 with expected crc 2023-09-23T20:10:55.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1095067800' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:10:55.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:54 smithi144 ceph-mon[194833]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T20:10:55.310 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:55.354 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:55.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:55.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:55.460 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:10:55.494 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:56.319 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:10:56.320 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:10:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:55 smithi144 ceph-mon[194833]: failed to encode map e371 with expected crc 2023-09-23T20:10:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:55 smithi144 ceph-mon[194833]: pgmap v1043: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 93 op/s; 6731/132384 objects degraded (5.084%); 779/132384 objects misplaced (0.588%); 6.9 MiB/s, 225 objects/s recovering 2023-09-23T20:10:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:55 smithi135 ceph-mon[213871]: failed to encode map e371 with expected crc 2023-09-23T20:10:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:55 smithi135 ceph-mon[213871]: pgmap v1043: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 93 op/s; 6731/132384 objects degraded (5.084%); 779/132384 objects misplaced (0.588%); 6.9 MiB/s, 225 objects/s recovering 2023-09-23T20:10:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:55 smithi135 ceph-mon[221113]: failed to encode map e371 with expected crc 2023-09-23T20:10:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:55 smithi135 ceph-mon[221113]: pgmap v1043: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 93 op/s; 6731/132384 objects degraded (5.084%); 779/132384 objects misplaced (0.588%); 6.9 MiB/s, 225 objects/s recovering 2023-09-23T20:10:57.115 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:10:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:57.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:10:57.220 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:57.252 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:57.843 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:10:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:57.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:10:57.849 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:10:57.948 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:10:57.982 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:58.088 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:10:58.112 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:10:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[221113]: pgmap v1044: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 9033/132234 objects degraded (6.831%); 779/132234 objects misplaced (0.589%); 5.6 MiB/s, 182 objects/s recovering 2023-09-23T20:10:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[221113]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[221113]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T20:10:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[221113]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[221113]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9033/132234 objects degraded (6.831%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[213871]: pgmap v1044: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 9033/132234 objects degraded (6.831%); 779/132234 objects misplaced (0.589%); 5.6 MiB/s, 182 objects/s recovering 2023-09-23T20:10:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[213871]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[213871]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T20:10:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[213871]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[213871]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:57 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9033/132234 objects degraded (6.831%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:58.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:58.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:10:58.220 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:10:58.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:57 smithi144 ceph-mon[194833]: pgmap v1044: 109 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 9033/132234 objects degraded (6.831%); 779/132234 objects misplaced (0.589%); 5.6 MiB/s, 182 objects/s recovering 2023-09-23T20:10:58.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:57 smithi144 ceph-mon[194833]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:57 smithi144 ceph-mon[194833]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T20:10:58.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:57 smithi144 ceph-mon[194833]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:57 smithi144 ceph-mon[194833]: failed to encode map e372 with expected crc 2023-09-23T20:10:58.249 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:57 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9033/132234 objects degraded (6.831%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:10:58.252 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:58.300 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:10:58.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:58.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:10:58.407 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:10:58.435 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:58.436 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:10:58.480 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:10:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:58.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:10:58.567 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:10:58.588 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:10:58.614 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:58.622 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:10:58.659 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:10:58.664 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:10:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:10:58.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:10:58.714 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:10:58.763 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:10:58.765 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:10:58.795 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:10:58.834 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:10:58.843 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:10:58.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:10:58.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:10:58.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:10:58.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:10:58.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:10:58.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:10:58.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:10:58.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:10:59.022 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:10:59.033 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:10:59.078 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[213871]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.078 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[213871]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T20:10:59.078 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[213871]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.078 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[213871]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.078 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4180618122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:59.078 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3339436880' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:10:59.078 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[213871]: pgmap v1047: 109 pgs: 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 11879/132231 objects degraded (8.984%); 1102/132231 objects misplaced (0.833%); 10 MiB/s, 159 objects/s recovering 2023-09-23T20:10:59.079 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[221113]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.079 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[221113]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T20:10:59.079 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[221113]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.079 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[221113]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.079 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4180618122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:59.079 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3339436880' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:10:59.079 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:10:59 smithi135 ceph-mon[221113]: pgmap v1047: 109 pgs: 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 11879/132231 objects degraded (8.984%); 1102/132231 objects misplaced (0.833%); 10 MiB/s, 159 objects/s recovering 2023-09-23T20:10:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:59 smithi144 ceph-mon[194833]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:59 smithi144 ceph-mon[194833]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T20:10:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:59 smithi144 ceph-mon[194833]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:59 smithi144 ceph-mon[194833]: failed to encode map e373 with expected crc 2023-09-23T20:10:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4180618122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:10:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3339436880' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:10:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:10:59 smithi144 ceph-mon[194833]: pgmap v1047: 109 pgs: 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 11879/132231 objects degraded (8.984%); 1102/132231 objects misplaced (0.833%); 10 MiB/s, 159 objects/s recovering 2023-09-23T20:11:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:00 smithi144 ceph-mon[194833]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:00 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3339436880' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:11:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:00 smithi144 ceph-mon[194833]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T20:11:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:00 smithi144 ceph-mon[194833]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:00 smithi144 ceph-mon[194833]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:00 smithi144 ceph-mon[194833]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[213871]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3339436880' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:11:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[213871]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T20:11:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[213871]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[213871]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[213871]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[221113]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3339436880' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:11:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[221113]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T20:11:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[221113]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[221113]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:00 smithi135 ceph-mon[221113]: failed to encode map e374 with expected crc 2023-09-23T20:11:00.678 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:00.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:00.782 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:00.812 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:01.349 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:01 smithi135 ceph-mon[221113]: pgmap v1049: 109 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 9525/131808 objects degraded (7.226%); 2636/131808 objects misplaced (2.000%); 9.5 MiB/s, 157 objects/s recovering 2023-09-23T20:11:01.349 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:01 smithi135 ceph-mon[213871]: pgmap v1049: 109 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 9525/131808 objects degraded (7.226%); 2636/131808 objects misplaced (2.000%); 9.5 MiB/s, 157 objects/s recovering 2023-09-23T20:11:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:01 smithi144 ceph-mon[194833]: pgmap v1049: 109 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 9525/131808 objects degraded (7.226%); 2636/131808 objects misplaced (2.000%); 9.5 MiB/s, 157 objects/s recovering 2023-09-23T20:11:01.403 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:01.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:01.506 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:11:01.509 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:01.535 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:01.577 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:01.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:01.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:01.682 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:01.700 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:11:01.701 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:11:01.701 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T20:11:01.701 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:11:01.730 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:01.732 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:01.974 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:01.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:02.082 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:02.107 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:02.150 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:02.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:02.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:02.255 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:02.282 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:02.764 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:11:02.797 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:02.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:02.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:02.909 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:02.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:02 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9525/131808 objects degraded (7.226%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:02.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:02 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9525/131808 objects degraded (7.226%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:03.046 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:03.107 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:03.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:03.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:03.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:02 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9525/131808 objects degraded (7.226%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:03.214 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:03.239 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:03.281 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:03.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:03.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:03.351 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:11:03.390 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:03.455 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:03.625 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:03.688 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:11:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:11:03] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-09-23T20:11:03.708 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:03.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:03.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:03.801 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:03.815 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:03.862 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:03.951 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:03.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:03.960 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:03 smithi135 ceph-mon[213871]: pgmap v1050: 109 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 43 op/s; 10521/131454 objects degraded (8.004%); 1534/131454 objects misplaced (1.167%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:11:03.961 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:03 smithi135 ceph-mon[221113]: pgmap v1050: 109 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 43 op/s; 10521/131454 objects degraded (8.004%); 1534/131454 objects misplaced (1.167%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:11:03.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:04.062 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:04.107 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:04.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:03 smithi144 ceph-mon[194833]: pgmap v1050: 109 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 43 op/s; 10521/131454 objects degraded (8.004%); 1534/131454 objects misplaced (1.167%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:11:04.307 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:04.307 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:04.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:04.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:04.417 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:04.443 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:04.485 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:04.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:04.590 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:04.616 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:04.660 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:04.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:04.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:04.765 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:04.808 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:05.942 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:05.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:05.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:06.020 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:11:06.056 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:06.094 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:06.150 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:05 smithi135 ceph-mon[221113]: pgmap v1051: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 10436/128841 objects degraded (8.100%); 1534/128841 objects misplaced (1.191%); 8.7 MiB/s, 142 objects/s recovering 2023-09-23T20:11:06.150 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3600984024' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:06.150 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:05 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:06.151 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:05 smithi135 ceph-mon[213871]: pgmap v1051: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 10436/128841 objects degraded (8.100%); 1534/128841 objects misplaced (1.191%); 8.7 MiB/s, 142 objects/s recovering 2023-09-23T20:11:06.152 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3600984024' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:06.152 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:05 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:06.287 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:06.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:06.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:05 smithi144 ceph-mon[194833]: pgmap v1051: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 10436/128841 objects degraded (8.100%); 1534/128841 objects misplaced (1.191%); 8.7 MiB/s, 142 objects/s recovering 2023-09-23T20:11:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3600984024' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:05 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:06.393 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:06.422 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:06.469 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:06.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:06.576 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:06.618 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:07.052 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:07.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:07.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:07.157 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:07.202 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:07.280 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:07.280 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[221113]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.280 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[221113]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[221113]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[221113]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[221113]: 1.1 scrub starts 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[213871]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[213871]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:11:07.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[213871]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T20:11:07.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[213871]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[213871]: 1.1 scrub starts 2023-09-23T20:11:07.282 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:07 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:11:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:07 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:07 smithi144 ceph-mon[194833]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:07 smithi144 ceph-mon[194833]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:07 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:11:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:07 smithi144 ceph-mon[194833]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T20:11:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:07 smithi144 ceph-mon[194833]: failed to encode map e375 with expected crc 2023-09-23T20:11:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:07 smithi144 ceph-mon[194833]: 1.1 scrub starts 2023-09-23T20:11:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:07 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:11:07.623 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:07.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:07.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:07.727 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:07.761 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:07.812 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:07.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:07.919 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:07.969 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:08.158 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:11:08.159 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:11:08.159 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:11:08.159 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:11:08.159 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:11:08.190 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:08.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[221113]: pgmap v1053: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 10436/127644 objects degraded (8.176%); 1453/127644 objects misplaced (1.138%); 2.7 MiB/s, 55 objects/s recovering 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[221113]: 1.11 scrub starts 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[221113]: 1.e scrub starts 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 10436/127644 objects degraded (8.176%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[213871]: pgmap v1053: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 10436/127644 objects degraded (8.176%); 1453/127644 objects misplaced (1.138%); 2.7 MiB/s, 55 objects/s recovering 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[213871]: 1.11 scrub starts 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:08.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[213871]: 1.e scrub starts 2023-09-23T20:11:08.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:08 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 10436/127644 objects degraded (8.176%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:08.298 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:08.344 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:08 smithi144 ceph-mon[194833]: pgmap v1053: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 10436/127644 objects degraded (8.176%); 1453/127644 objects misplaced (1.138%); 2.7 MiB/s, 55 objects/s recovering 2023-09-23T20:11:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:08 smithi144 ceph-mon[194833]: 1.11 scrub starts 2023-09-23T20:11:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:08 smithi144 ceph-mon[194833]: 1.e scrub starts 2023-09-23T20:11:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:08 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 10436/127644 objects degraded (8.176%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:08.627 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:08.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:08.734 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:08.760 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:08.802 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:08.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:08.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:08.907 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:08.959 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:09.061 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:11:09.132 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:11:09.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3491333472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:11:09.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:09 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:11:09.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:09 smithi135 ceph-mon[221113]: pgmap v1054: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 10351/124791 objects degraded (8.295%); 1453/124791 objects misplaced (1.164%); 2.7 MiB/s, 53 objects/s recovering 2023-09-23T20:11:09.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3491333472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:11:09.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:09 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:11:09.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:09 smithi135 ceph-mon[213871]: pgmap v1054: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 10351/124791 objects degraded (8.295%); 1453/124791 objects misplaced (1.164%); 2.7 MiB/s, 53 objects/s recovering 2023-09-23T20:11:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3491333472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:11:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:09 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:11:09.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:09 smithi144 ceph-mon[194833]: pgmap v1054: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 10351/124791 objects degraded (8.295%); 1453/124791 objects misplaced (1.164%); 2.7 MiB/s, 53 objects/s recovering 2023-09-23T20:11:09.616 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:11:09.792 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:09.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:09.898 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:09.925 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:09.972 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:09.976 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:11:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:09.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:10.078 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:10.141 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[221113]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[221113]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[221113]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[221113]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[213871]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[213871]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T20:11:10.143 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[213871]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.143 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:10 smithi135 ceph-mon[213871]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.148 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:10.278 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:10.284 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:11:10.300 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:10.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:10.327 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:11:10.380 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:11:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:10 smithi144 ceph-mon[194833]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:11:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:10 smithi144 ceph-mon[194833]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T20:11:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:10 smithi144 ceph-mon[194833]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:10 smithi144 ceph-mon[194833]: failed to encode map e376 with expected crc 2023-09-23T20:11:10.415 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:11:10.416 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:10.444 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:10.447 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:11:10.463 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:10.482 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:11:10.491 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:10.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:10.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:10.546 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:11:10.570 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:11:10.589 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:11:10.589 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:11:10.589 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:11:10.589 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:11:10.589 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:11:10.589 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:11:10.589 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:11:10.589 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:11:10.598 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:10.648 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/959217760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: pgmap v1057: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10351/123504 objects degraded (8.381%); 1355/123504 objects misplaced (1.097%); 2.8 MiB/s, 58 objects/s recovering 2023-09-23T20:11:11.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[221113]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/959217760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: pgmap v1057: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10351/123504 objects degraded (8.381%); 1355/123504 objects misplaced (1.097%); 2.8 MiB/s, 58 objects/s recovering 2023-09-23T20:11:11.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:11 smithi135 ceph-mon[213871]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T20:11:11.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:11:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T20:11:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: failed to encode map e377 with expected crc 2023-09-23T20:11:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/959217760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: pgmap v1057: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10351/123504 objects degraded (8.381%); 1355/123504 objects misplaced (1.097%); 2.8 MiB/s, 58 objects/s recovering 2023-09-23T20:11:11.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:11 smithi144 ceph-mon[194833]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T20:11:11.817 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:11.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:11.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:11.928 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:11.960 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:12.001 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:12.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:12.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:12.106 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:12.170 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:12 smithi144 ceph-mon[194833]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:12 smithi144 ceph-mon[194833]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:12 smithi144 ceph-mon[194833]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:12 smithi144 ceph-mon[194833]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:12 smithi135 ceph-mon[213871]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:12 smithi135 ceph-mon[213871]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:12 smithi135 ceph-mon[213871]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:12 smithi135 ceph-mon[213871]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:12 smithi135 ceph-mon[221113]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:12 smithi135 ceph-mon[221113]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:12 smithi135 ceph-mon[221113]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:12 smithi135 ceph-mon[221113]: failed to encode map e378 with expected crc 2023-09-23T20:11:12.699 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:12.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:12.804 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:12.832 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:13.171 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:13.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:13.279 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:13.305 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:13.306 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:13.348 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:13.353 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:13 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 10351/123504 objects degraded (8.381%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:13.353 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:13 smithi144 ceph-mon[194833]: pgmap v1059: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 10351/122880 objects degraded (8.424%); 1355/122880 objects misplaced (1.103%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T20:11:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:13.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:13.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:13 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 10351/123504 objects degraded (8.381%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:13.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:13 smithi135 ceph-mon[221113]: pgmap v1059: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 10351/122880 objects degraded (8.424%); 1355/122880 objects misplaced (1.103%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T20:11:13.407 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:13 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 10351/123504 objects degraded (8.381%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:13.407 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:13 smithi135 ceph-mon[213871]: pgmap v1059: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 10351/122880 objects degraded (8.424%); 1355/122880 objects misplaced (1.103%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T20:11:13.455 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:13.514 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:13.719 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:13.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:13.779 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:11:13.832 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:13.927 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:11:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:11:13] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-09-23T20:11:13.950 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:14.710 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:14.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:14.755 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:11:14.835 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:14.883 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:14 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:11:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2126137097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:14.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:14 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:14.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:14 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:11:14.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2126137097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:14.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:14 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:14.998 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:15.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:15.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:15.104 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:15.128 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:14 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:11:15.129 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2126137097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:15.129 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:14 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:15.131 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:15.174 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:15.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:15.280 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:15.350 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:15.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:11:15.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[213871]: failed to encode map e379 with expected crc 2023-09-23T20:11:15.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[213871]: failed to encode map e379 with expected crc 2023-09-23T20:11:15.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:15.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[213871]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T20:11:15.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[213871]: failed to encode map e379 with expected crc 2023-09-23T20:11:15.709 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[213871]: pgmap v1061: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 10266/119979 objects degraded (8.556%); 1355/119979 objects misplaced (1.129%); 2.0 MiB/s, 34 objects/s recovering 2023-09-23T20:11:15.875 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:11:15.875 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:11:15.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:11:15.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[221113]: failed to encode map e379 with expected crc 2023-09-23T20:11:15.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[221113]: failed to encode map e379 with expected crc 2023-09-23T20:11:15.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:15.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[221113]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T20:11:15.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[221113]: failed to encode map e379 with expected crc 2023-09-23T20:11:15.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:15 smithi135 ceph-mon[221113]: pgmap v1061: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 10266/119979 objects degraded (8.556%); 1355/119979 objects misplaced (1.129%); 2.0 MiB/s, 34 objects/s recovering 2023-09-23T20:11:16.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:15 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:11:16.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:15 smithi144 ceph-mon[194833]: failed to encode map e379 with expected crc 2023-09-23T20:11:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:15 smithi144 ceph-mon[194833]: failed to encode map e379 with expected crc 2023-09-23T20:11:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:15 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:15 smithi144 ceph-mon[194833]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T20:11:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:15 smithi144 ceph-mon[194833]: failed to encode map e379 with expected crc 2023-09-23T20:11:16.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:15 smithi144 ceph-mon[194833]: pgmap v1061: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 10266/119979 objects degraded (8.556%); 1355/119979 objects misplaced (1.129%); 2.0 MiB/s, 34 objects/s recovering 2023-09-23T20:11:16.172 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:16.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:16.282 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:16.314 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:16.356 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:16.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:16.462 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:16.494 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:16.544 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:16.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:16.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:16.652 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:16.691 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:16.749 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:16.779 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:16.783 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:11:16.783 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:11:16.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:16.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:16.892 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:16.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:16 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:11:16.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2139461068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:11:16.939 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:16 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:11:16.940 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2139461068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:11:16.942 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:17.025 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:17.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:17.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:17.131 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:16 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:11:17.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2139461068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:11:17.167 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:17.243 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:17.661 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:17.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:17.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:17.767 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:17.801 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:17.823 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:11:17.823 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:11:17.863 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:17.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:17.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[221113]: failed to encode map e380 with expected crc 2023-09-23T20:11:17.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[221113]: failed to encode map e380 with expected crc 2023-09-23T20:11:17.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2139461068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:11:17.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[221113]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T20:11:17.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:11:17.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[221113]: pgmap v1063: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 10266/118491 objects degraded (8.664%); 1262/118491 objects misplaced (1.065%); 1.9 MiB/s, 38 objects/s recovering 2023-09-23T20:11:17.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 10266/119979 objects degraded (8.556%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:17.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[221113]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T20:11:17.947 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[213871]: failed to encode map e380 with expected crc 2023-09-23T20:11:17.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[213871]: failed to encode map e380 with expected crc 2023-09-23T20:11:17.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2139461068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:11:17.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[213871]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T20:11:17.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:11:17.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[213871]: pgmap v1063: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 10266/118491 objects degraded (8.664%); 1262/118491 objects misplaced (1.065%); 1.9 MiB/s, 38 objects/s recovering 2023-09-23T20:11:17.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 10266/119979 objects degraded (8.556%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:17.948 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:17 smithi135 ceph-mon[213871]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T20:11:17.970 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:17.995 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:17 smithi144 ceph-mon[194833]: failed to encode map e380 with expected crc 2023-09-23T20:11:17.996 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:17 smithi144 ceph-mon[194833]: failed to encode map e380 with expected crc 2023-09-23T20:11:17.996 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2139461068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:11:17.996 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:17 smithi144 ceph-mon[194833]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T20:11:17.996 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:17 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:11:17.996 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:17 smithi144 ceph-mon[194833]: pgmap v1063: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 10266/118491 objects degraded (8.664%); 1262/118491 objects misplaced (1.065%); 1.9 MiB/s, 38 objects/s recovering 2023-09-23T20:11:17.996 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 10266/119979 objects degraded (8.556%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:17.996 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:17 smithi144 ceph-mon[194833]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T20:11:17.997 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:18.144 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:18.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:18.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:18.247 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:18.277 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:18.471 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:18.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:18.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:18.578 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:18.604 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:18.698 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:18.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:18.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:18.777 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:18.802 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:18.829 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:18.877 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:18.878 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.879 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.879 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.879 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.879 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.879 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.879 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.879 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: 1.12 deep-scrub starts 2023-09-23T20:11:18.879 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:18 smithi144 ceph-mon[194833]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T20:11:18.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:18.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:18.926 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:18.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: 1.12 deep-scrub starts 2023-09-23T20:11:18.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[213871]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T20:11:18.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: failed to encode map e381 with expected crc 2023-09-23T20:11:18.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: 1.12 deep-scrub starts 2023-09-23T20:11:18.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:18 smithi135 ceph-mon[221113]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T20:11:18.982 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:19.006 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:19.048 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:19.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:19.151 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:19.272 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:19.485 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:11:19.521 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:11:19.553 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:11:19.585 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:11:19.610 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:11:19.635 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:11:19.667 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:11:19.699 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:11:19.707 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:11:19.707 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:11:19.707 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:11:19.708 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:11:19.708 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:11:19.708 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:11:19.708 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:11:19.708 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:11:19.730 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:19.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: 8.b starting backfill to osd.3 from (308'2400,340'7049] MIN to 379'7946 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: pgmap v1066: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 10181/116028 objects degraded (8.775%); 1262/116028 objects misplaced (1.088%); 2.7 MiB/s, 54 objects/s recovering 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3235755292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:19.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:19 smithi144 ceph-mon[194833]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T20:11:19.842 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:19.874 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:19.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: 8.b starting backfill to osd.3 from (308'2400,340'7049] MIN to 379'7946 2023-09-23T20:11:19.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: pgmap v1066: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 10181/116028 objects degraded (8.775%); 1262/116028 objects misplaced (1.088%); 2.7 MiB/s, 54 objects/s recovering 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3235755292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[213871]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: 8.b starting backfill to osd.3 from (308'2400,340'7049] MIN to 379'7946 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: failed to encode map e382 with expected crc 2023-09-23T20:11:19.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:11:19.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: pgmap v1066: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 10181/116028 objects degraded (8.775%); 1262/116028 objects misplaced (1.088%); 2.7 MiB/s, 54 objects/s recovering 2023-09-23T20:11:19.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3235755292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:19.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:19 smithi135 ceph-mon[221113]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T20:11:20.011 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:11:20.011 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:11:20.011 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:11:20.011 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:11:20.800 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:11:20.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[213871]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[213871]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[213871]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[213871]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[213871]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[213871]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[221113]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[221113]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[221113]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[221113]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[221113]: failed to encode map e383 with expected crc 2023-09-23T20:11:20.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:20 smithi135 ceph-mon[221113]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T20:11:21.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:20 smithi144 ceph-mon[194833]: failed to encode map e383 with expected crc 2023-09-23T20:11:21.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:20 smithi144 ceph-mon[194833]: failed to encode map e383 with expected crc 2023-09-23T20:11:21.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:20 smithi144 ceph-mon[194833]: failed to encode map e383 with expected crc 2023-09-23T20:11:21.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:20 smithi144 ceph-mon[194833]: failed to encode map e383 with expected crc 2023-09-23T20:11:21.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:20 smithi144 ceph-mon[194833]: failed to encode map e383 with expected crc 2023-09-23T20:11:21.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:20 smithi144 ceph-mon[194833]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T20:11:21.155 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:21.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:21.201 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:21.260 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:21.286 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:21.355 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:21.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:21.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:21.459 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:21.511 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:21.762 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[213871]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.762 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[213871]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.762 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[213871]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.762 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[213871]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.762 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[213871]: pgmap v1069: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 274 op/s; 10181/115194 objects degraded (8.838%); 1200/115194 objects misplaced (1.042%); 2.3 MiB/s, 81 objects/s recovering 2023-09-23T20:11:21.763 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[221113]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.763 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[221113]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.763 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[221113]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.763 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[221113]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.763 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:21 smithi135 ceph-mon[221113]: pgmap v1069: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 274 op/s; 10181/115194 objects degraded (8.838%); 1200/115194 objects misplaced (1.042%); 2.3 MiB/s, 81 objects/s recovering 2023-09-23T20:11:21.805 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:21.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:21.924 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:21.950 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:21 smithi144 ceph-mon[194833]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.950 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:21 smithi144 ceph-mon[194833]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.950 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:21 smithi144 ceph-mon[194833]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.950 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:21 smithi144 ceph-mon[194833]: failed to encode map e384 with expected crc 2023-09-23T20:11:21.950 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:21 smithi144 ceph-mon[194833]: pgmap v1069: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 274 op/s; 10181/115194 objects degraded (8.838%); 1200/115194 objects misplaced (1.042%); 2.3 MiB/s, 81 objects/s recovering 2023-09-23T20:11:21.956 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:22.004 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:22.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:22.107 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:22.173 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:22.273 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:22.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:22.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:22.385 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:22.411 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:22.518 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:22.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:22.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:22.623 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:22.689 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:22.797 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:22.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:22.905 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:22.947 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:23.001 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:23.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:23.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:23.016 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:11:23.016 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:11:23.017 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:11:23.017 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:11:23.017 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:11:23.105 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:23.137 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:23.182 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:11:23.200 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:11:23.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:23.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:23.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3970422893' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:11:23.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:11:23.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 10181/115194 objects degraded (8.838%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:23.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1668008724' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:11:23.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:11:23.209 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[221113]: pgmap v1070: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 10181/114714 objects degraded (8.875%); 1200/114714 objects misplaced (1.046%); 1.8 MiB/s, 62 objects/s recovering 2023-09-23T20:11:23.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:23.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:23.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3970422893' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:11:23.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:11:23.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 10181/115194 objects degraded (8.838%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:23.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1668008724' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:11:23.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:11:23.214 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:23 smithi135 ceph-mon[213871]: pgmap v1070: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 10181/114714 objects degraded (8.875%); 1200/114714 objects misplaced (1.046%); 1.8 MiB/s, 62 objects/s recovering 2023-09-23T20:11:23.241 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:23.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:23.345 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:23 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3970422893' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:11:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:23 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:11:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 10181/115194 objects degraded (8.838%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:23 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1668008724' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:11:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:23 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:11:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:23 smithi144 ceph-mon[194833]: pgmap v1070: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 10181/114714 objects degraded (8.875%); 1200/114714 objects misplaced (1.046%); 1.8 MiB/s, 62 objects/s recovering 2023-09-23T20:11:23.382 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:23.693 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:23.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:23.797 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:23.824 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:11:23.830 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:11:23.843 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:23.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:11:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:11:23] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T20:11:24.220 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:24.258 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:24.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:24.371 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:24.414 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:11:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:11:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T20:11:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[213871]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T20:11:24.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:24 smithi135 ceph-mon[221113]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.491 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:24.491 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.491 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:11:24.491 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:11:24.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T20:11:24.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.492 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:24 smithi144 ceph-mon[194833]: failed to encode map e385 with expected crc 2023-09-23T20:11:24.980 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:24.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:25.063 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:11:25.087 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:25.146 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:25.182 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:25.221 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:25.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:25.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:25.329 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:25.373 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:25 smithi135 ceph-mon[221113]: pgmap v1072: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 10083/111816 objects degraded (9.017%); 3503/111816 objects misplaced (3.133%); 1.6 MiB/s, 54 objects/s recovering 2023-09-23T20:11:25.374 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:25 smithi135 ceph-mon[213871]: pgmap v1072: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 10083/111816 objects degraded (9.017%); 3503/111816 objects misplaced (3.133%); 1.6 MiB/s, 54 objects/s recovering 2023-09-23T20:11:25.378 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:25.444 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:25.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:25.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:25.548 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:25.575 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:25 smithi144 ceph-mon[194833]: pgmap v1072: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 10083/111816 objects degraded (9.017%); 3503/111816 objects misplaced (3.133%); 1.6 MiB/s, 54 objects/s recovering 2023-09-23T20:11:25.579 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:25.679 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:25.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:25.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:25.785 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:25.815 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:25.864 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:25.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:25.881 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:11:25.881 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:11:25.881 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:11:25.881 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52248 8.5 8:aed026a3:::benchmark_data_smithi135_460343_object52247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.949589+0000", 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 265.929404847, 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1372650540000002, 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.882 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52248 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949589+0000", 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949589+0000", 2023-09-23T20:11:25.883 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949588+0000", 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949658+0000", 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0339999999999994e-05 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949669+0000", 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1156e-05 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949675+0000", 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3979999999999998e-06 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949734+0000", 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9015000000000003e-05 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949744+0000", 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0358e-05 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.926958+0000", 2023-09-23T20:11:25.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97721380000000002 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.926969+0000", 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1173e-05 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.070949+0000", 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.143979493 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.071095+0000", 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000146501 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.071258+0000", 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016240699999999999 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.075199+0000", 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0039416440000000002 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.075207+0000", 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0730000000000005e-06 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.084221+0000", 2023-09-23T20:11:25.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090141570000000001 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.084282+0000", 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0473000000000002e-05 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.086854+0000", 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025719039999999999 2023-09-23T20:11:25.890 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:65654 8.5 8:abe5195a:::benchmark_data_smithi135_460343_object65653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:17.919947+0000", 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 247.95904678100001, 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.190072045, 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.891 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 65654 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919947+0000", 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.892 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919947+0000", 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919946+0000", 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920012+0000", 2023-09-23T20:11:25.893 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6530000000000002e-05 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920030+0000", 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7927999999999998e-05 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920037+0000", 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8449999999999997e-06 2023-09-23T20:11:25.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920085+0000", 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7766000000000003e-05 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920214+0000", 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000129589 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920312+0000", 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7917000000000005e-05 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.922814+0000", 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025019669999999999 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.924821+0000", 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020062880000000002 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.945068+0000", 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.020247023999999999 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.945081+0000", 2023-09-23T20:11:25.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3132999999999999e-05 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.109930+0000", 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.164849646 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.110019+0000", 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8443999999999995e-05 2023-09-23T20:11:25.898 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:65661 8.5 8:a319973d:::benchmark_data_smithi135_460343_object65660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:17.963073+0000", 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 247.91592103299999, 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1591320270000001, 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.899 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 65661 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963073+0000", 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.900 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963073+0000", 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963071+0000", 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963115+0000", 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4014999999999998e-05 2023-09-23T20:11:25.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963131+0000", 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5480000000000001e-05 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963137+0000", 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0739999999999998e-06 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963201+0000", 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4103999999999995e-05 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.963212+0000", 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0712e-05 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.903 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.110045+0000", 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.146833177 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.110156+0000", 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000110993 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.110267+0000", 2023-09-23T20:11:25.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111184 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.112139+0000", 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018723139999999999 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.115899+0000", 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003759877 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.119057+0000", 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031581249999999999 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.119101+0000", 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3328000000000003e-05 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.122205+0000", 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003103916 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77498 8.5 8:a5b293b3:::benchmark_data_smithi135_460343_object77497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.694200+0000", 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 221.18479395, 2023-09-23T20:11:25.907 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.006215106, 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77498 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694200+0000", 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.908 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694200+0000", 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694199+0000", 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694254+0000", 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5170000000000002e-05 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694271+0000", 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7184e-05 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.694278+0000", 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7870000000000004e-06 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698780+0000", 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045027890000000001 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698792+0000", 2023-09-23T20:11:25.911 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1778e-05 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.694233+0000", 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99544110799999996 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.694325+0000", 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1958000000000003e-05 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.912 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.694433+0000", 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000107877 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700191+0000", 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0057577390000000004 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700198+0000", 2023-09-23T20:11:25.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0609999999999998e-06 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700324+0000", 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000125535 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700376+0000", 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2336000000000002e-05 2023-09-23T20:11:25.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700415+0000", 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8923e-05 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77501 8.5 8:ad6d7fa4:::benchmark_data_smithi135_460343_object77500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:11:25.915 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.698362+0000", 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 221.18063175899999, 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0068878059999999, 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77501 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.916 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698362+0000", 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698362+0000", 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698361+0000", 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698392+0000", 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1779000000000002e-05 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698404+0000", 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1348999999999999e-05 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698409+0000", 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0799999999999996e-06 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698881+0000", 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000472029 2023-09-23T20:11:25.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.698888+0000", 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1720000000000001e-06 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.697427+0000", 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99853948599999998 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.920 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.697530+0000", 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010283100000000001 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.697659+0000", 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000129064 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.702943+0000", 2023-09-23T20:11:25.921 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052838679999999997 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705153+0000", 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022095389999999999 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705191+0000", 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7898999999999997e-05 2023-09-23T20:11:25.922 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705225+0000", 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4557999999999997e-05 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705250+0000", 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4436000000000001e-05 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.923 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77504 8.5 8:aa509932:::benchmark_data_smithi135_460343_object77503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.711477+0000", 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 221.167516592, 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.998780051, 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.924 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77504 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711477+0000", 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711477+0000", 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711475+0000", 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711503+0000", 2023-09-23T20:11:25.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8280999999999999e-05 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711523+0000", 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9758999999999998e-05 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711530+0000", 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3559999999999998e-06 2023-09-23T20:11:25.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711578+0000", 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8469e-05 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.711586+0000", 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2150000000000001e-06 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700445+0000", 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98885881099999995 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700548+0000", 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102729 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.700726+0000", 2023-09-23T20:11:25.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017792599999999999 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705297+0000", 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045714950000000001 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.707825+0000", 2023-09-23T20:11:25.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025279289999999999 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710195+0000", 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002369894 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710231+0000", 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6044999999999998e-05 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710257+0000", 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6038999999999999e-05 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77527 8.5 8:a80e791e:::benchmark_data_smithi135_460343_object77526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.758268+0000", 2023-09-23T20:11:25.932 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 221.120725779, 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.95206617400000004, 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77527 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758268+0000", 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758268+0000", 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758266+0000", 2023-09-23T20:11:25.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758296+0000", 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9828e-05 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758315+0000", 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8927999999999999e-05 2023-09-23T20:11:25.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758321+0000", 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9900000000000002e-06 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758388+0000", 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6811999999999997e-05 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.758401+0000", 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3227999999999999e-05 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.702999+0000", 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.94459813199999998 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.703108+0000", 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000108748 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.703205+0000", 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7096999999999995e-05 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710088+0000", 2023-09-23T20:11:25.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.006883134 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710132+0000", 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4391000000000002e-05 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710292+0000", 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015950099999999999 2023-09-23T20:11:25.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710315+0000", 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2713000000000001e-05 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710334+0000", 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9429e-05 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.940 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77549 8.5 8:a49adce5:::benchmark_data_smithi135_460343_object77548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.798013+0000", 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 221.08098092500001, 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.924997548, 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.941 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77549 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798013+0000", 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798013+0000", 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798011+0000", 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798051+0000", 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9983e-05 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798069+0000", 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7640000000000001e-05 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798076+0000", 2023-09-23T20:11:25.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.413e-06 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798143+0000", 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6481000000000003e-05 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.798154+0000", 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0956e-05 2023-09-23T20:11:25.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705321+0000", 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.90716749699999999 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705404+0000", 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3079999999999997e-05 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:11:25.947 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.705494+0000", 2023-09-23T20:11:25.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9672000000000005e-05 2023-09-23T20:11:25.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.947 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.947 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.947 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710157+0000", 2023-09-23T20:11:25.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046633819999999998 2023-09-23T20:11:25.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.710166+0000", 2023-09-23T20:11:25.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7220000000000005e-06 2023-09-23T20:11:25.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.722950+0000", 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012784617999999999 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.722985+0000", 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4872000000000001e-05 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.723010+0000", 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4885999999999999e-05 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:77558 8.5 8:a6a5e11a:::benchmark_data_smithi135_460343_object77557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:44.814035+0000", 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 221.06495902200001, 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.90909048400000003, 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.951 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 77558 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814035+0000", 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814035+0000", 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814033+0000", 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.953 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814079+0000", 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5802e-05 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814099+0000", 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0177999999999999e-05 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814107+0000", 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1820000000000006e-06 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814202+0000", 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5297999999999995e-05 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:44.814214+0000", 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1311999999999999e-05 2023-09-23T20:11:25.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.707855+0000", 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.89364169800000004 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.707982+0000", 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012681899999999999 2023-09-23T20:11:25.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.708127+0000", 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000145325 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.712213+0000", 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040858919999999998 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.712247+0000", 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3890999999999998e-05 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.723050+0000", 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010802959000000001 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.723097+0000", 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.672e-05 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:45.723125+0000", 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8221999999999998e-05 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:115187 8.b 8:ddd6b759:::benchmark_data_smithi135_460343_object30954:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e342)", 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:02.643566+0000", 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 143.23542783400001, 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2096566420000001, 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.960 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 115187 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643566+0000", 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643566+0000", 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643564+0000", 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643571+0000", 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.114e-06 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643589+0000", 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7634000000000001e-05 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643597+0000", 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.303e-06 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643662+0000", 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4257999999999994e-05 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643673+0000", 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1575e-05 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.650917+0000", 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0072438399999999997 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.650925+0000", 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2230000000000007e-06 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.827062+0000", 2023-09-23T20:11:25.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1761365800000001 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.827706+0000", 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00064398799999999996 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.837267+0000", 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0095610520000000004 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.838831+0000", 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001564466 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.838932+0000", 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101042 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851044+0000", 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012111709999999999 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851081+0000", 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6539999999999999e-05 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851972+0000", 2023-09-23T20:11:25.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00089110199999999995 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.852008+0000", 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6337999999999997e-05 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.853222+0000", 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001214302 2023-09-23T20:11:25.969 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:115197 8.b 8:d990aab7:::benchmark_data_smithi135_460343_object30964:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e342)", 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:02.643610+0000", 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 143.235383689, 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2161859939999999, 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.970 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 115197 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643610+0000", 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.971 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643610+0000", 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643609+0000", 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643613+0000", 2023-09-23T20:11:25.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3869999999999999e-06 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643620+0000", 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7240000000000003e-06 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643624+0000", 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2980000000000002e-06 2023-09-23T20:11:25.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643758+0000", 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000133769 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643768+0000", 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0671e-05 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.651097+0000", 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0073282039999999996 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.651106+0000", 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6639999999999993e-06 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.829201+0000", 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1780947209999999 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.829783+0000", 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000581997 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.844348+0000", 2023-09-23T20:11:25.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014565058000000001 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.844431+0000", 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3151000000000004e-05 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.844522+0000", 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1051000000000006e-05 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851914+0000", 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0073918050000000004 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.853255+0000", 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001341096 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.857930+0000", 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046753999999999997 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.857971+0000", 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.049e-05 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.859796+0000", 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018249539999999999 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:115254 8.b 8:d147ebd0:::benchmark_data_smithi135_460343_object31021:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e342)", 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:02.643628+0000", 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 143.235365395, 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2309227030000001, 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.980 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 115254 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643628+0000", 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643628+0000", 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643628+0000", 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:25.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643630+0000", 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1050000000000002e-06 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643636+0000", 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.105e-06 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643640+0000", 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2939999999999999e-06 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643799+0000", 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000158932 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:25.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.643810+0000", 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0139999999999999e-05 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.651130+0000", 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0073204899999999998 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:02.651137+0000", 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2150000000000004e-06 2023-09-23T20:11:25.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.829847+0000", 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.178710275 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.830512+0000", 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00066481899999999998 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.848989+0000", 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.018476328 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.849062+0000", 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3139000000000006e-05 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.849154+0000", 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2279000000000002e-05 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.851919+0000", 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002764533 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.859839+0000", 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0079203439999999993 2023-09-23T20:11:25.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.871956+0000", 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012117146000000001 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.872034+0000", 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7768999999999994e-05 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:03.874551+0000", 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025170689999999998 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:116078 8.5 8:ad389036:::benchmark_data_smithi135_460343_object31845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:28.841168+0000", 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 117.03782568299999, 2023-09-23T20:11:25.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.055073830999999997, 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116078 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841168+0000", 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841168+0000", 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841167+0000", 2023-09-23T20:11:25.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841173+0000", 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5030000000000002e-06 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841187+0000", 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3896999999999999e-05 2023-09-23T20:11:25.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841192+0000", 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2839999999999997e-06 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841237+0000", 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4274999999999999e-05 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841341+0000", 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103953 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.841422+0000", 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1631999999999994e-05 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:25.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.844545+0000", 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031223710000000001 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.844907+0000", 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036229099999999999 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.896146+0000", 2023-09-23T20:11:25.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.051239543999999998 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.896203+0000", 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6385000000000003e-05 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:28.896242+0000", 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8949999999999998e-05 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:25.997 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:116267 8.b 8:d95dfefb:::benchmark_data_smithi135_460343_object32034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:09:30.769210+0000", 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 115.10978381699999, 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.094167972000000003, 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:25.998 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 116267 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.769210+0000", 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.769210+0000", 2023-09-23T20:11:25.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.769209+0000", 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.769213+0000", 2023-09-23T20:11:26.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3499999999999999e-06 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.769222+0000", 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1139999999999995e-06 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.769228+0000", 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6160000000000001e-06 2023-09-23T20:11:26.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.769262+0000", 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3674000000000003e-05 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.769809+0000", 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00054710200000000005 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.775371+0000", 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055624589999999996 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.775474+0000", 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102852 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.775626+0000", 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015189399999999999 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.778610+0000", 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029836070000000001 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.780067+0000", 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001457135 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.860978+0000", 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.080911439000000002 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.861078+0000", 2023-09-23T20:11:26.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9977999999999997e-05 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:09:30.863378+0000", 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002299677 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.006 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:119775 8.b 8:de5f8f39:::benchmark_data_smithi135_460343_object35542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:10:10.805242+0000", 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 75.073751520000002, 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.055122485999999998, 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 119775 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.007 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.805242+0000", 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.805242+0000", 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.805241+0000", 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.805246+0000", 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9849999999999997e-06 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:26.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.805259+0000", 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2659e-05 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.805265+0000", 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7300000000000002e-06 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.809395+0000", 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0041301940000000002 2023-09-23T20:11:26.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.809508+0000", 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011316599999999999 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.809586+0000", 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7917999999999995e-05 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.822366+0000", 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012780325 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.855668+0000", 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.033301921999999998 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.859265+0000", 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035967970000000001 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.859303+0000", 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7663999999999997e-05 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:10:10.860365+0000", 2023-09-23T20:11:26.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0010620619999999999 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:126496 8.b 8:dbe0ee28:::benchmark_data_smithi135_460343_object42263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:06.341738+0000", 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 19.537255692999999, 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.073747823000000004, 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:26.014 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 126496 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.341738+0000", 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.341738+0000", 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.341737+0000", 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.341742+0000", 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5169999999999999e-06 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.341754+0000", 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1795e-05 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:26.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.341759+0000", 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8180000000000003e-06 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.341822+0000", 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2626999999999995e-05 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.341961+0000", 2023-09-23T20:11:26.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013971200000000001 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.342078+0000", 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011654099999999999 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.344273+0000", 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021950910000000001 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.344473+0000", 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000199858 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.415369+0000", 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.070896635999999999 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:26.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.415432+0000", 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2187000000000006e-05 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:06.415486+0000", 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4160000000000003e-05 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:127302 8.b 8:dfdc6f44:::benchmark_data_smithi135_460343_object43069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:08.735777+0000", 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 17.143216351, 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.125646753, 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127302 2023-09-23T20:11:26.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.735777+0000", 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.735777+0000", 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.735775+0000", 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999976 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.735788+0000", 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3083e-05 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.735823+0000", 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5219000000000003e-05 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.735841+0000", 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8099999999999999e-05 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:26.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.735961+0000", 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000119265 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.736168+0000", 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020713900000000001 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.736233+0000", 2023-09-23T20:11:26.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5099999999999997e-05 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.738491+0000", 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022578350000000001 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.738626+0000", 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013552599999999999 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.861357+0000", 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.122730166 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.861394+0000", 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7298000000000001e-05 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:26.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:08.861424+0000", 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0192000000000002e-05 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:127504 8.b 8:d4b8cd61:::benchmark_data_smithi135_460343_object43271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e376)", 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:09.371043+0000", 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.507950490999999, 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.085254896999999996, 2023-09-23T20:11:26.029 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 127504 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371043+0000", 2023-09-23T20:11:26.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371043+0000", 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371042+0000", 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:26.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371049+0000", 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3359999999999997e-06 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371066+0000", 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6657999999999999e-05 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371072+0000", 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3119999999999999e-06 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371123+0000", 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1505000000000002e-05 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:26.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371247+0000", 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012406599999999999 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.371358+0000", 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000110129 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.373626+0000", 2023-09-23T20:11:26.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022689530000000002 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.379231+0000", 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005604609 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.456194+0000", 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.076962814000000004 2023-09-23T20:11:26.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.456249+0000", 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5356000000000003e-05 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:09.456298+0000", 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8733999999999998e-05 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.036 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:131957 8.2 8:40d6a324:::benchmark_data_smithi135_460343_object47724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:24.891306+0000", 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.98768748500000003, 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.10492140799999999, 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:26.037 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 131957 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.891306+0000", 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:26.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.891306+0000", 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.891305+0000", 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.891310+0000", 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9819999999999999e-06 2023-09-23T20:11:26.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.891320+0000", 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4790000000000002e-06 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.891324+0000", 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4909999999999999e-06 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.891366+0000", 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2110000000000002e-05 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.983526+0000", 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.092159772000000001 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:11:26.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.983625+0000", 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9244000000000001e-05 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.986525+0000", 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002899301 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.986596+0000", 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1015999999999999e-05 2023-09-23T20:11:26.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.996160+0000", 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0095645369999999997 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.996197+0000", 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6847999999999997e-05 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.996228+0000", 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0614000000000002e-05 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.35643.0:131961 8.b e385/381 8:d7871ec2:::benchmark_data_smithi135_460343_object47728:head v 385'8092, mlcod=385'8090)", 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:24.927962+0000", 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.95103139400000003, 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.058101320999999997, 2023-09-23T20:11:26.044 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.927962+0000", 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.927962+0000", 2023-09-23T20:11:26.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.927953+0000", 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999909 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.927974+0000", 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0647000000000001e-05 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.983489+0000", 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.055514833999999999 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.983496+0000", 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.593e-06 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:26.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.983544+0000", 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7806999999999998e-05 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.983564+0000", 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9984e-05 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.986031+0000", 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002467431 2023-09-23T20:11:26.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:24.986064+0000", 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2119000000000002e-05 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:26.049 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:11:26.050 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:26.077 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:26.135 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:26.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:26.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:26.239 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:26.292 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:26.493 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:26.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:26.515 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:26.600 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:26.617 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:11:26.629 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:26.686 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:26.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:26.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:26.792 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:26.799 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:11:26.821 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:26.878 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:26.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:26.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:26.985 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:27.049 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:27.118 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:27.275 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:27.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:27.382 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:27.409 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:27.415 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:11:27.450 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:11:27.482 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:11:27.514 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:11:27.540 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:11:27.574 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:11:27.607 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:11:27.632 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:11:27.632 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:11:27.632 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:11:27.632 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:11:27.632 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:11:27.632 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:11:27.633 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:11:27.633 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:11:27.633 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:11:27.658 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:27.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:27.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:27.762 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:27.789 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:27.830 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:27.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:27.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:27.933 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:27.961 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:28.193 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:28.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:28.254 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:28 smithi144 ceph-mon[194833]: pgmap v1073: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 10083/110511 objects degraded (9.124%); 3417/110511 objects misplaced (3.092%); 2.0 MiB/s, 75 objects/s recovering 2023-09-23T20:11:28.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4024923895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:28.255 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 10083/110511 objects degraded (9.124%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:28.259 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:28 smithi135 ceph-mon[221113]: pgmap v1073: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 10083/110511 objects degraded (9.124%); 3417/110511 objects misplaced (3.092%); 2.0 MiB/s, 75 objects/s recovering 2023-09-23T20:11:28.259 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4024923895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:28.259 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 10083/110511 objects degraded (9.124%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:28.259 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:28 smithi135 ceph-mon[213871]: pgmap v1073: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 10083/110511 objects degraded (9.124%); 3417/110511 objects misplaced (3.092%); 2.0 MiB/s, 75 objects/s recovering 2023-09-23T20:11:28.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4024923895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:28.260 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 10083/110511 objects degraded (9.124%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:28.297 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:28.338 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:28.793 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:28.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:28.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:28.807 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:11:28.807 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:11:28.897 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:28.954 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:28.959 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:29.042 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:29.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:29.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:29.148 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:29.204 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:29.426 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:29.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:29.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:29.532 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:29.598 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:11:29.607 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:29.670 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:29.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:29.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:29.778 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:29.805 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:29.855 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:29.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:29.964 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:29.991 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:30.019 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:11:30.019 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:30.097 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:30.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:30.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:30.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:30 smithi144 ceph-mon[194833]: pgmap v1074: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 9985/107253 objects degraded (9.310%); 3318/107253 objects misplaced (3.094%); 2.1 MiB/s, 49 objects/s recovering 2023-09-23T20:11:30.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3715481500' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:11:30.202 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:30.246 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:30 smithi135 ceph-mon[221113]: pgmap v1074: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 9985/107253 objects degraded (9.310%); 3318/107253 objects misplaced (3.094%); 2.1 MiB/s, 49 objects/s recovering 2023-09-23T20:11:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3715481500' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:11:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:30 smithi135 ceph-mon[213871]: pgmap v1074: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 9985/107253 objects degraded (9.310%); 3318/107253 objects misplaced (3.094%); 2.1 MiB/s, 49 objects/s recovering 2023-09-23T20:11:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3715481500' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:11:30.863 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:30.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:30.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:30.927 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:11:30.967 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:31.059 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:31.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[221113]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3715481500' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:11:31.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[221113]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[221113]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[221113]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[221113]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[221113]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[221113]: pgmap v1076: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 362 op/s; 9985/106008 objects degraded (9.419%); 3217/106008 objects misplaced (3.035%); 2.9 MiB/s, 76 objects/s recovering 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[213871]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3715481500' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[213871]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[213871]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[213871]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[213871]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[213871]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:31 smithi135 ceph-mon[213871]: pgmap v1076: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 362 op/s; 9985/106008 objects degraded (9.419%); 3217/106008 objects misplaced (3.035%); 2.9 MiB/s, 76 objects/s recovering 2023-09-23T20:11:31.240 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:31.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:31.282 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:11:31.282 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:11:31.352 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:31.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:31 smithi144 ceph-mon[194833]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:31 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3715481500' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:11:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:31 smithi144 ceph-mon[194833]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T20:11:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:31 smithi144 ceph-mon[194833]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:31 smithi144 ceph-mon[194833]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:31 smithi144 ceph-mon[194833]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:31 smithi144 ceph-mon[194833]: failed to encode map e386 with expected crc 2023-09-23T20:11:31.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:31 smithi144 ceph-mon[194833]: pgmap v1076: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 362 op/s; 9985/106008 objects degraded (9.419%); 3217/106008 objects misplaced (3.035%); 2.9 MiB/s, 76 objects/s recovering 2023-09-23T20:11:31.389 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:31.462 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:31.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:31.566 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:31.600 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:31.706 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:31.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:31.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:31.712 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:31.795 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:11:31.795 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:11:31.810 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:31.864 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:32.030 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:32.036 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:32.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:32.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:32.147 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:32.170 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3261934227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:32.170 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:32 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:32.177 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:11:32.197 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:32.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3261934227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:32.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:32 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:32.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3261934227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:32.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:32 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:32.669 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:11:33.058 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:33.365 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:33.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:33.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:33.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:33.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9985/106008 objects degraded (9.419%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[221113]: pgmap v1078: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 268 op/s; 9985/105357 objects degraded (9.477%); 3217/105357 objects misplaced (3.053%); 2.3 MiB/s, 65 objects/s recovering 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T20:11:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9985/106008 objects degraded (9.419%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:33.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:33 smithi135 ceph-mon[213871]: pgmap v1078: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 268 op/s; 9985/105357 objects degraded (9.477%); 3217/105357 objects misplaced (3.053%); 2.3 MiB/s, 65 objects/s recovering 2023-09-23T20:11:33.474 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:33.505 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:33.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:33.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:33.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T20:11:33.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.507 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.508 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: failed to encode map e387 with expected crc 2023-09-23T20:11:33.508 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9985/106008 objects degraded (9.419%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:33.508 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:33 smithi144 ceph-mon[194833]: pgmap v1078: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 268 op/s; 9985/105357 objects degraded (9.477%); 3217/105357 objects misplaced (3.053%); 2.3 MiB/s, 65 objects/s recovering 2023-09-23T20:11:33.862 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:11:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:11:33] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T20:11:34.337 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:34.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:34.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:34.444 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:34.479 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:34.731 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:34.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:34.834 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:34.867 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:34.922 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:34.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:34.937 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:11:34.937 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:11:35.026 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:35.071 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:35.410 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:35.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:35.514 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:35.520 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:11:35.562 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:35.564 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:11:35.603 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:11:35.637 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:11:35.658 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:11:35.666 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:11:35.698 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:11:35.747 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:11:35.755 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:35.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:35.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:35.786 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:11:35.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:11:35.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:11:35.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:11:35.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:11:35.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:11:35.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:11:35.792 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:11:35.793 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:11:35.859 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:35.887 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:35.936 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:35.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:36.012 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:36 smithi135 ceph-mon[213871]: pgmap v1079: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 9864/101922 objects degraded (9.678%); 3106/101922 objects misplaced (3.047%); 2.5 MiB/s, 52 objects/s recovering 2023-09-23T20:11:36.044 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:36.092 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:36.108 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:11:36.112 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:36.145 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:11:36.150 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:11:36.183 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:36.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:36.286 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:36.309 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:36 smithi135 ceph-mon[221113]: pgmap v1079: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 9864/101922 objects degraded (9.678%); 3106/101922 objects misplaced (3.047%); 2.5 MiB/s, 52 objects/s recovering 2023-09-23T20:11:36.309 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2835044350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:36.310 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2835044350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:36.311 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:36.353 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:36.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:36.364 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:36 smithi144 ceph-mon[194833]: pgmap v1079: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 9864/101922 objects degraded (9.678%); 3106/101922 objects misplaced (3.047%); 2.5 MiB/s, 52 objects/s recovering 2023-09-23T20:11:36.364 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2835044350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:36.457 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:36.524 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:37.536 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:37.537 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1079,"stamp":"2023-09-23T20:11:34.945466+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2229361730,"num_objects":33974,"num_object_clones":0,"num_object_copies":101922,"num_objects_missing_on_primary":2755,"num_objects_missing":2755,"num_objects_degraded":9864,"num_objects_misplaced":3106,"num_objects_unfound":0,"num_objects_dirty":33974,"num_whiteouts":0,"num_read":6158,"num_read_kb":13539,"num_write":135510,"num_write_kb":5404324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29267,"num_bytes_recovered":1501565183,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93850,"ondisk_log_size":93850,"up":327,"acting":322,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11597096,"kb_used_data":7991280,"kb_used_omap":32,"kb_used_meta":3605727,"kb_avail":644618968,"statfs":{"total":671965249536,"available":660089823232,"internally_reserved":0,"allocated":8183070720,"data_stored":8171844511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3692264966},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-187629568,"num_objects":-2863,"num_object_clones":0,"num_object_copies":-8589,"num_objects_missing_on_primary":-320,"num_objects_missing":-320,"num_objects_degraded":-219,"num_objects_misplaced":-311,"num_objects_unfound":0,"num_objects_dirty":-2863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2863,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":419,"num_bytes_recovered":20774912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003772"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":385,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025394+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:11:30.025394+0000","last_peered":"2023-09-23T20:11:30.025394+0000","last_clean":"2023-09-23T20:11:30.025394+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:11:30.025394+0000","last_undegraded":"2023-09-23T20:11:30.025394+0000","last_fullsized":"2023-09-23T20:11:30.025394+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":215,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036605+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:11:30.036605+0000","last_peered":"2023-09-23T20:11:30.036605+0000","last_clean":"2023-09-23T20:11:30.036605+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:11:30.036605+0000","last_undegraded":"2023-09-23T20:11:30.036605+0000","last_fullsized":"2023-09-23T20:11:30.036605+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037244+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:11:30.037244+0000","last_peered":"2023-09-23T20:11:30.037244+0000","last_clean":"2023-09-23T20:11:30.037244+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:11:30.037244+0000","last_undegraded":"2023-09-23T20:11:30.037244+0000","last_fullsized":"2023-09-23T20:11:30.037244+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":95,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034990+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:11:30.034990+0000","last_peered":"2023-09-23T20:11:30.034990+0000","last_clean":"2023-09-23T20:11:30.034990+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:11:30.034990+0000","last_undegraded":"2023-09-23T20:11:30.034990+0000","last_fullsized":"2023-09-23T20:11:30.034990+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":215,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036788+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:11:30.036788+0000","last_peered":"2023-09-23T20:11:30.036788+0000","last_clean":"2023-09-23T20:11:30.036788+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:11:30.036788+0000","last_undegraded":"2023-09-23T20:11:30.036788+0000","last_fullsized":"2023-09-23T20:11:30.036788+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":332,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023422+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:11:30.023422+0000","last_peered":"2023-09-23T20:11:30.023422+0000","last_clean":"2023-09-23T20:11:30.023422+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:11:30.023422+0000","last_undegraded":"2023-09-23T20:11:30.023422+0000","last_fullsized":"2023-09-23T20:11:30.023422+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":313,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023331+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:11:30.023331+0000","last_peered":"2023-09-23T20:11:30.023331+0000","last_clean":"2023-09-23T20:11:30.023331+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:11:30.023331+0000","last_undegraded":"2023-09-23T20:11:30.023331+0000","last_fullsized":"2023-09-23T20:11:30.023331+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":229,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.162580+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:11:23.162580+0000","last_peered":"2023-09-23T20:11:23.162580+0000","last_clean":"2023-09-23T20:11:23.162580+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:11:23.162580+0000","last_undegraded":"2023-09-23T20:11:23.162580+0000","last_fullsized":"2023-09-23T20:11:23.162580+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":695,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.163900+0000","last_change":"2023-09-23T20:09:56.042750+0000","last_active":"2023-09-23T20:11:23.163900+0000","last_peered":"2023-09-23T20:11:23.163900+0000","last_clean":"2023-09-23T20:11:23.163900+0000","last_became_active":"2023-09-23T20:09:56.042455+0000","last_became_peered":"2023-09-23T20:09:56.042455+0000","last_unstale":"2023-09-23T20:11:23.163900+0000","last_undegraded":"2023-09-23T20:11:23.163900+0000","last_fullsized":"2023-09-23T20:11:23.163900+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":386,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023175+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:11:30.023175+0000","last_peered":"2023-09-23T20:11:30.023175+0000","last_clean":"2023-09-23T20:11:30.023175+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:11:30.023175+0000","last_undegraded":"2023-09-23T20:11:30.023175+0000","last_fullsized":"2023-09-23T20:11:30.023175+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":248,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.163727+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:11:23.163727+0000","last_peered":"2023-09-23T20:11:23.163727+0000","last_clean":"2023-09-23T20:11:23.163727+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:11:23.163727+0000","last_undegraded":"2023-09-23T20:11:23.163727+0000","last_fullsized":"2023-09-23T20:11:23.163727+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":80,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036132+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:11:30.036132+0000","last_peered":"2023-09-23T20:11:30.036132+0000","last_clean":"2023-09-23T20:11:30.036132+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:11:30.036132+0000","last_undegraded":"2023-09-23T20:11:30.036132+0000","last_fullsized":"2023-09-23T20:11:30.036132+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037771+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:11:30.037771+0000","last_peered":"2023-09-23T20:11:30.037771+0000","last_clean":"2023-09-23T20:11:30.037771+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:11:30.037771+0000","last_undegraded":"2023-09-23T20:11:30.037771+0000","last_fullsized":"2023-09-23T20:11:30.037771+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":441,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025240+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:11:30.025240+0000","last_peered":"2023-09-23T20:11:30.025240+0000","last_clean":"2023-09-23T20:11:30.025240+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:11:30.025240+0000","last_undegraded":"2023-09-23T20:11:30.025240+0000","last_fullsized":"2023-09-23T20:11:30.025240+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":383,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025251+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:11:30.025251+0000","last_peered":"2023-09-23T20:11:30.025251+0000","last_clean":"2023-09-23T20:11:30.025251+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:11:30.025251+0000","last_undegraded":"2023-09-23T20:11:30.025251+0000","last_fullsized":"2023-09-23T20:11:30.025251+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":716,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022848+0000","last_change":"2023-09-23T20:11:07.067995+0000","last_active":"2023-09-23T20:11:30.022848+0000","last_peered":"2023-09-23T20:11:30.022848+0000","last_clean":"2023-09-23T20:11:30.022848+0000","last_became_active":"2023-09-23T20:09:56.043288+0000","last_became_peered":"2023-09-23T20:09:56.043288+0000","last_unstale":"2023-09-23T20:11:30.022848+0000","last_undegraded":"2023-09-23T20:11:30.022848+0000","last_fullsized":"2023-09-23T20:11:30.022848+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":80,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034954+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:11:30.034954+0000","last_peered":"2023-09-23T20:11:30.034954+0000","last_clean":"2023-09-23T20:11:30.034954+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:11:30.034954+0000","last_undegraded":"2023-09-23T20:11:30.034954+0000","last_fullsized":"2023-09-23T20:11:30.034954+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":239,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037941+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:11:30.037941+0000","last_peered":"2023-09-23T20:11:30.037941+0000","last_clean":"2023-09-23T20:11:30.037941+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:11:30.037941+0000","last_undegraded":"2023-09-23T20:11:30.037941+0000","last_fullsized":"2023-09-23T20:11:30.037941+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":533,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034713+0000","last_change":"2023-09-23T20:10:40.721575+0000","last_active":"2023-09-23T20:11:30.034713+0000","last_peered":"2023-09-23T20:11:30.034713+0000","last_clean":"2023-09-23T20:11:30.034713+0000","last_became_active":"2023-09-23T20:09:56.050784+0000","last_became_peered":"2023-09-23T20:09:56.050784+0000","last_unstale":"2023-09-23T20:11:30.034713+0000","last_undegraded":"2023-09-23T20:11:30.034713+0000","last_fullsized":"2023-09-23T20:11:30.034713+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":95,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034917+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:11:30.034917+0000","last_peered":"2023-09-23T20:11:30.034917+0000","last_clean":"2023-09-23T20:11:30.034917+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:11:30.034917+0000","last_undegraded":"2023-09-23T20:11:30.034917+0000","last_fullsized":"2023-09-23T20:11:30.034917+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":385,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023651+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:11:30.023651+0000","last_peered":"2023-09-23T20:11:30.023651+0000","last_clean":"2023-09-23T20:11:30.023651+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:11:30.023651+0000","last_undegraded":"2023-09-23T20:11:30.023651+0000","last_fullsized":"2023-09-23T20:11:30.023651+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":512,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037276+0000","last_change":"2023-09-23T20:09:56.062364+0000","last_active":"2023-09-23T20:11:30.037276+0000","last_peered":"2023-09-23T20:11:30.037276+0000","last_clean":"2023-09-23T20:11:30.037276+0000","last_became_active":"2023-09-23T20:09:56.061865+0000","last_became_peered":"2023-09-23T20:09:56.061865+0000","last_unstale":"2023-09-23T20:11:30.037276+0000","last_undegraded":"2023-09-23T20:11:30.037276+0000","last_fullsized":"2023-09-23T20:11:30.037276+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034812+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:11:30.034812+0000","last_peered":"2023-09-23T20:11:30.034812+0000","last_clean":"2023-09-23T20:11:30.034812+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:11:30.034812+0000","last_undegraded":"2023-09-23T20:11:30.034812+0000","last_fullsized":"2023-09-23T20:11:30.034812+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":372,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.163241+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:11:23.163241+0000","last_peered":"2023-09-23T20:11:23.163241+0000","last_clean":"2023-09-23T20:11:23.163241+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:11:23.163241+0000","last_undegraded":"2023-09-23T20:11:23.163241+0000","last_fullsized":"2023-09-23T20:11:23.163241+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":608,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034776+0000","last_change":"2023-09-23T20:11:18.639154+0000","last_active":"2023-09-23T20:11:30.034776+0000","last_peered":"2023-09-23T20:11:30.034776+0000","last_clean":"2023-09-23T20:11:30.034776+0000","last_became_active":"2023-09-23T20:09:56.043157+0000","last_became_peered":"2023-09-23T20:09:56.043157+0000","last_unstale":"2023-09-23T20:11:30.034776+0000","last_undegraded":"2023-09-23T20:11:30.034776+0000","last_fullsized":"2023-09-23T20:11:30.034776+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":18,"reported_epoch":386,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:11:30.022843+0000","last_change":"2023-09-23T20:11:10.067603+0000","last_active":"2023-09-23T20:11:30.022843+0000","last_peered":"2023-09-23T20:11:30.022843+0000","last_clean":"2023-09-23T20:00:51.694506+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:11:30.022843+0000","last_undegraded":"2023-09-23T20:11:10.056843+0000","last_fullsized":"2023-09-23T20:11:30.022843+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":4}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":16,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.164299+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:11:23.164299+0000","last_peered":"2023-09-23T20:11:23.164299+0000","last_clean":"2023-09-23T20:11:23.164299+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:11:23.164299+0000","last_undegraded":"2023-09-23T20:11:23.164299+0000","last_fullsized":"2023-09-23T20:11:23.164299+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":807,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026442+0000","last_change":"2023-09-23T20:11:16.752974+0000","last_active":"2023-09-23T20:11:30.026442+0000","last_peered":"2023-09-23T20:11:30.026442+0000","last_clean":"2023-09-23T20:11:30.026442+0000","last_became_active":"2023-09-23T20:10:52.687280+0000","last_became_peered":"2023-09-23T20:10:52.687280+0000","last_unstale":"2023-09-23T20:11:30.026442+0000","last_undegraded":"2023-09-23T20:11:30.026442+0000","last_fullsized":"2023-09-23T20:11:30.026442+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.4","version":"387'8589","reported_seq":8685,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:33.260341+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:11:33.260341+0000","last_peered":"2023-09-23T20:11:33.260341+0000","last_clean":"2023-09-23T20:11:33.260341+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:11:33.260341+0000","last_undegraded":"2023-09-23T20:11:33.260341+0000","last_fullsized":"2023-09-23T20:11:33.260341+0000","mapping_epoch":299,"log_start":"309'2665","ondisk_log_start":"309'2665","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5924,"log_dups_size":0,"ondisk_log_size":5924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140312576,"num_objects":2141,"num_object_clones":0,"num_object_copies":6423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8589,"num_write_kb":343360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":78,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036070+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:11:30.036070+0000","last_peered":"2023-09-23T20:11:30.036070+0000","last_clean":"2023-09-23T20:11:30.036070+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:11:30.036070+0000","last_undegraded":"2023-09-23T20:11:30.036070+0000","last_fullsized":"2023-09-23T20:11:30.036070+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":395,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022807+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:11:30.022807+0000","last_peered":"2023-09-23T20:11:30.022807+0000","last_clean":"2023-09-23T20:11:30.022807+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:11:30.022807+0000","last_undegraded":"2023-09-23T20:11:30.022807+0000","last_fullsized":"2023-09-23T20:11:30.022807+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":696,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.163841+0000","last_change":"2023-09-23T20:09:56.064782+0000","last_active":"2023-09-23T20:11:23.163841+0000","last_peered":"2023-09-23T20:11:23.163841+0000","last_clean":"2023-09-23T20:11:23.163841+0000","last_became_active":"2023-09-23T20:09:56.064411+0000","last_became_peered":"2023-09-23T20:09:56.064411+0000","last_unstale":"2023-09-23T20:11:23.163841+0000","last_undegraded":"2023-09-23T20:11:23.163841+0000","last_fullsized":"2023-09-23T20:11:23.163841+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218454,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.5","version":"387'8208","reported_seq":10898,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:33.395890+0000","last_change":"2023-09-23T20:07:45.697314+0000","last_active":"2023-09-23T20:11:33.395890+0000","last_peered":"2023-09-23T20:11:33.395890+0000","last_clean":"2023-09-23T20:11:33.395890+0000","last_became_active":"2023-09-23T20:07:45.690354+0000","last_became_peered":"2023-09-23T20:07:45.690354+0000","last_unstale":"2023-09-23T20:11:33.395890+0000","last_undegraded":"2023-09-23T20:11:33.395890+0000","last_fullsized":"2023-09-23T20:11:33.395890+0000","mapping_epoch":327,"log_start":"309'2518","ondisk_log_start":"309'2518","created":299,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.926631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138018816,"num_objects":2106,"num_object_clones":0,"num_object_copies":6318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8208,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":79,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.035036+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:11:30.035036+0000","last_peered":"2023-09-23T20:11:30.035036+0000","last_clean":"2023-09-23T20:11:30.035036+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:11:30.035036+0000","last_undegraded":"2023-09-23T20:11:30.035036+0000","last_fullsized":"2023-09-23T20:11:30.035036+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":443,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025005+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:11:30.025005+0000","last_peered":"2023-09-23T20:11:30.025005+0000","last_clean":"2023-09-23T20:11:30.025005+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:11:30.025005+0000","last_undegraded":"2023-09-23T20:11:30.025005+0000","last_fullsized":"2023-09-23T20:11:30.025005+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":654,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.164397+0000","last_change":"2023-09-23T20:11:20.856999+0000","last_active":"2023-09-23T20:11:23.164397+0000","last_peered":"2023-09-23T20:11:23.164397+0000","last_clean":"2023-09-23T20:11:23.164397+0000","last_became_active":"2023-09-23T20:11:20.856640+0000","last_became_peered":"2023-09-23T20:11:20.856640+0000","last_unstale":"2023-09-23T20:11:23.164397+0000","last_undegraded":"2023-09-23T20:11:23.164397+0000","last_fullsized":"2023-09-23T20:11:23.164397+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.6","version":"386'8564","reported_seq":10184,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:30.505043+0000","last_change":"2023-09-23T20:07:37.064417+0000","last_active":"2023-09-23T20:11:30.505043+0000","last_peered":"2023-09-23T20:11:30.505043+0000","last_clean":"2023-09-23T20:06:59.831272+0000","last_became_active":"2023-09-23T20:07:37.059682+0000","last_became_peered":"2023-09-23T20:07:37.059682+0000","last_unstale":"2023-09-23T20:11:30.505043+0000","last_undegraded":"2023-09-23T20:07:36.983730+0000","last_fullsized":"2023-09-23T20:07:36.974121+0000","mapping_epoch":322,"log_start":"314'3465","ondisk_log_start":"314'3465","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5099,"log_dups_size":0,"ondisk_log_size":5099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:33.575626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147062784,"num_objects":2244,"num_object_clones":0,"num_object_copies":6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8564,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":853},{"shards":"0,2,3","objects":1391}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":78,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.035962+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:11:30.035962+0000","last_peered":"2023-09-23T20:11:30.035962+0000","last_clean":"2023-09-23T20:11:30.035962+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:11:30.035962+0000","last_undegraded":"2023-09-23T20:11:30.035962+0000","last_fullsized":"2023-09-23T20:11:30.035962+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037851+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:11:30.037851+0000","last_peered":"2023-09-23T20:11:30.037851+0000","last_clean":"2023-09-23T20:11:30.037851+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:11:30.037851+0000","last_undegraded":"2023-09-23T20:11:30.037851+0000","last_fullsized":"2023-09-23T20:11:30.037851+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":576,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025983+0000","last_change":"2023-09-23T20:11:07.538010+0000","last_active":"2023-09-23T20:11:30.025983+0000","last_peered":"2023-09-23T20:11:30.025983+0000","last_clean":"2023-09-23T20:11:30.025983+0000","last_became_active":"2023-09-23T20:09:56.043491+0000","last_became_peered":"2023-09-23T20:09:56.043491+0000","last_unstale":"2023-09-23T20:11:30.025983+0000","last_undegraded":"2023-09-23T20:11:30.025983+0000","last_fullsized":"2023-09-23T20:11:30.025983+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.7","version":"387'8437","reported_seq":11206,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:33.686951+0000","last_change":"2023-09-23T20:10:58.582053+0000","last_active":"2023-09-23T20:11:33.686951+0000","last_peered":"2023-09-23T20:11:33.686951+0000","last_clean":"2023-09-23T20:07:17.891274+0000","last_became_active":"2023-09-23T20:10:58.299550+0000","last_became_peered":"2023-09-23T20:10:58.299550+0000","last_unstale":"2023-09-23T20:11:33.686951+0000","last_undegraded":"2023-09-23T20:10:56.195777+0000","last_fullsized":"2023-09-23T20:10:53.692217+0000","mapping_epoch":369,"log_start":"309'2929","ondisk_log_start":"309'2929","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5508,"log_dups_size":0,"ondisk_log_size":5508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:48.194019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137691136,"num_objects":2101,"num_object_clones":0,"num_object_copies":6303,"num_objects_missing_on_primary":2751,"num_objects_missing":2751,"num_objects_degraded":3345,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8437,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1318,"num_bytes_recovered":86376448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2],"avail_no_missing":["3"],"object_location_counts":[{"shards":"1,2","objects":1181},{"shards":"1,2,3,4","objects":14},{"shards":"2,3,4","objects":906}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":425,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023409+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:11:30.023409+0000","last_peered":"2023-09-23T20:11:30.023409+0000","last_clean":"2023-09-23T20:11:30.023409+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:11:30.023409+0000","last_undegraded":"2023-09-23T20:11:30.023409+0000","last_fullsized":"2023-09-23T20:11:30.023409+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":82,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037882+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:11:30.037882+0000","last_peered":"2023-09-23T20:11:30.037882+0000","last_clean":"2023-09-23T20:11:30.037882+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:11:30.037882+0000","last_undegraded":"2023-09-23T20:11:30.037882+0000","last_fullsized":"2023-09-23T20:11:30.037882+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"386'8200","reported_seq":16326,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:31.128677+0000","last_change":"2023-09-23T20:10:10.729751+0000","last_active":"2023-09-23T20:11:31.128677+0000","last_peered":"2023-09-23T20:11:31.128677+0000","last_clean":"2023-09-23T20:11:31.128677+0000","last_became_active":"2023-09-23T20:10:10.708263+0000","last_became_peered":"2023-09-23T20:10:10.708263+0000","last_unstale":"2023-09-23T20:11:31.128677+0000","last_undegraded":"2023-09-23T20:11:31.128677+0000","last_fullsized":"2023-09-23T20:11:31.128677+0000","mapping_epoch":361,"log_start":"309'2810","ondisk_log_start":"309'2810","created":299,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5390,"log_dups_size":0,"ondisk_log_size":5390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:15.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138149888,"num_objects":2108,"num_object_clones":0,"num_object_copies":6324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8200,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4003,"num_bytes_recovered":136118272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":769,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034732+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:11:30.034732+0000","last_peered":"2023-09-23T20:11:30.034732+0000","last_clean":"2023-09-23T20:11:30.034732+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:11:30.034732+0000","last_undegraded":"2023-09-23T20:11:30.034732+0000","last_fullsized":"2023-09-23T20:11:30.034732+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":77,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025248+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:11:30.025248+0000","last_peered":"2023-09-23T20:11:30.025248+0000","last_clean":"2023-09-23T20:11:30.025248+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:11:30.025248+0000","last_undegraded":"2023-09-23T20:11:30.025248+0000","last_fullsized":"2023-09-23T20:11:30.025248+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":413,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025321+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:11:30.025321+0000","last_peered":"2023-09-23T20:11:30.025321+0000","last_clean":"2023-09-23T20:11:30.025321+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:11:30.025321+0000","last_undegraded":"2023-09-23T20:11:30.025321+0000","last_fullsized":"2023-09-23T20:11:30.025321+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"387'8479","reported_seq":14878,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:32.859032+0000","last_change":"2023-09-23T20:10:58.556423+0000","last_active":"2023-09-23T20:11:32.859032+0000","last_peered":"2023-09-23T20:11:32.859032+0000","last_clean":"2023-09-23T20:07:17.847486+0000","last_became_active":"2023-09-23T20:07:20.666556+0000","last_became_peered":"2023-09-23T20:07:20.666556+0000","last_unstale":"2023-09-23T20:11:32.859032+0000","last_undegraded":"2023-09-23T20:10:58.555698+0000","last_fullsized":"2023-09-23T20:07:19.932561+0000","mapping_epoch":317,"log_start":"314'3546","ondisk_log_start":"314'3546","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4933,"log_dups_size":0,"ondisk_log_size":4933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:38.441814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132448256,"num_objects":2021,"num_object_clones":0,"num_object_copies":6063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":996,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8479,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3145,"num_bytes_recovered":206110720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":1025},{"shards":"0,2,5","objects":912}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":721,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022917+0000","last_change":"2023-09-23T20:11:11.229144+0000","last_active":"2023-09-23T20:11:30.022917+0000","last_peered":"2023-09-23T20:11:30.022917+0000","last_clean":"2023-09-23T20:11:30.022917+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:11:30.022917+0000","last_undegraded":"2023-09-23T20:11:30.022917+0000","last_fullsized":"2023-09-23T20:11:30.022917+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":353,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025311+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:11:30.025311+0000","last_peered":"2023-09-23T20:11:30.025311+0000","last_clean":"2023-09-23T20:11:30.025311+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:11:30.025311+0000","last_undegraded":"2023-09-23T20:11:30.025311+0000","last_fullsized":"2023-09-23T20:11:30.025311+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":383,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026525+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:11:30.026525+0000","last_peered":"2023-09-23T20:11:30.026525+0000","last_clean":"2023-09-23T20:11:30.026525+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:11:30.026525+0000","last_undegraded":"2023-09-23T20:11:30.026525+0000","last_fullsized":"2023-09-23T20:11:30.026525+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":512,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037368+0000","last_change":"2023-09-23T20:09:56.062162+0000","last_active":"2023-09-23T20:11:30.037368+0000","last_peered":"2023-09-23T20:11:30.037368+0000","last_clean":"2023-09-23T20:11:30.037368+0000","last_became_active":"2023-09-23T20:09:56.061518+0000","last_became_peered":"2023-09-23T20:09:56.061518+0000","last_unstale":"2023-09-23T20:11:30.037368+0000","last_undegraded":"2023-09-23T20:11:30.037368+0000","last_fullsized":"2023-09-23T20:11:30.037368+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"387'8410","reported_seq":8506,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:33.445877+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:11:33.445877+0000","last_peered":"2023-09-23T20:11:33.445877+0000","last_clean":"2023-09-23T20:11:33.445877+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:11:33.445877+0000","last_undegraded":"2023-09-23T20:11:33.445877+0000","last_fullsized":"2023-09-23T20:11:33.445877+0000","mapping_epoch":299,"log_start":"309'2685","ondisk_log_start":"309'2685","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134348800,"num_objects":2050,"num_object_clones":0,"num_object_copies":6150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8410,"num_write_kb":334720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":377,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.024908+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:11:30.024908+0000","last_peered":"2023-09-23T20:11:30.024908+0000","last_clean":"2023-09-23T20:11:30.024908+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:11:30.024908+0000","last_undegraded":"2023-09-23T20:11:30.024908+0000","last_fullsized":"2023-09-23T20:11:30.024908+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":413,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026119+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:11:30.026119+0000","last_peered":"2023-09-23T20:11:30.026119+0000","last_clean":"2023-09-23T20:11:30.026119+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:11:30.026119+0000","last_undegraded":"2023-09-23T20:11:30.026119+0000","last_fullsized":"2023-09-23T20:11:30.026119+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"387'8256","reported_seq":16411,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:33.348832+0000","last_change":"2023-09-23T20:09:37.763558+0000","last_active":"2023-09-23T20:11:33.348832+0000","last_peered":"2023-09-23T20:11:33.348832+0000","last_clean":"2023-09-23T20:11:33.348832+0000","last_became_active":"2023-09-23T20:09:37.732414+0000","last_became_peered":"2023-09-23T20:09:37.732414+0000","last_unstale":"2023-09-23T20:11:33.348832+0000","last_undegraded":"2023-09-23T20:11:33.348832+0000","last_fullsized":"2023-09-23T20:11:33.348832+0000","mapping_epoch":352,"log_start":"309'2781","ondisk_log_start":"309'2781","created":299,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:39.792571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133955584,"num_objects":2044,"num_object_clones":0,"num_object_copies":6132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8256,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4021,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":725,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023714+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:11:30.023714+0000","last_peered":"2023-09-23T20:11:30.023714+0000","last_clean":"2023-09-23T20:11:30.023714+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:11:30.023714+0000","last_undegraded":"2023-09-23T20:11:30.023714+0000","last_fullsized":"2023-09-23T20:11:30.023714+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":16,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.162299+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:11:23.162299+0000","last_peered":"2023-09-23T20:11:23.162299+0000","last_clean":"2023-09-23T20:11:23.162299+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:11:23.162299+0000","last_undegraded":"2023-09-23T20:11:23.162299+0000","last_fullsized":"2023-09-23T20:11:23.162299+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":77,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.057519+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:11:30.057519+0000","last_peered":"2023-09-23T20:11:30.057519+0000","last_clean":"2023-09-23T20:11:30.057519+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:11:30.057519+0000","last_undegraded":"2023-09-23T20:11:30.057519+0000","last_fullsized":"2023-09-23T20:11:30.057519+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"386'8390","reported_seq":8488,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.426582+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:11:30.426582+0000","last_peered":"2023-09-23T20:11:30.426582+0000","last_clean":"2023-09-23T20:11:30.426582+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:11:30.426582+0000","last_undegraded":"2023-09-23T20:11:30.426582+0000","last_fullsized":"2023-09-23T20:11:30.426582+0000","mapping_epoch":299,"log_start":"309'3283","ondisk_log_start":"309'3283","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5107,"log_dups_size":0,"ondisk_log_size":5107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142344192,"num_objects":2172,"num_object_clones":0,"num_object_copies":6516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8390,"num_write_kb":337984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":654,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.164349+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:11:23.164349+0000","last_peered":"2023-09-23T20:11:23.164349+0000","last_clean":"2023-09-23T20:11:23.164349+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:11:23.164349+0000","last_undegraded":"2023-09-23T20:11:23.164349+0000","last_fullsized":"2023-09-23T20:11:23.164349+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":519,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.038341+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:11:30.038341+0000","last_peered":"2023-09-23T20:11:30.038341+0000","last_clean":"2023-09-23T20:11:30.038341+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:11:30.038341+0000","last_undegraded":"2023-09-23T20:11:30.038341+0000","last_fullsized":"2023-09-23T20:11:30.038341+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":387,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023405+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:11:30.023405+0000","last_peered":"2023-09-23T20:11:30.023405+0000","last_clean":"2023-09-23T20:11:30.023405+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:11:30.023405+0000","last_undegraded":"2023-09-23T20:11:30.023405+0000","last_fullsized":"2023-09-23T20:11:30.023405+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":83,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037641+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:11:30.037641+0000","last_peered":"2023-09-23T20:11:30.037641+0000","last_clean":"2023-09-23T20:11:30.037641+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:11:30.037641+0000","last_undegraded":"2023-09-23T20:11:30.037641+0000","last_fullsized":"2023-09-23T20:11:30.037641+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"387'8401","reported_seq":13226,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:33.214337+0000","last_change":"2023-09-23T20:07:52.783215+0000","last_active":"2023-09-23T20:11:33.214337+0000","last_peered":"2023-09-23T20:11:33.214337+0000","last_clean":"2023-09-23T20:11:33.214337+0000","last_became_active":"2023-09-23T20:07:52.778874+0000","last_became_peered":"2023-09-23T20:07:52.778874+0000","last_unstale":"2023-09-23T20:11:33.214337+0000","last_undegraded":"2023-09-23T20:11:33.214337+0000","last_fullsized":"2023-09-23T20:11:33.214337+0000","mapping_epoch":330,"log_start":"308'2400","ondisk_log_start":"308'2400","created":299,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:31.036337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139132928,"num_objects":2123,"num_object_clones":0,"num_object_copies":6369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8401,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2349,"num_bytes_recovered":153944064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":720,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022960+0000","last_change":"2023-09-23T20:11:06.026338+0000","last_active":"2023-09-23T20:11:30.022960+0000","last_peered":"2023-09-23T20:11:30.022960+0000","last_clean":"2023-09-23T20:11:30.022960+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:11:30.022960+0000","last_undegraded":"2023-09-23T20:11:30.022960+0000","last_fullsized":"2023-09-23T20:11:30.022960+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1007,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023554+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:11:30.023554+0000","last_peered":"2023-09-23T20:11:30.023554+0000","last_clean":"2023-09-23T20:11:30.023554+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:11:30.023554+0000","last_undegraded":"2023-09-23T20:11:30.023554+0000","last_fullsized":"2023-09-23T20:11:30.023554+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":406,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025409+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:11:30.025409+0000","last_peered":"2023-09-23T20:11:30.025409+0000","last_clean":"2023-09-23T20:11:30.025409+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:11:30.025409+0000","last_undegraded":"2023-09-23T20:11:30.025409+0000","last_fullsized":"2023-09-23T20:11:30.025409+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.189021+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:11:32.189021+0000","last_peered":"2023-09-23T20:11:32.189021+0000","last_clean":"2023-09-23T20:11:32.189021+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:11:32.189021+0000","last_undegraded":"2023-09-23T20:11:32.189021+0000","last_fullsized":"2023-09-23T20:11:32.189021+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"387'8271","reported_seq":20634,"reported_epoch":387,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:11:33.513054+0000","last_change":"2023-09-23T20:11:18.839874+0000","last_active":"2023-09-23T20:11:33.513054+0000","last_peered":"2023-09-23T20:11:33.513054+0000","last_clean":"2023-09-23T20:11:16.510239+0000","last_became_active":"2023-09-23T20:11:18.809696+0000","last_became_peered":"2023-09-23T20:11:18.809696+0000","last_unstale":"2023-09-23T20:11:33.513054+0000","last_undegraded":"2023-09-23T20:11:33.513054+0000","last_fullsized":"2023-09-23T20:11:33.513054+0000","mapping_epoch":381,"log_start":"309'2800","ondisk_log_start":"309'2800","created":299,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5471,"log_dups_size":0,"ondisk_log_size":5471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:58.376473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137166848,"num_objects":2093,"num_object_clones":0,"num_object_copies":6279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2093,"num_objects_unfound":0,"num_objects_dirty":2093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8271,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":217251840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[1,4,2],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":2093}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":606,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034926+0000","last_change":"2023-09-23T20:11:15.663436+0000","last_active":"2023-09-23T20:11:30.034926+0000","last_peered":"2023-09-23T20:11:30.034926+0000","last_clean":"2023-09-23T20:11:30.034926+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:11:30.034926+0000","last_undegraded":"2023-09-23T20:11:30.034926+0000","last_fullsized":"2023-09-23T20:11:30.034926+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1022,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.180370+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:11:23.180370+0000","last_peered":"2023-09-23T20:11:23.180370+0000","last_clean":"2023-09-23T20:11:23.180370+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:11:23.180370+0000","last_undegraded":"2023-09-23T20:11:23.180370+0000","last_fullsized":"2023-09-23T20:11:23.180370+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037008+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:11:30.037008+0000","last_peered":"2023-09-23T20:11:30.037008+0000","last_clean":"2023-09-23T20:11:30.037008+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:11:30.037008+0000","last_undegraded":"2023-09-23T20:11:30.037008+0000","last_fullsized":"2023-09-23T20:11:30.037008+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.057432+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:11:30.057432+0000","last_peered":"2023-09-23T20:11:30.057432+0000","last_clean":"2023-09-23T20:11:30.057432+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:11:30.057432+0000","last_undegraded":"2023-09-23T20:11:30.057432+0000","last_fullsized":"2023-09-23T20:11:30.057432+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"387'8453","reported_seq":8549,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:33.239930+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:11:33.239930+0000","last_peered":"2023-09-23T20:11:33.239930+0000","last_clean":"2023-09-23T20:11:33.239930+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:11:33.239930+0000","last_undegraded":"2023-09-23T20:11:33.239930+0000","last_fullsized":"2023-09-23T20:11:33.239930+0000","mapping_epoch":299,"log_start":"308'2565","ondisk_log_start":"308'2565","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138608640,"num_objects":2115,"num_object_clones":0,"num_object_copies":6345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8453,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1295,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023082+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:11:30.023082+0000","last_peered":"2023-09-23T20:11:30.023082+0000","last_clean":"2023-09-23T20:11:30.023082+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:11:30.023082+0000","last_undegraded":"2023-09-23T20:11:30.023082+0000","last_fullsized":"2023-09-23T20:11:30.023082+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":540,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034667+0000","last_change":"2023-09-23T20:11:09.775272+0000","last_active":"2023-09-23T20:11:30.034667+0000","last_peered":"2023-09-23T20:11:30.034667+0000","last_clean":"2023-09-23T20:11:30.034667+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:11:30.034667+0000","last_undegraded":"2023-09-23T20:11:30.034667+0000","last_fullsized":"2023-09-23T20:11:30.034667+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":252,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023657+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:11:30.023657+0000","last_peered":"2023-09-23T20:11:30.023657+0000","last_clean":"2023-09-23T20:11:30.023657+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:11:30.023657+0000","last_undegraded":"2023-09-23T20:11:30.023657+0000","last_fullsized":"2023-09-23T20:11:30.023657+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":282,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026671+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:11:30.026671+0000","last_peered":"2023-09-23T20:11:30.026671+0000","last_clean":"2023-09-23T20:11:30.026671+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:11:30.026671+0000","last_undegraded":"2023-09-23T20:11:30.026671+0000","last_fullsized":"2023-09-23T20:11:30.026671+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"386'8444","reported_seq":8653,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:30.385966+0000","last_change":"2023-09-23T20:10:57.512712+0000","last_active":"2023-09-23T20:11:30.385966+0000","last_peered":"2023-09-23T20:11:30.385966+0000","last_clean":"2023-09-23T20:07:17.907438+0000","last_became_active":"2023-09-23T20:07:20.693608+0000","last_became_peered":"2023-09-23T20:07:20.693608+0000","last_unstale":"2023-09-23T20:11:30.385966+0000","last_undegraded":"2023-09-23T20:07:19.959595+0000","last_fullsized":"2023-09-23T20:07:19.924202+0000","mapping_epoch":317,"log_start":"309'3288","ondisk_log_start":"309'3288","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5156,"log_dups_size":0,"ondisk_log_size":5156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:21.022797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143654912,"num_objects":2192,"num_object_clones":0,"num_object_copies":6576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8444,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":3407872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1054},{"shards":"2,5","objects":-1936}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":512,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.038002+0000","last_change":"2023-09-23T20:09:56.051517+0000","last_active":"2023-09-23T20:11:30.038002+0000","last_peered":"2023-09-23T20:11:30.038002+0000","last_clean":"2023-09-23T20:11:30.038002+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:11:30.038002+0000","last_undegraded":"2023-09-23T20:11:30.038002+0000","last_fullsized":"2023-09-23T20:11:30.038002+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":823,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034546+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:11:30.034546+0000","last_peered":"2023-09-23T20:11:30.034546+0000","last_clean":"2023-09-23T20:11:30.034546+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:11:30.034546+0000","last_undegraded":"2023-09-23T20:11:30.034546+0000","last_fullsized":"2023-09-23T20:11:30.034546+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":373,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025059+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:11:30.025059+0000","last_peered":"2023-09-23T20:11:30.025059+0000","last_clean":"2023-09-23T20:11:30.025059+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:11:30.025059+0000","last_undegraded":"2023-09-23T20:11:30.025059+0000","last_fullsized":"2023-09-23T20:11:30.025059+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":82,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.038818+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:11:30.038818+0000","last_peered":"2023-09-23T20:11:30.038818+0000","last_clean":"2023-09-23T20:11:30.038818+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:11:30.038818+0000","last_undegraded":"2023-09-23T20:11:30.038818+0000","last_fullsized":"2023-09-23T20:11:30.038818+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"386'8477","reported_seq":13935,"reported_epoch":386,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:11:30.507089+0000","last_change":"2023-09-23T20:10:58.722697+0000","last_active":"2023-09-23T20:11:30.507089+0000","last_peered":"2023-09-23T20:11:30.507089+0000","last_clean":"2023-09-23T20:07:17.917130+0000","last_became_active":"2023-09-23T20:10:56.953707+0000","last_became_peered":"2023-09-23T20:10:56.953707+0000","last_unstale":"2023-09-23T20:11:30.507089+0000","last_undegraded":"2023-09-23T20:11:30.507089+0000","last_fullsized":"2023-09-23T20:10:53.691838+0000","mapping_epoch":369,"log_start":"312'3346","ondisk_log_start":"312'3346","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5131,"log_dups_size":0,"ondisk_log_size":5131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:08.588715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144375808,"num_objects":2203,"num_object_clones":0,"num_object_copies":6609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1013,"num_objects_unfound":0,"num_objects_dirty":2203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8477,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3557,"num_bytes_recovered":194904064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,1],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"1,2","objects":1190},{"shards":"2,3,4","objects":1013}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":580,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026257+0000","last_change":"2023-09-23T20:11:18.808960+0000","last_active":"2023-09-23T20:11:30.026257+0000","last_peered":"2023-09-23T20:11:30.026257+0000","last_clean":"2023-09-23T20:11:30.026257+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:11:30.026257+0000","last_undegraded":"2023-09-23T20:11:30.026257+0000","last_fullsized":"2023-09-23T20:11:30.026257+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":87,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036018+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:11:30.036018+0000","last_peered":"2023-09-23T20:11:30.036018+0000","last_clean":"2023-09-23T20:11:30.036018+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:11:30.036018+0000","last_undegraded":"2023-09-23T20:11:30.036018+0000","last_fullsized":"2023-09-23T20:11:30.036018+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.188637+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:11:32.188637+0000","last_peered":"2023-09-23T20:11:32.188637+0000","last_clean":"2023-09-23T20:11:32.188637+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:11:32.188637+0000","last_undegraded":"2023-09-23T20:11:32.188637+0000","last_fullsized":"2023-09-23T20:11:32.188637+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022981+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:11:30.022981+0000","last_peered":"2023-09-23T20:11:30.022981+0000","last_clean":"2023-09-23T20:11:30.022981+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:11:30.022981+0000","last_undegraded":"2023-09-23T20:11:30.022981+0000","last_fullsized":"2023-09-23T20:11:30.022981+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"387'8517","reported_seq":12811,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:33.518024+0000","last_change":"2023-09-23T20:08:33.396812+0000","last_active":"2023-09-23T20:11:33.518024+0000","last_peered":"2023-09-23T20:11:33.518024+0000","last_clean":"2023-09-23T20:11:33.518024+0000","last_became_active":"2023-09-23T20:08:05.774750+0000","last_became_peered":"2023-09-23T20:08:05.774750+0000","last_unstale":"2023-09-23T20:11:33.518024+0000","last_undegraded":"2023-09-23T20:11:33.518024+0000","last_fullsized":"2023-09-23T20:11:33.518024+0000","mapping_epoch":334,"log_start":"306'1600","ondisk_log_start":"306'1600","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6917,"log_dups_size":0,"ondisk_log_size":6917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.633506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139526144,"num_objects":2129,"num_object_clones":0,"num_object_copies":6387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8517,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2086,"num_bytes_recovered":136708096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":808,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026447+0000","last_change":"2023-09-23T20:11:13.677585+0000","last_active":"2023-09-23T20:11:30.026447+0000","last_peered":"2023-09-23T20:11:30.026447+0000","last_clean":"2023-09-23T20:11:30.026447+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:11:30.026447+0000","last_undegraded":"2023-09-23T20:11:30.026447+0000","last_fullsized":"2023-09-23T20:11:30.026447+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":229,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.035878+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:11:30.035878+0000","last_peered":"2023-09-23T20:11:30.035878+0000","last_clean":"2023-09-23T20:11:30.035878+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:11:30.035878+0000","last_undegraded":"2023-09-23T20:11:30.035878+0000","last_fullsized":"2023-09-23T20:11:30.035878+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":424,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.038417+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:11:30.038417+0000","last_peered":"2023-09-23T20:11:30.038417+0000","last_clean":"2023-09-23T20:11:30.038417+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:11:30.038417+0000","last_undegraded":"2023-09-23T20:11:30.038417+0000","last_fullsized":"2023-09-23T20:11:30.038417+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":16,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.162077+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:11:23.162077+0000","last_peered":"2023-09-23T20:11:23.162077+0000","last_clean":"2023-09-23T20:11:23.162077+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:11:23.162077+0000","last_undegraded":"2023-09-23T20:11:23.162077+0000","last_fullsized":"2023-09-23T20:11:23.162077+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"387'8420","reported_seq":11185,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:33.224157+0000","last_change":"2023-09-23T20:09:08.683160+0000","last_active":"2023-09-23T20:11:33.224157+0000","last_peered":"2023-09-23T20:11:33.224157+0000","last_clean":"2023-09-23T20:11:33.224157+0000","last_became_active":"2023-09-23T20:09:08.670527+0000","last_became_peered":"2023-09-23T20:09:08.670527+0000","last_unstale":"2023-09-23T20:11:33.224157+0000","last_undegraded":"2023-09-23T20:11:33.224157+0000","last_fullsized":"2023-09-23T20:11:33.224157+0000","mapping_epoch":346,"log_start":"308'2400","ondisk_log_start":"308'2400","created":299,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6020,"log_dups_size":0,"ondisk_log_size":6020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:47.531381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137822232,"num_objects":2104,"num_object_clones":0,"num_object_copies":6312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2104,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8420,"num_write_kb":336705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":696,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.164244+0000","last_change":"2023-09-23T20:09:56.036037+0000","last_active":"2023-09-23T20:11:23.164244+0000","last_peered":"2023-09-23T20:11:23.164244+0000","last_clean":"2023-09-23T20:11:23.164244+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:11:23.164244+0000","last_undegraded":"2023-09-23T20:11:23.164244+0000","last_fullsized":"2023-09-23T20:11:23.164244+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":88,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036030+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:11:30.036030+0000","last_peered":"2023-09-23T20:11:30.036030+0000","last_clean":"2023-09-23T20:11:30.036030+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:11:30.036030+0000","last_undegraded":"2023-09-23T20:11:30.036030+0000","last_fullsized":"2023-09-23T20:11:30.036030+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":86,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.038713+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:11:30.038713+0000","last_peered":"2023-09-23T20:11:30.038713+0000","last_clean":"2023-09-23T20:11:30.038713+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:11:30.038713+0000","last_undegraded":"2023-09-23T20:11:30.038713+0000","last_fullsized":"2023-09-23T20:11:30.038713+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":346,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.024914+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:11:30.024914+0000","last_peered":"2023-09-23T20:11:30.024914+0000","last_clean":"2023-09-23T20:11:30.024914+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:11:30.024914+0000","last_undegraded":"2023-09-23T20:11:30.024914+0000","last_fullsized":"2023-09-23T20:11:30.024914+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":161,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.185988+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:11:23.185988+0000","last_peered":"2023-09-23T20:11:23.185988+0000","last_clean":"2023-09-23T20:11:23.185988+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:11:23.185988+0000","last_undegraded":"2023-09-23T20:11:23.185988+0000","last_fullsized":"2023-09-23T20:11:23.185988+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":152,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.185424+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:11:23.185424+0000","last_peered":"2023-09-23T20:11:23.185424+0000","last_clean":"2023-09-23T20:11:23.185424+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:11:23.185424+0000","last_undegraded":"2023-09-23T20:11:23.185424+0000","last_fullsized":"2023-09-23T20:11:23.185424+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":390,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023309+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:11:30.023309+0000","last_peered":"2023-09-23T20:11:30.023309+0000","last_clean":"2023-09-23T20:11:30.023309+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:11:30.023309+0000","last_undegraded":"2023-09-23T20:11:30.023309+0000","last_fullsized":"2023-09-23T20:11:30.023309+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":403,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025987+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:11:30.025987+0000","last_peered":"2023-09-23T20:11:30.025987+0000","last_clean":"2023-09-23T20:11:30.025987+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:11:30.025987+0000","last_undegraded":"2023-09-23T20:11:30.025987+0000","last_fullsized":"2023-09-23T20:11:30.025987+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":380,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023313+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:11:30.023313+0000","last_peered":"2023-09-23T20:11:30.023313+0000","last_clean":"2023-09-23T20:11:30.023313+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:11:30.023313+0000","last_undegraded":"2023-09-23T20:11:30.023313+0000","last_fullsized":"2023-09-23T20:11:30.023313+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":383,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025224+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:11:30.025224+0000","last_peered":"2023-09-23T20:11:30.025224+0000","last_clean":"2023-09-23T20:11:30.025224+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:11:30.025224+0000","last_undegraded":"2023-09-23T20:11:30.025224+0000","last_fullsized":"2023-09-23T20:11:30.025224+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":86,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.189062+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:11:23.189062+0000","last_peered":"2023-09-23T20:11:23.189062+0000","last_clean":"2023-09-23T20:11:23.189062+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:11:23.189062+0000","last_undegraded":"2023-09-23T20:11:23.189062+0000","last_fullsized":"2023-09-23T20:11:23.189062+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":243,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T20:11:23.180450+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:11:23.180450+0000","last_peered":"2023-09-23T20:11:23.180450+0000","last_clean":"2023-09-23T20:11:23.180450+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:11:23.180450+0000","last_undegraded":"2023-09-23T20:11:23.180450+0000","last_fullsized":"2023-09-23T20:11:23.180450+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":232,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023285+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:11:30.023285+0000","last_peered":"2023-09-23T20:11:30.023285+0000","last_clean":"2023-09-23T20:11:30.023285+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:11:30.023285+0000","last_undegraded":"2023-09-23T20:11:30.023285+0000","last_fullsized":"2023-09-23T20:11:30.023285+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":433,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.024922+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:11:30.024922+0000","last_peered":"2023-09-23T20:11:30.024922+0000","last_clean":"2023-09-23T20:11:30.024922+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:11:30.024922+0000","last_undegraded":"2023-09-23T20:11:30.024922+0000","last_fullsized":"2023-09-23T20:11:30.024922+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":26174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2224619544,"num_objects":33946,"num_object_clones":0,"num_object_copies":101838,"num_objects_missing_on_primary":2751,"num_objects_missing":2751,"num_objects_degraded":9860,"num_objects_misplaced":3106,"num_objects_unfound":0,"num_objects_dirty":33946,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134516,"num_write_kb":5390721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29060,"num_bytes_recovered":1491861504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8096985088,"data_stored":8096972872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89435,"ondisk_log_size":89435,"up":48,"acting":43,"num_store_stats":7},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":542359,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":177,"num_read_kb":331,"num_write":87,"num_write_kb":878,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1299398,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234365,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":937832,"kb_used_data":604896,"kb_used_omap":0,"kb_used_meta":332928,"kb_avail":92807320,"statfs":{"total":95995035648,"available":95034695680,"internally_reserved":0,"allocated":619413504,"data_stored":618171796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340918272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":3.0049999999999999,"5min":2.161,"15min":2.5369999999999999},"min":{"1min":0.252,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":55.390000000000001,"5min":55.390000000000001,"15min":62.061999999999998},"last":0.33500000000000002},{"interface":"front","average":{"1min":3.0840000000000001,"5min":2.052,"15min":2.71},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":56.664000000000001,"5min":56.664000000000001,"15min":62.389000000000003},"last":0.56499999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9940000000000002,"5min":2.0739999999999998,"15min":3.5129999999999999},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":54.945999999999998,"5min":54.945999999999998,"15min":104.16800000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.044,"5min":2.1030000000000002,"15min":3.6379999999999999},"min":{"1min":0.28100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":55.094000000000001,"5min":55.094000000000001,"15min":104.003},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:10:48 2023","interfaces":[{"interface":"back","average":{"1min":3.6440000000000001,"5min":4.8570000000000002,"15min":3.6520000000000001},"min":{"1min":0.32400000000000001,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":57.746000000000002,"5min":92.319000000000003,"15min":105.842},"last":0.51000000000000001},{"interface":"front","average":{"1min":3.504,"5min":5.0229999999999997,"15min":4.1230000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":56.259999999999998,"5min":92.049000000000007,"15min":105.66},"last":0.45100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":3.79,"15min":4.4630000000000001},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":1.6000000000000001,"5min":94.953000000000003,"15min":146.38800000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.70599999999999996,"5min":3.6859999999999999,"15min":4.5019999999999998},"min":{"1min":0.38,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.633,"5min":93.423000000000002,"15min":146.851},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.53500000000000003,"15min":1.3089999999999999},"min":{"1min":0.26400000000000001,"5min":0.25900000000000001,"15min":0.22500000000000001},"max":{"1min":1.048,"5min":5.431,"15min":143.255},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.60199999999999998,"15min":1.3480000000000001},"min":{"1min":0.29899999999999999,"5min":0.246,"15min":0.246},"max":{"1min":1.1020000000000001,"5min":4.1459999999999999,"15min":143.41200000000001},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.94699999999999995,"15min":1.056},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":0.60999999999999999,"5min":24.876000000000001,"15min":35.488},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.874,"15min":0.879},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":0.98699999999999999,"5min":21.623000000000001,"15min":34.896000000000001},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.62,"15min":0.81699999999999995},"min":{"1min":0.308,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":1.454,"5min":4.3710000000000004,"15min":41.902999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.64600000000000002,"15min":0.64800000000000002},"min":{"1min":0.39100000000000001,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":0.70499999999999996,"5min":3.9889999999999999,"15min":6.3620000000000001},"last":0.57499999999999996}]}]},{"osd":6,"up_from":0,"seq":1078036791406,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762564,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1254508,"kb_used_data":704804,"kb_used_omap":7,"kb_used_meta":549688,"kb_avail":92490644,"statfs":{"total":95995035648,"available":94710419456,"internally_reserved":0,"allocated":721719296,"data_stored":719891399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":562881338},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.1339999999999999,"15min":3.698},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.151},"max":{"1min":9.9410000000000007,"5min":28.510000000000002,"15min":186.84},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.776,"5min":2.4209999999999998,"15min":3.9929999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":8.6660000000000004,"5min":14.781000000000001,"15min":191.322},"last":0.49099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":4.6180000000000003,"15min":5.508},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":17.785,"5min":98.531000000000006,"15min":180.25200000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":2.2589999999999999,"5min":4.2859999999999996,"15min":5.548},"min":{"1min":0.23300000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":28.469999999999999,"5min":98.346000000000004,"15min":180.05000000000001},"last":0.69199999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.5459999999999998,"15min":3.7879999999999998},"min":{"1min":0.25700000000000001,"5min":0.255,"15min":0.22},"max":{"1min":3.9849999999999999,"5min":25.963999999999999,"15min":68.180999999999997},"last":0.439},{"interface":"front","average":{"1min":1.4019999999999999,"5min":2.4430000000000001,"15min":3.6909999999999998},"min":{"1min":0.29299999999999998,"5min":0.28199999999999997,"15min":0.24299999999999999},"max":{"1min":6.4320000000000004,"5min":40.716000000000001,"15min":66.509},"last":0.26600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.6140000000000001,"15min":1.9430000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23599999999999999},"max":{"1min":6.6029999999999998,"5min":18.087,"15min":60.951000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.573,"5min":1.764,"15min":2.1190000000000002},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":8.1669999999999998,"5min":14.890000000000001,"15min":60.841000000000001},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.8300000000000001,"15min":2.952},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.25900000000000001},"max":{"1min":6.5679999999999996,"5min":14.83,"15min":219.048},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.3440000000000001,"5min":2.1819999999999999,"15min":3.0950000000000002},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28399999999999997},"max":{"1min":6.0190000000000001,"5min":25.893000000000001,"15min":218.69499999999999},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:10:42 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.782,"15min":2.0499999999999998},"min":{"1min":0.38500000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":18.117999999999999,"5min":18.117999999999999,"15min":18.117999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.84099999999999997,"5min":1.708,"15min":2.7810000000000001},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.54,"5min":14.862,"15min":20.734999999999999},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":2.1869999999999998,"15min":2.3239999999999998},"min":{"1min":0.42999999999999999,"5min":0.33300000000000002,"15min":0.26800000000000002},"max":{"1min":4.5899999999999999,"5min":25.937000000000001,"15min":60.213000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.556,"5min":1.7749999999999999,"15min":2.0619999999999998},"min":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.30199999999999999},"max":{"1min":6.649,"5min":18.053999999999998,"15min":61.298999999999999},"last":0.66000000000000003}]}]},{"osd":0,"up_from":94,"seq":403726926101,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1736960,"kb_used_data":1110584,"kb_used_omap":8,"kb_used_meta":626359,"kb_avail":92008192,"statfs":{"total":95995035648,"available":94216388608,"internally_reserved":0,"allocated":1137238016,"data_stored":1135394469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":641392225},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":2.855,"15min":3.907},"min":{"1min":0.217,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":4.3179999999999996,"5min":63.639000000000003,"15min":94.055999999999997},"last":0.216},{"interface":"front","average":{"1min":0.70199999999999996,"5min":2.9140000000000001,"15min":4.2000000000000002},"min":{"1min":0.23200000000000001,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":1.6639999999999999,"5min":62.802,"15min":91.632999999999996},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:11:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":3.5880000000000001,"15min":5.0199999999999996},"min":{"1min":0.255,"5min":0.217,"15min":0.19800000000000001},"max":{"1min":9.0700000000000003,"5min":102.65300000000001,"15min":270.51100000000002},"last":19.486999999999998},{"interface":"front","average":{"1min":2.0139999999999998,"5min":3.6150000000000002,"15min":4.8810000000000002},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.2599999999999998,"5min":96.578999999999994,"15min":269.69299999999998},"last":20.693999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":4.5670000000000002,"15min":5.9219999999999997},"min":{"1min":0.27700000000000002,"5min":0.252,"15min":0.21299999999999999},"max":{"1min":1.6120000000000001,"5min":126.95399999999999,"15min":145.72800000000001},"last":0.316},{"interface":"front","average":{"1min":1.4590000000000001,"5min":4.9480000000000004,"15min":6.0919999999999996},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":9.3439999999999994,"5min":127.294,"15min":145.50700000000001},"last":0.29199999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.5629999999999999,"15min":2.2629999999999999},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":6.1369999999999996,"5min":15.688000000000001,"15min":148.20599999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.099,"5min":1.4039999999999999,"15min":2.0379999999999998},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":5.3940000000000001,"5min":13.428000000000001,"15min":148.11000000000001},"last":0.38300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.298,"15min":2.464},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":1.1459999999999999,"5min":9.5719999999999992,"15min":131.34399999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.163,"5min":1.718,"15min":2.468},"min":{"1min":0.377,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.1059999999999999,"5min":15.717000000000001,"15min":131.386},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.758,"15min":1.6779999999999999},"min":{"1min":0.433,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":6.1660000000000004,"5min":15.744999999999999,"15min":15.744999999999999},"last":0.496},{"interface":"front","average":{"1min":0.79800000000000004,"5min":1.579,"15min":1.873},"min":{"1min":0.40400000000000003,"5min":0.33900000000000002,"15min":0.309},"max":{"1min":1.6160000000000001,"5min":13.470000000000001,"15min":13.766999999999999},"last":0.36799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.327,"15min":1.891},"min":{"1min":0.38900000000000001,"5min":0.38100000000000001,"15min":0.31900000000000001},"max":{"1min":1.0980000000000001,"5min":9.6069999999999993,"15min":65.081999999999994},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.724,"15min":2.0190000000000001},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.1920000000000002,"5min":15.771000000000001,"15min":15.771000000000001},"last":0.46999999999999997}]}]},{"osd":2,"up_from":104,"seq":446676599039,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2156024,"kb_used_data":1439088,"kb_used_omap":12,"kb_used_meta":716915,"kb_avail":91589128,"statfs":{"total":95995035648,"available":93787267072,"internally_reserved":0,"allocated":1473626112,"data_stored":1471782970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":734121982},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.9119999999999999,"15min":3.1739999999999999},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.154},"max":{"1min":3.714,"5min":17.876999999999999,"15min":76.251999999999995},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.323,"5min":2.4609999999999999,"15min":3.3149999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.192},"max":{"1min":10.808999999999999,"5min":18.561,"15min":76.590000000000003},"last":0.65100000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":2.3159999999999998,"15min":3.6699999999999999},"min":{"1min":0.24199999999999999,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":2.4729999999999999,"5min":19.562999999999999,"15min":94.768000000000001},"last":2.778},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.9079999999999999,"15min":3.6589999999999998},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":3.6320000000000001,"5min":19.314,"15min":96.808000000000007},"last":2.6629999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":3.5840000000000001,"15min":4.4859999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.214},"max":{"1min":1.45,"5min":84.207999999999998,"15min":107.381},"last":2.8180000000000001},{"interface":"front","average":{"1min":0.71499999999999997,"5min":2.3860000000000001,"15min":3.8650000000000002},"min":{"1min":0.29199999999999998,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.681,"5min":20.308,"15min":106.301},"last":1.536}]},{"osd":4,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.609,"15min":2.2170000000000001},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.26900000000000002},"max":{"1min":3.742,"5min":75.608000000000004,"15min":75.608000000000004},"last":3.2839999999999998},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.7839999999999998,"15min":2.2330000000000001},"min":{"1min":0.34599999999999997,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":2.544,"5min":76.161000000000001,"15min":76.161000000000001},"last":2.8540000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.1549999999999998,"15min":2.726},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.28100000000000003},"max":{"1min":3.77,"5min":65.921999999999997,"15min":172.505},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.5489999999999999,"15min":2.8959999999999999},"min":{"1min":0.39500000000000002,"5min":0.38100000000000001,"15min":0.29099999999999998},"max":{"1min":10.696,"5min":63.792000000000002,"15min":172.375},"last":0.502}]},{"osd":6,"last update":"Sat Sep 23 20:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.6180000000000001,"15min":1.6359999999999999},"min":{"1min":0.40500000000000003,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":3.8210000000000002,"5min":10.209,"15min":10.209},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.819,"15min":1.9099999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.36799999999999999},"max":{"1min":2.3820000000000001,"5min":14.279999999999999,"15min":14.279999999999999},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.6599999999999999,"15min":1.9379999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.28799999999999998},"max":{"1min":2.5699999999999998,"5min":8.1519999999999992,"15min":50.118000000000002},"last":3.5169999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.619,"15min":2.0419999999999998},"min":{"1min":0.41699999999999998,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":3.794,"5min":10.244,"15min":50.664999999999999},"last":3.5019999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435513,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2122976,"kb_used_data":1462680,"kb_used_omap":4,"kb_used_meta":660283,"kb_avail":91622176,"statfs":{"total":95995035648,"available":93821108224,"internally_reserved":0,"allocated":1497784320,"data_stored":1495781140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":676129901},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":4.7380000000000004,"15min":3.673},"min":{"1min":0.186,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":5.9989999999999997,"5min":74.323999999999998,"15min":74.323999999999998},"last":0.19700000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":4.5720000000000001,"15min":4.0350000000000001},"min":{"1min":0.215,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":3.1779999999999999,"5min":74.447000000000003,"15min":74.447000000000003},"last":1.2030000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":4.6959999999999997,"5min":4.6200000000000001,"15min":3.6309999999999998},"min":{"1min":0.23799999999999999,"5min":0.20300000000000001,"15min":0.19600000000000001},"max":{"1min":73.572000000000003,"5min":112.122,"15min":112.122},"last":1.4259999999999999},{"interface":"front","average":{"1min":4.4100000000000001,"5min":5.3609999999999998,"15min":3.9500000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.19900000000000001},"max":{"1min":73.414000000000001,"5min":111.786,"15min":111.786},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":4.3310000000000004,"15min":4.1369999999999996},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":7.7640000000000002,"5min":88.486999999999995,"15min":88.486999999999995},"last":2.5539999999999998},{"interface":"front","average":{"1min":1.2090000000000001,"5min":4.6550000000000002,"15min":4.04},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":7.9059999999999997,"5min":88.387,"15min":88.387},"last":0.45700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":2.6459999999999999,"15min":3.1899999999999999},"min":{"1min":0.34100000000000003,"5min":0.29899999999999999,"15min":0.25700000000000001},"max":{"1min":6.0510000000000002,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":2.9969999999999999,"15min":3.593},"min":{"1min":0.28299999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.2149999999999999,"5min":58.408000000000001,"15min":110.295},"last":1.2589999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":3.6779999999999999,"15min":3.4409999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":3.2469999999999999,"5min":77.206000000000003,"15min":113.881},"last":1.2390000000000001},{"interface":"front","average":{"1min":1.107,"5min":3.9100000000000001,"15min":3.3620000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":5.8689999999999998,"5min":77.968000000000004,"15min":114.598},"last":0.65400000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:10:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":2.746,"15min":4.2320000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":5.9080000000000004,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":2.9820000000000002,"15min":4.3959999999999999},"min":{"1min":0.443,"5min":0.375,"15min":0.33400000000000002},"max":{"1min":6.0730000000000004,"5min":58.701000000000001,"15min":82.233000000000004},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":3.0059999999999998,"15min":3.5760000000000001},"min":{"1min":0.377,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":5.9340000000000002,"5min":58.686,"15min":138.035},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.98599999999999999,"5min":3.0920000000000001,"15min":3.5099999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.28699999999999998},"max":{"1min":6.1020000000000003,"5min":58.667999999999999,"15min":138.054},"last":0.53400000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019275,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1651616,"kb_used_data":1321112,"kb_used_omap":0,"kb_used_meta":330496,"kb_avail":92093536,"statfs":{"total":95995035648,"available":94303780864,"internally_reserved":0,"allocated":1352818688,"data_stored":1351585136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":338427904},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.54,"15min":2.5699999999999998},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.189},"max":{"1min":3.2000000000000002,"5min":56.061999999999998,"15min":104.604},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.5409999999999999,"15min":2.5339999999999998},"min":{"1min":0.28899999999999998,"5min":0.218,"15min":0.218},"max":{"1min":3.1899999999999999,"5min":56.075000000000003,"15min":104.687},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":2.9489999999999998,"15min":5.5389999999999997},"min":{"1min":0.252,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":4.1669999999999998,"5min":119.892,"15min":161.613},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.96499999999999997,"5min":3.2109999999999999,"15min":5.6139999999999999},"min":{"1min":0.28899999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.125,"5min":119.351,"15min":162.017},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":6.6200000000000001,"15min":6.75},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.245},"max":{"1min":17.948,"5min":182.96700000000001,"15min":182.96700000000001},"last":76.076999999999998},{"interface":"front","average":{"1min":1.349,"5min":6.2729999999999997,"15min":6.5039999999999996},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":17.978000000000002,"5min":183.244,"15min":183.244},"last":76.100999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":4.843,"15min":6.8440000000000003},"min":{"1min":0.30499999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":2.5259999999999998,"5min":184.37100000000001,"15min":266.54700000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":4.9130000000000003,"15min":5.798},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":5.407,"5min":184.22300000000001,"15min":184.22300000000001},"last":0.40400000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":2.1640000000000001,"15min":2.1160000000000001},"min":{"1min":0.25800000000000001,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":0.622,"5min":176.81899999999999,"15min":266.59899999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":2.2149999999999999,"15min":2.1880000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":0.56599999999999995,"5min":176.172,"15min":266.72899999999998},"last":0.36799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.49099999999999999,"15min":0.50800000000000001},"min":{"1min":0.33100000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.56399999999999995,"5min":2.5489999999999999,"15min":4.0410000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.59899999999999998,"15min":0.58299999999999996},"min":{"1min":0.36599999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.74199999999999999,"5min":4.2889999999999997,"15min":4.2889999999999997},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.337,"15min":1.446},"min":{"1min":0.36799999999999999,"5min":0.32000000000000001,"15min":0.26100000000000001},"max":{"1min":0.71699999999999997,"5min":74.040000000000006,"15min":76.507000000000005},"last":0.442},{"interface":"front","average":{"1min":0.5,"5min":1.3049999999999999,"15min":1.5629999999999999},"min":{"1min":0.36399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.61099999999999999,"5min":74.021000000000001,"15min":76.462000000000003},"last":0.43099999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757639,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1737180,"kb_used_data":1348116,"kb_used_omap":0,"kb_used_meta":389056,"kb_avail":92007972,"statfs":{"total":95995035648,"available":94216163328,"internally_reserved":0,"allocated":1380470784,"data_stored":1379237601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":398393344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.88700000000000001,"15min":1.881},"min":{"1min":0.23000000000000001,"5min":0.224,"15min":0.216},"max":{"1min":0.625,"5min":8.6739999999999995,"15min":121.94799999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.436,"5min":1.1399999999999999,"15min":2.0750000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":1.28,"5min":10.327,"15min":121.964},"last":0.39700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":4.4189999999999996,"5min":3.6259999999999999,"15min":6.0190000000000001},"min":{"1min":0.255,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":75.204999999999998,"5min":127.41800000000001,"15min":266.197},"last":0.5},{"interface":"front","average":{"1min":4.4720000000000004,"5min":3.7799999999999998,"15min":6.194},"min":{"1min":0.248,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":75.231999999999999,"5min":127.37,"15min":266.14999999999998},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:10:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":5.6289999999999996,"15min":6.1059999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":25.227,"5min":240.57300000000001,"15min":240.57300000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.415,"5min":5.6699999999999999,"15min":6.1520000000000001},"min":{"1min":0.39800000000000002,"5min":0.38600000000000001,"15min":0.313},"max":{"1min":25.286999999999999,"5min":240.464,"15min":240.464},"last":0.52100000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":3.206,"15min":4.9610000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.7359999999999998,"5min":127.83799999999999,"15min":201.50899999999999},"last":0.379},{"interface":"front","average":{"1min":0.93999999999999995,"5min":2.9609999999999999,"15min":4.7910000000000004},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":7.149,"5min":128.291,"15min":201.58099999999999},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.71199999999999997,"15min":1.4490000000000001},"min":{"1min":0.27700000000000002,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":0.58799999999999997,"5min":9.7240000000000002,"15min":191.78899999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.59299999999999997,"15min":1.3360000000000001},"min":{"1min":0.311,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":1.3879999999999999,"5min":8.7360000000000007,"15min":191.821},"last":0.437}]},{"osd":6,"last update":"Sat Sep 23 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.67600000000000005,"15min":0.63600000000000001},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":0.749,"5min":9.9009999999999998,"15min":9.9009999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.60999999999999999,"15min":0.51800000000000002},"min":{"1min":0.316,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63900000000000001,"5min":5.0119999999999996,"15min":5.0119999999999996},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.59599999999999997,"15min":1.5329999999999999},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":0.57399999999999995,"5min":5.2670000000000003,"15min":191.84299999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":1.7549999999999999},"min":{"1min":0.41699999999999998,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":0.78600000000000003,"5min":9.6920000000000002,"15min":191.69},"last":0.42599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1123876864,"data_stored":1123876864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":708640768,"data_stored":708640768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1463947264,"data_stored":1463943192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1485111296,"data_stored":1485111296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1340735488,"data_stored":1340735488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1370034176,"data_stored":1370030104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":604639232,"data_stored":604635160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:11:37.545 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:11:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:37.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:37.646 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:37.647 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:37.676 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:37.722 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:37.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:37.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:37.826 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:37.857 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:37.858 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:11:37.858 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:11:37.858 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:11:37.858 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:11:37.905 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:37.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:37.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:38.008 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:38.068 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:38.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[221113]: from='client.55807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:38.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[221113]: pgmap v1080: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 9864/100113 objects degraded (9.853%); 2992/100113 objects misplaced (2.989%); 2.6 MiB/s, 53 objects/s recovering 2023-09-23T20:11:38.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:38.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:38.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9864/100113 objects degraded (9.853%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:38.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[213871]: from='client.55807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:38.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[213871]: pgmap v1080: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 9864/100113 objects degraded (9.853%); 2992/100113 objects misplaced (2.989%); 2.6 MiB/s, 53 objects/s recovering 2023-09-23T20:11:38.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:38.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:38.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9864/100113 objects degraded (9.853%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:38.364 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:38.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:38 smithi144 ceph-mon[194833]: from='client.55807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:38 smithi144 ceph-mon[194833]: pgmap v1080: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 9864/100113 objects degraded (9.853%); 2992/100113 objects misplaced (2.989%); 2.6 MiB/s, 53 objects/s recovering 2023-09-23T20:11:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9864/100113 objects degraded (9.853%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:38.474 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:38.501 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:38.541 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:38.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:38.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:38.622 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:11:38.644 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:38.692 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:38.972 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:38.989 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:38.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:38.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:39.025 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:39.097 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:39.169 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:39.298 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:11:39.336 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:39.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:39.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:39.446 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:39.475 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:39.579 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:39.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:39.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:39.682 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:39.709 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:39.752 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:39.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:39.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:39.856 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:39.883 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:40.094 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:40.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:40.202 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:40.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:40 smithi135 ceph-mon[213871]: pgmap v1081: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 9756/96621 objects degraded (10.097%); 2883/96621 objects misplaced (2.984%); 3.1 MiB/s, 60 objects/s recovering 2023-09-23T20:11:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:40 smithi135 ceph-mon[221113]: pgmap v1081: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 9756/96621 objects degraded (10.097%); 2883/96621 objects misplaced (2.984%); 3.1 MiB/s, 60 objects/s recovering 2023-09-23T20:11:40.233 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:40.332 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:40 smithi144 ceph-mon[194833]: pgmap v1081: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 9756/96621 objects degraded (10.097%); 2883/96621 objects misplaced (2.984%); 3.1 MiB/s, 60 objects/s recovering 2023-09-23T20:11:40.347 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:40.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:40.455 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:40.483 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:40.575 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:40.636 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:40.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:40.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:40.740 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:40.773 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:40.823 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:40.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:40.837 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:11:40.837 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:11:40.837 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:11:40.837 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:11:40.837 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:11:40.927 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:40.956 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:41.044 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/168309432' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:41.044 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/168309432' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:41.056 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:41.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:41.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:41.062 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:11:41.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/168309432' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:11:41.161 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:41.194 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:41.261 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:41.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:41.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:41.372 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:41.376 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:11:41.384 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:11:41.405 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:41.660 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:41.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:41.723 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:11:41.765 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:41.792 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:41.855 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:41.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:41.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:41.960 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:42.029 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:42.139 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:42.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:42 smithi144 ceph-mon[194833]: pgmap v1082: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9756/95166 objects degraded (10.252%); 2758/95166 objects misplaced (2.898%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:11:42.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:42 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:42.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:42 smithi144 ceph-mon[194833]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:42 smithi144 ceph-mon[194833]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/168309432' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:11:42.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:42 smithi144 ceph-mon[194833]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T20:11:42.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:42 smithi144 ceph-mon[194833]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:42 smithi144 ceph-mon[194833]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[221113]: pgmap v1082: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9756/95166 objects degraded (10.252%); 2758/95166 objects misplaced (2.898%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:11:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[221113]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[221113]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/168309432' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[221113]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[221113]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[221113]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[213871]: pgmap v1082: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9756/95166 objects degraded (10.252%); 2758/95166 objects misplaced (2.898%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[213871]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[213871]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/168309432' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:11:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[213871]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T20:11:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[213871]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:42 smithi135 ceph-mon[213871]: failed to encode map e388 with expected crc 2023-09-23T20:11:42.561 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:42.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:42.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:42.665 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:42.699 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:43.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:43 smithi144 ceph-mon[194833]: from='client.45929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:43.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:43 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:11:43.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:43 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9756/95166 objects degraded (10.252%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:43.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:43 smithi144 ceph-mon[194833]: pgmap v1084: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 9756/94575 objects degraded (10.316%); 2758/94575 objects misplaced (2.916%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:11:43.193 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:43.195 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:43 smithi135 ceph-mon[221113]: from='client.45929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:43.195 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:43 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:11:43.195 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:43 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9756/95166 objects degraded (10.252%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:43.195 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:43 smithi135 ceph-mon[221113]: pgmap v1084: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 9756/94575 objects degraded (10.316%); 2758/94575 objects misplaced (2.916%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:11:43.196 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:43 smithi135 ceph-mon[213871]: from='client.45929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:43.196 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:43 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:11:43.196 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:43 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9756/95166 objects degraded (10.252%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:43.196 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:43 smithi135 ceph-mon[213871]: pgmap v1084: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 9756/94575 objects degraded (10.316%); 2758/94575 objects misplaced (2.916%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:11:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:43.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:43.297 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:43.336 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:43.358 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1083,"stamp":"2023-09-23T20:11:41.045169+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2081774658,"num_objects":31722,"num_object_clones":0,"num_object_copies":95166,"num_objects_missing_on_primary":2647,"num_objects_missing":2647,"num_objects_degraded":9756,"num_objects_misplaced":2758,"num_objects_unfound":0,"num_objects_dirty":31722,"num_whiteouts":0,"num_read":6158,"num_read_kb":13539,"num_write":137762,"num_write_kb":5404324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29615,"num_bytes_recovered":1524371711,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94302,"ondisk_log_size":94302,"up":327,"acting":322,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11208584,"kb_used_data":7593040,"kb_used_omap":32,"kb_used_meta":3615455,"kb_avail":645007480,"statfs":{"total":671965249536,"available":660487659520,"internally_reserved":0,"allocated":7775272960,"data_stored":7764031089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3702226438},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,3,1,1,2,8,2,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-230686720,"num_objects":-3520,"num_object_clones":0,"num_object_copies":-10560,"num_objects_missing_on_primary":-229,"num_objects_missing":-229,"num_objects_degraded":-229,"num_objects_misplaced":-459,"num_objects_unfound":0,"num_objects_dirty":-3520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3520,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":461,"num_bytes_recovered":30212096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.877806"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":385,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025394+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:11:30.025394+0000","last_peered":"2023-09-23T20:11:30.025394+0000","last_clean":"2023-09-23T20:11:30.025394+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:11:30.025394+0000","last_undegraded":"2023-09-23T20:11:30.025394+0000","last_fullsized":"2023-09-23T20:11:30.025394+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":215,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036605+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:11:30.036605+0000","last_peered":"2023-09-23T20:11:30.036605+0000","last_clean":"2023-09-23T20:11:30.036605+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:11:30.036605+0000","last_undegraded":"2023-09-23T20:11:30.036605+0000","last_fullsized":"2023-09-23T20:11:30.036605+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037244+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:11:30.037244+0000","last_peered":"2023-09-23T20:11:30.037244+0000","last_clean":"2023-09-23T20:11:30.037244+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:11:30.037244+0000","last_undegraded":"2023-09-23T20:11:30.037244+0000","last_fullsized":"2023-09-23T20:11:30.037244+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":95,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034990+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:11:30.034990+0000","last_peered":"2023-09-23T20:11:30.034990+0000","last_clean":"2023-09-23T20:11:30.034990+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:11:30.034990+0000","last_undegraded":"2023-09-23T20:11:30.034990+0000","last_fullsized":"2023-09-23T20:11:30.034990+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":215,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036788+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:11:30.036788+0000","last_peered":"2023-09-23T20:11:30.036788+0000","last_clean":"2023-09-23T20:11:30.036788+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:11:30.036788+0000","last_undegraded":"2023-09-23T20:11:30.036788+0000","last_fullsized":"2023-09-23T20:11:30.036788+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":332,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023422+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:11:30.023422+0000","last_peered":"2023-09-23T20:11:30.023422+0000","last_clean":"2023-09-23T20:11:30.023422+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:11:30.023422+0000","last_undegraded":"2023-09-23T20:11:30.023422+0000","last_fullsized":"2023-09-23T20:11:30.023422+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":313,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023331+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:11:30.023331+0000","last_peered":"2023-09-23T20:11:30.023331+0000","last_clean":"2023-09-23T20:11:30.023331+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:11:30.023331+0000","last_undegraded":"2023-09-23T20:11:30.023331+0000","last_fullsized":"2023-09-23T20:11:30.023331+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":230,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.039675+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:11:30.039675+0000","last_peered":"2023-09-23T20:11:30.039675+0000","last_clean":"2023-09-23T20:11:30.039675+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:11:30.039675+0000","last_undegraded":"2023-09-23T20:11:30.039675+0000","last_fullsized":"2023-09-23T20:11:30.039675+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":696,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.040104+0000","last_change":"2023-09-23T20:09:56.042750+0000","last_active":"2023-09-23T20:11:30.040104+0000","last_peered":"2023-09-23T20:11:30.040104+0000","last_clean":"2023-09-23T20:11:30.040104+0000","last_became_active":"2023-09-23T20:09:56.042455+0000","last_became_peered":"2023-09-23T20:09:56.042455+0000","last_unstale":"2023-09-23T20:11:30.040104+0000","last_undegraded":"2023-09-23T20:11:30.040104+0000","last_fullsized":"2023-09-23T20:11:30.040104+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":386,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023175+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:11:30.023175+0000","last_peered":"2023-09-23T20:11:30.023175+0000","last_clean":"2023-09-23T20:11:30.023175+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:11:30.023175+0000","last_undegraded":"2023-09-23T20:11:30.023175+0000","last_fullsized":"2023-09-23T20:11:30.023175+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":249,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.041023+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:11:30.041023+0000","last_peered":"2023-09-23T20:11:30.041023+0000","last_clean":"2023-09-23T20:11:30.041023+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:11:30.041023+0000","last_undegraded":"2023-09-23T20:11:30.041023+0000","last_fullsized":"2023-09-23T20:11:30.041023+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":80,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036132+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:11:30.036132+0000","last_peered":"2023-09-23T20:11:30.036132+0000","last_clean":"2023-09-23T20:11:30.036132+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:11:30.036132+0000","last_undegraded":"2023-09-23T20:11:30.036132+0000","last_fullsized":"2023-09-23T20:11:30.036132+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037771+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:11:30.037771+0000","last_peered":"2023-09-23T20:11:30.037771+0000","last_clean":"2023-09-23T20:11:30.037771+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:11:30.037771+0000","last_undegraded":"2023-09-23T20:11:30.037771+0000","last_fullsized":"2023-09-23T20:11:30.037771+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":441,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025240+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:11:30.025240+0000","last_peered":"2023-09-23T20:11:30.025240+0000","last_clean":"2023-09-23T20:11:30.025240+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:11:30.025240+0000","last_undegraded":"2023-09-23T20:11:30.025240+0000","last_fullsized":"2023-09-23T20:11:30.025240+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":383,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025251+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:11:30.025251+0000","last_peered":"2023-09-23T20:11:30.025251+0000","last_clean":"2023-09-23T20:11:30.025251+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:11:30.025251+0000","last_undegraded":"2023-09-23T20:11:30.025251+0000","last_fullsized":"2023-09-23T20:11:30.025251+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":716,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022848+0000","last_change":"2023-09-23T20:11:07.067995+0000","last_active":"2023-09-23T20:11:30.022848+0000","last_peered":"2023-09-23T20:11:30.022848+0000","last_clean":"2023-09-23T20:11:30.022848+0000","last_became_active":"2023-09-23T20:09:56.043288+0000","last_became_peered":"2023-09-23T20:09:56.043288+0000","last_unstale":"2023-09-23T20:11:30.022848+0000","last_undegraded":"2023-09-23T20:11:30.022848+0000","last_fullsized":"2023-09-23T20:11:30.022848+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":80,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034954+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:11:30.034954+0000","last_peered":"2023-09-23T20:11:30.034954+0000","last_clean":"2023-09-23T20:11:30.034954+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:11:30.034954+0000","last_undegraded":"2023-09-23T20:11:30.034954+0000","last_fullsized":"2023-09-23T20:11:30.034954+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":239,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037941+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:11:30.037941+0000","last_peered":"2023-09-23T20:11:30.037941+0000","last_clean":"2023-09-23T20:11:30.037941+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:11:30.037941+0000","last_undegraded":"2023-09-23T20:11:30.037941+0000","last_fullsized":"2023-09-23T20:11:30.037941+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":533,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034713+0000","last_change":"2023-09-23T20:10:40.721575+0000","last_active":"2023-09-23T20:11:30.034713+0000","last_peered":"2023-09-23T20:11:30.034713+0000","last_clean":"2023-09-23T20:11:30.034713+0000","last_became_active":"2023-09-23T20:09:56.050784+0000","last_became_peered":"2023-09-23T20:09:56.050784+0000","last_unstale":"2023-09-23T20:11:30.034713+0000","last_undegraded":"2023-09-23T20:11:30.034713+0000","last_fullsized":"2023-09-23T20:11:30.034713+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":95,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034917+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:11:30.034917+0000","last_peered":"2023-09-23T20:11:30.034917+0000","last_clean":"2023-09-23T20:11:30.034917+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:11:30.034917+0000","last_undegraded":"2023-09-23T20:11:30.034917+0000","last_fullsized":"2023-09-23T20:11:30.034917+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":385,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023651+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:11:30.023651+0000","last_peered":"2023-09-23T20:11:30.023651+0000","last_clean":"2023-09-23T20:11:30.023651+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:11:30.023651+0000","last_undegraded":"2023-09-23T20:11:30.023651+0000","last_fullsized":"2023-09-23T20:11:30.023651+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":512,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037276+0000","last_change":"2023-09-23T20:09:56.062364+0000","last_active":"2023-09-23T20:11:30.037276+0000","last_peered":"2023-09-23T20:11:30.037276+0000","last_clean":"2023-09-23T20:11:30.037276+0000","last_became_active":"2023-09-23T20:09:56.061865+0000","last_became_peered":"2023-09-23T20:09:56.061865+0000","last_unstale":"2023-09-23T20:11:30.037276+0000","last_undegraded":"2023-09-23T20:11:30.037276+0000","last_fullsized":"2023-09-23T20:11:30.037276+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034812+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:11:30.034812+0000","last_peered":"2023-09-23T20:11:30.034812+0000","last_clean":"2023-09-23T20:11:30.034812+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:11:30.034812+0000","last_undegraded":"2023-09-23T20:11:30.034812+0000","last_fullsized":"2023-09-23T20:11:30.034812+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":373,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.039659+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:11:30.039659+0000","last_peered":"2023-09-23T20:11:30.039659+0000","last_clean":"2023-09-23T20:11:30.039659+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:11:30.039659+0000","last_undegraded":"2023-09-23T20:11:30.039659+0000","last_fullsized":"2023-09-23T20:11:30.039659+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":608,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034776+0000","last_change":"2023-09-23T20:11:18.639154+0000","last_active":"2023-09-23T20:11:30.034776+0000","last_peered":"2023-09-23T20:11:30.034776+0000","last_clean":"2023-09-23T20:11:30.034776+0000","last_became_active":"2023-09-23T20:09:56.043157+0000","last_became_peered":"2023-09-23T20:09:56.043157+0000","last_unstale":"2023-09-23T20:11:30.034776+0000","last_undegraded":"2023-09-23T20:11:30.034776+0000","last_fullsized":"2023-09-23T20:11:30.034776+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":18,"reported_epoch":386,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:11:30.022843+0000","last_change":"2023-09-23T20:11:10.067603+0000","last_active":"2023-09-23T20:11:30.022843+0000","last_peered":"2023-09-23T20:11:30.022843+0000","last_clean":"2023-09-23T20:00:51.694506+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:11:30.022843+0000","last_undegraded":"2023-09-23T20:11:10.056843+0000","last_fullsized":"2023-09-23T20:11:30.022843+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":4}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.039604+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:11:30.039604+0000","last_peered":"2023-09-23T20:11:30.039604+0000","last_clean":"2023-09-23T20:11:30.039604+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:11:30.039604+0000","last_undegraded":"2023-09-23T20:11:30.039604+0000","last_fullsized":"2023-09-23T20:11:30.039604+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":807,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026442+0000","last_change":"2023-09-23T20:11:16.752974+0000","last_active":"2023-09-23T20:11:30.026442+0000","last_peered":"2023-09-23T20:11:30.026442+0000","last_clean":"2023-09-23T20:11:30.026442+0000","last_became_active":"2023-09-23T20:10:52.687280+0000","last_became_peered":"2023-09-23T20:10:52.687280+0000","last_unstale":"2023-09-23T20:11:30.026442+0000","last_undegraded":"2023-09-23T20:11:30.026442+0000","last_fullsized":"2023-09-23T20:11:30.026442+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.4","version":"387'8700","reported_seq":8796,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:38.253370+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:11:38.253370+0000","last_peered":"2023-09-23T20:11:38.253370+0000","last_clean":"2023-09-23T20:11:38.253370+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:11:38.253370+0000","last_undegraded":"2023-09-23T20:11:38.253370+0000","last_fullsized":"2023-09-23T20:11:38.253370+0000","mapping_epoch":299,"log_start":"309'2665","ondisk_log_start":"309'2665","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6035,"log_dups_size":0,"ondisk_log_size":6035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133038080,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8700,"num_write_kb":343360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":78,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036070+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:11:30.036070+0000","last_peered":"2023-09-23T20:11:30.036070+0000","last_clean":"2023-09-23T20:11:30.036070+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:11:30.036070+0000","last_undegraded":"2023-09-23T20:11:30.036070+0000","last_fullsized":"2023-09-23T20:11:30.036070+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":395,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022807+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:11:30.022807+0000","last_peered":"2023-09-23T20:11:30.022807+0000","last_clean":"2023-09-23T20:11:30.022807+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:11:30.022807+0000","last_undegraded":"2023-09-23T20:11:30.022807+0000","last_fullsized":"2023-09-23T20:11:30.022807+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":697,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.040010+0000","last_change":"2023-09-23T20:09:56.064782+0000","last_active":"2023-09-23T20:11:30.040010+0000","last_peered":"2023-09-23T20:11:30.040010+0000","last_clean":"2023-09-23T20:11:30.040010+0000","last_became_active":"2023-09-23T20:09:56.064411+0000","last_became_peered":"2023-09-23T20:09:56.064411+0000","last_unstale":"2023-09-23T20:11:30.040010+0000","last_undegraded":"2023-09-23T20:11:30.040010+0000","last_fullsized":"2023-09-23T20:11:30.040010+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218454,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.5","version":"387'8307","reported_seq":10997,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:38.356298+0000","last_change":"2023-09-23T20:07:45.697314+0000","last_active":"2023-09-23T20:11:38.356298+0000","last_peered":"2023-09-23T20:11:38.356298+0000","last_clean":"2023-09-23T20:11:38.356298+0000","last_became_active":"2023-09-23T20:07:45.690354+0000","last_became_peered":"2023-09-23T20:07:45.690354+0000","last_unstale":"2023-09-23T20:11:38.356298+0000","last_undegraded":"2023-09-23T20:11:38.356298+0000","last_fullsized":"2023-09-23T20:11:38.356298+0000","mapping_epoch":327,"log_start":"309'2618","ondisk_log_start":"309'2618","created":299,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.926631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131530752,"num_objects":2007,"num_object_clones":0,"num_object_copies":6021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8307,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":79,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.035036+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:11:30.035036+0000","last_peered":"2023-09-23T20:11:30.035036+0000","last_clean":"2023-09-23T20:11:30.035036+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:11:30.035036+0000","last_undegraded":"2023-09-23T20:11:30.035036+0000","last_fullsized":"2023-09-23T20:11:30.035036+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":443,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025005+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:11:30.025005+0000","last_peered":"2023-09-23T20:11:30.025005+0000","last_clean":"2023-09-23T20:11:30.025005+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:11:30.025005+0000","last_undegraded":"2023-09-23T20:11:30.025005+0000","last_fullsized":"2023-09-23T20:11:30.025005+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":655,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.039790+0000","last_change":"2023-09-23T20:11:20.856999+0000","last_active":"2023-09-23T20:11:30.039790+0000","last_peered":"2023-09-23T20:11:30.039790+0000","last_clean":"2023-09-23T20:11:30.039790+0000","last_became_active":"2023-09-23T20:11:20.856640+0000","last_became_peered":"2023-09-23T20:11:20.856640+0000","last_unstale":"2023-09-23T20:11:30.039790+0000","last_undegraded":"2023-09-23T20:11:30.039790+0000","last_fullsized":"2023-09-23T20:11:30.039790+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.6","version":"387'8804","reported_seq":10425,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:40.420674+0000","last_change":"2023-09-23T20:07:37.064417+0000","last_active":"2023-09-23T20:11:40.420674+0000","last_peered":"2023-09-23T20:11:40.420674+0000","last_clean":"2023-09-23T20:06:59.831272+0000","last_became_active":"2023-09-23T20:07:37.059682+0000","last_became_peered":"2023-09-23T20:07:37.059682+0000","last_unstale":"2023-09-23T20:11:40.420674+0000","last_undegraded":"2023-09-23T20:07:36.983730+0000","last_fullsized":"2023-09-23T20:07:36.974121+0000","mapping_epoch":322,"log_start":"314'3665","ondisk_log_start":"314'3665","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5139,"log_dups_size":0,"ondisk_log_size":5139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:33.575626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131334144,"num_objects":2004,"num_object_clones":0,"num_object_copies":6012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8804,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":613},{"shards":"0,2,3","objects":1310}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":78,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.035962+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:11:30.035962+0000","last_peered":"2023-09-23T20:11:30.035962+0000","last_clean":"2023-09-23T20:11:30.035962+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:11:30.035962+0000","last_undegraded":"2023-09-23T20:11:30.035962+0000","last_fullsized":"2023-09-23T20:11:30.035962+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037851+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:11:30.037851+0000","last_peered":"2023-09-23T20:11:30.037851+0000","last_clean":"2023-09-23T20:11:30.037851+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:11:30.037851+0000","last_undegraded":"2023-09-23T20:11:30.037851+0000","last_fullsized":"2023-09-23T20:11:30.037851+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":576,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025983+0000","last_change":"2023-09-23T20:11:07.538010+0000","last_active":"2023-09-23T20:11:30.025983+0000","last_peered":"2023-09-23T20:11:30.025983+0000","last_clean":"2023-09-23T20:11:30.025983+0000","last_became_active":"2023-09-23T20:09:56.043491+0000","last_became_peered":"2023-09-23T20:09:56.043491+0000","last_unstale":"2023-09-23T20:11:30.025983+0000","last_undegraded":"2023-09-23T20:11:30.025983+0000","last_fullsized":"2023-09-23T20:11:30.025983+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.7","version":"387'8546","reported_seq":11533,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:38.708980+0000","last_change":"2023-09-23T20:10:58.582053+0000","last_active":"2023-09-23T20:11:38.708980+0000","last_peered":"2023-09-23T20:11:38.708980+0000","last_clean":"2023-09-23T20:07:17.891274+0000","last_became_active":"2023-09-23T20:10:58.299550+0000","last_became_peered":"2023-09-23T20:10:58.299550+0000","last_unstale":"2023-09-23T20:11:38.708980+0000","last_undegraded":"2023-09-23T20:10:56.195777+0000","last_fullsized":"2023-09-23T20:10:53.692217+0000","mapping_epoch":369,"log_start":"309'3029","ondisk_log_start":"309'3029","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5517,"log_dups_size":0,"ondisk_log_size":5517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:48.194019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130547712,"num_objects":1992,"num_object_clones":0,"num_object_copies":5976,"num_objects_missing_on_primary":2643,"num_objects_missing":2643,"num_objects_degraded":3237,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8546,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1427,"num_bytes_recovered":93519872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2],"avail_no_missing":["3"],"object_location_counts":[{"shards":"1,2","objects":1181},{"shards":"1,2,3,4","objects":13},{"shards":"2,3,4","objects":798}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":425,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023409+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:11:30.023409+0000","last_peered":"2023-09-23T20:11:30.023409+0000","last_clean":"2023-09-23T20:11:30.023409+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:11:30.023409+0000","last_undegraded":"2023-09-23T20:11:30.023409+0000","last_fullsized":"2023-09-23T20:11:30.023409+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":82,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037882+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:11:30.037882+0000","last_peered":"2023-09-23T20:11:30.037882+0000","last_clean":"2023-09-23T20:11:30.037882+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:11:30.037882+0000","last_undegraded":"2023-09-23T20:11:30.037882+0000","last_fullsized":"2023-09-23T20:11:30.037882+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"387'8320","reported_seq":16447,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:36.011048+0000","last_change":"2023-09-23T20:10:10.729751+0000","last_active":"2023-09-23T20:11:36.011048+0000","last_peered":"2023-09-23T20:11:36.011048+0000","last_clean":"2023-09-23T20:11:36.011048+0000","last_became_active":"2023-09-23T20:10:10.708263+0000","last_became_peered":"2023-09-23T20:10:10.708263+0000","last_unstale":"2023-09-23T20:11:36.011048+0000","last_undegraded":"2023-09-23T20:11:36.011048+0000","last_fullsized":"2023-09-23T20:11:36.011048+0000","mapping_epoch":361,"log_start":"309'2910","ondisk_log_start":"309'2910","created":299,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5410,"log_dups_size":0,"ondisk_log_size":5410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:15.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130285568,"num_objects":1988,"num_object_clones":0,"num_object_copies":5964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8320,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4003,"num_bytes_recovered":136118272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":769,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034732+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:11:30.034732+0000","last_peered":"2023-09-23T20:11:30.034732+0000","last_clean":"2023-09-23T20:11:30.034732+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:11:30.034732+0000","last_undegraded":"2023-09-23T20:11:30.034732+0000","last_fullsized":"2023-09-23T20:11:30.034732+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":77,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025248+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:11:30.025248+0000","last_peered":"2023-09-23T20:11:30.025248+0000","last_clean":"2023-09-23T20:11:30.025248+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:11:30.025248+0000","last_undegraded":"2023-09-23T20:11:30.025248+0000","last_fullsized":"2023-09-23T20:11:30.025248+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":413,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025321+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:11:30.025321+0000","last_peered":"2023-09-23T20:11:30.025321+0000","last_clean":"2023-09-23T20:11:30.025321+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:11:30.025321+0000","last_undegraded":"2023-09-23T20:11:30.025321+0000","last_fullsized":"2023-09-23T20:11:30.025321+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"387'8581","reported_seq":14980,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:37.831021+0000","last_change":"2023-09-23T20:10:58.556423+0000","last_active":"2023-09-23T20:11:37.831021+0000","last_peered":"2023-09-23T20:11:37.831021+0000","last_clean":"2023-09-23T20:07:17.847486+0000","last_became_active":"2023-09-23T20:07:20.666556+0000","last_became_peered":"2023-09-23T20:07:20.666556+0000","last_unstale":"2023-09-23T20:11:37.831021+0000","last_undegraded":"2023-09-23T20:10:58.555698+0000","last_fullsized":"2023-09-23T20:07:19.932561+0000","mapping_epoch":317,"log_start":"314'3646","ondisk_log_start":"314'3646","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:38.441814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125763584,"num_objects":1919,"num_object_clones":0,"num_object_copies":5757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":996,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8581,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3145,"num_bytes_recovered":206110720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":923},{"shards":"0,2,5","objects":810}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":721,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022917+0000","last_change":"2023-09-23T20:11:11.229144+0000","last_active":"2023-09-23T20:11:30.022917+0000","last_peered":"2023-09-23T20:11:30.022917+0000","last_clean":"2023-09-23T20:11:30.022917+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:11:30.022917+0000","last_undegraded":"2023-09-23T20:11:30.022917+0000","last_fullsized":"2023-09-23T20:11:30.022917+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":353,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025311+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:11:30.025311+0000","last_peered":"2023-09-23T20:11:30.025311+0000","last_clean":"2023-09-23T20:11:30.025311+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:11:30.025311+0000","last_undegraded":"2023-09-23T20:11:30.025311+0000","last_fullsized":"2023-09-23T20:11:30.025311+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":383,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026525+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:11:30.026525+0000","last_peered":"2023-09-23T20:11:30.026525+0000","last_clean":"2023-09-23T20:11:30.026525+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:11:30.026525+0000","last_undegraded":"2023-09-23T20:11:30.026525+0000","last_fullsized":"2023-09-23T20:11:30.026525+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":512,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037368+0000","last_change":"2023-09-23T20:09:56.062162+0000","last_active":"2023-09-23T20:11:30.037368+0000","last_peered":"2023-09-23T20:11:30.037368+0000","last_clean":"2023-09-23T20:11:30.037368+0000","last_became_active":"2023-09-23T20:09:56.061518+0000","last_became_peered":"2023-09-23T20:09:56.061518+0000","last_unstale":"2023-09-23T20:11:30.037368+0000","last_undegraded":"2023-09-23T20:11:30.037368+0000","last_fullsized":"2023-09-23T20:11:30.037368+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"387'8514","reported_seq":8610,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:38.337910+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:11:38.337910+0000","last_peered":"2023-09-23T20:11:38.337910+0000","last_clean":"2023-09-23T20:11:38.337910+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:11:38.337910+0000","last_undegraded":"2023-09-23T20:11:38.337910+0000","last_fullsized":"2023-09-23T20:11:38.337910+0000","mapping_epoch":299,"log_start":"309'2785","ondisk_log_start":"309'2785","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127533056,"num_objects":1946,"num_object_clones":0,"num_object_copies":5838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8514,"num_write_kb":334720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":377,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.024908+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:11:30.024908+0000","last_peered":"2023-09-23T20:11:30.024908+0000","last_clean":"2023-09-23T20:11:30.024908+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:11:30.024908+0000","last_undegraded":"2023-09-23T20:11:30.024908+0000","last_fullsized":"2023-09-23T20:11:30.024908+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":413,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026119+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:11:30.026119+0000","last_peered":"2023-09-23T20:11:30.026119+0000","last_clean":"2023-09-23T20:11:30.026119+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:11:30.026119+0000","last_undegraded":"2023-09-23T20:11:30.026119+0000","last_fullsized":"2023-09-23T20:11:30.026119+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"387'8352","reported_seq":16507,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:38.389220+0000","last_change":"2023-09-23T20:09:37.763558+0000","last_active":"2023-09-23T20:11:38.389220+0000","last_peered":"2023-09-23T20:11:38.389220+0000","last_clean":"2023-09-23T20:11:38.389220+0000","last_became_active":"2023-09-23T20:09:37.732414+0000","last_became_peered":"2023-09-23T20:09:37.732414+0000","last_unstale":"2023-09-23T20:11:38.389220+0000","last_undegraded":"2023-09-23T20:11:38.389220+0000","last_fullsized":"2023-09-23T20:11:38.389220+0000","mapping_epoch":352,"log_start":"309'2881","ondisk_log_start":"309'2881","created":299,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5471,"log_dups_size":0,"ondisk_log_size":5471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:39.792571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127664128,"num_objects":1948,"num_object_clones":0,"num_object_copies":5844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8352,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4021,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":725,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023714+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:11:30.023714+0000","last_peered":"2023-09-23T20:11:30.023714+0000","last_clean":"2023-09-23T20:11:30.023714+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:11:30.023714+0000","last_undegraded":"2023-09-23T20:11:30.023714+0000","last_fullsized":"2023-09-23T20:11:30.023714+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":17,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.039331+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:11:30.039331+0000","last_peered":"2023-09-23T20:11:30.039331+0000","last_clean":"2023-09-23T20:11:30.039331+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:11:30.039331+0000","last_undegraded":"2023-09-23T20:11:30.039331+0000","last_fullsized":"2023-09-23T20:11:30.039331+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":77,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.057519+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:11:30.057519+0000","last_peered":"2023-09-23T20:11:30.057519+0000","last_clean":"2023-09-23T20:11:30.057519+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:11:30.057519+0000","last_undegraded":"2023-09-23T20:11:30.057519+0000","last_fullsized":"2023-09-23T20:11:30.057519+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"387'8628","reported_seq":8727,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:40.509928+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:11:40.509928+0000","last_peered":"2023-09-23T20:11:40.509928+0000","last_clean":"2023-09-23T20:11:40.509928+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:11:40.509928+0000","last_undegraded":"2023-09-23T20:11:40.509928+0000","last_fullsized":"2023-09-23T20:11:40.509928+0000","mapping_epoch":299,"log_start":"314'3483","ondisk_log_start":"314'3483","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5145,"log_dups_size":0,"ondisk_log_size":5145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126746624,"num_objects":1934,"num_object_clones":0,"num_object_copies":5802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8628,"num_write_kb":337984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":655,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.039698+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:11:30.039698+0000","last_peered":"2023-09-23T20:11:30.039698+0000","last_clean":"2023-09-23T20:11:30.039698+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:11:30.039698+0000","last_undegraded":"2023-09-23T20:11:30.039698+0000","last_fullsized":"2023-09-23T20:11:30.039698+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":520,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.189065+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:11:32.189065+0000","last_peered":"2023-09-23T20:11:32.189065+0000","last_clean":"2023-09-23T20:11:32.189065+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:11:32.189065+0000","last_undegraded":"2023-09-23T20:11:32.189065+0000","last_fullsized":"2023-09-23T20:11:32.189065+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":387,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023405+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:11:30.023405+0000","last_peered":"2023-09-23T20:11:30.023405+0000","last_clean":"2023-09-23T20:11:30.023405+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:11:30.023405+0000","last_undegraded":"2023-09-23T20:11:30.023405+0000","last_fullsized":"2023-09-23T20:11:30.023405+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":83,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037641+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:11:30.037641+0000","last_peered":"2023-09-23T20:11:30.037641+0000","last_clean":"2023-09-23T20:11:30.037641+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:11:30.037641+0000","last_undegraded":"2023-09-23T20:11:30.037641+0000","last_fullsized":"2023-09-23T20:11:30.037641+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"387'8509","reported_seq":13334,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:38.229326+0000","last_change":"2023-09-23T20:07:52.783215+0000","last_active":"2023-09-23T20:11:38.229326+0000","last_peered":"2023-09-23T20:11:38.229326+0000","last_clean":"2023-09-23T20:11:38.229326+0000","last_became_active":"2023-09-23T20:07:52.778874+0000","last_became_peered":"2023-09-23T20:07:52.778874+0000","last_unstale":"2023-09-23T20:11:38.229326+0000","last_undegraded":"2023-09-23T20:11:38.229326+0000","last_fullsized":"2023-09-23T20:11:38.229326+0000","mapping_epoch":330,"log_start":"308'2400","ondisk_log_start":"308'2400","created":299,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6109,"log_dups_size":0,"ondisk_log_size":6109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:31.036337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132055040,"num_objects":2015,"num_object_clones":0,"num_object_copies":6045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8509,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2349,"num_bytes_recovered":153944064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":720,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022960+0000","last_change":"2023-09-23T20:11:06.026338+0000","last_active":"2023-09-23T20:11:30.022960+0000","last_peered":"2023-09-23T20:11:30.022960+0000","last_clean":"2023-09-23T20:11:30.022960+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:11:30.022960+0000","last_undegraded":"2023-09-23T20:11:30.022960+0000","last_fullsized":"2023-09-23T20:11:30.022960+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1007,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023554+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:11:30.023554+0000","last_peered":"2023-09-23T20:11:30.023554+0000","last_clean":"2023-09-23T20:11:30.023554+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:11:30.023554+0000","last_undegraded":"2023-09-23T20:11:30.023554+0000","last_fullsized":"2023-09-23T20:11:30.023554+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":406,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025409+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:11:30.025409+0000","last_peered":"2023-09-23T20:11:30.025409+0000","last_clean":"2023-09-23T20:11:30.025409+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:11:30.025409+0000","last_undegraded":"2023-09-23T20:11:30.025409+0000","last_fullsized":"2023-09-23T20:11:30.025409+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.189021+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:11:32.189021+0000","last_peered":"2023-09-23T20:11:32.189021+0000","last_clean":"2023-09-23T20:11:32.189021+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:11:32.189021+0000","last_undegraded":"2023-09-23T20:11:32.189021+0000","last_fullsized":"2023-09-23T20:11:32.189021+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"387'8380","reported_seq":20743,"reported_epoch":387,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:11:38.696375+0000","last_change":"2023-09-23T20:11:18.839874+0000","last_active":"2023-09-23T20:11:38.696375+0000","last_peered":"2023-09-23T20:11:38.696375+0000","last_clean":"2023-09-23T20:11:16.510239+0000","last_became_active":"2023-09-23T20:11:18.809696+0000","last_became_peered":"2023-09-23T20:11:18.809696+0000","last_unstale":"2023-09-23T20:11:38.696375+0000","last_undegraded":"2023-09-23T20:11:38.696375+0000","last_fullsized":"2023-09-23T20:11:38.696375+0000","mapping_epoch":381,"log_start":"309'2900","ondisk_log_start":"309'2900","created":299,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:58.376473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130023424,"num_objects":1984,"num_object_clones":0,"num_object_copies":5952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1984,"num_objects_unfound":0,"num_objects_dirty":1984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8380,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":217251840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[1,4,2],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":1984}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":606,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034926+0000","last_change":"2023-09-23T20:11:15.663436+0000","last_active":"2023-09-23T20:11:30.034926+0000","last_peered":"2023-09-23T20:11:30.034926+0000","last_clean":"2023-09-23T20:11:30.034926+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:11:30.034926+0000","last_undegraded":"2023-09-23T20:11:30.034926+0000","last_fullsized":"2023-09-23T20:11:30.034926+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1023,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025261+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:11:30.025261+0000","last_peered":"2023-09-23T20:11:30.025261+0000","last_clean":"2023-09-23T20:11:30.025261+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:11:30.025261+0000","last_undegraded":"2023-09-23T20:11:30.025261+0000","last_fullsized":"2023-09-23T20:11:30.025261+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.037008+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:11:30.037008+0000","last_peered":"2023-09-23T20:11:30.037008+0000","last_clean":"2023-09-23T20:11:30.037008+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:11:30.037008+0000","last_undegraded":"2023-09-23T20:11:30.037008+0000","last_fullsized":"2023-09-23T20:11:30.037008+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.221879+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:11:32.221879+0000","last_peered":"2023-09-23T20:11:32.221879+0000","last_clean":"2023-09-23T20:11:32.221879+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:11:32.221879+0000","last_undegraded":"2023-09-23T20:11:32.221879+0000","last_fullsized":"2023-09-23T20:11:32.221879+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"387'8571","reported_seq":8667,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:38.260076+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:11:38.260076+0000","last_peered":"2023-09-23T20:11:38.260076+0000","last_clean":"2023-09-23T20:11:38.260076+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:11:38.260076+0000","last_undegraded":"2023-09-23T20:11:38.260076+0000","last_fullsized":"2023-09-23T20:11:38.260076+0000","mapping_epoch":299,"log_start":"308'2565","ondisk_log_start":"308'2565","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130875392,"num_objects":1997,"num_object_clones":0,"num_object_copies":5991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8571,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1295,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023082+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:11:30.023082+0000","last_peered":"2023-09-23T20:11:30.023082+0000","last_clean":"2023-09-23T20:11:30.023082+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:11:30.023082+0000","last_undegraded":"2023-09-23T20:11:30.023082+0000","last_fullsized":"2023-09-23T20:11:30.023082+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":540,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034667+0000","last_change":"2023-09-23T20:11:09.775272+0000","last_active":"2023-09-23T20:11:30.034667+0000","last_peered":"2023-09-23T20:11:30.034667+0000","last_clean":"2023-09-23T20:11:30.034667+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:11:30.034667+0000","last_undegraded":"2023-09-23T20:11:30.034667+0000","last_fullsized":"2023-09-23T20:11:30.034667+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":252,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023657+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:11:30.023657+0000","last_peered":"2023-09-23T20:11:30.023657+0000","last_clean":"2023-09-23T20:11:30.023657+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:11:30.023657+0000","last_undegraded":"2023-09-23T20:11:30.023657+0000","last_fullsized":"2023-09-23T20:11:30.023657+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":282,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026671+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:11:30.026671+0000","last_peered":"2023-09-23T20:11:30.026671+0000","last_clean":"2023-09-23T20:11:30.026671+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:11:30.026671+0000","last_undegraded":"2023-09-23T20:11:30.026671+0000","last_fullsized":"2023-09-23T20:11:30.026671+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"387'8690","reported_seq":8900,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:40.481468+0000","last_change":"2023-09-23T20:10:57.512712+0000","last_active":"2023-09-23T20:11:40.481468+0000","last_peered":"2023-09-23T20:11:40.481468+0000","last_clean":"2023-09-23T20:07:17.907438+0000","last_became_active":"2023-09-23T20:07:20.693608+0000","last_became_peered":"2023-09-23T20:07:20.693608+0000","last_unstale":"2023-09-23T20:11:40.481468+0000","last_undegraded":"2023-09-23T20:07:19.959595+0000","last_fullsized":"2023-09-23T20:07:19.924202+0000","mapping_epoch":317,"log_start":"314'3588","ondisk_log_start":"314'3588","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5102,"log_dups_size":0,"ondisk_log_size":5102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:21.022797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127533056,"num_objects":1946,"num_object_clones":0,"num_object_copies":5838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8690,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":3407872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":808},{"shards":"2,5","objects":-2182}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":513,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.189280+0000","last_change":"2023-09-23T20:09:56.051517+0000","last_active":"2023-09-23T20:11:32.189280+0000","last_peered":"2023-09-23T20:11:32.189280+0000","last_clean":"2023-09-23T20:11:32.189280+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:11:32.189280+0000","last_undegraded":"2023-09-23T20:11:32.189280+0000","last_fullsized":"2023-09-23T20:11:32.189280+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":823,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.034546+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:11:30.034546+0000","last_peered":"2023-09-23T20:11:30.034546+0000","last_clean":"2023-09-23T20:11:30.034546+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:11:30.034546+0000","last_undegraded":"2023-09-23T20:11:30.034546+0000","last_fullsized":"2023-09-23T20:11:30.034546+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":373,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025059+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:11:30.025059+0000","last_peered":"2023-09-23T20:11:30.025059+0000","last_clean":"2023-09-23T20:11:30.025059+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:11:30.025059+0000","last_undegraded":"2023-09-23T20:11:30.025059+0000","last_fullsized":"2023-09-23T20:11:30.025059+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.190890+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:11:32.190890+0000","last_peered":"2023-09-23T20:11:32.190890+0000","last_clean":"2023-09-23T20:11:32.190890+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:11:32.190890+0000","last_undegraded":"2023-09-23T20:11:32.190890+0000","last_fullsized":"2023-09-23T20:11:32.190890+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"387'8721","reported_seq":14657,"reported_epoch":387,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:11:40.538620+0000","last_change":"2023-09-23T20:10:58.722697+0000","last_active":"2023-09-23T20:11:40.538620+0000","last_peered":"2023-09-23T20:11:40.538620+0000","last_clean":"2023-09-23T20:07:17.917130+0000","last_became_active":"2023-09-23T20:10:56.953707+0000","last_became_peered":"2023-09-23T20:10:56.953707+0000","last_unstale":"2023-09-23T20:11:40.538620+0000","last_undegraded":"2023-09-23T20:11:40.538620+0000","last_fullsized":"2023-09-23T20:10:53.691838+0000","mapping_epoch":369,"log_start":"314'3546","ondisk_log_start":"314'3546","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5175,"log_dups_size":0,"ondisk_log_size":5175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:08.588715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128385024,"num_objects":1959,"num_object_clones":0,"num_object_copies":5877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":774,"num_objects_unfound":0,"num_objects_dirty":1959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8721,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3796,"num_bytes_recovered":210567168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,1],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"1,2","objects":1185},{"shards":"2,3,4","objects":774}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":580,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026257+0000","last_change":"2023-09-23T20:11:18.808960+0000","last_active":"2023-09-23T20:11:30.026257+0000","last_peered":"2023-09-23T20:11:30.026257+0000","last_clean":"2023-09-23T20:11:30.026257+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:11:30.026257+0000","last_undegraded":"2023-09-23T20:11:30.026257+0000","last_fullsized":"2023-09-23T20:11:30.026257+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":87,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036018+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:11:30.036018+0000","last_peered":"2023-09-23T20:11:30.036018+0000","last_clean":"2023-09-23T20:11:30.036018+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:11:30.036018+0000","last_undegraded":"2023-09-23T20:11:30.036018+0000","last_fullsized":"2023-09-23T20:11:30.036018+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.188637+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:11:32.188637+0000","last_peered":"2023-09-23T20:11:32.188637+0000","last_clean":"2023-09-23T20:11:32.188637+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:11:32.188637+0000","last_undegraded":"2023-09-23T20:11:32.188637+0000","last_fullsized":"2023-09-23T20:11:32.188637+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.022981+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:11:30.022981+0000","last_peered":"2023-09-23T20:11:30.022981+0000","last_clean":"2023-09-23T20:11:30.022981+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:11:30.022981+0000","last_undegraded":"2023-09-23T20:11:30.022981+0000","last_fullsized":"2023-09-23T20:11:30.022981+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"387'8623","reported_seq":12917,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:38.330906+0000","last_change":"2023-09-23T20:08:33.396812+0000","last_active":"2023-09-23T20:11:38.330906+0000","last_peered":"2023-09-23T20:11:38.330906+0000","last_clean":"2023-09-23T20:11:38.330906+0000","last_became_active":"2023-09-23T20:08:05.774750+0000","last_became_peered":"2023-09-23T20:08:05.774750+0000","last_unstale":"2023-09-23T20:11:38.330906+0000","last_undegraded":"2023-09-23T20:11:38.330906+0000","last_fullsized":"2023-09-23T20:11:38.330906+0000","mapping_epoch":334,"log_start":"307'1800","ondisk_log_start":"307'1800","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6823,"log_dups_size":0,"ondisk_log_size":6823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.633506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8623,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2086,"num_bytes_recovered":136708096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":808,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026447+0000","last_change":"2023-09-23T20:11:13.677585+0000","last_active":"2023-09-23T20:11:30.026447+0000","last_peered":"2023-09-23T20:11:30.026447+0000","last_clean":"2023-09-23T20:11:30.026447+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:11:30.026447+0000","last_undegraded":"2023-09-23T20:11:30.026447+0000","last_fullsized":"2023-09-23T20:11:30.026447+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":229,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.035878+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:11:30.035878+0000","last_peered":"2023-09-23T20:11:30.035878+0000","last_clean":"2023-09-23T20:11:30.035878+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:11:30.035878+0000","last_undegraded":"2023-09-23T20:11:30.035878+0000","last_fullsized":"2023-09-23T20:11:30.035878+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":424,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.038417+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:11:30.038417+0000","last_peered":"2023-09-23T20:11:30.038417+0000","last_clean":"2023-09-23T20:11:30.038417+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:11:30.038417+0000","last_undegraded":"2023-09-23T20:11:30.038417+0000","last_fullsized":"2023-09-23T20:11:30.038417+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.039065+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:11:30.039065+0000","last_peered":"2023-09-23T20:11:30.039065+0000","last_clean":"2023-09-23T20:11:30.039065+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:11:30.039065+0000","last_undegraded":"2023-09-23T20:11:30.039065+0000","last_fullsized":"2023-09-23T20:11:30.039065+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"387'8522","reported_seq":11287,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:38.087833+0000","last_change":"2023-09-23T20:09:08.683160+0000","last_active":"2023-09-23T20:11:38.087833+0000","last_peered":"2023-09-23T20:11:38.087833+0000","last_clean":"2023-09-23T20:11:38.087833+0000","last_became_active":"2023-09-23T20:09:08.670527+0000","last_became_peered":"2023-09-23T20:09:08.670527+0000","last_unstale":"2023-09-23T20:11:38.087833+0000","last_undegraded":"2023-09-23T20:11:38.087833+0000","last_fullsized":"2023-09-23T20:11:38.087833+0000","mapping_epoch":346,"log_start":"308'2400","ondisk_log_start":"308'2400","created":299,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6122,"log_dups_size":0,"ondisk_log_size":6122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:47.531381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131137560,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8522,"num_write_kb":336705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":697,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.040443+0000","last_change":"2023-09-23T20:09:56.036037+0000","last_active":"2023-09-23T20:11:30.040443+0000","last_peered":"2023-09-23T20:11:30.040443+0000","last_clean":"2023-09-23T20:11:30.040443+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:11:30.040443+0000","last_undegraded":"2023-09-23T20:11:30.040443+0000","last_fullsized":"2023-09-23T20:11:30.040443+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":88,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.036030+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:11:30.036030+0000","last_peered":"2023-09-23T20:11:30.036030+0000","last_clean":"2023-09-23T20:11:30.036030+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:11:30.036030+0000","last_undegraded":"2023-09-23T20:11:30.036030+0000","last_fullsized":"2023-09-23T20:11:30.036030+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":87,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.190745+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:11:32.190745+0000","last_peered":"2023-09-23T20:11:32.190745+0000","last_clean":"2023-09-23T20:11:32.190745+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:11:32.190745+0000","last_undegraded":"2023-09-23T20:11:32.190745+0000","last_fullsized":"2023-09-23T20:11:32.190745+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":346,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.024914+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:11:30.024914+0000","last_peered":"2023-09-23T20:11:30.024914+0000","last_clean":"2023-09-23T20:11:30.024914+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:11:30.024914+0000","last_undegraded":"2023-09-23T20:11:30.024914+0000","last_fullsized":"2023-09-23T20:11:30.024914+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":162,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.027445+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:11:30.027445+0000","last_peered":"2023-09-23T20:11:30.027445+0000","last_clean":"2023-09-23T20:11:30.027445+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:11:30.027445+0000","last_undegraded":"2023-09-23T20:11:30.027445+0000","last_fullsized":"2023-09-23T20:11:30.027445+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":153,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.026187+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:11:30.026187+0000","last_peered":"2023-09-23T20:11:30.026187+0000","last_clean":"2023-09-23T20:11:30.026187+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:11:30.026187+0000","last_undegraded":"2023-09-23T20:11:30.026187+0000","last_fullsized":"2023-09-23T20:11:30.026187+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":390,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023309+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:11:30.023309+0000","last_peered":"2023-09-23T20:11:30.023309+0000","last_clean":"2023-09-23T20:11:30.023309+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:11:30.023309+0000","last_undegraded":"2023-09-23T20:11:30.023309+0000","last_fullsized":"2023-09-23T20:11:30.023309+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":403,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025987+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:11:30.025987+0000","last_peered":"2023-09-23T20:11:30.025987+0000","last_clean":"2023-09-23T20:11:30.025987+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:11:30.025987+0000","last_undegraded":"2023-09-23T20:11:30.025987+0000","last_fullsized":"2023-09-23T20:11:30.025987+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":380,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023313+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:11:30.023313+0000","last_peered":"2023-09-23T20:11:30.023313+0000","last_clean":"2023-09-23T20:11:30.023313+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:11:30.023313+0000","last_undegraded":"2023-09-23T20:11:30.023313+0000","last_fullsized":"2023-09-23T20:11:30.023313+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":383,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025224+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:11:30.025224+0000","last_peered":"2023-09-23T20:11:30.025224+0000","last_clean":"2023-09-23T20:11:30.025224+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:11:30.025224+0000","last_undegraded":"2023-09-23T20:11:30.025224+0000","last_fullsized":"2023-09-23T20:11:30.025224+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":87,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.038263+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:11:30.038263+0000","last_peered":"2023-09-23T20:11:30.038263+0000","last_clean":"2023-09-23T20:11:30.038263+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:11:30.038263+0000","last_undegraded":"2023-09-23T20:11:30.038263+0000","last_fullsized":"2023-09-23T20:11:30.038263+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":244,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.025306+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:11:30.025306+0000","last_peered":"2023-09-23T20:11:30.025306+0000","last_clean":"2023-09-23T20:11:30.025306+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:11:30.025306+0000","last_undegraded":"2023-09-23T20:11:30.025306+0000","last_fullsized":"2023-09-23T20:11:30.025306+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":232,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.023285+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:11:30.023285+0000","last_peered":"2023-09-23T20:11:30.023285+0000","last_clean":"2023-09-23T20:11:30.023285+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:11:30.023285+0000","last_undegraded":"2023-09-23T20:11:30.023285+0000","last_fullsized":"2023-09-23T20:11:30.023285+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":433,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T20:11:30.024922+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:11:30.024922+0000","last_peered":"2023-09-23T20:11:30.024922+0000","last_clean":"2023-09-23T20:11:30.024922+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:11:30.024922+0000","last_undegraded":"2023-09-23T20:11:30.024922+0000","last_fullsized":"2023-09-23T20:11:30.024922+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":26174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2077032472,"num_objects":31694,"num_object_clones":0,"num_object_copies":95082,"num_objects_missing_on_primary":2643,"num_objects_missing":2643,"num_objects_degraded":9752,"num_objects_misplaced":2758,"num_objects_unfound":0,"num_objects_dirty":31694,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136768,"num_write_kb":5390721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29408,"num_bytes_recovered":1514668032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7681945600,"data_stored":7681933384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89887,"ondisk_log_size":89887,"up":48,"acting":43,"num_store_stats":7},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":542359,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":177,"num_read_kb":331,"num_write":87,"num_write_kb":878,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1299398,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234366,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908840,"kb_used_data":571424,"kb_used_omap":0,"kb_used_meta":337408,"kb_avail":92836312,"statfs":{"total":95995035648,"available":95064383488,"internally_reserved":0,"allocated":585138176,"data_stored":583896468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":345505792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":3.0049999999999999,"5min":2.161,"15min":2.5369999999999999},"min":{"1min":0.252,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":55.390000000000001,"5min":55.390000000000001,"15min":62.061999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":3.0840000000000001,"5min":2.052,"15min":2.71},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":56.664000000000001,"5min":56.664000000000001,"15min":62.389000000000003},"last":0.248}]},{"osd":1,"last update":"Sat Sep 23 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9940000000000002,"5min":2.0739999999999998,"15min":3.5129999999999999},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":54.945999999999998,"5min":54.945999999999998,"15min":104.16800000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":3.044,"5min":2.1030000000000002,"15min":3.6379999999999999},"min":{"1min":0.28100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":55.094000000000001,"5min":55.094000000000001,"15min":104.003},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 20:10:48 2023","interfaces":[{"interface":"back","average":{"1min":3.6440000000000001,"5min":4.8570000000000002,"15min":3.6520000000000001},"min":{"1min":0.32400000000000001,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":57.746000000000002,"5min":92.319000000000003,"15min":105.842},"last":0.27700000000000002},{"interface":"front","average":{"1min":3.504,"5min":5.0229999999999997,"15min":4.1230000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":56.259999999999998,"5min":92.049000000000007,"15min":105.66},"last":0.376}]},{"osd":3,"last update":"Sat Sep 23 20:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":3.79,"15min":4.4630000000000001},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":1.6000000000000001,"5min":94.953000000000003,"15min":146.38800000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.70599999999999996,"5min":3.6859999999999999,"15min":4.5019999999999998},"min":{"1min":0.38,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.633,"5min":93.423000000000002,"15min":146.851},"last":0.45700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.53500000000000003,"15min":1.3089999999999999},"min":{"1min":0.26400000000000001,"5min":0.25900000000000001,"15min":0.22500000000000001},"max":{"1min":1.048,"5min":5.431,"15min":143.255},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.60199999999999998,"15min":1.3480000000000001},"min":{"1min":0.29899999999999999,"5min":0.246,"15min":0.246},"max":{"1min":1.1020000000000001,"5min":4.1459999999999999,"15min":143.41200000000001},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.94699999999999995,"15min":1.056},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":0.60999999999999999,"5min":24.876000000000001,"15min":35.488},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.874,"15min":0.879},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":0.98699999999999999,"5min":21.623000000000001,"15min":34.896000000000001},"last":0.44600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.62,"15min":0.81699999999999995},"min":{"1min":0.308,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":1.454,"5min":4.3710000000000004,"15min":41.902999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.64600000000000002,"15min":0.64800000000000002},"min":{"1min":0.39100000000000001,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":0.70499999999999996,"5min":3.9889999999999999,"15min":6.3620000000000001},"last":0.46999999999999997}]}]},{"osd":6,"up_from":0,"seq":1078036791407,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762565,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1235116,"kb_used_data":681316,"kb_used_omap":7,"kb_used_meta":553784,"kb_avail":92510036,"statfs":{"total":95995035648,"available":94730276864,"internally_reserved":0,"allocated":697667584,"data_stored":695839687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":567075642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,3,1,0,2,8,2,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.1339999999999999,"15min":3.698},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.151},"max":{"1min":9.9410000000000007,"5min":28.510000000000002,"15min":186.84},"last":0.20200000000000001},{"interface":"front","average":{"1min":1.776,"5min":2.4209999999999998,"15min":3.9929999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":8.6660000000000004,"5min":14.781000000000001,"15min":191.322},"last":0.252}]},{"osd":2,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":4.6180000000000003,"15min":5.508},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":17.785,"5min":98.531000000000006,"15min":180.25200000000001},"last":0.23100000000000001},{"interface":"front","average":{"1min":2.2589999999999999,"5min":4.2859999999999996,"15min":5.548},"min":{"1min":0.23300000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":28.469999999999999,"5min":98.346000000000004,"15min":180.05000000000001},"last":0.32200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.5459999999999998,"15min":3.7879999999999998},"min":{"1min":0.25700000000000001,"5min":0.255,"15min":0.22},"max":{"1min":3.9849999999999999,"5min":25.963999999999999,"15min":68.180999999999997},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.4019999999999999,"5min":2.4430000000000001,"15min":3.6909999999999998},"min":{"1min":0.29299999999999998,"5min":0.28199999999999997,"15min":0.24299999999999999},"max":{"1min":6.4320000000000004,"5min":40.716000000000001,"15min":66.509},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.6140000000000001,"15min":1.9430000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23599999999999999},"max":{"1min":6.6029999999999998,"5min":18.087,"15min":60.951000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.573,"5min":1.764,"15min":2.1190000000000002},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":8.1669999999999998,"5min":14.890000000000001,"15min":60.841000000000001},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.8300000000000001,"15min":2.952},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.25900000000000001},"max":{"1min":6.5679999999999996,"5min":14.83,"15min":219.048},"last":0.314},{"interface":"front","average":{"1min":1.3440000000000001,"5min":2.1819999999999999,"15min":3.0950000000000002},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28399999999999997},"max":{"1min":6.0190000000000001,"5min":25.893000000000001,"15min":218.69499999999999},"last":0.59899999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:10:42 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.782,"15min":2.0499999999999998},"min":{"1min":0.38500000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":18.117999999999999,"5min":18.117999999999999,"15min":18.117999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.84099999999999997,"5min":1.708,"15min":2.7810000000000001},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.54,"5min":14.862,"15min":20.734999999999999},"last":1.0469999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":2.1869999999999998,"15min":2.3239999999999998},"min":{"1min":0.42999999999999999,"5min":0.33300000000000002,"15min":0.26800000000000002},"max":{"1min":4.5899999999999999,"5min":25.937000000000001,"15min":60.213000000000001},"last":0.434},{"interface":"front","average":{"1min":1.556,"5min":1.7749999999999999,"15min":2.0619999999999998},"min":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.30199999999999999},"max":{"1min":6.649,"5min":18.053999999999998,"15min":61.298999999999999},"last":0.373}]}]},{"osd":0,"up_from":94,"seq":403726926102,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1686784,"kb_used_data":1060408,"kb_used_omap":8,"kb_used_meta":626359,"kb_avail":92058368,"statfs":{"total":95995035648,"available":94267768832,"internally_reserved":0,"allocated":1085857792,"data_stored":1084014245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":641392225},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":2.855,"15min":3.907},"min":{"1min":0.217,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":4.3179999999999996,"5min":63.639000000000003,"15min":94.055999999999997},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":2.9140000000000001,"15min":4.2000000000000002},"min":{"1min":0.23200000000000001,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":1.6639999999999999,"5min":62.802,"15min":91.632999999999996},"last":0.24099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":3.5880000000000001,"15min":5.0199999999999996},"min":{"1min":0.255,"5min":0.217,"15min":0.19800000000000001},"max":{"1min":9.0700000000000003,"5min":102.65300000000001,"15min":270.51100000000002},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.0139999999999998,"5min":3.6150000000000002,"15min":4.8810000000000002},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.2599999999999998,"5min":96.578999999999994,"15min":269.69299999999998},"last":1.044}]},{"osd":3,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":4.5670000000000002,"15min":5.9219999999999997},"min":{"1min":0.27700000000000002,"5min":0.252,"15min":0.21299999999999999},"max":{"1min":1.6120000000000001,"5min":126.95399999999999,"15min":145.72800000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.4590000000000001,"5min":4.9480000000000004,"15min":6.0919999999999996},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":9.3439999999999994,"5min":127.294,"15min":145.50700000000001},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.5629999999999999,"15min":2.2629999999999999},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":6.1369999999999996,"5min":15.688000000000001,"15min":148.20599999999999},"last":1.103},{"interface":"front","average":{"1min":1.099,"5min":1.4039999999999999,"15min":2.0379999999999998},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":5.3940000000000001,"5min":13.428000000000001,"15min":148.11000000000001},"last":0.51200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.298,"15min":2.464},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":1.1459999999999999,"5min":9.5719999999999992,"15min":131.34399999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.163,"5min":1.718,"15min":2.468},"min":{"1min":0.377,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.1059999999999999,"5min":15.717000000000001,"15min":131.386},"last":1.1399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.758,"15min":1.6779999999999999},"min":{"1min":0.433,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":6.1660000000000004,"5min":15.744999999999999,"15min":15.744999999999999},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.79800000000000004,"5min":1.579,"15min":1.873},"min":{"1min":0.40400000000000003,"5min":0.33900000000000002,"15min":0.309},"max":{"1min":1.6160000000000001,"5min":13.470000000000001,"15min":13.766999999999999},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.327,"15min":1.891},"min":{"1min":0.38900000000000001,"5min":0.38100000000000001,"15min":0.31900000000000001},"max":{"1min":1.0980000000000001,"5min":9.6069999999999993,"15min":65.081999999999994},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.724,"15min":2.0190000000000001},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.1920000000000002,"5min":15.771000000000001,"15min":15.771000000000001},"last":1.466}]}]},{"osd":2,"up_from":104,"seq":446676599041,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2013832,"kb_used_data":1305088,"kb_used_omap":12,"kb_used_meta":708723,"kb_avail":91731320,"statfs":{"total":95995035648,"available":93932871680,"internally_reserved":0,"allocated":1336410112,"data_stored":1334559139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":725733374},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.9119999999999999,"15min":3.1739999999999999},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.154},"max":{"1min":3.714,"5min":17.876999999999999,"15min":76.251999999999995},"last":1.149},{"interface":"front","average":{"1min":1.323,"5min":2.4609999999999999,"15min":3.3149999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.192},"max":{"1min":10.808999999999999,"5min":18.561,"15min":76.590000000000003},"last":0.88600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":2.3159999999999998,"15min":3.6699999999999999},"min":{"1min":0.24199999999999999,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":2.4729999999999999,"5min":19.562999999999999,"15min":94.768000000000001},"last":1.0669999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.9079999999999999,"15min":3.6589999999999998},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":3.6320000000000001,"5min":19.314,"15min":96.808000000000007},"last":1.298}]},{"osd":3,"last update":"Sat Sep 23 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.5169999999999999,"15min":4.4029999999999996},"min":{"1min":0.32800000000000001,"5min":0.25,"15min":0.214},"max":{"1min":8.5850000000000009,"5min":13.513,"15min":107.381},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":2.097,"15min":3.7610000000000001},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.23699999999999999},"max":{"1min":4.7919999999999998,"5min":20.308,"15min":106.301},"last":1.244}]},{"osd":4,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.609,"15min":2.2170000000000001},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.26900000000000002},"max":{"1min":3.742,"5min":75.608000000000004,"15min":75.608000000000004},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.7839999999999998,"15min":2.2330000000000001},"min":{"1min":0.34599999999999997,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":2.544,"5min":76.161000000000001,"15min":76.161000000000001},"last":0.95999999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.1549999999999998,"15min":2.726},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.28100000000000003},"max":{"1min":3.77,"5min":65.921999999999997,"15min":172.505},"last":1.22},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.5489999999999999,"15min":2.8959999999999999},"min":{"1min":0.39500000000000002,"5min":0.38100000000000001,"15min":0.29099999999999998},"max":{"1min":10.696,"5min":63.792000000000002,"15min":172.375},"last":1.113}]},{"osd":6,"last update":"Sat Sep 23 20:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.6180000000000001,"15min":1.6359999999999999},"min":{"1min":0.40500000000000003,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":3.8210000000000002,"5min":10.209,"15min":10.209},"last":1.425},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.819,"15min":1.9099999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.36799999999999999},"max":{"1min":2.3820000000000001,"5min":14.279999999999999,"15min":14.279999999999999},"last":1.137}]},{"osd":7,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.6599999999999999,"15min":1.9379999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.28799999999999998},"max":{"1min":2.5699999999999998,"5min":8.1519999999999992,"15min":50.118000000000002},"last":1.341},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.619,"15min":2.0419999999999998},"min":{"1min":0.41699999999999998,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":3.794,"5min":10.244,"15min":50.664999999999999},"last":1.401}]}]},{"osd":3,"up_from":109,"seq":468151435514,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2070192,"kb_used_data":1409896,"kb_used_omap":4,"kb_used_meta":660283,"kb_avail":91674960,"statfs":{"total":95995035648,"available":93875159040,"internally_reserved":0,"allocated":1443733504,"data_stored":1441722493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":676129901},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":4.7380000000000004,"15min":3.673},"min":{"1min":0.186,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":5.9989999999999997,"5min":74.323999999999998,"15min":74.323999999999998},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.68600000000000005,"5min":4.5720000000000001,"15min":4.0350000000000001},"min":{"1min":0.215,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":3.1779999999999999,"5min":74.447000000000003,"15min":74.447000000000003},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":4.6959999999999997,"5min":4.6200000000000001,"15min":3.6309999999999998},"min":{"1min":0.23799999999999999,"5min":0.20300000000000001,"15min":0.19600000000000001},"max":{"1min":73.572000000000003,"5min":112.122,"15min":112.122},"last":0.51500000000000001},{"interface":"front","average":{"1min":4.4100000000000001,"5min":5.3609999999999998,"15min":3.9500000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.19900000000000001},"max":{"1min":73.414000000000001,"5min":111.786,"15min":111.786},"last":1.0189999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":4.3310000000000004,"15min":4.1369999999999996},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":7.7640000000000002,"5min":88.486999999999995,"15min":88.486999999999995},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.2090000000000001,"5min":4.6550000000000002,"15min":4.04},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":7.9059999999999997,"5min":88.387,"15min":88.387},"last":1.052}]},{"osd":4,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":2.6459999999999999,"15min":3.1899999999999999},"min":{"1min":0.34100000000000003,"5min":0.29899999999999999,"15min":0.25700000000000001},"max":{"1min":6.0510000000000002,"5min":58.472999999999999,"15min":109.66800000000001},"last":1.115},{"interface":"front","average":{"1min":0.65900000000000003,"5min":2.9969999999999999,"15min":3.593},"min":{"1min":0.28299999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.2149999999999999,"5min":58.408000000000001,"15min":110.295},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":3.6779999999999999,"15min":3.4409999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":3.2469999999999999,"5min":77.206000000000003,"15min":113.881},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.107,"5min":3.9100000000000001,"15min":3.3620000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":5.8689999999999998,"5min":77.968000000000004,"15min":114.598},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:10:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":2.746,"15min":4.2320000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":5.9080000000000004,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.97599999999999998,"5min":2.9820000000000002,"15min":4.3959999999999999},"min":{"1min":0.443,"5min":0.375,"15min":0.33400000000000002},"max":{"1min":6.0730000000000004,"5min":58.701000000000001,"15min":82.233000000000004},"last":1.0840000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":3.0059999999999998,"15min":3.5760000000000001},"min":{"1min":0.377,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":5.9340000000000002,"5min":58.686,"15min":138.035},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.98599999999999999,"5min":3.0920000000000001,"15min":3.5099999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.28699999999999998},"max":{"1min":6.1020000000000003,"5min":58.667999999999999,"15min":138.054},"last":1.1439999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019276,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1609952,"kb_used_data":1274904,"kb_used_omap":0,"kb_used_meta":335040,"kb_avail":92135200,"statfs":{"total":95995035648,"available":94346444800,"internally_reserved":0,"allocated":1305501696,"data_stored":1304268144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":343080960},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.54,"15min":2.5699999999999998},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.189},"max":{"1min":3.2000000000000002,"5min":56.061999999999998,"15min":104.604},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.5409999999999999,"15min":2.5339999999999998},"min":{"1min":0.28899999999999998,"5min":0.218,"15min":0.218},"max":{"1min":3.1899999999999999,"5min":56.075000000000003,"15min":104.687},"last":0.28199999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":2.9489999999999998,"15min":5.5389999999999997},"min":{"1min":0.252,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":4.1669999999999998,"5min":119.892,"15min":161.613},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.96499999999999997,"5min":3.2109999999999999,"15min":5.6139999999999999},"min":{"1min":0.28899999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.125,"5min":119.351,"15min":162.017},"last":0.247}]},{"osd":2,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":6.6200000000000001,"15min":6.75},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.245},"max":{"1min":17.948,"5min":182.96700000000001,"15min":182.96700000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.349,"5min":6.2729999999999997,"15min":6.5039999999999996},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":17.978000000000002,"5min":183.244,"15min":183.244},"last":0.436}]},{"osd":3,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":4.843,"15min":6.8440000000000003},"min":{"1min":0.30499999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":2.5259999999999998,"5min":184.37100000000001,"15min":266.54700000000003},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.95299999999999996,"5min":4.9130000000000003,"15min":5.798},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":5.407,"5min":184.22300000000001,"15min":184.22300000000001},"last":0.61699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":2.1640000000000001,"15min":2.1160000000000001},"min":{"1min":0.25800000000000001,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":0.622,"5min":176.81899999999999,"15min":266.59899999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":2.2149999999999999,"15min":2.1880000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":0.56599999999999995,"5min":176.172,"15min":266.72899999999998},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.49099999999999999,"15min":0.50800000000000001},"min":{"1min":0.33100000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.56399999999999995,"5min":2.5489999999999999,"15min":4.0410000000000004},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.59899999999999998,"15min":0.58299999999999996},"min":{"1min":0.36599999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.74199999999999999,"5min":4.2889999999999997,"15min":4.2889999999999997},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.337,"15min":1.446},"min":{"1min":0.36799999999999999,"5min":0.32000000000000001,"15min":0.26100000000000001},"max":{"1min":0.71699999999999997,"5min":74.040000000000006,"15min":76.507000000000005},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.5,"5min":1.3049999999999999,"15min":1.5629999999999999},"min":{"1min":0.36399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.61099999999999999,"5min":74.021000000000001,"15min":76.462000000000003},"last":0.66900000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757640,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1683868,"kb_used_data":1290004,"kb_used_omap":0,"kb_used_meta":393856,"kb_avail":92061284,"statfs":{"total":95995035648,"available":94270754816,"internally_reserved":0,"allocated":1320964096,"data_stored":1319730913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":403308544},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.88700000000000001,"15min":1.881},"min":{"1min":0.23000000000000001,"5min":0.224,"15min":0.216},"max":{"1min":0.625,"5min":8.6739999999999995,"15min":121.94799999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.436,"5min":1.1399999999999999,"15min":2.0750000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":1.28,"5min":10.327,"15min":121.964},"last":0.47399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":4.4189999999999996,"5min":3.6259999999999999,"15min":6.0190000000000001},"min":{"1min":0.255,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":75.204999999999998,"5min":127.41800000000001,"15min":266.197},"last":0.34000000000000002},{"interface":"front","average":{"1min":4.4720000000000004,"5min":3.7799999999999998,"15min":6.194},"min":{"1min":0.248,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":75.231999999999999,"5min":127.37,"15min":266.14999999999998},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:10:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":5.6289999999999996,"15min":6.1059999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":25.227,"5min":240.57300000000001,"15min":240.57300000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.415,"5min":5.6699999999999999,"15min":6.1520000000000001},"min":{"1min":0.39800000000000002,"5min":0.38600000000000001,"15min":0.313},"max":{"1min":25.286999999999999,"5min":240.464,"15min":240.464},"last":0.48899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":3.206,"15min":4.9610000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.7359999999999998,"5min":127.83799999999999,"15min":201.50899999999999},"last":0.373},{"interface":"front","average":{"1min":0.93999999999999995,"5min":2.9609999999999999,"15min":4.7910000000000004},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":7.149,"5min":128.291,"15min":201.58099999999999},"last":1.871}]},{"osd":4,"last update":"Sat Sep 23 20:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.71199999999999997,"15min":1.4490000000000001},"min":{"1min":0.27700000000000002,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":0.58799999999999997,"5min":9.7240000000000002,"15min":191.78899999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.59299999999999997,"15min":1.3360000000000001},"min":{"1min":0.311,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":1.3879999999999999,"5min":8.7360000000000007,"15min":191.821},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.67600000000000005,"15min":0.63600000000000001},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":0.749,"5min":9.9009999999999998,"15min":9.9009999999999998},"last":0.435},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.60999999999999999,"15min":0.51800000000000002},"min":{"1min":0.316,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63900000000000001,"5min":5.0119999999999996,"15min":5.0119999999999996},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.59599999999999997,"15min":1.5329999999999999},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":0.57399999999999995,"5min":5.2670000000000003,"15min":191.84299999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":1.7549999999999999},"min":{"1min":0.41699999999999998,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":0.78600000000000003,"5min":9.6920000000000002,"15min":191.69},"last":0.51100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1073741824,"data_stored":1073741824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":688652288,"data_stored":688652288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1310134272,"data_stored":1310130200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1428815872,"data_stored":1428815872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1295450112,"data_stored":1295450112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1312952320,"data_stored":1312948248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":572198912,"data_stored":572194840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:11:43.365 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:11:43.439 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:43.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:43.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:43.545 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:43.573 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:43.625 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:43.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:43.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:43.728 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:43.763 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:43.867 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:43.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:43.946 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:11:43.946 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:11:43.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:11:43.946 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:11:43.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.946 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86850 8.c 8:349d498f:::benchmark_data_smithi135_460343_object2617:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:43.946 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739145+0000", 2023-09-23T20:11:43.946 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.204031023, 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.085194797, 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86850 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:43.947 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739145+0000", 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739145+0000", 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739139+0000", 2023-09-23T20:11:43.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999938 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739153+0000", 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3427e-05 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739174+0000", 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.109e-05 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739182+0000", 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4689999999999997e-06 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739246+0000", 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3938999999999999e-05 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op must wait for map", 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739258+0000", 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2038999999999999e-05 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.744912+0000", 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005654236 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.744923+0000", 2023-09-23T20:11:43.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0957e-05 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:05.822899+0000", 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.077975949 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:05.824340+0000", 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00144074 2023-09-23T20:11:43.952 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86863 8.c 8:380a5fea:::benchmark_data_smithi135_460343_object2630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739195+0000", 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.203980702, 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8269497810000002, 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:43.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86863 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739195+0000", 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739195+0000", 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739195+0000", 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739199+0000", 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7620000000000001e-06 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739206+0000", 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0180000000000004e-06 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:43.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739210+0000", 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2899999999999996e-06 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739360+0000", 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014967299999999999 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739366+0000", 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9220000000000002e-06 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.744973+0000", 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0056069969999999998 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.744980+0000", 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1969999999999999e-06 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.549574+0000", 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8045945909999999 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.551797+0000", 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022222050000000001 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.554752+0000", 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029555440000000001 2023-09-23T20:11:43.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.559261+0000", 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045087060000000003 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.559914+0000", 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000652935 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.560940+0000", 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001026662 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.561820+0000", 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00087918800000000004 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:43.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.566145+0000", 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043255960000000001 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86871 8.c 8:31b0809c:::benchmark_data_smithi135_460343_object2638:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739234+0000", 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.20394196399999, 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8548918140000001, 2023-09-23T20:11:43.962 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86871 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739234+0000", 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739234+0000", 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739234+0000", 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739237+0000", 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4000000000000001e-06 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739244+0000", 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9090000000000003e-06 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739250+0000", 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4940000000000003e-06 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739440+0000", 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019021199999999999 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:43.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739449+0000", 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4739999999999994e-06 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745023+0000", 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055736689999999998 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745033+0000", 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7259999999999997e-06 2023-09-23T20:11:43.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.566364+0000", 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.82133119 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.568556+0000", 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021923670000000002 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.571455+0000", 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028990069999999999 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.578549+0000", 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070936990000000002 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.579111+0000", 2023-09-23T20:11:43.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00056198900000000004 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.589552+0000", 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010441071 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.590517+0000", 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00096461099999999998 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.594126+0000", 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003609456 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86885 8.c 8:32317725:::benchmark_data_smithi135_460343_object2652:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739267+0000", 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.20390947499999, 2023-09-23T20:11:43.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8611840710000001, 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86885 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739267+0000", 2023-09-23T20:11:43.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739267+0000", 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739266+0000", 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:43.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739270+0000", 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.63e-06 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739276+0000", 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4579999999999998e-06 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739281+0000", 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8369999999999996e-06 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739527+0000", 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00024542900000000002 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739532+0000", 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1830000000000002e-06 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745069+0000", 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055371109999999999 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745076+0000", 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9589999999999998e-06 2023-09-23T20:11:43.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.579317+0000", 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.834241362 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.581453+0000", 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002135924 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.584274+0000", 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028212060000000001 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.594764+0000", 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010489986 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.595380+0000", 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00061564400000000004 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.595556+0000", 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017588099999999999 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.596442+0000", 2023-09-23T20:11:43.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00088592499999999997 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.600451+0000", 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004009047 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.980 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86890 8.c 8:307d7fb6:::benchmark_data_smithi135_460343_object2657:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739286+0000", 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.20389054099999, 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.893261627, 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86890 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:43.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739286+0000", 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739286+0000", 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739285+0000", 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739288+0000", 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.807e-06 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739294+0000", 2023-09-23T20:11:43.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1850000000000001e-06 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739298+0000", 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9940000000000002e-06 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739580+0000", 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000282226 2023-09-23T20:11:43.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739592+0000", 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1204e-05 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745098+0000", 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0055061060000000002 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745104+0000", 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5470000000000002e-06 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.600774+0000", 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.855669483 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:43.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.602963+0000", 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021891580000000001 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.605747+0000", 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027836179999999999 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.626390+0000", 2023-09-23T20:11:43.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.020643702999999999 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.627218+0000", 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000827927 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.627853+0000", 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00063512200000000001 2023-09-23T20:11:43.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.628838+0000", 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00098511600000000007 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.632547+0000", 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037087980000000001 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:43.989 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86908 8.c 8:32cc60b2:::benchmark_data_smithi135_460343_object2675:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739349+0000", 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.203827475, 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.898834269, 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:43.990 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86908 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739349+0000", 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:43.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739349+0000", 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739348+0000", 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739351+0000", 2023-09-23T20:11:43.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.345e-06 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739356+0000", 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8080000000000004e-06 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739359+0000", 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9569999999999998e-06 2023-09-23T20:11:43.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739628+0000", 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00026991800000000001 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739632+0000", 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9620000000000004e-06 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745126+0000", 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054939289999999998 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745134+0000", 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2590000000000003e-06 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.609209+0000", 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8640758420000001 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.611665+0000", 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0024555200000000001 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.614562+0000", 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002896844 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.626565+0000", 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012003363 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.633352+0000", 2023-09-23T20:11:43.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0067871950000000002 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.633894+0000", 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00054150400000000005 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.634773+0000", 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00087868799999999997 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.638183+0000", 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003410419 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86920 8.c 8:3ed02899:::benchmark_data_smithi135_460343_object2687:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739361+0000", 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.20381503600001, 2023-09-23T20:11:43.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9050515560000001, 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86920 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739361+0000", 2023-09-23T20:11:44.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739361+0000", 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739361+0000", 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967296 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739363+0000", 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6789999999999999e-06 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739366+0000", 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2160000000000002e-06 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739368+0000", 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.356e-06 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739674+0000", 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00030608300000000002 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:44.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739684+0000", 2023-09-23T20:11:44.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0084999999999999e-05 2023-09-23T20:11:44.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745166+0000", 2023-09-23T20:11:44.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054815239999999998 2023-09-23T20:11:44.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745173+0000", 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5170000000000002e-06 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.617808+0000", 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.872634696 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.620036+0000", 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022280609999999999 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.623068+0000", 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003031607 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.632728+0000", 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0096604940000000004 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.638860+0000", 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0061315099999999997 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.639660+0000", 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00080000299999999998 2023-09-23T20:11:44.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.640816+0000", 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0011565 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.644413+0000", 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035964320000000001 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.009 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86956 8.c 8:3f54cf6a:::benchmark_data_smithi135_460343_object2723:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739380+0000", 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.20379627899999, 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9306421629999999, 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.010 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86956 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739380+0000", 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739380+0000", 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739380+0000", 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739383+0000", 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6720000000000002e-06 2023-09-23T20:11:44.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739399+0000", 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5656999999999999e-05 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739404+0000", 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2429999999999996e-06 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739718+0000", 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00031407100000000002 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739722+0000", 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8639999999999998e-06 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745199+0000", 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054767610000000001 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745206+0000", 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7070000000000003e-06 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.644782+0000", 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8995764450000001 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.647638+0000", 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028560899999999999 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.650609+0000", 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002970767 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.664765+0000", 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014155753 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.664972+0000", 2023-09-23T20:11:44.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020733399999999999 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.665590+0000", 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00061814900000000002 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.666503+0000", 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00091274599999999998 2023-09-23T20:11:44.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.670022+0000", 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035192499999999998 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:86986 8.c 8:34297654:::benchmark_data_smithi135_460343_object2753:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:44.019 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739422+0000", 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.20375379000001, 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.936387968, 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 86986 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739422+0000", 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739422+0000", 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739422+0000", 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739426+0000", 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9639999999999997e-06 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739443+0000", 2023-09-23T20:11:44.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7269999999999999e-05 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739450+0000", 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4130000000000003e-06 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739764+0000", 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00031484200000000002 2023-09-23T20:11:44.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739775+0000", 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0380000000000001e-05 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745230+0000", 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054552330000000003 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745237+0000", 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4900000000000003e-06 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.654209+0000", 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.908971516 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.656817+0000", 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002608239 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.660560+0000", 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037429049999999999 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.670625+0000", 2023-09-23T20:11:44.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010065075999999999 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.670808+0000", 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018266199999999999 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.671366+0000", 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00055820200000000005 2023-09-23T20:11:44.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.672259+0000", 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00089319400000000004 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.675810+0000", 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035510759999999998 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.028 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:87008 8.c 8:3e426314:::benchmark_data_smithi135_460343_object2775:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:08:04.739455+0000", 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 219.20372112499999, 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9619393139999999, 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.029 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 87008 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739455+0000", 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739455+0000", 2023-09-23T20:11:44.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739455+0000", 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739459+0000", 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1200000000000004e-06 2023-09-23T20:11:44.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739466+0000", 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5449999999999998e-06 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739471+0000", 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4229999999999998e-06 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739791+0000", 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00032025199999999998 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.739795+0000", 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8290000000000001e-06 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745262+0000", 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005467253 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:04.745269+0000", 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0690000000000004e-06 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.676029+0000", 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9307603389999999 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.678302+0000", 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022723969999999998 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.681140+0000", 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028378449999999999 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.695255+0000", 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014115384 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.695838+0000", 2023-09-23T20:11:44.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00058260400000000002 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.696048+0000", 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00021043400000000001 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.697041+0000", 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00099287799999999999 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:08:06.701394+0000", 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043534610000000003 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:131069 8.c 8:35ceac34:::benchmark_data_smithi135_460343_object46836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e384)", 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:22.161484+0000", 2023-09-23T20:11:44.038 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 21.781692511999999, 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.036066039000000001, 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 131069 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161484+0000", 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161484+0000", 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161481+0000", 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999976 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161496+0000", 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4344e-05 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161532+0000", 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6162999999999999e-05 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161546+0000", 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4453e-05 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161639+0000", 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2305000000000005e-05 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161829+0000", 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019029600000000001 2023-09-23T20:11:44.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.161918+0000", 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9259999999999996e-05 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.164707+0000", 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027886529999999999 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.175204+0000", 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010497044000000001 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.197468+0000", 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022263812000000001 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.197512+0000", 2023-09-23T20:11:44.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3893999999999997e-05 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:22.197550+0000", 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8099000000000002e-05 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.045 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:132527 8.c 8:3199cc56:::benchmark_data_smithi135_460343_object48294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:26.809110+0000", 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 17.134065873000001, 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021675191999999999, 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 132527 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809110+0000", 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809110+0000", 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809109+0000", 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809114+0000", 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4790000000000004e-06 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809128+0000", 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3818999999999999e-05 2023-09-23T20:11:44.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809135+0000", 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4320000000000004e-06 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809183+0000", 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8572000000000002e-05 2023-09-23T20:11:44.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809311+0000", 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012816499999999999 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.809440+0000", 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000128591 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.812417+0000", 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029772280000000002 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.820630+0000", 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0082128740000000002 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.830681+0000", 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010050702 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.830737+0000", 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5949000000000003e-05 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:26.830786+0000", 2023-09-23T20:11:44.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8693000000000003e-05 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:132652 8.c 8:358d95c3:::benchmark_data_smithi135_460343_object48419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:27.086675+0000", 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.856500971999999, 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.028433663000000001, 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.053 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 132652 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086675+0000", 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.054 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086675+0000", 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086674+0000", 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086682+0000", 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7100000000000007e-06 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086700+0000", 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8162999999999999e-05 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086708+0000", 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0360000000000006e-06 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086749+0000", 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1674000000000001e-05 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086852+0000", 2023-09-23T20:11:44.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102688 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.086937+0000", 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4606999999999998e-05 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.090131+0000", 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031940100000000002 2023-09-23T20:11:44.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.097229+0000", 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070979679999999996 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115037+0000", 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.017808286999999999 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115079+0000", 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1647000000000003e-05 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115109+0000", 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0272000000000001e-05 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:132657 8.c 8:3250ea45:::benchmark_data_smithi135_460343_object48424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:27.115143+0000", 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.828033121000001, 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.021625379, 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.061 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 132657 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115143+0000", 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115143+0000", 2023-09-23T20:11:44.062 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115143+0000", 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967296 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115145+0000", 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7689999999999999e-06 2023-09-23T20:11:44.063 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115153+0000", 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9670000000000001e-06 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.115156+0000", 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.942e-06 2023-09-23T20:11:44.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.119016+0000", 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038603439999999999 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.119144+0000", 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012818500000000001 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.119277+0000", 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000132929 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.122955+0000", 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0036777260000000001 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.066 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.129946+0000", 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0069916470000000001 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.136679+0000", 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0067327489999999997 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.136728+0000", 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9045999999999998e-05 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.136768+0000", 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0312999999999998e-05 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:132660 8.c 8:30720a94:::benchmark_data_smithi135_460343_object48427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:27.118687+0000", 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 16.824489571000001, 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.032428356999999998, 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 132660 2023-09-23T20:11:44.069 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.118687+0000", 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.118687+0000", 2023-09-23T20:11:44.070 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.118686+0000", 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.118692+0000", 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0460000000000002e-06 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.118706+0000", 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4239e-05 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.118712+0000", 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6670000000000001e-06 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.121338+0000", 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0026266420000000002 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.121414+0000", 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5796000000000004e-05 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.121488+0000", 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4222999999999997e-05 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.123073+0000", 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0015843280000000001 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.130056+0000", 2023-09-23T20:11:44.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0069835640000000003 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.151049+0000", 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.020993041 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.151087+0000", 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7407000000000003e-05 2023-09-23T20:11:44.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:27.151115+0000", 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8408999999999999e-05 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:137355 8.c 8:35571db5:::benchmark_data_smithi135_460343_object53122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T20:11:44.076 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:40.367085+0000", 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 3.5760911699999998, 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022144258, 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 137355 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367085+0000", 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367085+0000", 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367084+0000", 2023-09-23T20:11:44.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367092+0000", 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3380000000000005e-06 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367109+0000", 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7628000000000001e-05 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367115+0000", 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.959e-06 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367188+0000", 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3010999999999996e-05 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367277+0000", 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8605999999999998e-05 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367348+0000", 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0570999999999999e-05 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.373217+0000", 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005869183 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.379014+0000", 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005797211 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.082 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.389160+0000", 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010146143 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.389199+0000", 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8707000000000001e-05 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.389229+0000", 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0399000000000002e-05 2023-09-23T20:11:44.083 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:137356 8.c 8:3bfc6045:::benchmark_data_smithi135_460343_object53123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:40.367125+0000", 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 3.5760516300000003, 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022262733999999999, 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.084 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 137356 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367125+0000", 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367125+0000", 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367124+0000", 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367127+0000", 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3939999999999999e-06 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367130+0000", 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.546e-06 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.367132+0000", 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1880000000000001e-06 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.370220+0000", 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0030877370000000001 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.370341+0000", 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000121029 2023-09-23T20:11:44.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.370419+0000", 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8275000000000005e-05 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.373367+0000", 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029479469999999998 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.379221+0000", 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0058540240000000002 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.389280+0000", 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010058269 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.389339+0000", 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9058000000000003e-05 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:40.389387+0000", 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8615000000000001e-05 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:137632 8.c 8:3d877841:::benchmark_data_smithi135_460343_object53399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:41.053331+0000", 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.8898456079999999, 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.195902837, 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 137632 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.092 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053331+0000", 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053331+0000", 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053329+0000", 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053338+0000", 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0569999999999999e-06 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053354+0000", 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6441999999999999e-05 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053361+0000", 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4620000000000001e-06 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053400+0000", 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9159999999999998e-05 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053493+0000", 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3277999999999997e-05 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.053574+0000", 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1119999999999996e-05 2023-09-23T20:11:44.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.056734+0000", 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031596110000000001 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249051+0000", 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.192317074 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249165+0000", 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000114014 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249209+0000", 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4348000000000003e-05 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249233+0000", 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3952999999999999e-05 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop_reply(client.35643.0:137632 8.c e387/334 ondisk, result = 0)", 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:41.060950+0000", 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.88222574, 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.188287923, 2023-09-23T20:11:44.099 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.060950+0000", 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.060950+0000", 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.060950+0000", 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.060953+0000", 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.224e-06 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.248838+0000", 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.187884358 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.248845+0000", 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.754e-06 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249153+0000", 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00030816699999999998 2023-09-23T20:11:44.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249160+0000", 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9009999999999997e-06 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249238+0000", 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8763000000000005e-05 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.103 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop_reply(client.35643.0:137632 8.c e388/334 ondisk, result = 0)", 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:11:41.061057+0000", 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 2.8821192739999999, 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18799986799999999, 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.061057+0000", 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.061057+0000", 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:11:44.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.061056+0000", 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.061061+0000", 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6639999999999997e-06 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.248828+0000", 2023-09-23T20:11:44.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.1877673 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.248838+0000", 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5529999999999998e-06 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249037+0000", 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019938200000000001 2023-09-23T20:11:44.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249044+0000", 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3089999999999998e-06 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:11:41.249057+0000", 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2369e-05 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.108 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:11:44.109 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:11:44.109 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:11:44.109 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:44.110 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:11:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:11:43] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-09-23T20:11:44.162 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:44.162 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:44.246 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:44.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:44.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:44.311 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:11:44.344 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:11:44.356 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:44.371 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:11:44.385 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:44.405 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:11:44.429 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:44.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:44.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:44.435 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:11:44.475 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:11:44.505 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:11:44.533 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:44.537 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:11:44.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:11:44.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:11:44.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:11:44.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:11:44.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:11:44.549 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:11:44.549 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:11:44.549 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:11:44.561 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:44 smithi135 ceph-mon[213871]: 1.e scrub starts 2023-09-23T20:11:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:44 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3458531093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:44 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:11:44.843 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:11:44.851 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:44.853 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:11:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:44.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:44.958 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:44.988 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:44 smithi144 ceph-mon[194833]: 1.e scrub starts 2023-09-23T20:11:44.988 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:44 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3458531093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:44.988 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:44.988 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:44 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:11:44.991 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:45.095 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:45.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:45.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:45.201 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:44 smithi135 ceph-mon[221113]: 1.e scrub starts 2023-09-23T20:11:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:44 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3458531093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:45.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:44 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:11:45.228 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:45.293 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:45.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:45.398 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:45.426 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:45.575 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:45.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:45.669 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:45.681 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:45.714 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:45.807 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:45.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:45.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:45.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:45 smithi144 ceph-mon[194833]: pgmap v1085: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9640/91065 objects degraded (10.586%); 2627/91065 objects misplaced (2.885%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T20:11:45.915 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:45.951 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:45.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:45 smithi135 ceph-mon[221113]: pgmap v1085: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9640/91065 objects degraded (10.586%); 2627/91065 objects misplaced (2.885%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T20:11:45.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:45 smithi135 ceph-mon[213871]: pgmap v1085: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9640/91065 objects degraded (10.586%); 2627/91065 objects misplaced (2.885%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T20:11:46.074 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:46.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:46.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:46.181 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:46.275 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:46.391 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:46.397 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:46.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:46.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:46.500 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:46.533 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:46.639 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:46.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:46.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:46.744 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:46.780 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:46 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:11:46.913 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:46 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:11:46.913 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:46 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:11:46.993 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:47.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:47.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:47.104 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:47.174 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:47.477 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:11:47.477 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 9640/91065 objects degraded (10.586%), 5 pgs degraded, 3 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1476,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":388,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":7,"osd_in_since":1695499655,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":102},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":109,"num_pools":5,"num_objects":30355,"data_bytes":1992186946,"bytes_used":11220365312,"bytes_avail":660744884224,"bytes_total":671965249536,"degraded_objects":9640,"degraded_total":91065,"degraded_ratio":0.1058584527535277,"misplaced_objects":2627,"misplaced_total":91065,"misplaced_ratio":0.028847526492066105,"recovering_objects_per_sec":47,"recovering_bytes_per_sec":3091316,"recovering_keys_per_sec":0,"num_objects_recovered":472,"num_bytes_recovered":30932992,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":361},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-23T20:06:56.817144+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"e76096c8-5249-4f94-9197-02047057b831":{"message":"Global Recovery Event (4m)\n [==========================..] (remaining: 19s)","progress":0.93577980995178223,"add_to_ceph_s":true}}} 2023-09-23T20:11:47.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:47 smithi135 ceph-mon[221113]: pgmap v1086: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9640/89274 objects degraded (10.798%); 2508/89274 objects misplaced (2.809%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:11:47.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3705577413' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:11:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9640/91065 objects degraded (10.586%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:47.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:47 smithi135 ceph-mon[213871]: pgmap v1086: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9640/89274 objects degraded (10.798%); 2508/89274 objects misplaced (2.809%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:11:47.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3705577413' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:11:47.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9640/91065 objects degraded (10.586%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:48.037 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:11:48.038 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:11:48.062 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:47 smithi144 ceph-mon[194833]: pgmap v1086: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 9640/89274 objects degraded (10.798%); 2508/89274 objects misplaced (2.809%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:11:48.062 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3705577413' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:11:48.062 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9640/91065 objects degraded (10.586%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:48.210 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:11:48.474 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:48.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:48.590 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:48.620 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:48.743 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:48 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:11:48.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:48 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:11:48.902 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:48.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:48.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:49.006 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:49.009 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:11:49.034 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:49.114 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:49.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:49.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:48 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:11:49.226 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:49.282 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:49.460 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:49.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:49.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:49.513 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:11:49.513 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:11:49.524 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:49.570 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:49.703 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:49.933 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:49.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:49.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:49.958 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:49 smithi135 ceph-mon[213871]: pgmap v1087: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 9509/84750 objects degraded (11.220%); 2378/84750 objects misplaced (2.806%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:11:49.958 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:49 smithi135 ceph-mon[221113]: pgmap v1087: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 9509/84750 objects degraded (11.220%); 2378/84750 objects misplaced (2.806%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:11:50.052 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:49 smithi144 ceph-mon[194833]: pgmap v1087: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 9509/84750 objects degraded (11.220%); 2378/84750 objects misplaced (2.806%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:11:50.053 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:50.058 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:50.141 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:50.352 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:50.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:50.473 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:50.546 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:50.642 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:50.645 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:50.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:50.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:50.750 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:50.787 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:50.804 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:11:50.965 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:50.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:50.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3545898364' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:50.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:50 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:50.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:50 smithi135 ceph-mon[213871]: 1.12 scrub starts 2023-09-23T20:11:50.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3545898364' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:50.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:50 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:50.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:50 smithi135 ceph-mon[221113]: 1.12 scrub starts 2023-09-23T20:11:51.036 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3545898364' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:51.037 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:50 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:11:51.037 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:50 smithi144 ceph-mon[194833]: 1.12 scrub starts 2023-09-23T20:11:51.072 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:51.125 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:51.530 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:11:51.531 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:11:51.770 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:51.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:51.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:51.878 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:51.906 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:52.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:11:52.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:52.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T20:11:52.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: pgmap v1089: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 9509/82884 objects degraded (11.473%); 2237/82884 objects misplaced (2.699%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:11:52.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:52.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:52.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:11:52.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:52.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T20:11:52.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: pgmap v1089: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 9509/82884 objects degraded (11.473%); 2237/82884 objects misplaced (2.699%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:11:52.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:52.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:52.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:11:52.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:11:52.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T20:11:52.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: failed to encode map e389 with expected crc 2023-09-23T20:11:52.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: pgmap v1089: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 9509/82884 objects degraded (11.473%); 2237/82884 objects misplaced (2.699%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:11:52.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:52.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:52.358 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:11:52.620 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:52.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:52.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:52.729 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:52.762 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:52.802 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:52.826 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:52.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:52.933 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:52.967 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:53.121 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:53.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:53.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:53.226 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:11:53.291 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:53 smithi144 ceph-mon[194833]: 1.10 scrub starts 2023-09-23T20:11:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9509/82884 objects degraded (11.473%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:53 smithi144 ceph-mon[194833]: pgmap v1090: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 418 op/s; 9509/82014 objects degraded (11.594%); 2237/82014 objects misplaced (2.728%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:11:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[213871]: 1.10 scrub starts 2023-09-23T20:11:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9509/82884 objects degraded (11.473%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[213871]: pgmap v1090: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 418 op/s; 9509/82014 objects degraded (11.594%); 2237/82014 objects misplaced (2.728%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:11:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[221113]: 1.10 scrub starts 2023-09-23T20:11:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9509/82884 objects degraded (11.473%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:53 smithi135 ceph-mon[221113]: pgmap v1090: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 418 op/s; 9509/82014 objects degraded (11.594%); 2237/82014 objects misplaced (2.728%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:11:53.570 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:11:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:53.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:11:53.677 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:53.749 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:11:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:11:53] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-09-23T20:11:54.220 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:54.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:54.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:54.326 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:54.360 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:54 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:11:54.360 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:54 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:11:54.401 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:54 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:11:54.402 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:54 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:11:54.402 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:54.402 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:54 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:11:54.402 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:54 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:11:54.468 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:11:54.580 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:11:54.580 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:11:54.683 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:11:54.707 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:54.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:54.731 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:11:54.769 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:11:54.811 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:11:54.826 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:54.861 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:11:54.862 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:54.886 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:11:54.911 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:54.913 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:11:54.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:54.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:54.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:11:54.994 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:11:54.994 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:11:54.995 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:11:54.995 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:11:54.995 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:11:54.995 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:11:54.995 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:11:54.995 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:11:55.015 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:55.044 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:55.090 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:55.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:55.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:55.195 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:55.259 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:55.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:55 smithi135 ceph-mon[213871]: 1.2 deep-scrub starts 2023-09-23T20:11:55.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:55 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1504414857' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:55.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:55 smithi135 ceph-mon[213871]: pgmap v1091: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 9371/77430 objects degraded (12.103%); 2084/77430 objects misplaced (2.691%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:11:55.357 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:55 smithi135 ceph-mon[221113]: 1.2 deep-scrub starts 2023-09-23T20:11:55.357 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:55 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1504414857' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:55.357 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:55 smithi135 ceph-mon[221113]: pgmap v1091: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 9371/77430 objects degraded (12.103%); 2084/77430 objects misplaced (2.691%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:11:55.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:55 smithi144 ceph-mon[194833]: 1.2 deep-scrub starts 2023-09-23T20:11:55.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:55 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1504414857' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:11:55.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:55 smithi144 ceph-mon[194833]: pgmap v1091: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 9371/77430 objects degraded (12.103%); 2084/77430 objects misplaced (2.691%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:11:55.418 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:55.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:55.445 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:11:55.471 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:11:55.508 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:11:55.522 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:55.550 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:55.594 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:55.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:55.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:55.701 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:55.727 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:55.768 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:55.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:55.876 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:11:55.951 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:56 smithi135 ceph-mon[213871]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:56.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:56 smithi135 ceph-mon[221113]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:56.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:56 smithi144 ceph-mon[194833]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:11:56.823 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:11:56.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:56.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:11:56.834 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1091,"stamp":"2023-09-23T20:11:54.956293+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1694325826,"num_objects":25810,"num_object_clones":0,"num_object_copies":77430,"num_objects_missing_on_primary":2262,"num_objects_missing":2262,"num_objects_degraded":9371,"num_objects_misplaced":2084,"num_objects_unfound":0,"num_objects_dirty":25810,"num_whiteouts":0,"num_read":6158,"num_read_kb":13539,"num_write":143674,"num_write_kb":5404324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30266,"num_bytes_recovered":1567035647,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95314,"ondisk_log_size":95314,"up":327,"acting":322,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10127660,"kb_used_data":6506420,"kb_used_omap":32,"kb_used_meta":3621087,"kb_avail":646088404,"statfs":{"total":671965249536,"available":661594525696,"internally_reserved":0,"allocated":6662574080,"data_stored":6651264795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3707993606},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-297861120,"num_objects":-4545,"num_object_clones":0,"num_object_copies":-13635,"num_objects_missing_on_primary":-269,"num_objects_missing":-269,"num_objects_degraded":-269,"num_objects_misplaced":-543,"num_objects_unfound":0,"num_objects_dirty":-4545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4545,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":527,"num_bytes_recovered":34537472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004411"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":387,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067406+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:11:41.067406+0000","last_peered":"2023-09-23T20:11:41.067406+0000","last_clean":"2023-09-23T20:11:41.067406+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:11:41.067406+0000","last_undegraded":"2023-09-23T20:11:41.067406+0000","last_fullsized":"2023-09-23T20:11:41.067406+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":217,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.052040+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:11:41.052040+0000","last_peered":"2023-09-23T20:11:41.052040+0000","last_clean":"2023-09-23T20:11:41.052040+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:11:41.052040+0000","last_undegraded":"2023-09-23T20:11:41.052040+0000","last_fullsized":"2023-09-23T20:11:41.052040+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.051804+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:11:41.051804+0000","last_peered":"2023-09-23T20:11:41.051804+0000","last_clean":"2023-09-23T20:11:41.051804+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:11:41.051804+0000","last_undegraded":"2023-09-23T20:11:41.051804+0000","last_fullsized":"2023-09-23T20:11:41.051804+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":97,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.248811+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:11:41.248811+0000","last_peered":"2023-09-23T20:11:41.248811+0000","last_clean":"2023-09-23T20:11:41.248811+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:11:41.248811+0000","last_undegraded":"2023-09-23T20:11:41.248811+0000","last_fullsized":"2023-09-23T20:11:41.248811+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":217,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.051107+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:11:41.051107+0000","last_peered":"2023-09-23T20:11:41.051107+0000","last_clean":"2023-09-23T20:11:41.051107+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:11:41.051107+0000","last_undegraded":"2023-09-23T20:11:41.051107+0000","last_fullsized":"2023-09-23T20:11:41.051107+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":334,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.251856+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:11:41.251856+0000","last_peered":"2023-09-23T20:11:41.251856+0000","last_clean":"2023-09-23T20:11:41.251856+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:11:41.251856+0000","last_undegraded":"2023-09-23T20:11:41.251856+0000","last_fullsized":"2023-09-23T20:11:41.251856+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":315,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.251731+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:11:41.251731+0000","last_peered":"2023-09-23T20:11:41.251731+0000","last_clean":"2023-09-23T20:11:41.251731+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:11:41.251731+0000","last_undegraded":"2023-09-23T20:11:41.251731+0000","last_fullsized":"2023-09-23T20:11:41.251731+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":232,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.059297+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:11:41.059297+0000","last_peered":"2023-09-23T20:11:41.059297+0000","last_clean":"2023-09-23T20:11:41.059297+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:11:41.059297+0000","last_undegraded":"2023-09-23T20:11:41.059297+0000","last_fullsized":"2023-09-23T20:11:41.059297+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":698,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.062343+0000","last_change":"2023-09-23T20:09:56.042750+0000","last_active":"2023-09-23T20:11:41.062343+0000","last_peered":"2023-09-23T20:11:41.062343+0000","last_clean":"2023-09-23T20:11:41.062343+0000","last_became_active":"2023-09-23T20:09:56.042455+0000","last_became_peered":"2023-09-23T20:09:56.042455+0000","last_unstale":"2023-09-23T20:11:41.062343+0000","last_undegraded":"2023-09-23T20:11:41.062343+0000","last_fullsized":"2023-09-23T20:11:41.062343+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":388,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049997+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:11:41.049997+0000","last_peered":"2023-09-23T20:11:41.049997+0000","last_clean":"2023-09-23T20:11:41.049997+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:11:41.049997+0000","last_undegraded":"2023-09-23T20:11:41.049997+0000","last_fullsized":"2023-09-23T20:11:41.049997+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":251,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.060387+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:11:41.060387+0000","last_peered":"2023-09-23T20:11:41.060387+0000","last_clean":"2023-09-23T20:11:41.060387+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:11:41.060387+0000","last_undegraded":"2023-09-23T20:11:41.060387+0000","last_fullsized":"2023-09-23T20:11:41.060387+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":82,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.248900+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:11:41.248900+0000","last_peered":"2023-09-23T20:11:41.248900+0000","last_clean":"2023-09-23T20:11:41.248900+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:11:41.248900+0000","last_undegraded":"2023-09-23T20:11:41.248900+0000","last_fullsized":"2023-09-23T20:11:41.248900+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.052647+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:11:41.052647+0000","last_peered":"2023-09-23T20:11:41.052647+0000","last_clean":"2023-09-23T20:11:41.052647+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:11:41.052647+0000","last_undegraded":"2023-09-23T20:11:41.052647+0000","last_fullsized":"2023-09-23T20:11:41.052647+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":443,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067330+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:11:41.067330+0000","last_peered":"2023-09-23T20:11:41.067330+0000","last_clean":"2023-09-23T20:11:41.067330+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:11:41.067330+0000","last_undegraded":"2023-09-23T20:11:41.067330+0000","last_fullsized":"2023-09-23T20:11:41.067330+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":385,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.250083+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:11:41.250083+0000","last_peered":"2023-09-23T20:11:41.250083+0000","last_clean":"2023-09-23T20:11:41.250083+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:11:41.250083+0000","last_undegraded":"2023-09-23T20:11:41.250083+0000","last_fullsized":"2023-09-23T20:11:41.250083+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":718,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049590+0000","last_change":"2023-09-23T20:11:07.067995+0000","last_active":"2023-09-23T20:11:41.049590+0000","last_peered":"2023-09-23T20:11:41.049590+0000","last_clean":"2023-09-23T20:11:41.049590+0000","last_became_active":"2023-09-23T20:09:56.043288+0000","last_became_peered":"2023-09-23T20:09:56.043288+0000","last_unstale":"2023-09-23T20:11:41.049590+0000","last_undegraded":"2023-09-23T20:11:41.049590+0000","last_fullsized":"2023-09-23T20:11:41.049590+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":82,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.057434+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:11:41.057434+0000","last_peered":"2023-09-23T20:11:41.057434+0000","last_clean":"2023-09-23T20:11:41.057434+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:11:41.057434+0000","last_undegraded":"2023-09-23T20:11:41.057434+0000","last_fullsized":"2023-09-23T20:11:41.057434+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":241,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.051541+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:11:41.051541+0000","last_peered":"2023-09-23T20:11:41.051541+0000","last_clean":"2023-09-23T20:11:41.051541+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:11:41.051541+0000","last_undegraded":"2023-09-23T20:11:41.051541+0000","last_fullsized":"2023-09-23T20:11:41.051541+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":538,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:52.408644+0000","last_change":"2023-09-23T20:11:52.408644+0000","last_active":"2023-09-23T20:11:52.408644+0000","last_peered":"2023-09-23T20:11:52.408644+0000","last_clean":"2023-09-23T20:11:52.408644+0000","last_became_active":"2023-09-23T20:09:56.050784+0000","last_became_peered":"2023-09-23T20:09:56.050784+0000","last_unstale":"2023-09-23T20:11:52.408644+0000","last_undegraded":"2023-09-23T20:11:52.408644+0000","last_fullsized":"2023-09-23T20:11:52.408644+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":97,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.057334+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:11:41.057334+0000","last_peered":"2023-09-23T20:11:41.057334+0000","last_clean":"2023-09-23T20:11:41.057334+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:11:41.057334+0000","last_undegraded":"2023-09-23T20:11:41.057334+0000","last_fullsized":"2023-09-23T20:11:41.057334+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":387,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.252602+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:11:41.252602+0000","last_peered":"2023-09-23T20:11:41.252602+0000","last_clean":"2023-09-23T20:11:41.252602+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:11:41.252602+0000","last_undegraded":"2023-09-23T20:11:41.252602+0000","last_fullsized":"2023-09-23T20:11:41.252602+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":514,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.052193+0000","last_change":"2023-09-23T20:09:56.062364+0000","last_active":"2023-09-23T20:11:41.052193+0000","last_peered":"2023-09-23T20:11:41.052193+0000","last_clean":"2023-09-23T20:11:41.052193+0000","last_became_active":"2023-09-23T20:09:56.061865+0000","last_became_peered":"2023-09-23T20:09:56.061865+0000","last_unstale":"2023-09-23T20:11:41.052193+0000","last_undegraded":"2023-09-23T20:11:41.052193+0000","last_fullsized":"2023-09-23T20:11:41.052193+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":76,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.057272+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:11:41.057272+0000","last_peered":"2023-09-23T20:11:41.057272+0000","last_clean":"2023-09-23T20:11:41.057272+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:11:41.057272+0000","last_undegraded":"2023-09-23T20:11:41.057272+0000","last_fullsized":"2023-09-23T20:11:41.057272+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":375,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.064477+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:11:41.064477+0000","last_peered":"2023-09-23T20:11:41.064477+0000","last_clean":"2023-09-23T20:11:41.064477+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:11:41.064477+0000","last_undegraded":"2023-09-23T20:11:41.064477+0000","last_fullsized":"2023-09-23T20:11:41.064477+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":613,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.445494+0000","last_change":"2023-09-23T20:11:50.445473+0000","last_active":"2023-09-23T20:11:50.445494+0000","last_peered":"2023-09-23T20:11:50.445494+0000","last_clean":"2023-09-23T20:11:50.445494+0000","last_became_active":"2023-09-23T20:09:56.043157+0000","last_became_peered":"2023-09-23T20:09:56.043157+0000","last_unstale":"2023-09-23T20:11:50.445494+0000","last_undegraded":"2023-09-23T20:11:50.445494+0000","last_fullsized":"2023-09-23T20:11:50.445494+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":388,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:11:41.049578+0000","last_change":"2023-09-23T20:11:10.067603+0000","last_active":"2023-09-23T20:11:41.049578+0000","last_peered":"2023-09-23T20:11:41.049578+0000","last_clean":"2023-09-23T20:00:51.694506+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:11:41.049578+0000","last_undegraded":"2023-09-23T20:11:10.056843+0000","last_fullsized":"2023-09-23T20:11:41.049578+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":4}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":19,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.062779+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:11:41.062779+0000","last_peered":"2023-09-23T20:11:41.062779+0000","last_clean":"2023-09-23T20:11:41.062779+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:11:41.062779+0000","last_undegraded":"2023-09-23T20:11:41.062779+0000","last_fullsized":"2023-09-23T20:11:41.062779+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":812,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:52.828453+0000","last_change":"2023-09-23T20:11:52.828453+0000","last_active":"2023-09-23T20:11:52.828453+0000","last_peered":"2023-09-23T20:11:52.828453+0000","last_clean":"2023-09-23T20:11:52.828453+0000","last_became_active":"2023-09-23T20:10:52.687280+0000","last_became_peered":"2023-09-23T20:10:52.687280+0000","last_unstale":"2023-09-23T20:11:52.828453+0000","last_undegraded":"2023-09-23T20:11:52.828453+0000","last_fullsized":"2023-09-23T20:11:52.828453+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.4","version":"389'9093","reported_seq":9191,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.261134+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:11:53.261134+0000","last_peered":"2023-09-23T20:11:53.261134+0000","last_clean":"2023-09-23T20:11:53.261134+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:11:53.261134+0000","last_undegraded":"2023-09-23T20:11:53.261134+0000","last_fullsized":"2023-09-23T20:11:53.261134+0000","mapping_epoch":299,"log_start":"309'2765","ondisk_log_start":"309'2765","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107282432,"num_objects":1637,"num_object_clones":0,"num_object_copies":4911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9093,"num_write_kb":343360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":80,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.249174+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:11:41.249174+0000","last_peered":"2023-09-23T20:11:41.249174+0000","last_clean":"2023-09-23T20:11:41.249174+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:11:41.249174+0000","last_undegraded":"2023-09-23T20:11:41.249174+0000","last_fullsized":"2023-09-23T20:11:41.249174+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":397,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049567+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:11:41.049567+0000","last_peered":"2023-09-23T20:11:41.049567+0000","last_clean":"2023-09-23T20:11:41.049567+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:11:41.049567+0000","last_undegraded":"2023-09-23T20:11:41.049567+0000","last_fullsized":"2023-09-23T20:11:41.049567+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":699,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.062082+0000","last_change":"2023-09-23T20:09:56.064782+0000","last_active":"2023-09-23T20:11:41.062082+0000","last_peered":"2023-09-23T20:11:41.062082+0000","last_clean":"2023-09-23T20:11:41.062082+0000","last_became_active":"2023-09-23T20:09:56.064411+0000","last_became_peered":"2023-09-23T20:09:56.064411+0000","last_unstale":"2023-09-23T20:11:41.062082+0000","last_undegraded":"2023-09-23T20:11:41.062082+0000","last_fullsized":"2023-09-23T20:11:41.062082+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218454,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.5","version":"389'8710","reported_seq":11402,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.347909+0000","last_change":"2023-09-23T20:07:45.697314+0000","last_active":"2023-09-23T20:11:53.347909+0000","last_peered":"2023-09-23T20:11:53.347909+0000","last_clean":"2023-09-23T20:11:53.347909+0000","last_became_active":"2023-09-23T20:07:45.690354+0000","last_became_peered":"2023-09-23T20:07:45.690354+0000","last_unstale":"2023-09-23T20:11:53.347909+0000","last_undegraded":"2023-09-23T20:11:53.347909+0000","last_fullsized":"2023-09-23T20:11:53.347909+0000","mapping_epoch":327,"log_start":"309'3018","ondisk_log_start":"309'3018","created":299,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.926631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105119744,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8710,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":81,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.248906+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:11:41.248906+0000","last_peered":"2023-09-23T20:11:41.248906+0000","last_clean":"2023-09-23T20:11:41.248906+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:11:41.248906+0000","last_undegraded":"2023-09-23T20:11:41.248906+0000","last_fullsized":"2023-09-23T20:11:41.248906+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":445,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067197+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:11:41.067197+0000","last_peered":"2023-09-23T20:11:41.067197+0000","last_clean":"2023-09-23T20:11:41.067197+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:11:41.067197+0000","last_undegraded":"2023-09-23T20:11:41.067197+0000","last_fullsized":"2023-09-23T20:11:41.067197+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":657,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.062969+0000","last_change":"2023-09-23T20:11:20.856999+0000","last_active":"2023-09-23T20:11:41.062969+0000","last_peered":"2023-09-23T20:11:41.062969+0000","last_clean":"2023-09-23T20:11:41.062969+0000","last_became_active":"2023-09-23T20:11:20.856640+0000","last_became_peered":"2023-09-23T20:11:20.856640+0000","last_unstale":"2023-09-23T20:11:41.062969+0000","last_undegraded":"2023-09-23T20:11:41.062969+0000","last_fullsized":"2023-09-23T20:11:41.062969+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.6","version":"388'9092","reported_seq":10714,"reported_epoch":388,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:50.530954+0000","last_change":"2023-09-23T20:07:37.064417+0000","last_active":"2023-09-23T20:11:50.530954+0000","last_peered":"2023-09-23T20:11:50.530954+0000","last_clean":"2023-09-23T20:06:59.831272+0000","last_became_active":"2023-09-23T20:07:37.059682+0000","last_became_peered":"2023-09-23T20:07:37.059682+0000","last_unstale":"2023-09-23T20:11:50.530954+0000","last_undegraded":"2023-09-23T20:07:36.983730+0000","last_fullsized":"2023-09-23T20:07:36.974121+0000","mapping_epoch":322,"log_start":"315'3965","ondisk_log_start":"315'3965","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5127,"log_dups_size":0,"ondisk_log_size":5127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:33.575626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112459776,"num_objects":1716,"num_object_clones":0,"num_object_copies":5148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9092,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":325},{"shards":"0,2,3","objects":1022}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":80,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.057554+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:11:41.057554+0000","last_peered":"2023-09-23T20:11:41.057554+0000","last_clean":"2023-09-23T20:11:41.057554+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:11:41.057554+0000","last_undegraded":"2023-09-23T20:11:41.057554+0000","last_fullsized":"2023-09-23T20:11:41.057554+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":94,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:43.731649+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:11:43.731649+0000","last_peered":"2023-09-23T20:11:43.731649+0000","last_clean":"2023-09-23T20:11:43.731649+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:11:43.731649+0000","last_undegraded":"2023-09-23T20:11:43.731649+0000","last_fullsized":"2023-09-23T20:11:43.731649+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":581,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:43.909993+0000","last_change":"2023-09-23T20:11:43.909971+0000","last_active":"2023-09-23T20:11:43.909993+0000","last_peered":"2023-09-23T20:11:43.909993+0000","last_clean":"2023-09-23T20:11:43.909993+0000","last_became_active":"2023-09-23T20:09:56.043491+0000","last_became_peered":"2023-09-23T20:09:56.043491+0000","last_unstale":"2023-09-23T20:11:43.909993+0000","last_undegraded":"2023-09-23T20:11:43.909993+0000","last_fullsized":"2023-09-23T20:11:43.909993+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.7","version":"389'8937","reported_seq":12702,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:53.746829+0000","last_change":"2023-09-23T20:10:58.582053+0000","last_active":"2023-09-23T20:11:53.746829+0000","last_peered":"2023-09-23T20:11:53.746829+0000","last_clean":"2023-09-23T20:07:17.891274+0000","last_became_active":"2023-09-23T20:10:58.299550+0000","last_became_peered":"2023-09-23T20:10:58.299550+0000","last_unstale":"2023-09-23T20:11:53.746829+0000","last_undegraded":"2023-09-23T20:10:56.195777+0000","last_fullsized":"2023-09-23T20:10:53.692217+0000","mapping_epoch":369,"log_start":"314'3429","ondisk_log_start":"314'3429","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5508,"log_dups_size":0,"ondisk_log_size":5508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:48.194019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104923136,"num_objects":1601,"num_object_clones":0,"num_object_copies":4803,"num_objects_missing_on_primary":2258,"num_objects_missing":2258,"num_objects_degraded":2852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8937,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1818,"num_bytes_recovered":119144448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2],"avail_no_missing":["3"],"object_location_counts":[{"shards":"1,2","objects":1181},{"shards":"1,2,3,4","objects":7},{"shards":"2,3,4","objects":413}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":427,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.252231+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:11:41.252231+0000","last_peered":"2023-09-23T20:11:41.252231+0000","last_clean":"2023-09-23T20:11:41.252231+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:11:41.252231+0000","last_undegraded":"2023-09-23T20:11:41.252231+0000","last_fullsized":"2023-09-23T20:11:41.252231+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":85,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:48.741946+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:11:48.741946+0000","last_peered":"2023-09-23T20:11:48.741946+0000","last_clean":"2023-09-23T20:11:48.741946+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:11:48.741946+0000","last_undegraded":"2023-09-23T20:11:48.741946+0000","last_fullsized":"2023-09-23T20:11:48.741946+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"389'8675","reported_seq":16804,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:51.114902+0000","last_change":"2023-09-23T20:10:10.729751+0000","last_active":"2023-09-23T20:11:51.114902+0000","last_peered":"2023-09-23T20:11:51.114902+0000","last_clean":"2023-09-23T20:11:51.114902+0000","last_became_active":"2023-09-23T20:10:10.708263+0000","last_became_peered":"2023-09-23T20:10:10.708263+0000","last_unstale":"2023-09-23T20:11:51.114902+0000","last_undegraded":"2023-09-23T20:11:51.114902+0000","last_fullsized":"2023-09-23T20:11:51.114902+0000","mapping_epoch":361,"log_start":"314'3310","ondisk_log_start":"314'3310","created":299,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5365,"log_dups_size":0,"ondisk_log_size":5365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:15.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107020288,"num_objects":1633,"num_object_clones":0,"num_object_copies":4899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8675,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4003,"num_bytes_recovered":136118272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":771,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.057267+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:11:41.057267+0000","last_peered":"2023-09-23T20:11:41.057267+0000","last_clean":"2023-09-23T20:11:41.057267+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:11:41.057267+0000","last_undegraded":"2023-09-23T20:11:41.057267+0000","last_fullsized":"2023-09-23T20:11:41.057267+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":79,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067406+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:11:41.067406+0000","last_peered":"2023-09-23T20:11:41.067406+0000","last_clean":"2023-09-23T20:11:41.067406+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:11:41.067406+0000","last_undegraded":"2023-09-23T20:11:41.067406+0000","last_fullsized":"2023-09-23T20:11:41.067406+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":415,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067343+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:11:41.067343+0000","last_peered":"2023-09-23T20:11:41.067343+0000","last_clean":"2023-09-23T20:11:41.067343+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:11:41.067343+0000","last_undegraded":"2023-09-23T20:11:41.067343+0000","last_fullsized":"2023-09-23T20:11:41.067343+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"389'8991","reported_seq":15392,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:52.778989+0000","last_change":"2023-09-23T20:10:58.556423+0000","last_active":"2023-09-23T20:11:52.778989+0000","last_peered":"2023-09-23T20:11:52.778989+0000","last_clean":"2023-09-23T20:07:17.847486+0000","last_became_active":"2023-09-23T20:07:20.666556+0000","last_became_peered":"2023-09-23T20:07:20.666556+0000","last_unstale":"2023-09-23T20:11:52.778989+0000","last_undegraded":"2023-09-23T20:10:58.555698+0000","last_fullsized":"2023-09-23T20:07:19.932561+0000","mapping_epoch":317,"log_start":"318'4146","ondisk_log_start":"318'4146","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4845,"log_dups_size":0,"ondisk_log_size":4845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:38.441814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98893824,"num_objects":1509,"num_object_clones":0,"num_object_copies":4527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":996,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8991,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3145,"num_bytes_recovered":206110720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":513},{"shards":"0,2,5","objects":400}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":723,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049650+0000","last_change":"2023-09-23T20:11:11.229144+0000","last_active":"2023-09-23T20:11:41.049650+0000","last_peered":"2023-09-23T20:11:41.049650+0000","last_clean":"2023-09-23T20:11:41.049650+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:11:41.049650+0000","last_undegraded":"2023-09-23T20:11:41.049650+0000","last_fullsized":"2023-09-23T20:11:41.049650+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":355,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067318+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:11:41.067318+0000","last_peered":"2023-09-23T20:11:41.067318+0000","last_clean":"2023-09-23T20:11:41.067318+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:11:41.067318+0000","last_undegraded":"2023-09-23T20:11:41.067318+0000","last_fullsized":"2023-09-23T20:11:41.067318+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":385,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.249676+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:11:41.249676+0000","last_peered":"2023-09-23T20:11:41.249676+0000","last_clean":"2023-09-23T20:11:41.249676+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:11:41.249676+0000","last_undegraded":"2023-09-23T20:11:41.249676+0000","last_fullsized":"2023-09-23T20:11:41.249676+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":514,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.052307+0000","last_change":"2023-09-23T20:09:56.062162+0000","last_active":"2023-09-23T20:11:41.052307+0000","last_peered":"2023-09-23T20:11:41.052307+0000","last_clean":"2023-09-23T20:11:41.052307+0000","last_became_active":"2023-09-23T20:09:56.061518+0000","last_became_peered":"2023-09-23T20:09:56.061518+0000","last_unstale":"2023-09-23T20:11:41.052307+0000","last_undegraded":"2023-09-23T20:11:41.052307+0000","last_fullsized":"2023-09-23T20:11:41.052307+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"389'8935","reported_seq":9033,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.394638+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:11:53.394638+0000","last_peered":"2023-09-23T20:11:53.394638+0000","last_clean":"2023-09-23T20:11:53.394638+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:11:53.394638+0000","last_undegraded":"2023-09-23T20:11:53.394638+0000","last_fullsized":"2023-09-23T20:11:53.394638+0000","mapping_epoch":299,"log_start":"309'3185","ondisk_log_start":"309'3185","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99942400,"num_objects":1525,"num_object_clones":0,"num_object_copies":4575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8935,"num_write_kb":334720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":379,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067061+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:11:41.067061+0000","last_peered":"2023-09-23T20:11:41.067061+0000","last_clean":"2023-09-23T20:11:41.067061+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:11:41.067061+0000","last_undegraded":"2023-09-23T20:11:41.067061+0000","last_fullsized":"2023-09-23T20:11:41.067061+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":415,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.249952+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:11:41.249952+0000","last_peered":"2023-09-23T20:11:41.249952+0000","last_clean":"2023-09-23T20:11:41.249952+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:11:41.249952+0000","last_undegraded":"2023-09-23T20:11:41.249952+0000","last_fullsized":"2023-09-23T20:11:41.249952+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"389'8766","reported_seq":16924,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.736921+0000","last_change":"2023-09-23T20:09:37.763558+0000","last_active":"2023-09-23T20:11:53.736921+0000","last_peered":"2023-09-23T20:11:53.736921+0000","last_clean":"2023-09-23T20:11:53.736921+0000","last_became_active":"2023-09-23T20:09:37.732414+0000","last_became_peered":"2023-09-23T20:09:37.732414+0000","last_unstale":"2023-09-23T20:11:53.736921+0000","last_undegraded":"2023-09-23T20:11:53.736921+0000","last_fullsized":"2023-09-23T20:11:53.736921+0000","mapping_epoch":352,"log_start":"314'3281","ondisk_log_start":"314'3281","created":299,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5485,"log_dups_size":0,"ondisk_log_size":5485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:39.792571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100532224,"num_objects":1534,"num_object_clones":0,"num_object_copies":4602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8766,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4021,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":727,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049932+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:11:41.049932+0000","last_peered":"2023-09-23T20:11:41.049932+0000","last_clean":"2023-09-23T20:11:41.049932+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:11:41.049932+0000","last_undegraded":"2023-09-23T20:11:41.049932+0000","last_fullsized":"2023-09-23T20:11:41.049932+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":19,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.059039+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:11:41.059039+0000","last_peered":"2023-09-23T20:11:41.059039+0000","last_clean":"2023-09-23T20:11:41.059039+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:11:41.059039+0000","last_undegraded":"2023-09-23T20:11:41.059039+0000","last_fullsized":"2023-09-23T20:11:41.059039+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":80,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.802831+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:11:50.802831+0000","last_peered":"2023-09-23T20:11:50.802831+0000","last_clean":"2023-09-23T20:11:50.802831+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:11:50.802831+0000","last_undegraded":"2023-09-23T20:11:50.802831+0000","last_fullsized":"2023-09-23T20:11:50.802831+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"388'8896","reported_seq":8996,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.531826+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:11:50.531826+0000","last_peered":"2023-09-23T20:11:50.531826+0000","last_clean":"2023-09-23T20:11:50.531826+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:11:50.531826+0000","last_undegraded":"2023-09-23T20:11:50.531826+0000","last_fullsized":"2023-09-23T20:11:50.531826+0000","mapping_epoch":299,"log_start":"315'3783","ondisk_log_start":"315'3783","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5113,"log_dups_size":0,"ondisk_log_size":5113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8896,"num_write_kb":337984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":657,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.062863+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:11:41.062863+0000","last_peered":"2023-09-23T20:11:41.062863+0000","last_clean":"2023-09-23T20:11:41.062863+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:11:41.062863+0000","last_undegraded":"2023-09-23T20:11:41.062863+0000","last_fullsized":"2023-09-23T20:11:41.062863+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":522,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.051944+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:11:41.051944+0000","last_peered":"2023-09-23T20:11:41.051944+0000","last_clean":"2023-09-23T20:11:41.051944+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:11:41.051944+0000","last_undegraded":"2023-09-23T20:11:41.051944+0000","last_fullsized":"2023-09-23T20:11:41.051944+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":389,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.252538+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:11:41.252538+0000","last_peered":"2023-09-23T20:11:41.252538+0000","last_clean":"2023-09-23T20:11:41.252538+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:11:41.252538+0000","last_undegraded":"2023-09-23T20:11:41.252538+0000","last_fullsized":"2023-09-23T20:11:41.252538+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.052502+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:11:41.052502+0000","last_peered":"2023-09-23T20:11:41.052502+0000","last_clean":"2023-09-23T20:11:41.052502+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:11:41.052502+0000","last_undegraded":"2023-09-23T20:11:41.052502+0000","last_fullsized":"2023-09-23T20:11:41.052502+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"389'8914","reported_seq":13741,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.230188+0000","last_change":"2023-09-23T20:07:52.783215+0000","last_active":"2023-09-23T20:11:53.230188+0000","last_peered":"2023-09-23T20:11:53.230188+0000","last_clean":"2023-09-23T20:11:53.230188+0000","last_became_active":"2023-09-23T20:07:52.778874+0000","last_became_peered":"2023-09-23T20:07:52.778874+0000","last_unstale":"2023-09-23T20:11:53.230188+0000","last_undegraded":"2023-09-23T20:11:53.230188+0000","last_fullsized":"2023-09-23T20:11:53.230188+0000","mapping_epoch":330,"log_start":"309'2600","ondisk_log_start":"309'2600","created":299,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6314,"log_dups_size":0,"ondisk_log_size":6314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:31.036337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105512960,"num_objects":1610,"num_object_clones":0,"num_object_copies":4830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8914,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2349,"num_bytes_recovered":153944064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":722,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049672+0000","last_change":"2023-09-23T20:11:06.026338+0000","last_active":"2023-09-23T20:11:41.049672+0000","last_peered":"2023-09-23T20:11:41.049672+0000","last_clean":"2023-09-23T20:11:41.049672+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:11:41.049672+0000","last_undegraded":"2023-09-23T20:11:41.049672+0000","last_fullsized":"2023-09-23T20:11:41.049672+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1009,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049769+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:11:41.049769+0000","last_peered":"2023-09-23T20:11:41.049769+0000","last_clean":"2023-09-23T20:11:41.049769+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:11:41.049769+0000","last_undegraded":"2023-09-23T20:11:41.049769+0000","last_fullsized":"2023-09-23T20:11:41.049769+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":408,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.249949+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:11:41.249949+0000","last_peered":"2023-09-23T20:11:41.249949+0000","last_clean":"2023-09-23T20:11:41.249949+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:11:41.249949+0000","last_undegraded":"2023-09-23T20:11:41.249949+0000","last_fullsized":"2023-09-23T20:11:41.249949+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":85,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.051860+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:11:41.051860+0000","last_peered":"2023-09-23T20:11:41.051860+0000","last_clean":"2023-09-23T20:11:41.051860+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:11:41.051860+0000","last_undegraded":"2023-09-23T20:11:41.051860+0000","last_fullsized":"2023-09-23T20:11:41.051860+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"389'8794","reported_seq":21159,"reported_epoch":389,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:11:53.729070+0000","last_change":"2023-09-23T20:11:18.839874+0000","last_active":"2023-09-23T20:11:53.729070+0000","last_peered":"2023-09-23T20:11:53.729070+0000","last_clean":"2023-09-23T20:11:16.510239+0000","last_became_active":"2023-09-23T20:11:18.809696+0000","last_became_peered":"2023-09-23T20:11:18.809696+0000","last_unstale":"2023-09-23T20:11:53.729070+0000","last_undegraded":"2023-09-23T20:11:53.729070+0000","last_fullsized":"2023-09-23T20:11:53.729070+0000","mapping_epoch":381,"log_start":"314'3300","ondisk_log_start":"314'3300","created":299,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5494,"log_dups_size":0,"ondisk_log_size":5494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:58.376473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102891520,"num_objects":1570,"num_object_clones":0,"num_object_copies":4710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1570,"num_objects_unfound":0,"num_objects_dirty":1570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8794,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":217251840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[1,4,2],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":1570}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":611,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:48.488043+0000","last_change":"2023-09-23T20:11:48.488016+0000","last_active":"2023-09-23T20:11:48.488043+0000","last_peered":"2023-09-23T20:11:48.488043+0000","last_clean":"2023-09-23T20:11:48.488043+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:11:48.488043+0000","last_undegraded":"2023-09-23T20:11:48.488043+0000","last_fullsized":"2023-09-23T20:11:48.488043+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1024,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.224169+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:11:32.224169+0000","last_peered":"2023-09-23T20:11:32.224169+0000","last_clean":"2023-09-23T20:11:32.224169+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:11:32.224169+0000","last_undegraded":"2023-09-23T20:11:32.224169+0000","last_fullsized":"2023-09-23T20:11:32.224169+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":85,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.051341+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:11:41.051341+0000","last_peered":"2023-09-23T20:11:41.051341+0000","last_clean":"2023-09-23T20:11:41.051341+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:11:41.051341+0000","last_undegraded":"2023-09-23T20:11:41.051341+0000","last_fullsized":"2023-09-23T20:11:41.051341+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":80,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.055736+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:11:41.055736+0000","last_peered":"2023-09-23T20:11:41.055736+0000","last_clean":"2023-09-23T20:11:41.055736+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:11:41.055736+0000","last_undegraded":"2023-09-23T20:11:41.055736+0000","last_fullsized":"2023-09-23T20:11:41.055736+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"389'8963","reported_seq":9061,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.239385+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:11:53.239385+0000","last_peered":"2023-09-23T20:11:53.239385+0000","last_clean":"2023-09-23T20:11:53.239385+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:11:53.239385+0000","last_undegraded":"2023-09-23T20:11:53.239385+0000","last_fullsized":"2023-09-23T20:11:53.239385+0000","mapping_epoch":299,"log_start":"309'2665","ondisk_log_start":"309'2665","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6298,"log_dups_size":0,"ondisk_log_size":6298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105185280,"num_objects":1605,"num_object_clones":0,"num_object_copies":4815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8963,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1297,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049945+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:11:41.049945+0000","last_peered":"2023-09-23T20:11:41.049945+0000","last_clean":"2023-09-23T20:11:41.049945+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:11:41.049945+0000","last_undegraded":"2023-09-23T20:11:41.049945+0000","last_fullsized":"2023-09-23T20:11:41.049945+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":548,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.362155+0000","last_change":"2023-09-23T20:11:53.362155+0000","last_active":"2023-09-23T20:11:53.362155+0000","last_peered":"2023-09-23T20:11:53.362155+0000","last_clean":"2023-09-23T20:11:53.362155+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:11:53.362155+0000","last_undegraded":"2023-09-23T20:11:53.362155+0000","last_fullsized":"2023-09-23T20:11:53.362155+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":254,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.050037+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:11:41.050037+0000","last_peered":"2023-09-23T20:11:41.050037+0000","last_clean":"2023-09-23T20:11:41.050037+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:11:41.050037+0000","last_undegraded":"2023-09-23T20:11:41.050037+0000","last_fullsized":"2023-09-23T20:11:41.050037+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":284,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.249900+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:11:41.249900+0000","last_peered":"2023-09-23T20:11:41.249900+0000","last_clean":"2023-09-23T20:11:41.249900+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:11:41.249900+0000","last_undegraded":"2023-09-23T20:11:41.249900+0000","last_fullsized":"2023-09-23T20:11:41.249900+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"388'8970","reported_seq":9181,"reported_epoch":388,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:11:50.516826+0000","last_change":"2023-09-23T20:10:57.512712+0000","last_active":"2023-09-23T20:11:50.516826+0000","last_peered":"2023-09-23T20:11:50.516826+0000","last_clean":"2023-09-23T20:07:17.907438+0000","last_became_active":"2023-09-23T20:07:20.693608+0000","last_became_peered":"2023-09-23T20:07:20.693608+0000","last_unstale":"2023-09-23T20:11:50.516826+0000","last_undegraded":"2023-09-23T20:07:19.959595+0000","last_fullsized":"2023-09-23T20:07:19.924202+0000","mapping_epoch":317,"log_start":"315'3788","ondisk_log_start":"315'3788","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5182,"log_dups_size":0,"ondisk_log_size":5182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:21.022797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8970,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":3407872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":528},{"shards":"2,5","objects":-2462}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":515,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.053110+0000","last_change":"2023-09-23T20:09:56.051517+0000","last_active":"2023-09-23T20:11:41.053110+0000","last_peered":"2023-09-23T20:11:41.053110+0000","last_clean":"2023-09-23T20:11:41.053110+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:11:41.053110+0000","last_undegraded":"2023-09-23T20:11:41.053110+0000","last_fullsized":"2023-09-23T20:11:41.053110+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":825,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.057117+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:11:41.057117+0000","last_peered":"2023-09-23T20:11:41.057117+0000","last_clean":"2023-09-23T20:11:41.057117+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:11:41.057117+0000","last_undegraded":"2023-09-23T20:11:41.057117+0000","last_fullsized":"2023-09-23T20:11:41.057117+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":375,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067210+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:11:41.067210+0000","last_peered":"2023-09-23T20:11:41.067210+0000","last_clean":"2023-09-23T20:11:41.067210+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:11:41.067210+0000","last_undegraded":"2023-09-23T20:11:41.067210+0000","last_fullsized":"2023-09-23T20:11:41.067210+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":85,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.052726+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:11:41.052726+0000","last_peered":"2023-09-23T20:11:41.052726+0000","last_clean":"2023-09-23T20:11:41.052726+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:11:41.052726+0000","last_undegraded":"2023-09-23T20:11:41.052726+0000","last_fullsized":"2023-09-23T20:11:41.052726+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"388'8984","reported_seq":15441,"reported_epoch":388,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:11:50.512266+0000","last_change":"2023-09-23T20:10:58.722697+0000","last_active":"2023-09-23T20:11:50.512266+0000","last_peered":"2023-09-23T20:11:50.512266+0000","last_clean":"2023-09-23T20:07:17.917130+0000","last_became_active":"2023-09-23T20:10:56.953707+0000","last_became_peered":"2023-09-23T20:10:56.953707+0000","last_unstale":"2023-09-23T20:11:50.512266+0000","last_undegraded":"2023-09-23T20:11:50.512266+0000","last_fullsized":"2023-09-23T20:10:53.691838+0000","mapping_epoch":369,"log_start":"315'3846","ondisk_log_start":"315'3846","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5138,"log_dups_size":0,"ondisk_log_size":5138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:08.588715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111149056,"num_objects":1696,"num_object_clones":0,"num_object_copies":5088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":514,"num_objects_unfound":0,"num_objects_dirty":1696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8984,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4056,"num_bytes_recovered":227606528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,1],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"1,2","objects":1182},{"shards":"2,3,4","objects":514}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":585,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.914323+0000","last_change":"2023-09-23T20:11:41.914302+0000","last_active":"2023-09-23T20:11:41.914323+0000","last_peered":"2023-09-23T20:11:41.914323+0000","last_clean":"2023-09-23T20:11:41.914323+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:11:41.914323+0000","last_undegraded":"2023-09-23T20:11:41.914323+0000","last_fullsized":"2023-09-23T20:11:41.914323+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":89,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.057565+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:11:41.057565+0000","last_peered":"2023-09-23T20:11:41.057565+0000","last_clean":"2023-09-23T20:11:41.057565+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:11:41.057565+0000","last_undegraded":"2023-09-23T20:11:41.057565+0000","last_fullsized":"2023-09-23T20:11:41.057565+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":85,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.051787+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:11:41.051787+0000","last_peered":"2023-09-23T20:11:41.051787+0000","last_clean":"2023-09-23T20:11:41.051787+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:11:41.051787+0000","last_undegraded":"2023-09-23T20:11:41.051787+0000","last_fullsized":"2023-09-23T20:11:41.051787+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":215,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.049839+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:11:41.049839+0000","last_peered":"2023-09-23T20:11:41.049839+0000","last_clean":"2023-09-23T20:11:41.049839+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:11:41.049839+0000","last_undegraded":"2023-09-23T20:11:41.049839+0000","last_fullsized":"2023-09-23T20:11:41.049839+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"389'9013","reported_seq":13309,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.576673+0000","last_change":"2023-09-23T20:08:33.396812+0000","last_active":"2023-09-23T20:11:53.576673+0000","last_peered":"2023-09-23T20:11:53.576673+0000","last_clean":"2023-09-23T20:11:53.576673+0000","last_became_active":"2023-09-23T20:08:05.774750+0000","last_became_peered":"2023-09-23T20:08:05.774750+0000","last_unstale":"2023-09-23T20:11:53.576673+0000","last_undegraded":"2023-09-23T20:11:53.576673+0000","last_fullsized":"2023-09-23T20:11:53.576673+0000","mapping_epoch":334,"log_start":"307'2100","ondisk_log_start":"307'2100","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6913,"log_dups_size":0,"ondisk_log_size":6913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.633506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107020288,"num_objects":1633,"num_object_clones":0,"num_object_copies":4899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9013,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2086,"num_bytes_recovered":136708096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":813,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:45.860655+0000","last_change":"2023-09-23T20:11:45.860623+0000","last_active":"2023-09-23T20:11:45.860655+0000","last_peered":"2023-09-23T20:11:45.860655+0000","last_clean":"2023-09-23T20:11:45.860655+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:11:45.860655+0000","last_undegraded":"2023-09-23T20:11:45.860655+0000","last_fullsized":"2023-09-23T20:11:45.860655+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":231,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.250096+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:11:41.250096+0000","last_peered":"2023-09-23T20:11:41.250096+0000","last_clean":"2023-09-23T20:11:41.250096+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:11:41.250096+0000","last_undegraded":"2023-09-23T20:11:41.250096+0000","last_fullsized":"2023-09-23T20:11:41.250096+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":426,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.052651+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:11:41.052651+0000","last_peered":"2023-09-23T20:11:41.052651+0000","last_clean":"2023-09-23T20:11:41.052651+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:11:41.052651+0000","last_undegraded":"2023-09-23T20:11:41.052651+0000","last_fullsized":"2023-09-23T20:11:41.052651+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.058795+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:11:41.058795+0000","last_peered":"2023-09-23T20:11:41.058795+0000","last_clean":"2023-09-23T20:11:41.058795+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:11:41.058795+0000","last_undegraded":"2023-09-23T20:11:41.058795+0000","last_fullsized":"2023-09-23T20:11:41.058795+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"389'8947","reported_seq":11714,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.261227+0000","last_change":"2023-09-23T20:09:08.683160+0000","last_active":"2023-09-23T20:11:53.261227+0000","last_peered":"2023-09-23T20:11:53.261227+0000","last_clean":"2023-09-23T20:11:53.261227+0000","last_became_active":"2023-09-23T20:09:08.670527+0000","last_became_peered":"2023-09-23T20:09:08.670527+0000","last_unstale":"2023-09-23T20:11:53.261227+0000","last_undegraded":"2023-09-23T20:11:53.261227+0000","last_fullsized":"2023-09-23T20:11:53.261227+0000","mapping_epoch":346,"log_start":"309'2600","ondisk_log_start":"309'2600","created":299,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6347,"log_dups_size":0,"ondisk_log_size":6347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:47.531381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103284760,"num_objects":1577,"num_object_clones":0,"num_object_copies":4731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1577,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8947,"num_write_kb":336705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":699,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.062618+0000","last_change":"2023-09-23T20:09:56.036037+0000","last_active":"2023-09-23T20:11:41.062618+0000","last_peered":"2023-09-23T20:11:41.062618+0000","last_clean":"2023-09-23T20:11:41.062618+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:11:41.062618+0000","last_undegraded":"2023-09-23T20:11:41.062618+0000","last_fullsized":"2023-09-23T20:11:41.062618+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":90,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.249087+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:11:41.249087+0000","last_peered":"2023-09-23T20:11:41.249087+0000","last_clean":"2023-09-23T20:11:41.249087+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:11:41.249087+0000","last_undegraded":"2023-09-23T20:11:41.249087+0000","last_fullsized":"2023-09-23T20:11:41.249087+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":89,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.052766+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:11:41.052766+0000","last_peered":"2023-09-23T20:11:41.052766+0000","last_clean":"2023-09-23T20:11:41.052766+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:11:41.052766+0000","last_undegraded":"2023-09-23T20:11:41.052766+0000","last_fullsized":"2023-09-23T20:11:41.052766+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":348,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067112+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:11:41.067112+0000","last_peered":"2023-09-23T20:11:41.067112+0000","last_clean":"2023-09-23T20:11:41.067112+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:11:41.067112+0000","last_undegraded":"2023-09-23T20:11:41.067112+0000","last_fullsized":"2023-09-23T20:11:41.067112+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":163,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.224614+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:11:32.224614+0000","last_peered":"2023-09-23T20:11:32.224614+0000","last_clean":"2023-09-23T20:11:32.224614+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:11:32.224614+0000","last_undegraded":"2023-09-23T20:11:32.224614+0000","last_fullsized":"2023-09-23T20:11:32.224614+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.228052+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:11:32.228052+0000","last_peered":"2023-09-23T20:11:32.228052+0000","last_clean":"2023-09-23T20:11:32.228052+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:11:32.228052+0000","last_undegraded":"2023-09-23T20:11:32.228052+0000","last_fullsized":"2023-09-23T20:11:32.228052+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":392,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.252075+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:11:41.252075+0000","last_peered":"2023-09-23T20:11:41.252075+0000","last_clean":"2023-09-23T20:11:41.252075+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:11:41.252075+0000","last_undegraded":"2023-09-23T20:11:41.252075+0000","last_fullsized":"2023-09-23T20:11:41.252075+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":405,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.250027+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:11:41.250027+0000","last_peered":"2023-09-23T20:11:41.250027+0000","last_clean":"2023-09-23T20:11:41.250027+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:11:41.250027+0000","last_undegraded":"2023-09-23T20:11:41.250027+0000","last_fullsized":"2023-09-23T20:11:41.250027+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":382,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.252468+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:11:41.252468+0000","last_peered":"2023-09-23T20:11:41.252468+0000","last_clean":"2023-09-23T20:11:41.252468+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:11:41.252468+0000","last_undegraded":"2023-09-23T20:11:41.252468+0000","last_fullsized":"2023-09-23T20:11:41.252468+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":385,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067279+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:11:41.067279+0000","last_peered":"2023-09-23T20:11:41.067279+0000","last_clean":"2023-09-23T20:11:41.067279+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:11:41.067279+0000","last_undegraded":"2023-09-23T20:11:41.067279+0000","last_fullsized":"2023-09-23T20:11:41.067279+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":88,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.226590+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:11:32.226590+0000","last_peered":"2023-09-23T20:11:32.226590+0000","last_clean":"2023-09-23T20:11:32.226590+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:11:32.226590+0000","last_undegraded":"2023-09-23T20:11:32.226590+0000","last_fullsized":"2023-09-23T20:11:32.226590+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":245,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T20:11:32.224195+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:11:32.224195+0000","last_peered":"2023-09-23T20:11:32.224195+0000","last_clean":"2023-09-23T20:11:32.224195+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:11:32.224195+0000","last_undegraded":"2023-09-23T20:11:32.224195+0000","last_fullsized":"2023-09-23T20:11:32.224195+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":234,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.251939+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:11:41.251939+0000","last_peered":"2023-09-23T20:11:41.251939+0000","last_clean":"2023-09-23T20:11:41.251939+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:11:41.251939+0000","last_undegraded":"2023-09-23T20:11:41.251939+0000","last_fullsized":"2023-09-23T20:11:41.251939+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":435,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T20:11:41.067075+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:11:41.067075+0000","last_peered":"2023-09-23T20:11:41.067075+0000","last_clean":"2023-09-23T20:11:41.067075+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:11:41.067075+0000","last_undegraded":"2023-09-23T20:11:41.067075+0000","last_fullsized":"2023-09-23T20:11:41.067075+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":26174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1689583640,"num_objects":25782,"num_object_clones":0,"num_object_copies":77346,"num_objects_missing_on_primary":2258,"num_objects_missing":2258,"num_objects_degraded":9367,"num_objects_misplaced":2084,"num_objects_unfound":0,"num_objects_dirty":25782,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142680,"num_write_kb":5390721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30059,"num_bytes_recovered":1557331968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6578647040,"data_stored":6578634824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90899,"ondisk_log_size":90899,"up":48,"acting":43,"num_store_stats":7},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":542359,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":177,"num_read_kb":331,"num_write":87,"num_write_kb":878,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1299398,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234369,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":794944,"kb_used_data":463928,"kb_used_omap":0,"kb_used_meta":331008,"kb_avail":92950208,"statfs":{"total":95995035648,"available":95181012992,"internally_reserved":0,"allocated":475062272,"data_stored":473812490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":338952192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.0449999999999999,"15min":2.2400000000000002},"min":{"1min":0.23499999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.5030000000000001,"5min":55.390000000000001,"15min":62.061999999999998},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.9870000000000001,"15min":2.3780000000000001},"min":{"1min":0.248,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":3.726,"5min":56.664000000000001,"15min":62.389000000000003},"last":0.34499999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.907,"15min":3.4039999999999999},"min":{"1min":0.29299999999999998,"5min":0.28299999999999997,"15min":0.22700000000000001},"max":{"1min":3.7080000000000002,"5min":54.945999999999998,"15min":104.16800000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.84299999999999997,"5min":2,"15min":3.4929999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.223},"max":{"1min":2.9390000000000001,"5min":55.094000000000001,"15min":104.003},"last":0.59599999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":4.0800000000000001,"15min":3.6440000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":28.707000000000001,"5min":92.319000000000003,"15min":105.842},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.8919999999999999,"5min":4.4699999999999998,"15min":4.1390000000000002},"min":{"1min":0.34300000000000003,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":28.596,"5min":92.049000000000007,"15min":105.66},"last":0.40899999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":3.1070000000000002,"15min":4.3840000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":4.0359999999999996,"5min":94.953000000000003,"15min":146.38800000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.159,"5min":3.0390000000000001,"15min":4.4020000000000001},"min":{"1min":0.41399999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.6949999999999998,"5min":93.423000000000002,"15min":146.851},"last":0.79800000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.53500000000000003,"15min":1.3089999999999999},"min":{"1min":0.26400000000000001,"5min":0.25900000000000001,"15min":0.22500000000000001},"max":{"1min":1.048,"5min":5.431,"15min":143.255},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.501,"5min":0.60199999999999998,"15min":1.3480000000000001},"min":{"1min":0.29899999999999999,"5min":0.246,"15min":0.246},"max":{"1min":1.1020000000000001,"5min":4.1459999999999999,"15min":143.41200000000001},"last":0.55000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.94699999999999995,"15min":1.056},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":0.60999999999999999,"5min":24.876000000000001,"15min":35.488},"last":0.38},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.874,"15min":0.879},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":0.98699999999999999,"5min":21.623000000000001,"15min":34.896000000000001},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.625,"15min":0.80800000000000005},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":1.4650000000000001,"5min":4.3710000000000004,"15min":41.902999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.63200000000000001,"15min":0.64200000000000002},"min":{"1min":0.38400000000000001,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":0.85599999999999998,"5min":3.9889999999999999,"15min":6.3620000000000001},"last":0.53200000000000003}]}]},{"osd":6,"up_from":0,"seq":1078036791410,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762568,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1157324,"kb_used_data":603524,"kb_used_omap":7,"kb_used_meta":553784,"kb_avail":92587828,"statfs":{"total":95995035648,"available":94809935872,"internally_reserved":0,"allocated":618008576,"data_stored":616165017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":567075642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.1339999999999999,"15min":3.698},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.151},"max":{"1min":9.9410000000000007,"5min":28.510000000000002,"15min":186.84},"last":2.9820000000000002},{"interface":"front","average":{"1min":1.776,"5min":2.4209999999999998,"15min":3.9929999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":8.6660000000000004,"5min":14.781000000000001,"15min":191.322},"last":0.42199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":4.6180000000000003,"15min":5.508},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":17.785,"5min":98.531000000000006,"15min":180.25200000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.2589999999999999,"5min":4.2859999999999996,"15min":5.548},"min":{"1min":0.23300000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":28.469999999999999,"5min":98.346000000000004,"15min":180.05000000000001},"last":9.8279999999999994}]},{"osd":3,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.5459999999999998,"15min":3.7879999999999998},"min":{"1min":0.25700000000000001,"5min":0.255,"15min":0.22},"max":{"1min":3.9849999999999999,"5min":25.963999999999999,"15min":68.180999999999997},"last":0.62},{"interface":"front","average":{"1min":1.4019999999999999,"5min":2.4430000000000001,"15min":3.6909999999999998},"min":{"1min":0.29299999999999998,"5min":0.28199999999999997,"15min":0.24299999999999999},"max":{"1min":6.4320000000000004,"5min":40.716000000000001,"15min":66.509},"last":0.45600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.6140000000000001,"15min":1.9430000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23599999999999999},"max":{"1min":6.6029999999999998,"5min":18.087,"15min":60.951000000000001},"last":1.9870000000000001},{"interface":"front","average":{"1min":1.573,"5min":1.764,"15min":2.1190000000000002},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":8.1669999999999998,"5min":14.890000000000001,"15min":60.841000000000001},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.8300000000000001,"15min":2.952},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.25900000000000001},"max":{"1min":6.5679999999999996,"5min":14.83,"15min":219.048},"last":0.496},{"interface":"front","average":{"1min":1.3440000000000001,"5min":2.1819999999999999,"15min":3.0950000000000002},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28399999999999997},"max":{"1min":6.0190000000000001,"5min":25.893000000000001,"15min":218.69499999999999},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:11:43 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.7649999999999999,"15min":1.994},"min":{"1min":0.38800000000000001,"5min":0.38500000000000001,"15min":0.35899999999999999},"max":{"1min":6.6749999999999998,"5min":18.117999999999999,"15min":18.117999999999999},"last":2.0550000000000002},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.8089999999999999,"15min":2.6579999999999999},"min":{"1min":0.42299999999999999,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":6.5069999999999997,"5min":14.862,"15min":20.734999999999999},"last":0.69199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":2.1869999999999998,"15min":2.3239999999999998},"min":{"1min":0.42999999999999999,"5min":0.33300000000000002,"15min":0.26800000000000002},"max":{"1min":4.5899999999999999,"5min":25.937000000000001,"15min":60.213000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.556,"5min":1.7749999999999999,"15min":2.0619999999999998},"min":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.30199999999999999},"max":{"1min":6.649,"5min":18.053999999999998,"15min":61.298999999999999},"last":2.0270000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926105,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1500444,"kb_used_data":878164,"kb_used_omap":8,"kb_used_meta":622263,"kb_avail":92244708,"statfs":{"total":95995035648,"available":94458580992,"internally_reserved":0,"allocated":899239936,"data_stored":897384521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":637197921},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":2.855,"15min":3.907},"min":{"1min":0.217,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":4.3179999999999996,"5min":63.639000000000003,"15min":94.055999999999997},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.70199999999999996,"5min":2.9140000000000001,"15min":4.2000000000000002},"min":{"1min":0.23200000000000001,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":1.6639999999999999,"5min":62.802,"15min":91.632999999999996},"last":1.657}]},{"osd":2,"last update":"Sat Sep 23 20:11:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":3.5880000000000001,"15min":5.0199999999999996},"min":{"1min":0.255,"5min":0.217,"15min":0.19800000000000001},"max":{"1min":9.0700000000000003,"5min":102.65300000000001,"15min":270.51100000000002},"last":0.20999999999999999},{"interface":"front","average":{"1min":2.0139999999999998,"5min":3.6150000000000002,"15min":4.8810000000000002},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.2599999999999998,"5min":96.578999999999994,"15min":269.69299999999998},"last":0.31900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":4.5670000000000002,"15min":5.9219999999999997},"min":{"1min":0.27700000000000002,"5min":0.252,"15min":0.21299999999999999},"max":{"1min":1.6120000000000001,"5min":126.95399999999999,"15min":145.72800000000001},"last":3.9510000000000001},{"interface":"front","average":{"1min":1.4590000000000001,"5min":4.9480000000000004,"15min":6.0919999999999996},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":9.3439999999999994,"5min":127.294,"15min":145.50700000000001},"last":1.76}]},{"osd":4,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.5629999999999999,"15min":2.2629999999999999},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":6.1369999999999996,"5min":15.688000000000001,"15min":148.20599999999999},"last":1.415},{"interface":"front","average":{"1min":1.099,"5min":1.4039999999999999,"15min":2.0379999999999998},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":5.3940000000000001,"5min":13.428000000000001,"15min":148.11000000000001},"last":1.712}]},{"osd":5,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.298,"15min":2.464},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":1.1459999999999999,"5min":9.5719999999999992,"15min":131.34399999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.163,"5min":1.718,"15min":2.468},"min":{"1min":0.377,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.1059999999999999,"5min":15.717000000000001,"15min":131.386},"last":1.462}]},{"osd":6,"last update":"Sat Sep 23 20:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.758,"15min":1.6779999999999999},"min":{"1min":0.433,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":6.1660000000000004,"5min":15.744999999999999,"15min":15.744999999999999},"last":1.4379999999999999},{"interface":"front","average":{"1min":0.79800000000000004,"5min":1.579,"15min":1.873},"min":{"1min":0.40400000000000003,"5min":0.33900000000000002,"15min":0.309},"max":{"1min":1.6160000000000001,"5min":13.470000000000001,"15min":13.766999999999999},"last":1.6890000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.327,"15min":1.891},"min":{"1min":0.38900000000000001,"5min":0.38100000000000001,"15min":0.31900000000000001},"max":{"1min":1.0980000000000001,"5min":9.6069999999999993,"15min":65.081999999999994},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.724,"15min":2.0190000000000001},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.1920000000000002,"5min":15.771000000000001,"15min":15.771000000000001},"last":1.3799999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599043,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1838552,"kb_used_data":1117520,"kb_used_omap":12,"kb_used_meta":721011,"kb_avail":91906600,"statfs":{"total":95995035648,"available":94112358400,"internally_reserved":0,"allocated":1144340480,"data_stored":1142481676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":738316286},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.9119999999999999,"15min":3.1739999999999999},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.154},"max":{"1min":3.714,"5min":17.876999999999999,"15min":76.251999999999995},"last":14.794},{"interface":"front","average":{"1min":1.323,"5min":2.4609999999999999,"15min":3.3149999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.192},"max":{"1min":10.808999999999999,"5min":18.561,"15min":76.590000000000003},"last":2.714}]},{"osd":1,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":2.3159999999999998,"15min":3.6699999999999999},"min":{"1min":0.24199999999999999,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":2.4729999999999999,"5min":19.562999999999999,"15min":94.768000000000001},"last":13.574},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.9079999999999999,"15min":3.6589999999999998},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":3.6320000000000001,"5min":19.314,"15min":96.808000000000007},"last":15.156000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.5169999999999999,"15min":4.4029999999999996},"min":{"1min":0.32800000000000001,"5min":0.25,"15min":0.214},"max":{"1min":8.5850000000000009,"5min":13.513,"15min":107.381},"last":13.351000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":2.097,"15min":3.7610000000000001},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.23699999999999999},"max":{"1min":4.7919999999999998,"5min":20.308,"15min":106.301},"last":14.778}]},{"osd":4,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.609,"15min":2.2170000000000001},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.26900000000000002},"max":{"1min":3.742,"5min":75.608000000000004,"15min":75.608000000000004},"last":14.757999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.7839999999999998,"15min":2.2330000000000001},"min":{"1min":0.34599999999999997,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":2.544,"5min":76.161000000000001,"15min":76.161000000000001},"last":13.387}]},{"osd":5,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.1549999999999998,"15min":2.726},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.28100000000000003},"max":{"1min":3.77,"5min":65.921999999999997,"15min":172.505},"last":14.728999999999999},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.5489999999999999,"15min":2.8959999999999999},"min":{"1min":0.39500000000000002,"5min":0.38100000000000001,"15min":0.29099999999999998},"max":{"1min":10.696,"5min":63.792000000000002,"15min":172.375},"last":0.59399999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.5900000000000001,"15min":1.601},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":4.718,"5min":10.209,"15min":10.209},"last":14.664},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.8220000000000001,"15min":1.909},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.776,"5min":14.279999999999999,"15min":14.279999999999999},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.6599999999999999,"15min":1.9379999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.28799999999999998},"max":{"1min":2.5699999999999998,"5min":8.1519999999999992,"15min":50.118000000000002},"last":13.481999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.619,"15min":2.0419999999999998},"min":{"1min":0.41699999999999998,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":3.794,"5min":10.244,"15min":50.664999999999999},"last":14.698}]}]},{"osd":3,"up_from":109,"seq":468151435517,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1895168,"kb_used_data":1238968,"kb_used_omap":4,"kb_used_meta":656187,"kb_avail":91849984,"statfs":{"total":95995035648,"available":94054383616,"internally_reserved":0,"allocated":1268703232,"data_stored":1266684390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":671935597},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":4.2430000000000003,"15min":3.4049999999999998},"min":{"1min":0.19700000000000001,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":7.6440000000000001,"5min":74.323999999999998,"15min":74.323999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":4.0170000000000003,"15min":3.758},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":4.9000000000000004,"5min":74.447000000000003,"15min":74.447000000000003},"last":0.55000000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":8.7829999999999995,"5min":4.7240000000000002,"15min":4.0599999999999996},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":163.49700000000001,"5min":163.49700000000001,"15min":163.49700000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":8.5839999999999996,"5min":5.2400000000000002,"15min":4.351},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":163.512,"5min":163.512,"15min":163.512},"last":0.88900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":12.541,"5min":6.2560000000000002,"15min":4.5679999999999996},"min":{"1min":0.33800000000000002,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":163.83500000000001,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":11.702,"5min":6.319,"15min":4.4500000000000002},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":163.92099999999999,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.68700000000000006}]},{"osd":4,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":2.4489999999999998,"15min":3.1349999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.3740000000000001,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.7669999999999999,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":5.4820000000000002,"5min":58.408000000000001,"15min":110.295},"last":0.622}]},{"osd":5,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":3.452,"15min":3.3799999999999999},"min":{"1min":0.316,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.5270000000000001,"5min":77.206000000000003,"15min":113.881},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.80400000000000005,"5min":3.7250000000000001,"15min":3.2650000000000001},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":2.2930000000000001,"5min":77.968000000000004,"15min":114.598},"last":0.71899999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.629,"15min":3.8999999999999999},"min":{"1min":0.41599999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":2.2759999999999998,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.8439999999999999,"15min":4.0839999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":3.9279999999999999,"5min":58.701000000000001,"15min":82.233000000000004},"last":0.84299999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.8210000000000002,"15min":3.4740000000000002},"min":{"1min":0.40000000000000002,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":2.2559999999999998,"5min":58.686,"15min":138.035},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.8679999999999999,"15min":3.4510000000000001},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.28699999999999998},"max":{"1min":3.4260000000000002,"5min":58.667999999999999,"15min":138.054},"last":4.0030000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019279,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1464248,"kb_used_data":1119344,"kb_used_omap":0,"kb_used_meta":344832,"kb_avail":92280904,"statfs":{"total":95995035648,"available":94495645696,"internally_reserved":0,"allocated":1146208256,"data_stored":1144966630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":353107968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.54,"15min":2.5699999999999998},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.189},"max":{"1min":3.2000000000000002,"5min":56.061999999999998,"15min":104.604},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.5409999999999999,"15min":2.5339999999999998},"min":{"1min":0.28899999999999998,"5min":0.218,"15min":0.218},"max":{"1min":3.1899999999999999,"5min":56.075000000000003,"15min":104.687},"last":0.90200000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":2.9489999999999998,"15min":5.5389999999999997},"min":{"1min":0.252,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":4.1669999999999998,"5min":119.892,"15min":161.613},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":3.2109999999999999,"15min":5.6139999999999999},"min":{"1min":0.28899999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.125,"5min":119.351,"15min":162.017},"last":4.3959999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":6.6200000000000001,"15min":6.75},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.245},"max":{"1min":17.948,"5min":182.96700000000001,"15min":182.96700000000001},"last":2.3500000000000001},{"interface":"front","average":{"1min":1.349,"5min":6.2729999999999997,"15min":6.5039999999999996},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":17.978000000000002,"5min":183.244,"15min":183.244},"last":1.0680000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":4.843,"15min":6.8440000000000003},"min":{"1min":0.30499999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":2.5259999999999998,"5min":184.37100000000001,"15min":266.54700000000003},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.95299999999999996,"5min":4.9130000000000003,"15min":5.798},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":5.407,"5min":184.22300000000001,"15min":184.22300000000001},"last":0.98499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":2.1640000000000001,"15min":2.1160000000000001},"min":{"1min":0.25800000000000001,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":0.622,"5min":176.81899999999999,"15min":266.59899999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":2.2149999999999999,"15min":2.1880000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":0.56599999999999995,"5min":176.172,"15min":266.72899999999998},"last":0.48999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.502,"15min":0.51000000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.804,"5min":2.5489999999999999,"15min":4.0410000000000004},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60799999999999998,"15min":0.58899999999999997},"min":{"1min":0.38300000000000001,"5min":0.247,"15min":0.247},"max":{"1min":1.5880000000000001,"5min":4.2889999999999997,"15min":4.2889999999999997},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.337,"15min":1.446},"min":{"1min":0.36799999999999999,"5min":0.32000000000000001,"15min":0.26100000000000001},"max":{"1min":0.71699999999999997,"5min":74.040000000000006,"15min":76.507000000000005},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.5,"5min":1.3049999999999999,"15min":1.5629999999999999},"min":{"1min":0.36399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.61099999999999999,"5min":74.021000000000001,"15min":76.462000000000003},"last":0.505}]}]},{"osd":5,"up_from":38,"seq":163208757643,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1476980,"kb_used_data":1084972,"kb_used_omap":0,"kb_used_meta":392000,"kb_avail":92268172,"statfs":{"total":95995035648,"available":94482608128,"internally_reserved":0,"allocated":1111011328,"data_stored":1109770071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":401408000},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.88700000000000001,"15min":1.881},"min":{"1min":0.23000000000000001,"5min":0.224,"15min":0.216},"max":{"1min":0.625,"5min":8.6739999999999995,"15min":121.94799999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.436,"5min":1.1399999999999999,"15min":2.0750000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":1.28,"5min":10.327,"15min":121.964},"last":0.375}]},{"osd":1,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":2.093,"15min":5.8949999999999996},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":3.52,"5min":75.204999999999998,"15min":266.197},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.1480000000000001,"15min":6.0700000000000003},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1399999999999997,"5min":75.231999999999999,"15min":266.14999999999998},"last":1.6599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:10:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":5.6289999999999996,"15min":6.1059999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":25.227,"5min":240.57300000000001,"15min":240.57300000000001},"last":1.875},{"interface":"front","average":{"1min":2.415,"5min":5.6699999999999999,"15min":6.1520000000000001},"min":{"1min":0.39800000000000002,"5min":0.38600000000000001,"15min":0.313},"max":{"1min":25.286999999999999,"5min":240.464,"15min":240.464},"last":1.903}]},{"osd":3,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.7030000000000001,"15min":4.8390000000000004},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.742,"5min":14.808,"15min":201.50899999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.407,"15min":4.6959999999999997},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":3.4500000000000002,"5min":14.893000000000001,"15min":201.58099999999999},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 20:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.71199999999999997,"15min":1.4490000000000001},"min":{"1min":0.27700000000000002,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":0.58799999999999997,"5min":9.7240000000000002,"15min":191.78899999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.59299999999999997,"15min":1.3360000000000001},"min":{"1min":0.311,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":1.3879999999999999,"5min":8.7360000000000007,"15min":191.821},"last":0.34899999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.69299999999999995,"15min":0.64100000000000001},"min":{"1min":0.377,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":3.278,"5min":9.9009999999999998,"15min":9.9009999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.54600000000000004,"15min":0.51700000000000002},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.72799999999999998,"5min":5.0119999999999996,"15min":5.0119999999999996},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.59599999999999997,"15min":1.5329999999999999},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":0.57399999999999995,"5min":5.2670000000000003,"15min":191.84299999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":1.7549999999999999},"min":{"1min":0.41699999999999998,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":0.78600000000000003,"5min":9.6920000000000002,"15min":191.69},"last":0.55600000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":887947264,"data_stored":887947264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":608043008,"data_stored":608043008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1134759936,"data_stored":1134755864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1251606528,"data_stored":1251606528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1131282432,"data_stored":1131282432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1099763712,"data_stored":1099759640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":465244160,"data_stored":465240088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:11:56.876 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:56.934 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:57.029 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:57.286 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:57.292 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:57 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:11:57.293 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:57 smithi135 ceph-mon[213871]: 1.12 scrub starts 2023-09-23T20:11:57.293 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:57 smithi135 ceph-mon[213871]: pgmap v1092: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 9371/75111 objects degraded (12.476%); 1935/75111 objects misplaced (2.576%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:11:57.293 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:57 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:11:57.293 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:57 smithi135 ceph-mon[221113]: 1.12 scrub starts 2023-09-23T20:11:57.294 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:57 smithi135 ceph-mon[221113]: pgmap v1092: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 9371/75111 objects degraded (12.476%); 1935/75111 objects misplaced (2.576%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:11:57.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:57.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:57.395 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:57.459 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:57 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:11:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:57 smithi144 ceph-mon[194833]: 1.12 scrub starts 2023-09-23T20:11:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:57 smithi144 ceph-mon[194833]: pgmap v1092: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 9371/75111 objects degraded (12.476%); 1935/75111 objects misplaced (2.576%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:11:57.653 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:11:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:57.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:11:57.758 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:11:57.782 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:57.838 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:58.050 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:11:58.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:58.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:11:58.155 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:11:58.186 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:58.194 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:58 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9371/75111 objects degraded (12.476%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:58.194 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:58.195 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:58.245 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:11:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:58.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:11:58.353 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:58.382 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:58.431 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:58.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:58 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9371/75111 objects degraded (12.476%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:58.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:58.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:58.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:58.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:58.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:58 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9371/75111 objects degraded (12.476%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:11:58.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:58.539 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:58.568 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:58.613 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:58.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:58.718 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:11:58.745 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:58.784 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:11:58.800 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:11:58.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:58.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:11:58.906 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:58.932 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:58.978 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:58.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:58.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:59.083 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:11:59.112 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:59.162 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:11:59.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:59.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:11:59.269 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:11:59.299 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:59.350 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:11:59.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:11:59.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:11:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:59.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:11:59 smithi144 ceph-mon[194833]: pgmap v1093: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 9220/69888 objects degraded (13.193%); 1763/69888 objects misplaced (2.523%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:11:59.440 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:59.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:59.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:59.441 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:11:59 smithi135 ceph-mon[213871]: pgmap v1093: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 9220/69888 objects degraded (13.193%); 1763/69888 objects misplaced (2.523%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:11:59.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:59.442 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:59.442 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:11:59.442 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:11:59 smithi135 ceph-mon[221113]: pgmap v1093: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 9220/69888 objects degraded (13.193%); 1763/69888 objects misplaced (2.523%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:11:59.456 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:11:59.536 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:11:59.841 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:12:00.139 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:00.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:00.247 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:00.277 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:00.334 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:00.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:00.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:00.441 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:00.469 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:00.515 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:00.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:00.620 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:00.655 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:00.705 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:00.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:00.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:00 smithi135 ceph-mon[221113]: 1.10 deep-scrub starts 2023-09-23T20:12:00.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:00 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1860399156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:12:00.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:00 smithi135 ceph-mon[213871]: 1.10 deep-scrub starts 2023-09-23T20:12:00.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:00 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1860399156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:12:00.731 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:12:00.743 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:00.779 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:00.810 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:12:00.888 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:01.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:00 smithi144 ceph-mon[194833]: 1.10 deep-scrub starts 2023-09-23T20:12:01.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:00 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1860399156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:12:01.421 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:12:01.421 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35643.0:147252 8.c e390/334)", 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:01.337966+0000", 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.001229508, 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012495539999999999, 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:01.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.337966+0000", 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.337966+0000", 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.337965+0000", 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.337980+0000", 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4935e-05 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.337994+0000", 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3964e-05 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.338002+0000", 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1159999999999999e-06 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:01.425 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35643.0:147253 8.6 e390/322 8:684d8ffb:::benchmark_data_smithi135_460343_object63020:head v 390'9463, mlcod=309'3319)", 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:01.339082+0000", 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.000113791, 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017118499999999999, 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:12:01.426 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.339082+0000", 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.339082+0000", 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:01.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.339081+0000", 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.339088+0000", 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9099999999999999e-06 2023-09-23T20:12:01.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.339099+0000", 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0611999999999999e-05 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.339105+0000", 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6889999999999998e-06 2023-09-23T20:12:01.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.339154+0000", 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8588000000000003e-05 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:01.339188+0000", 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4204e-05 2023-09-23T20:12:01.430 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:01.431 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:01.431 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:01.431 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:01.431 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:12:01.431 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 2 2023-09-23T20:12:01.431 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:12:01.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:01.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[221113]: failed to encode map e390 with expected crc 2023-09-23T20:12:01.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[221113]: failed to encode map e390 with expected crc 2023-09-23T20:12:01.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1860399156' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:12:01.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[221113]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T20:12:01.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[221113]: pgmap v1095: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 9220/67749 objects degraded (13.609%); 1583/67749 objects misplaced (2.337%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:12:01.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:01.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[213871]: failed to encode map e390 with expected crc 2023-09-23T20:12:01.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[213871]: failed to encode map e390 with expected crc 2023-09-23T20:12:01.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1860399156' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:12:01.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[213871]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T20:12:01.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:01 smithi135 ceph-mon[213871]: pgmap v1095: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 9220/67749 objects degraded (13.609%); 1583/67749 objects misplaced (2.337%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:12:02.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:01 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:02.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:01 smithi144 ceph-mon[194833]: failed to encode map e390 with expected crc 2023-09-23T20:12:02.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:01 smithi144 ceph-mon[194833]: failed to encode map e390 with expected crc 2023-09-23T20:12:02.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:01 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1860399156' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:12:02.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:01 smithi144 ceph-mon[194833]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T20:12:02.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:01 smithi144 ceph-mon[194833]: pgmap v1095: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 9220/67749 objects degraded (13.609%); 1583/67749 objects misplaced (2.337%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:12:02.913 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:02 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9220/67749 objects degraded (13.609%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:02.913 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:02 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9220/67749 objects degraded (13.609%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:02.932 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:12:02.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:02.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:12:02.954 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:12:03.041 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:03.067 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:02 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9220/67749 objects degraded (13.609%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:03.069 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:03.118 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:03.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:03.223 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:03.310 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:03.575 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:03.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:03.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:03.685 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:03.727 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:12:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:12:03] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-09-23T20:12:03.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:03 smithi135 ceph-mon[221113]: pgmap v1096: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9220/66765 objects degraded (13.810%); 1583/66765 objects misplaced (2.371%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:12:03.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:03 smithi135 ceph-mon[213871]: pgmap v1096: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9220/66765 objects degraded (13.810%); 1583/66765 objects misplaced (2.371%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:12:04.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:03 smithi144 ceph-mon[194833]: pgmap v1096: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 9220/66765 objects degraded (13.810%); 1583/66765 objects misplaced (2.371%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:12:04.386 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:04.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:04.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:04.493 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:04.557 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:04.913 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:12:05.286 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:05.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:05.399 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:05.468 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:05.600 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:05.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:05.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:05.708 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:05.742 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:05.795 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:05.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:05.837 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:05.901 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:06.000 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:06.023 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:12:06.069 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:12:06.100 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:12:06.131 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:12:06.169 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:12:06.210 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:12:06.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:06 smithi135 ceph-mon[221113]: pgmap v1097: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 9044/61812 objects degraded (14.631%); 1430/61812 objects misplaced (2.313%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:12:06.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1791834434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:06.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:06 smithi135 ceph-mon[213871]: pgmap v1097: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 9044/61812 objects degraded (14.631%); 1430/61812 objects misplaced (2.313%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:12:06.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1791834434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:06.239 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:12:06.280 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:12:06.282 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:06.291 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:12:06.292 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:12:06.292 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:12:06.292 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:12:06.292 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:12:06.292 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:12:06.292 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:12:06.292 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:12:06.319 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:06.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:06 smithi144 ceph-mon[194833]: pgmap v1097: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 9044/61812 objects degraded (14.631%); 1430/61812 objects misplaced (2.313%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:12:06.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1791834434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:06.428 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:06.515 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:06.693 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:12:06.726 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:12:07.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:07 smithi144 ceph-mon[194833]: from='client.55852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:07.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:07 smithi144 ceph-mon[194833]: pgmap v1098: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 9044/59247 objects degraded (15.265%); 1256/59247 objects misplaced (2.120%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:12:07.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:07 smithi135 ceph-mon[213871]: from='client.55852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:07.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:07 smithi135 ceph-mon[213871]: pgmap v1098: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 9044/59247 objects degraded (15.265%); 1256/59247 objects misplaced (2.120%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:12:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:07 smithi135 ceph-mon[221113]: from='client.55852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:07 smithi135 ceph-mon[221113]: pgmap v1098: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 9044/59247 objects degraded (15.265%); 1256/59247 objects misplaced (2.120%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:12:07.832 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:07.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:07.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:07.940 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:07.968 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:08.011 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:08.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:08.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:08.117 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:08.152 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:08.152 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:08.152 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:08 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9044/59247 objects degraded (15.265%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:08.152 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:08.153 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:08.153 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:08 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9044/59247 objects degraded (15.265%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:08.213 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:08 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9044/59247 objects degraded (15.265%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:08.477 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:08.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:08.588 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:08.697 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:09.073 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:09.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:09.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:09.179 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:09.206 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:09.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[221113]: failed to encode map e391 with expected crc 2023-09-23T20:12:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[221113]: failed to encode map e391 with expected crc 2023-09-23T20:12:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[221113]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T20:12:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[221113]: failed to encode map e391 with expected crc 2023-09-23T20:12:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[221113]: failed to encode map e391 with expected crc 2023-09-23T20:12:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[221113]: pgmap v1100: 109 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 6519/53166 objects degraded (12.262%); 1086/53166 objects misplaced (2.043%); 4.7 MiB/s, 85 objects/s recovering 2023-09-23T20:12:09.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[213871]: failed to encode map e391 with expected crc 2023-09-23T20:12:09.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[213871]: failed to encode map e391 with expected crc 2023-09-23T20:12:09.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[213871]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T20:12:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[213871]: failed to encode map e391 with expected crc 2023-09-23T20:12:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[213871]: failed to encode map e391 with expected crc 2023-09-23T20:12:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:09 smithi135 ceph-mon[213871]: pgmap v1100: 109 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 6519/53166 objects degraded (12.262%); 1086/53166 objects misplaced (2.043%); 4.7 MiB/s, 85 objects/s recovering 2023-09-23T20:12:10.067 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:10.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:10.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:10.121 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1097,"stamp":"2023-09-23T20:12:04.961259+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1353145410,"num_objects":20604,"num_object_clones":0,"num_object_copies":61812,"num_objects_missing_on_primary":1935,"num_objects_missing":1935,"num_objects_degraded":9044,"num_objects_misplaced":1430,"num_objects_unfound":0,"num_objects_dirty":20604,"num_whiteouts":0,"num_read":6158,"num_read_kb":13539,"num_write":148880,"num_write_kb":5404324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30925,"num_bytes_recovered":1610223871,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95520,"ondisk_log_size":95520,"up":327,"acting":322,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":376,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9272656,"kb_used_data":5606680,"kb_used_omap":32,"kb_used_meta":3665823,"kb_avail":646943408,"statfs":{"total":671965249536,"available":662470049792,"internally_reserved":0,"allocated":5741240320,"data_stored":5729894945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3753803270},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,8,6,2,3,2,3,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-341180416,"num_objects":-5206,"num_object_clones":0,"num_object_copies":-15618,"num_objects_missing_on_primary":-327,"num_objects_missing":-327,"num_objects_degraded":-327,"num_objects_misplaced":-654,"num_objects_unfound":0,"num_objects_dirty":-5206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5206,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":659,"num_bytes_recovered":43188224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004966"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":388,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789063+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:11:50.789063+0000","last_peered":"2023-09-23T20:11:50.789063+0000","last_clean":"2023-09-23T20:11:50.789063+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:11:50.789063+0000","last_undegraded":"2023-09-23T20:11:50.789063+0000","last_fullsized":"2023-09-23T20:11:50.789063+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":218,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796461+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:11:50.796461+0000","last_peered":"2023-09-23T20:11:50.796461+0000","last_clean":"2023-09-23T20:11:50.796461+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:11:50.796461+0000","last_undegraded":"2023-09-23T20:11:50.796461+0000","last_fullsized":"2023-09-23T20:11:50.796461+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":85,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.795711+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:11:50.795711+0000","last_peered":"2023-09-23T20:11:50.795711+0000","last_clean":"2023-09-23T20:11:50.795711+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:11:50.795711+0000","last_undegraded":"2023-09-23T20:11:50.795711+0000","last_fullsized":"2023-09-23T20:11:50.795711+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.797647+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:11:50.797647+0000","last_peered":"2023-09-23T20:11:50.797647+0000","last_clean":"2023-09-23T20:11:50.797647+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:11:50.797647+0000","last_undegraded":"2023-09-23T20:11:50.797647+0000","last_fullsized":"2023-09-23T20:11:50.797647+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":218,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.795602+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:11:50.795602+0000","last_peered":"2023-09-23T20:11:50.795602+0000","last_clean":"2023-09-23T20:11:50.795602+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:11:50.795602+0000","last_undegraded":"2023-09-23T20:11:50.795602+0000","last_fullsized":"2023-09-23T20:11:50.795602+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":335,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.792010+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:11:50.792010+0000","last_peered":"2023-09-23T20:11:50.792010+0000","last_clean":"2023-09-23T20:11:50.792010+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:11:50.792010+0000","last_undegraded":"2023-09-23T20:11:50.792010+0000","last_fullsized":"2023-09-23T20:11:50.792010+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":316,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791898+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:11:50.791898+0000","last_peered":"2023-09-23T20:11:50.791898+0000","last_clean":"2023-09-23T20:11:50.791898+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:11:50.791898+0000","last_undegraded":"2023-09-23T20:11:50.791898+0000","last_fullsized":"2023-09-23T20:11:50.791898+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":233,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.787764+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:11:50.787764+0000","last_peered":"2023-09-23T20:11:50.787764+0000","last_clean":"2023-09-23T20:11:50.787764+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:11:50.787764+0000","last_undegraded":"2023-09-23T20:11:50.787764+0000","last_fullsized":"2023-09-23T20:11:50.787764+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":699,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.784718+0000","last_change":"2023-09-23T20:09:56.042750+0000","last_active":"2023-09-23T20:11:50.784718+0000","last_peered":"2023-09-23T20:11:50.784718+0000","last_clean":"2023-09-23T20:11:50.784718+0000","last_became_active":"2023-09-23T20:09:56.042455+0000","last_became_peered":"2023-09-23T20:09:56.042455+0000","last_unstale":"2023-09-23T20:11:50.784718+0000","last_undegraded":"2023-09-23T20:11:50.784718+0000","last_fullsized":"2023-09-23T20:11:50.784718+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":389,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791540+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:11:50.791540+0000","last_peered":"2023-09-23T20:11:50.791540+0000","last_clean":"2023-09-23T20:11:50.791540+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:11:50.791540+0000","last_undegraded":"2023-09-23T20:11:50.791540+0000","last_fullsized":"2023-09-23T20:11:50.791540+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":252,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.783105+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:11:50.783105+0000","last_peered":"2023-09-23T20:11:50.783105+0000","last_clean":"2023-09-23T20:11:50.783105+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:11:50.783105+0000","last_undegraded":"2023-09-23T20:11:50.783105+0000","last_fullsized":"2023-09-23T20:11:50.783105+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":83,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.797467+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:11:50.797467+0000","last_peered":"2023-09-23T20:11:50.797467+0000","last_clean":"2023-09-23T20:11:50.797467+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:11:50.797467+0000","last_undegraded":"2023-09-23T20:11:50.797467+0000","last_fullsized":"2023-09-23T20:11:50.797467+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":86,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.714650+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:12:00.714650+0000","last_peered":"2023-09-23T20:12:00.714650+0000","last_clean":"2023-09-23T20:12:00.714650+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:12:00.714650+0000","last_undegraded":"2023-09-23T20:12:00.714650+0000","last_fullsized":"2023-09-23T20:12:00.714650+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":444,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.788657+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:11:50.788657+0000","last_peered":"2023-09-23T20:11:50.788657+0000","last_clean":"2023-09-23T20:11:50.788657+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:11:50.788657+0000","last_undegraded":"2023-09-23T20:11:50.788657+0000","last_fullsized":"2023-09-23T20:11:50.788657+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":386,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789145+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:11:50.789145+0000","last_peered":"2023-09-23T20:11:50.789145+0000","last_clean":"2023-09-23T20:11:50.789145+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:11:50.789145+0000","last_undegraded":"2023-09-23T20:11:50.789145+0000","last_fullsized":"2023-09-23T20:11:50.789145+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":719,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791011+0000","last_change":"2023-09-23T20:11:07.067995+0000","last_active":"2023-09-23T20:11:50.791011+0000","last_peered":"2023-09-23T20:11:50.791011+0000","last_clean":"2023-09-23T20:11:50.791011+0000","last_became_active":"2023-09-23T20:09:56.043288+0000","last_became_peered":"2023-09-23T20:09:56.043288+0000","last_unstale":"2023-09-23T20:11:50.791011+0000","last_undegraded":"2023-09-23T20:11:50.791011+0000","last_fullsized":"2023-09-23T20:11:50.791011+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":83,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796135+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:11:50.796135+0000","last_peered":"2023-09-23T20:11:50.796135+0000","last_clean":"2023-09-23T20:11:50.796135+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:11:50.796135+0000","last_undegraded":"2023-09-23T20:11:50.796135+0000","last_fullsized":"2023-09-23T20:11:50.796135+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":242,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796019+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:11:50.796019+0000","last_peered":"2023-09-23T20:11:50.796019+0000","last_clean":"2023-09-23T20:11:50.796019+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:11:50.796019+0000","last_undegraded":"2023-09-23T20:11:50.796019+0000","last_fullsized":"2023-09-23T20:11:50.796019+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":542,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.290379+0000","last_change":"2023-09-23T20:12:00.290344+0000","last_active":"2023-09-23T20:12:00.290379+0000","last_peered":"2023-09-23T20:12:00.290379+0000","last_clean":"2023-09-23T20:12:00.290379+0000","last_became_active":"2023-09-23T20:09:56.050784+0000","last_became_peered":"2023-09-23T20:09:56.050784+0000","last_unstale":"2023-09-23T20:12:00.290379+0000","last_undegraded":"2023-09-23T20:12:00.290379+0000","last_fullsized":"2023-09-23T20:12:00.290379+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":98,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.795333+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:11:50.795333+0000","last_peered":"2023-09-23T20:11:50.795333+0000","last_clean":"2023-09-23T20:11:50.795333+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:11:50.795333+0000","last_undegraded":"2023-09-23T20:11:50.795333+0000","last_fullsized":"2023-09-23T20:11:50.795333+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":388,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791886+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:11:50.791886+0000","last_peered":"2023-09-23T20:11:50.791886+0000","last_clean":"2023-09-23T20:11:50.791886+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:11:50.791886+0000","last_undegraded":"2023-09-23T20:11:50.791886+0000","last_fullsized":"2023-09-23T20:11:50.791886+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":515,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.800585+0000","last_change":"2023-09-23T20:09:56.062364+0000","last_active":"2023-09-23T20:11:50.800585+0000","last_peered":"2023-09-23T20:11:50.800585+0000","last_clean":"2023-09-23T20:11:50.800585+0000","last_became_active":"2023-09-23T20:09:56.061865+0000","last_became_peered":"2023-09-23T20:09:56.061865+0000","last_unstale":"2023-09-23T20:11:50.800585+0000","last_undegraded":"2023-09-23T20:11:50.800585+0000","last_fullsized":"2023-09-23T20:11:50.800585+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":77,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.795187+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:11:50.795187+0000","last_peered":"2023-09-23T20:11:50.795187+0000","last_clean":"2023-09-23T20:11:50.795187+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:11:50.795187+0000","last_undegraded":"2023-09-23T20:11:50.795187+0000","last_fullsized":"2023-09-23T20:11:50.795187+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":376,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.784376+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:11:50.784376+0000","last_peered":"2023-09-23T20:11:50.784376+0000","last_clean":"2023-09-23T20:11:50.784376+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:11:50.784376+0000","last_undegraded":"2023-09-23T20:11:50.784376+0000","last_fullsized":"2023-09-23T20:11:50.784376+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":617,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:56.308410+0000","last_change":"2023-09-23T20:11:56.308383+0000","last_active":"2023-09-23T20:11:56.308410+0000","last_peered":"2023-09-23T20:11:56.308410+0000","last_clean":"2023-09-23T20:11:56.308410+0000","last_became_active":"2023-09-23T20:09:56.043157+0000","last_became_peered":"2023-09-23T20:09:56.043157+0000","last_unstale":"2023-09-23T20:11:56.308410+0000","last_undegraded":"2023-09-23T20:11:56.308410+0000","last_fullsized":"2023-09-23T20:11:56.308410+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":389,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:11:50.790996+0000","last_change":"2023-09-23T20:11:10.067603+0000","last_active":"2023-09-23T20:11:50.790996+0000","last_peered":"2023-09-23T20:11:50.790996+0000","last_clean":"2023-09-23T20:00:51.694506+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:11:50.790996+0000","last_undegraded":"2023-09-23T20:11:10.056843+0000","last_fullsized":"2023-09-23T20:11:50.790996+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":4}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":20,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.785425+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:11:50.785425+0000","last_peered":"2023-09-23T20:11:50.785425+0000","last_clean":"2023-09-23T20:11:50.785425+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:11:50.785425+0000","last_undegraded":"2023-09-23T20:11:50.785425+0000","last_fullsized":"2023-09-23T20:11:50.785425+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":813,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:52.828486+0000","last_change":"2023-09-23T20:11:52.828453+0000","last_active":"2023-09-23T20:11:52.828486+0000","last_peered":"2023-09-23T20:11:52.828486+0000","last_clean":"2023-09-23T20:11:52.828486+0000","last_became_active":"2023-09-23T20:10:52.687280+0000","last_became_peered":"2023-09-23T20:10:52.687280+0000","last_unstale":"2023-09-23T20:11:52.828486+0000","last_undegraded":"2023-09-23T20:11:52.828486+0000","last_fullsized":"2023-09-23T20:11:52.828486+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.4","version":"390'9448","reported_seq":9547,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:03.252884+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:12:03.252884+0000","last_peered":"2023-09-23T20:12:03.252884+0000","last_clean":"2023-09-23T20:12:03.252884+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:12:03.252884+0000","last_undegraded":"2023-09-23T20:12:03.252884+0000","last_fullsized":"2023-09-23T20:12:03.252884+0000","mapping_epoch":299,"log_start":"309'3165","ondisk_log_start":"309'3165","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84017152,"num_objects":1282,"num_object_clones":0,"num_object_copies":3846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9448,"num_write_kb":343360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":81,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.797707+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:11:50.797707+0000","last_peered":"2023-09-23T20:11:50.797707+0000","last_clean":"2023-09-23T20:11:50.797707+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:11:50.797707+0000","last_undegraded":"2023-09-23T20:11:50.797707+0000","last_fullsized":"2023-09-23T20:11:50.797707+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":398,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791151+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:11:50.791151+0000","last_peered":"2023-09-23T20:11:50.791151+0000","last_clean":"2023-09-23T20:11:50.791151+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:11:50.791151+0000","last_undegraded":"2023-09-23T20:11:50.791151+0000","last_fullsized":"2023-09-23T20:11:50.791151+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":700,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.782716+0000","last_change":"2023-09-23T20:09:56.064782+0000","last_active":"2023-09-23T20:11:50.782716+0000","last_peered":"2023-09-23T20:11:50.782716+0000","last_clean":"2023-09-23T20:11:50.782716+0000","last_became_active":"2023-09-23T20:09:56.064411+0000","last_became_peered":"2023-09-23T20:09:56.064411+0000","last_unstale":"2023-09-23T20:11:50.782716+0000","last_undegraded":"2023-09-23T20:11:50.782716+0000","last_fullsized":"2023-09-23T20:11:50.782716+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218454,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.5","version":"390'9036","reported_seq":11729,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:03.610778+0000","last_change":"2023-09-23T20:07:45.697314+0000","last_active":"2023-09-23T20:12:03.610778+0000","last_peered":"2023-09-23T20:12:03.610778+0000","last_clean":"2023-09-23T20:12:03.610778+0000","last_became_active":"2023-09-23T20:07:45.690354+0000","last_became_peered":"2023-09-23T20:07:45.690354+0000","last_unstale":"2023-09-23T20:12:03.610778+0000","last_undegraded":"2023-09-23T20:12:03.610778+0000","last_fullsized":"2023-09-23T20:12:03.610778+0000","mapping_epoch":327,"log_start":"314'3318","ondisk_log_start":"314'3318","created":299,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.926631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9036,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":82,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.797725+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:11:50.797725+0000","last_peered":"2023-09-23T20:11:50.797725+0000","last_clean":"2023-09-23T20:11:50.797725+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:11:50.797725+0000","last_undegraded":"2023-09-23T20:11:50.797725+0000","last_fullsized":"2023-09-23T20:11:50.797725+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":446,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789407+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:11:50.789407+0000","last_peered":"2023-09-23T20:11:50.789407+0000","last_clean":"2023-09-23T20:11:50.789407+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:11:50.789407+0000","last_undegraded":"2023-09-23T20:11:50.789407+0000","last_fullsized":"2023-09-23T20:11:50.789407+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":658,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.785620+0000","last_change":"2023-09-23T20:11:20.856999+0000","last_active":"2023-09-23T20:11:50.785620+0000","last_peered":"2023-09-23T20:11:50.785620+0000","last_clean":"2023-09-23T20:11:50.785620+0000","last_became_active":"2023-09-23T20:11:20.856640+0000","last_became_peered":"2023-09-23T20:11:20.856640+0000","last_unstale":"2023-09-23T20:11:50.785620+0000","last_undegraded":"2023-09-23T20:11:50.785620+0000","last_fullsized":"2023-09-23T20:11:50.785620+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.6","version":"390'9439","reported_seq":11063,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:00.792985+0000","last_change":"2023-09-23T20:07:37.064417+0000","last_active":"2023-09-23T20:12:00.792985+0000","last_peered":"2023-09-23T20:12:00.792985+0000","last_clean":"2023-09-23T20:06:59.831272+0000","last_became_active":"2023-09-23T20:07:37.059682+0000","last_became_peered":"2023-09-23T20:07:37.059682+0000","last_unstale":"2023-09-23T20:12:00.792985+0000","last_undegraded":"2023-09-23T20:07:36.983730+0000","last_fullsized":"2023-09-23T20:07:36.974121+0000","mapping_epoch":322,"log_start":"319'4265","ondisk_log_start":"319'4265","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5174,"log_dups_size":0,"ondisk_log_size":5174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:33.575626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89718784,"num_objects":1369,"num_object_clones":0,"num_object_copies":4107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9439,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-22},{"shards":"0,2,3","objects":675}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":81,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.795350+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:11:50.795350+0000","last_peered":"2023-09-23T20:11:50.795350+0000","last_clean":"2023-09-23T20:11:50.795350+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:11:50.795350+0000","last_undegraded":"2023-09-23T20:11:50.795350+0000","last_fullsized":"2023-09-23T20:11:50.795350+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796107+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:11:50.796107+0000","last_peered":"2023-09-23T20:11:50.796107+0000","last_clean":"2023-09-23T20:11:50.796107+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:11:50.796107+0000","last_undegraded":"2023-09-23T20:11:50.796107+0000","last_fullsized":"2023-09-23T20:11:50.796107+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":582,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.788974+0000","last_change":"2023-09-23T20:11:43.909971+0000","last_active":"2023-09-23T20:11:50.788974+0000","last_peered":"2023-09-23T20:11:50.788974+0000","last_clean":"2023-09-23T20:11:50.788974+0000","last_became_active":"2023-09-23T20:09:56.043491+0000","last_became_peered":"2023-09-23T20:09:56.043491+0000","last_unstale":"2023-09-23T20:11:50.788974+0000","last_undegraded":"2023-09-23T20:11:50.788974+0000","last_fullsized":"2023-09-23T20:11:50.788974+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.7","version":"390'9267","reported_seq":13690,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:03.713956+0000","last_change":"2023-09-23T20:10:58.582053+0000","last_active":"2023-09-23T20:12:03.713956+0000","last_peered":"2023-09-23T20:12:03.713956+0000","last_clean":"2023-09-23T20:07:17.891274+0000","last_became_active":"2023-09-23T20:10:58.299550+0000","last_became_peered":"2023-09-23T20:10:58.299550+0000","last_unstale":"2023-09-23T20:12:03.713956+0000","last_undegraded":"2023-09-23T20:10:56.195777+0000","last_fullsized":"2023-09-23T20:10:53.692217+0000","mapping_epoch":369,"log_start":"315'3729","ondisk_log_start":"315'3729","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:48.194019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":1931,"num_objects_missing":1931,"num_objects_degraded":2525,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9267,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2148,"num_bytes_recovered":140771328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2],"avail_no_missing":["3"],"object_location_counts":[{"shards":"1,2","objects":1181},{"shards":"1,2,3,4","objects":4},{"shards":"2,3,4","objects":86}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":428,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791757+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:11:50.791757+0000","last_peered":"2023-09-23T20:11:50.791757+0000","last_clean":"2023-09-23T20:11:50.791757+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:11:50.791757+0000","last_undegraded":"2023-09-23T20:11:50.791757+0000","last_fullsized":"2023-09-23T20:11:50.791757+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":86,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.800871+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:11:50.800871+0000","last_peered":"2023-09-23T20:11:50.800871+0000","last_clean":"2023-09-23T20:11:50.800871+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:11:50.800871+0000","last_undegraded":"2023-09-23T20:11:50.800871+0000","last_fullsized":"2023-09-23T20:11:50.800871+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"390'8996","reported_seq":17126,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.144106+0000","last_change":"2023-09-23T20:10:10.729751+0000","last_active":"2023-09-23T20:12:01.144106+0000","last_peered":"2023-09-23T20:12:01.144106+0000","last_clean":"2023-09-23T20:12:01.144106+0000","last_became_active":"2023-09-23T20:10:10.708263+0000","last_became_peered":"2023-09-23T20:10:10.708263+0000","last_unstale":"2023-09-23T20:12:01.144106+0000","last_undegraded":"2023-09-23T20:12:01.144106+0000","last_fullsized":"2023-09-23T20:12:01.144106+0000","mapping_epoch":361,"log_start":"314'3510","ondisk_log_start":"314'3510","created":299,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:15.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85983232,"num_objects":1312,"num_object_clones":0,"num_object_copies":3936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8996,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4003,"num_bytes_recovered":136118272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":772,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.795188+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:11:50.795188+0000","last_peered":"2023-09-23T20:11:50.795188+0000","last_clean":"2023-09-23T20:11:50.795188+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:11:50.795188+0000","last_undegraded":"2023-09-23T20:11:50.795188+0000","last_fullsized":"2023-09-23T20:11:50.795188+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":80,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789061+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:11:50.789061+0000","last_peered":"2023-09-23T20:11:50.789061+0000","last_clean":"2023-09-23T20:11:50.789061+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:11:50.789061+0000","last_undegraded":"2023-09-23T20:11:50.789061+0000","last_fullsized":"2023-09-23T20:11:50.789061+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":416,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.788746+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:11:50.788746+0000","last_peered":"2023-09-23T20:11:50.788746+0000","last_clean":"2023-09-23T20:11:50.788746+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:11:50.788746+0000","last_undegraded":"2023-09-23T20:11:50.788746+0000","last_fullsized":"2023-09-23T20:11:50.788746+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"390'9277","reported_seq":15679,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:02.820017+0000","last_change":"2023-09-23T20:10:58.556423+0000","last_active":"2023-09-23T20:12:02.820017+0000","last_peered":"2023-09-23T20:12:02.820017+0000","last_clean":"2023-09-23T20:07:17.847486+0000","last_became_active":"2023-09-23T20:07:20.666556+0000","last_became_peered":"2023-09-23T20:07:20.666556+0000","last_unstale":"2023-09-23T20:12:02.820017+0000","last_undegraded":"2023-09-23T20:10:58.555698+0000","last_fullsized":"2023-09-23T20:07:19.932561+0000","mapping_epoch":317,"log_start":"321'4346","ondisk_log_start":"321'4346","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4931,"log_dups_size":0,"ondisk_log_size":4931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:38.441814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80150528,"num_objects":1223,"num_object_clones":0,"num_object_copies":3669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":996,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9277,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3145,"num_bytes_recovered":206110720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":227},{"shards":"0,2,5","objects":114}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":724,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791227+0000","last_change":"2023-09-23T20:11:11.229144+0000","last_active":"2023-09-23T20:11:50.791227+0000","last_peered":"2023-09-23T20:11:50.791227+0000","last_clean":"2023-09-23T20:11:50.791227+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:11:50.791227+0000","last_undegraded":"2023-09-23T20:11:50.791227+0000","last_fullsized":"2023-09-23T20:11:50.791227+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":356,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.788983+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:11:50.788983+0000","last_peered":"2023-09-23T20:11:50.788983+0000","last_clean":"2023-09-23T20:11:50.788983+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:11:50.788983+0000","last_undegraded":"2023-09-23T20:11:50.788983+0000","last_fullsized":"2023-09-23T20:11:50.788983+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":386,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789298+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:11:50.789298+0000","last_peered":"2023-09-23T20:11:50.789298+0000","last_clean":"2023-09-23T20:11:50.789298+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:11:50.789298+0000","last_undegraded":"2023-09-23T20:11:50.789298+0000","last_fullsized":"2023-09-23T20:11:50.789298+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":515,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.800630+0000","last_change":"2023-09-23T20:09:56.062162+0000","last_active":"2023-09-23T20:11:50.800630+0000","last_peered":"2023-09-23T20:11:50.800630+0000","last_clean":"2023-09-23T20:11:50.800630+0000","last_became_active":"2023-09-23T20:09:56.061518+0000","last_became_peered":"2023-09-23T20:09:56.061518+0000","last_unstale":"2023-09-23T20:11:50.800630+0000","last_undegraded":"2023-09-23T20:11:50.800630+0000","last_fullsized":"2023-09-23T20:11:50.800630+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"390'9243","reported_seq":9342,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:03.602634+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:12:03.602634+0000","last_peered":"2023-09-23T20:12:03.602634+0000","last_clean":"2023-09-23T20:12:03.602634+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:12:03.602634+0000","last_undegraded":"2023-09-23T20:12:03.602634+0000","last_fullsized":"2023-09-23T20:12:03.602634+0000","mapping_epoch":299,"log_start":"314'3485","ondisk_log_start":"314'3485","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79757312,"num_objects":1217,"num_object_clones":0,"num_object_copies":3651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9243,"num_write_kb":334720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":380,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.788522+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:11:50.788522+0000","last_peered":"2023-09-23T20:11:50.788522+0000","last_clean":"2023-09-23T20:11:50.788522+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:11:50.788522+0000","last_undegraded":"2023-09-23T20:11:50.788522+0000","last_fullsized":"2023-09-23T20:11:50.788522+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":416,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789165+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:11:50.789165+0000","last_peered":"2023-09-23T20:11:50.789165+0000","last_clean":"2023-09-23T20:11:50.789165+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:11:50.789165+0000","last_undegraded":"2023-09-23T20:11:50.789165+0000","last_fullsized":"2023-09-23T20:11:50.789165+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"390'9089","reported_seq":17248,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:03.825960+0000","last_change":"2023-09-23T20:09:37.763558+0000","last_active":"2023-09-23T20:12:03.825960+0000","last_peered":"2023-09-23T20:12:03.825960+0000","last_clean":"2023-09-23T20:12:03.825960+0000","last_became_active":"2023-09-23T20:09:37.732414+0000","last_became_peered":"2023-09-23T20:09:37.732414+0000","last_unstale":"2023-09-23T20:12:03.825960+0000","last_undegraded":"2023-09-23T20:12:03.825960+0000","last_fullsized":"2023-09-23T20:12:03.825960+0000","mapping_epoch":352,"log_start":"315'3581","ondisk_log_start":"315'3581","created":299,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5508,"log_dups_size":0,"ondisk_log_size":5508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:39.792571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79364096,"num_objects":1211,"num_object_clones":0,"num_object_copies":3633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9089,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4021,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":728,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791470+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:11:50.791470+0000","last_peered":"2023-09-23T20:11:50.791470+0000","last_clean":"2023-09-23T20:11:50.791470+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:11:50.791470+0000","last_undegraded":"2023-09-23T20:11:50.791470+0000","last_fullsized":"2023-09-23T20:11:50.791470+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.787459+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:11:50.787459+0000","last_peered":"2023-09-23T20:11:50.787459+0000","last_clean":"2023-09-23T20:11:50.787459+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:11:50.787459+0000","last_undegraded":"2023-09-23T20:11:50.787459+0000","last_fullsized":"2023-09-23T20:11:50.787459+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":81,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.774226+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:11:53.774226+0000","last_peered":"2023-09-23T20:11:53.774226+0000","last_clean":"2023-09-23T20:11:53.774226+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:11:53.774226+0000","last_undegraded":"2023-09-23T20:11:53.774226+0000","last_fullsized":"2023-09-23T20:11:53.774226+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"390'9216","reported_seq":9318,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.744649+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:12:00.744649+0000","last_peered":"2023-09-23T20:12:00.744649+0000","last_clean":"2023-09-23T20:12:00.744649+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:12:00.744649+0000","last_undegraded":"2023-09-23T20:12:00.744649+0000","last_fullsized":"2023-09-23T20:12:00.744649+0000","mapping_epoch":299,"log_start":"315'4083","ondisk_log_start":"315'4083","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5133,"log_dups_size":0,"ondisk_log_size":5133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88211456,"num_objects":1346,"num_object_clones":0,"num_object_copies":4038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9216,"num_write_kb":337984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":658,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.785530+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:11:50.785530+0000","last_peered":"2023-09-23T20:11:50.785530+0000","last_clean":"2023-09-23T20:11:50.785530+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:11:50.785530+0000","last_undegraded":"2023-09-23T20:11:50.785530+0000","last_fullsized":"2023-09-23T20:11:50.785530+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":523,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796406+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:11:50.796406+0000","last_peered":"2023-09-23T20:11:50.796406+0000","last_clean":"2023-09-23T20:11:50.796406+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:11:50.796406+0000","last_undegraded":"2023-09-23T20:11:50.796406+0000","last_fullsized":"2023-09-23T20:11:50.796406+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":390,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791809+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:11:50.791809+0000","last_peered":"2023-09-23T20:11:50.791809+0000","last_clean":"2023-09-23T20:11:50.791809+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:11:50.791809+0000","last_undegraded":"2023-09-23T20:11:50.791809+0000","last_fullsized":"2023-09-23T20:11:50.791809+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":86,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796013+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:11:50.796013+0000","last_peered":"2023-09-23T20:11:50.796013+0000","last_clean":"2023-09-23T20:11:50.796013+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:11:50.796013+0000","last_undegraded":"2023-09-23T20:11:50.796013+0000","last_fullsized":"2023-09-23T20:11:50.796013+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"390'9246","reported_seq":14074,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:03.282083+0000","last_change":"2023-09-23T20:07:52.783215+0000","last_active":"2023-09-23T20:12:03.282083+0000","last_peered":"2023-09-23T20:12:03.282083+0000","last_clean":"2023-09-23T20:12:03.282083+0000","last_became_active":"2023-09-23T20:07:52.778874+0000","last_became_peered":"2023-09-23T20:07:52.778874+0000","last_unstale":"2023-09-23T20:12:03.282083+0000","last_undegraded":"2023-09-23T20:12:03.282083+0000","last_fullsized":"2023-09-23T20:12:03.282083+0000","mapping_epoch":330,"log_start":"309'2900","ondisk_log_start":"309'2900","created":299,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6346,"log_dups_size":0,"ondisk_log_size":6346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:31.036337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9246,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2349,"num_bytes_recovered":153944064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":723,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791207+0000","last_change":"2023-09-23T20:11:06.026338+0000","last_active":"2023-09-23T20:11:50.791207+0000","last_peered":"2023-09-23T20:11:50.791207+0000","last_clean":"2023-09-23T20:11:50.791207+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:11:50.791207+0000","last_undegraded":"2023-09-23T20:11:50.791207+0000","last_fullsized":"2023-09-23T20:11:50.791207+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1010,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791204+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:11:50.791204+0000","last_peered":"2023-09-23T20:11:50.791204+0000","last_clean":"2023-09-23T20:11:50.791204+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:11:50.791204+0000","last_undegraded":"2023-09-23T20:11:50.791204+0000","last_fullsized":"2023-09-23T20:11:50.791204+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":409,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789278+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:11:50.789278+0000","last_peered":"2023-09-23T20:11:50.789278+0000","last_clean":"2023-09-23T20:11:50.789278+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:11:50.789278+0000","last_undegraded":"2023-09-23T20:11:50.789278+0000","last_fullsized":"2023-09-23T20:11:50.789278+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":86,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796355+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:11:50.796355+0000","last_peered":"2023-09-23T20:11:50.796355+0000","last_clean":"2023-09-23T20:11:50.796355+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:11:50.796355+0000","last_undegraded":"2023-09-23T20:11:50.796355+0000","last_fullsized":"2023-09-23T20:11:50.796355+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"390'9119","reported_seq":21485,"reported_epoch":390,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:12:03.835615+0000","last_change":"2023-09-23T20:11:18.839874+0000","last_active":"2023-09-23T20:12:03.835615+0000","last_peered":"2023-09-23T20:12:03.835615+0000","last_clean":"2023-09-23T20:11:16.510239+0000","last_became_active":"2023-09-23T20:11:18.809696+0000","last_became_peered":"2023-09-23T20:11:18.809696+0000","last_unstale":"2023-09-23T20:12:03.835615+0000","last_undegraded":"2023-09-23T20:12:03.835615+0000","last_fullsized":"2023-09-23T20:12:03.835615+0000","mapping_epoch":381,"log_start":"315'3600","ondisk_log_start":"315'3600","created":299,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:58.376473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81592320,"num_objects":1245,"num_object_clones":0,"num_object_copies":3735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1245,"num_objects_unfound":0,"num_objects_dirty":1245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9119,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":217251840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[1,4,2],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":1245}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":615,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:54.316429+0000","last_change":"2023-09-23T20:11:54.316411+0000","last_active":"2023-09-23T20:11:54.316429+0000","last_peered":"2023-09-23T20:11:54.316429+0000","last_clean":"2023-09-23T20:11:54.316429+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:11:54.316429+0000","last_undegraded":"2023-09-23T20:11:54.316429+0000","last_fullsized":"2023-09-23T20:11:54.316429+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1026,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.798824+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:11:50.798824+0000","last_peered":"2023-09-23T20:11:50.798824+0000","last_clean":"2023-09-23T20:11:50.798824+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:11:50.798824+0000","last_undegraded":"2023-09-23T20:11:50.798824+0000","last_fullsized":"2023-09-23T20:11:50.798824+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":86,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.795732+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:11:50.795732+0000","last_peered":"2023-09-23T20:11:50.795732+0000","last_clean":"2023-09-23T20:11:50.795732+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:11:50.795732+0000","last_undegraded":"2023-09-23T20:11:50.795732+0000","last_fullsized":"2023-09-23T20:11:50.795732+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":81,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.802711+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:11:50.802711+0000","last_peered":"2023-09-23T20:11:50.802711+0000","last_clean":"2023-09-23T20:11:50.802711+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:11:50.802711+0000","last_undegraded":"2023-09-23T20:11:50.802711+0000","last_fullsized":"2023-09-23T20:11:50.802711+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"390'9269","reported_seq":9368,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:03.206423+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:12:03.206423+0000","last_peered":"2023-09-23T20:12:03.206423+0000","last_clean":"2023-09-23T20:12:03.206423+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:12:03.206423+0000","last_undegraded":"2023-09-23T20:12:03.206423+0000","last_fullsized":"2023-09-23T20:12:03.206423+0000","mapping_epoch":299,"log_start":"309'2965","ondisk_log_start":"309'2965","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6304,"log_dups_size":0,"ondisk_log_size":6304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85131264,"num_objects":1299,"num_object_clones":0,"num_object_copies":3897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9269,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1298,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791389+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:11:50.791389+0000","last_peered":"2023-09-23T20:11:50.791389+0000","last_clean":"2023-09-23T20:11:50.791389+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:11:50.791389+0000","last_undegraded":"2023-09-23T20:11:50.791389+0000","last_fullsized":"2023-09-23T20:11:50.791389+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":549,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:53.362175+0000","last_change":"2023-09-23T20:11:53.362155+0000","last_active":"2023-09-23T20:11:53.362175+0000","last_peered":"2023-09-23T20:11:53.362175+0000","last_clean":"2023-09-23T20:11:53.362175+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:11:53.362175+0000","last_undegraded":"2023-09-23T20:11:53.362175+0000","last_fullsized":"2023-09-23T20:11:53.362175+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":255,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791345+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:11:50.791345+0000","last_peered":"2023-09-23T20:11:50.791345+0000","last_clean":"2023-09-23T20:11:50.791345+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:11:50.791345+0000","last_undegraded":"2023-09-23T20:11:50.791345+0000","last_fullsized":"2023-09-23T20:11:50.791345+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":285,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789558+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:11:50.789558+0000","last_peered":"2023-09-23T20:11:50.789558+0000","last_clean":"2023-09-23T20:11:50.789558+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:11:50.789558+0000","last_undegraded":"2023-09-23T20:11:50.789558+0000","last_fullsized":"2023-09-23T20:11:50.789558+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"390'9309","reported_seq":9521,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:00.772867+0000","last_change":"2023-09-23T20:10:57.512712+0000","last_active":"2023-09-23T20:12:00.772867+0000","last_peered":"2023-09-23T20:12:00.772867+0000","last_clean":"2023-09-23T20:07:17.907438+0000","last_became_active":"2023-09-23T20:07:20.693608+0000","last_became_peered":"2023-09-23T20:07:20.693608+0000","last_unstale":"2023-09-23T20:12:00.772867+0000","last_undegraded":"2023-09-23T20:07:19.959595+0000","last_fullsized":"2023-09-23T20:07:19.924202+0000","mapping_epoch":317,"log_start":"318'4188","ondisk_log_start":"318'4188","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5121,"log_dups_size":0,"ondisk_log_size":5121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:21.022797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86966272,"num_objects":1327,"num_object_clones":0,"num_object_copies":3981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9309,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":3407872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":189},{"shards":"2,5","objects":-2801}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":516,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.800887+0000","last_change":"2023-09-23T20:09:56.051517+0000","last_active":"2023-09-23T20:11:50.800887+0000","last_peered":"2023-09-23T20:11:50.800887+0000","last_clean":"2023-09-23T20:11:50.800887+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:11:50.800887+0000","last_undegraded":"2023-09-23T20:11:50.800887+0000","last_fullsized":"2023-09-23T20:11:50.800887+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":826,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796986+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:11:50.796986+0000","last_peered":"2023-09-23T20:11:50.796986+0000","last_clean":"2023-09-23T20:11:50.796986+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:11:50.796986+0000","last_undegraded":"2023-09-23T20:11:50.796986+0000","last_fullsized":"2023-09-23T20:11:50.796986+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":376,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789476+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:11:50.789476+0000","last_peered":"2023-09-23T20:11:50.789476+0000","last_clean":"2023-09-23T20:11:50.789476+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:11:50.789476+0000","last_undegraded":"2023-09-23T20:11:50.789476+0000","last_fullsized":"2023-09-23T20:11:50.789476+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":86,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.800477+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:11:50.800477+0000","last_peered":"2023-09-23T20:11:50.800477+0000","last_clean":"2023-09-23T20:11:50.800477+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:11:50.800477+0000","last_undegraded":"2023-09-23T20:11:50.800477+0000","last_fullsized":"2023-09-23T20:11:50.800477+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"390'9313","reported_seq":16430,"reported_epoch":390,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:12:00.804862+0000","last_change":"2023-09-23T20:10:58.722697+0000","last_active":"2023-09-23T20:12:00.804862+0000","last_peered":"2023-09-23T20:12:00.804862+0000","last_clean":"2023-09-23T20:07:17.917130+0000","last_became_active":"2023-09-23T20:10:56.953707+0000","last_became_peered":"2023-09-23T20:10:56.953707+0000","last_unstale":"2023-09-23T20:12:00.804862+0000","last_undegraded":"2023-09-23T20:12:00.804862+0000","last_fullsized":"2023-09-23T20:10:53.691838+0000","mapping_epoch":369,"log_start":"315'4146","ondisk_log_start":"315'4146","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5167,"log_dups_size":0,"ondisk_log_size":5167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:08.588715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89587712,"num_objects":1367,"num_object_clones":0,"num_object_copies":4101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":185,"num_objects_unfound":0,"num_objects_dirty":1367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9313,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4385,"num_bytes_recovered":249167872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,1],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"1,2","objects":1182},{"shards":"2,3,4","objects":185}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":589,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:55.924231+0000","last_change":"2023-09-23T20:11:55.924213+0000","last_active":"2023-09-23T20:11:55.924231+0000","last_peered":"2023-09-23T20:11:55.924231+0000","last_clean":"2023-09-23T20:11:55.924231+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:11:55.924231+0000","last_undegraded":"2023-09-23T20:11:55.924231+0000","last_fullsized":"2023-09-23T20:11:55.924231+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":90,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.797358+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:11:50.797358+0000","last_peered":"2023-09-23T20:11:50.797358+0000","last_clean":"2023-09-23T20:11:50.797358+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:11:50.797358+0000","last_undegraded":"2023-09-23T20:11:50.797358+0000","last_fullsized":"2023-09-23T20:11:50.797358+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":86,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.796292+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:11:50.796292+0000","last_peered":"2023-09-23T20:11:50.796292+0000","last_clean":"2023-09-23T20:11:50.796292+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:11:50.796292+0000","last_undegraded":"2023-09-23T20:11:50.796292+0000","last_fullsized":"2023-09-23T20:11:50.796292+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":216,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791084+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:11:50.791084+0000","last_peered":"2023-09-23T20:11:50.791084+0000","last_clean":"2023-09-23T20:11:50.791084+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:11:50.791084+0000","last_undegraded":"2023-09-23T20:11:50.791084+0000","last_fullsized":"2023-09-23T20:11:50.791084+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"390'9341","reported_seq":13638,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:03.585495+0000","last_change":"2023-09-23T20:08:33.396812+0000","last_active":"2023-09-23T20:12:03.585495+0000","last_peered":"2023-09-23T20:12:03.585495+0000","last_clean":"2023-09-23T20:12:03.585495+0000","last_became_active":"2023-09-23T20:08:05.774750+0000","last_became_peered":"2023-09-23T20:08:05.774750+0000","last_unstale":"2023-09-23T20:12:03.585495+0000","last_undegraded":"2023-09-23T20:12:03.585495+0000","last_fullsized":"2023-09-23T20:12:03.585495+0000","mapping_epoch":334,"log_start":"308'2500","ondisk_log_start":"308'2500","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6841,"log_dups_size":0,"ondisk_log_size":6841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.633506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85524480,"num_objects":1305,"num_object_clones":0,"num_object_copies":3915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9341,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2086,"num_bytes_recovered":136708096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":814,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789169+0000","last_change":"2023-09-23T20:11:45.860623+0000","last_active":"2023-09-23T20:11:50.789169+0000","last_peered":"2023-09-23T20:11:50.789169+0000","last_clean":"2023-09-23T20:11:50.789169+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:11:50.789169+0000","last_undegraded":"2023-09-23T20:11:50.789169+0000","last_fullsized":"2023-09-23T20:11:50.789169+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":232,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.797540+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:11:50.797540+0000","last_peered":"2023-09-23T20:11:50.797540+0000","last_clean":"2023-09-23T20:11:50.797540+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:11:50.797540+0000","last_undegraded":"2023-09-23T20:11:50.797540+0000","last_fullsized":"2023-09-23T20:11:50.797540+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":428,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:58.790512+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:11:58.790512+0000","last_peered":"2023-09-23T20:11:58.790512+0000","last_clean":"2023-09-23T20:11:58.790512+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:11:58.790512+0000","last_undegraded":"2023-09-23T20:11:58.790512+0000","last_fullsized":"2023-09-23T20:11:58.790512+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":20,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.782585+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:11:50.782585+0000","last_peered":"2023-09-23T20:11:50.782585+0000","last_clean":"2023-09-23T20:11:50.782585+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:11:50.782585+0000","last_undegraded":"2023-09-23T20:11:50.782585+0000","last_fullsized":"2023-09-23T20:11:50.782585+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"390'9278","reported_seq":12046,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T20:12:03.210667+0000","last_change":"2023-09-23T20:09:08.683160+0000","last_active":"2023-09-23T20:12:03.210667+0000","last_peered":"2023-09-23T20:12:03.210667+0000","last_clean":"2023-09-23T20:12:03.210667+0000","last_became_active":"2023-09-23T20:09:08.670527+0000","last_became_peered":"2023-09-23T20:09:08.670527+0000","last_unstale":"2023-09-23T20:12:03.210667+0000","last_undegraded":"2023-09-23T20:12:03.210667+0000","last_fullsized":"2023-09-23T20:12:03.210667+0000","mapping_epoch":346,"log_start":"309'3000","ondisk_log_start":"309'3000","created":299,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6278,"log_dups_size":0,"ondisk_log_size":6278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:47.531381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81592344,"num_objects":1246,"num_object_clones":0,"num_object_copies":3738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1246,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9278,"num_write_kb":336705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":700,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.785083+0000","last_change":"2023-09-23T20:09:56.036037+0000","last_active":"2023-09-23T20:11:50.785083+0000","last_peered":"2023-09-23T20:11:50.785083+0000","last_clean":"2023-09-23T20:11:50.785083+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:11:50.785083+0000","last_undegraded":"2023-09-23T20:11:50.785083+0000","last_fullsized":"2023-09-23T20:11:50.785083+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":91,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.797655+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:11:50.797655+0000","last_peered":"2023-09-23T20:11:50.797655+0000","last_clean":"2023-09-23T20:11:50.797655+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:11:50.797655+0000","last_undegraded":"2023-09-23T20:11:50.797655+0000","last_fullsized":"2023-09-23T20:11:50.797655+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":90,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.800221+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:11:50.800221+0000","last_peered":"2023-09-23T20:11:50.800221+0000","last_clean":"2023-09-23T20:11:50.800221+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:11:50.800221+0000","last_undegraded":"2023-09-23T20:11:50.800221+0000","last_fullsized":"2023-09-23T20:11:50.800221+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":349,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.788593+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:11:50.788593+0000","last_peered":"2023-09-23T20:11:50.788593+0000","last_clean":"2023-09-23T20:11:50.788593+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:11:50.788593+0000","last_undegraded":"2023-09-23T20:11:50.788593+0000","last_fullsized":"2023-09-23T20:11:50.788593+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":165,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.799178+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:11:50.799178+0000","last_peered":"2023-09-23T20:11:50.799178+0000","last_clean":"2023-09-23T20:11:50.799178+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:11:50.799178+0000","last_undegraded":"2023-09-23T20:11:50.799178+0000","last_fullsized":"2023-09-23T20:11:50.799178+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":156,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.799569+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:11:50.799569+0000","last_peered":"2023-09-23T20:11:50.799569+0000","last_clean":"2023-09-23T20:11:50.799569+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:11:50.799569+0000","last_undegraded":"2023-09-23T20:11:50.799569+0000","last_fullsized":"2023-09-23T20:11:50.799569+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":393,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791679+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:11:50.791679+0000","last_peered":"2023-09-23T20:11:50.791679+0000","last_clean":"2023-09-23T20:11:50.791679+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:11:50.791679+0000","last_undegraded":"2023-09-23T20:11:50.791679+0000","last_fullsized":"2023-09-23T20:11:50.791679+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":406,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.789346+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:11:50.789346+0000","last_peered":"2023-09-23T20:11:50.789346+0000","last_clean":"2023-09-23T20:11:50.789346+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:11:50.789346+0000","last_undegraded":"2023-09-23T20:11:50.789346+0000","last_fullsized":"2023-09-23T20:11:50.789346+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":383,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791724+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:11:50.791724+0000","last_peered":"2023-09-23T20:11:50.791724+0000","last_clean":"2023-09-23T20:11:50.791724+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:11:50.791724+0000","last_undegraded":"2023-09-23T20:11:50.791724+0000","last_fullsized":"2023-09-23T20:11:50.791724+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":386,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.788908+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:11:50.788908+0000","last_peered":"2023-09-23T20:11:50.788908+0000","last_clean":"2023-09-23T20:11:50.788908+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:11:50.788908+0000","last_undegraded":"2023-09-23T20:11:50.788908+0000","last_fullsized":"2023-09-23T20:11:50.788908+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":90,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.799855+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:11:50.799855+0000","last_peered":"2023-09-23T20:11:50.799855+0000","last_clean":"2023-09-23T20:11:50.799855+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:11:50.799855+0000","last_undegraded":"2023-09-23T20:11:50.799855+0000","last_fullsized":"2023-09-23T20:11:50.799855+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":247,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.798897+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:11:50.798897+0000","last_peered":"2023-09-23T20:11:50.798897+0000","last_clean":"2023-09-23T20:11:50.798897+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:11:50.798897+0000","last_undegraded":"2023-09-23T20:11:50.798897+0000","last_fullsized":"2023-09-23T20:11:50.798897+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":235,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.791820+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:11:50.791820+0000","last_peered":"2023-09-23T20:11:50.791820+0000","last_clean":"2023-09-23T20:11:50.791820+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:11:50.791820+0000","last_undegraded":"2023-09-23T20:11:50.791820+0000","last_fullsized":"2023-09-23T20:11:50.791820+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":436,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T20:11:50.788534+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:11:50.788534+0000","last_peered":"2023-09-23T20:11:50.788534+0000","last_clean":"2023-09-23T20:11:50.788534+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:11:50.788534+0000","last_undegraded":"2023-09-23T20:11:50.788534+0000","last_fullsized":"2023-09-23T20:11:50.788534+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":26174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1348403224,"num_objects":20576,"num_object_clones":0,"num_object_copies":61728,"num_objects_missing_on_primary":1931,"num_objects_missing":1931,"num_objects_degraded":9040,"num_objects_misplaced":1430,"num_objects_unfound":0,"num_objects_dirty":20576,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147886,"num_write_kb":5390721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30718,"num_bytes_recovered":1600520192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5644234752,"data_stored":5644222536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91105,"ondisk_log_size":91105,"up":48,"acting":43,"num_store_stats":7},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":542359,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":177,"num_read_kb":331,"num_write":87,"num_write_kb":878,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1299398,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234371,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":722060,"kb_used_data":381508,"kb_used_omap":0,"kb_used_meta":340544,"kb_avail":93023092,"statfs":{"total":95995035648,"available":95255646208,"internally_reserved":0,"allocated":390664192,"data_stored":389410373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":348717056},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.0449999999999999,"15min":2.2400000000000002},"min":{"1min":0.23499999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.5030000000000001,"5min":55.390000000000001,"15min":62.061999999999998},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.034,"5min":1.9870000000000001,"15min":2.3780000000000001},"min":{"1min":0.248,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":3.726,"5min":56.664000000000001,"15min":62.389000000000003},"last":0.59199999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.907,"15min":3.4039999999999999},"min":{"1min":0.29299999999999998,"5min":0.28299999999999997,"15min":0.22700000000000001},"max":{"1min":3.7080000000000002,"5min":54.945999999999998,"15min":104.16800000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.84299999999999997,"5min":2,"15min":3.4929999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.223},"max":{"1min":2.9390000000000001,"5min":55.094000000000001,"15min":104.003},"last":0.34999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":4.0800000000000001,"15min":3.6440000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":28.707000000000001,"5min":92.319000000000003,"15min":105.842},"last":3.0539999999999998},{"interface":"front","average":{"1min":2.8919999999999999,"5min":4.4699999999999998,"15min":4.1390000000000002},"min":{"1min":0.34300000000000003,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":28.596,"5min":92.049000000000007,"15min":105.66},"last":0.52500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":3.1070000000000002,"15min":4.3840000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":4.0359999999999996,"5min":94.953000000000003,"15min":146.38800000000001},"last":3.2970000000000002},{"interface":"front","average":{"1min":1.159,"5min":3.0390000000000001,"15min":4.4020000000000001},"min":{"1min":0.41399999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.6949999999999998,"5min":93.423000000000002,"15min":146.851},"last":2.0139999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.53500000000000003,"15min":1.3089999999999999},"min":{"1min":0.26400000000000001,"5min":0.25900000000000001,"15min":0.22500000000000001},"max":{"1min":1.048,"5min":5.431,"15min":143.255},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.60199999999999998,"15min":1.3480000000000001},"min":{"1min":0.29899999999999999,"5min":0.246,"15min":0.246},"max":{"1min":1.1020000000000001,"5min":4.1459999999999999,"15min":143.41200000000001},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.94699999999999995,"15min":1.056},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":0.60999999999999999,"5min":24.876000000000001,"15min":35.488},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.874,"15min":0.879},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":0.98699999999999999,"5min":21.623000000000001,"15min":34.896000000000001},"last":0.33000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.625,"15min":0.80800000000000005},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":1.4650000000000001,"5min":4.3710000000000004,"15min":41.902999999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.63200000000000001,"15min":0.64200000000000002},"min":{"1min":0.38400000000000001,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":0.85599999999999998,"5min":3.9889999999999999,"15min":6.3620000000000001},"last":0.36499999999999999}]}]},{"osd":6,"up_from":0,"seq":1078036791412,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762570,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1104536,"kb_used_data":540880,"kb_used_omap":7,"kb_used_meta":563640,"kb_avail":92640616,"statfs":{"total":95995035648,"available":94863990784,"internally_reserved":0,"allocated":553861120,"data_stored":552013524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":577168186},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,0,2,1,3,2,3,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.1339999999999999,"15min":3.698},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.151},"max":{"1min":9.9410000000000007,"5min":28.510000000000002,"15min":186.84},"last":0.218},{"interface":"front","average":{"1min":1.776,"5min":2.4209999999999998,"15min":3.9929999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":8.6660000000000004,"5min":14.781000000000001,"15min":191.322},"last":2.4510000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":4.6180000000000003,"15min":5.508},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":17.785,"5min":98.531000000000006,"15min":180.25200000000001},"last":1.016},{"interface":"front","average":{"1min":2.2589999999999999,"5min":4.2859999999999996,"15min":5.548},"min":{"1min":0.23300000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":28.469999999999999,"5min":98.346000000000004,"15min":180.05000000000001},"last":0.48999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.5459999999999998,"15min":3.7879999999999998},"min":{"1min":0.25700000000000001,"5min":0.255,"15min":0.22},"max":{"1min":3.9849999999999999,"5min":25.963999999999999,"15min":68.180999999999997},"last":0.255},{"interface":"front","average":{"1min":1.4019999999999999,"5min":2.4430000000000001,"15min":3.6909999999999998},"min":{"1min":0.29299999999999998,"5min":0.28199999999999997,"15min":0.24299999999999999},"max":{"1min":6.4320000000000004,"5min":40.716000000000001,"15min":66.509},"last":2.5059999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.6140000000000001,"15min":1.9430000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23599999999999999},"max":{"1min":6.6029999999999998,"5min":18.087,"15min":60.951000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.573,"5min":1.764,"15min":2.1190000000000002},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":8.1669999999999998,"5min":14.890000000000001,"15min":60.841000000000001},"last":2.3620000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.8300000000000001,"15min":2.952},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.25900000000000001},"max":{"1min":6.5679999999999996,"5min":14.83,"15min":219.048},"last":2.4009999999999998},{"interface":"front","average":{"1min":1.3440000000000001,"5min":2.1819999999999999,"15min":3.0950000000000002},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28399999999999997},"max":{"1min":6.0190000000000001,"5min":25.893000000000001,"15min":218.69499999999999},"last":0.40200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:11:43 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.7649999999999999,"15min":1.994},"min":{"1min":0.38800000000000001,"5min":0.38500000000000001,"15min":0.35899999999999999},"max":{"1min":6.6749999999999998,"5min":18.117999999999999,"15min":18.117999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.8089999999999999,"15min":2.6579999999999999},"min":{"1min":0.42299999999999999,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":6.5069999999999997,"5min":14.862,"15min":20.734999999999999},"last":2.427}]},{"osd":7,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":2.1869999999999998,"15min":2.3239999999999998},"min":{"1min":0.42999999999999999,"5min":0.33300000000000002,"15min":0.26800000000000002},"max":{"1min":4.5899999999999999,"5min":25.937000000000001,"15min":60.213000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.556,"5min":1.7749999999999999,"15min":2.0619999999999998},"min":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.30199999999999999},"max":{"1min":6.649,"5min":18.053999999999998,"15min":61.298999999999999},"last":0.45800000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926107,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375336,"kb_used_data":744864,"kb_used_omap":8,"kb_used_meta":630455,"kb_avail":92369816,"statfs":{"total":95995035648,"available":94586691584,"internally_reserved":0,"allocated":762740736,"data_stored":760881284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":645586529},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5190000000000001,"5min":3.3839999999999999,"15min":3.8860000000000001},"min":{"1min":0.21299999999999999,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":72.534000000000006,"5min":72.534000000000006,"15min":94.055999999999997},"last":1.3069999999999999},{"interface":"front","average":{"1min":5.5250000000000004,"5min":3.6579999999999999,"15min":4.0819999999999999},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":72.667000000000002,"5min":72.667000000000002,"15min":91.632999999999996},"last":2.4129999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:11:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":3.5880000000000001,"15min":5.0199999999999996},"min":{"1min":0.255,"5min":0.217,"15min":0.19800000000000001},"max":{"1min":9.0700000000000003,"5min":102.65300000000001,"15min":270.51100000000002},"last":1.4670000000000001},{"interface":"front","average":{"1min":2.0139999999999998,"5min":3.6150000000000002,"15min":4.8810000000000002},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.2599999999999998,"5min":96.578999999999994,"15min":269.69299999999998},"last":2.516}]},{"osd":3,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":4.4909999999999997,"15min":5.5590000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":8.6489999999999991,"5min":126.95399999999999,"15min":145.72800000000001},"last":10.082000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":4.8780000000000001,"15min":5.7240000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.23999999999999999},"max":{"1min":5.4690000000000003,"5min":127.294,"15min":145.50700000000001},"last":0.89300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.5629999999999999,"15min":2.2629999999999999},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":6.1369999999999996,"5min":15.688000000000001,"15min":148.20599999999999},"last":2.3860000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.4039999999999999,"15min":2.0379999999999998},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":5.3940000000000001,"5min":13.428000000000001,"15min":148.11000000000001},"last":0.92000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.298,"15min":2.464},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":1.1459999999999999,"5min":9.5719999999999992,"15min":131.34399999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.163,"5min":1.718,"15min":2.468},"min":{"1min":0.377,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.1059999999999999,"5min":15.717000000000001,"15min":131.386},"last":2.4340000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.7270000000000001,"15min":1.6439999999999999},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":4.9800000000000004,"5min":15.744999999999999,"15min":15.744999999999999},"last":2.4660000000000002},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.5549999999999999,"15min":1.841},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":5.4390000000000001,"5min":13.470000000000001,"15min":13.766999999999999},"last":0.93899999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.327,"15min":1.891},"min":{"1min":0.38900000000000001,"5min":0.38100000000000001,"15min":0.31900000000000001},"max":{"1min":1.0980000000000001,"5min":9.6069999999999993,"15min":65.081999999999994},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.724,"15min":2.0190000000000001},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.1920000000000002,"5min":15.771000000000001,"15min":15.771000000000001},"last":2.492}]}]},{"osd":2,"up_from":104,"seq":446676599045,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1637936,"kb_used_data":907048,"kb_used_omap":12,"kb_used_meta":730867,"kb_avail":92107216,"statfs":{"total":95995035648,"available":94317789184,"internally_reserved":0,"allocated":928817152,"data_stored":926950274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":748408830},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.9119999999999999,"15min":3.1739999999999999},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.154},"max":{"1min":3.714,"5min":17.876999999999999,"15min":76.251999999999995},"last":9.6989999999999998},{"interface":"front","average":{"1min":1.323,"5min":2.4609999999999999,"15min":3.3149999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.192},"max":{"1min":10.808999999999999,"5min":18.561,"15min":76.590000000000003},"last":3.3199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":2.3159999999999998,"15min":3.6699999999999999},"min":{"1min":0.24199999999999999,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":2.4729999999999999,"5min":19.562999999999999,"15min":94.768000000000001},"last":1.704},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.9079999999999999,"15min":3.6589999999999998},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":3.6320000000000001,"5min":19.314,"15min":96.808000000000007},"last":6.7110000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.5169999999999999,"15min":4.4029999999999996},"min":{"1min":0.32800000000000001,"5min":0.25,"15min":0.214},"max":{"1min":8.5850000000000009,"5min":13.513,"15min":107.381},"last":4.1909999999999998},{"interface":"front","average":{"1min":1.0900000000000001,"5min":2.097,"15min":3.7610000000000001},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.23699999999999999},"max":{"1min":4.7919999999999998,"5min":20.308,"15min":106.301},"last":9.9019999999999992}]},{"osd":4,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.609,"15min":2.2170000000000001},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.26900000000000002},"max":{"1min":3.742,"5min":75.608000000000004,"15min":75.608000000000004},"last":9.7370000000000001},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.7839999999999998,"15min":2.2330000000000001},"min":{"1min":0.34599999999999997,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":2.544,"5min":76.161000000000001,"15min":76.161000000000001},"last":1.7310000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.1549999999999998,"15min":2.726},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.28100000000000003},"max":{"1min":3.77,"5min":65.921999999999997,"15min":172.505},"last":9.8040000000000003},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.5489999999999999,"15min":2.8959999999999999},"min":{"1min":0.39500000000000002,"5min":0.38100000000000001,"15min":0.29099999999999998},"max":{"1min":10.696,"5min":63.792000000000002,"15min":172.375},"last":3.52}]},{"osd":6,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.5900000000000001,"15min":1.601},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":4.718,"5min":10.209,"15min":10.209},"last":9.7680000000000007},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.8220000000000001,"15min":1.909},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.776,"5min":14.279999999999999,"15min":14.279999999999999},"last":3.5569999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.6599999999999999,"15min":1.9379999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.28799999999999998},"max":{"1min":2.5699999999999998,"5min":8.1519999999999992,"15min":50.118000000000002},"last":1.7509999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.619,"15min":2.0419999999999998},"min":{"1min":0.41699999999999998,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":3.794,"5min":10.244,"15min":50.664999999999999},"last":9.8740000000000006}]}]},{"osd":3,"up_from":109,"seq":468151435519,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1783760,"kb_used_data":1119368,"kb_used_omap":4,"kb_used_meta":664379,"kb_avail":91961392,"statfs":{"total":95995035648,"available":94168465408,"internally_reserved":0,"allocated":1146232832,"data_stored":1144206159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":680324205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":4.2430000000000003,"15min":3.4049999999999998},"min":{"1min":0.19700000000000001,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":7.6440000000000001,"5min":74.323999999999998,"15min":74.323999999999998},"last":2.8879999999999999},{"interface":"front","average":{"1min":0.94599999999999995,"5min":4.0170000000000003,"15min":3.758},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":4.9000000000000004,"5min":74.447000000000003,"15min":74.447000000000003},"last":3.8300000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":8.7829999999999995,"5min":4.7240000000000002,"15min":4.0599999999999996},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":163.49700000000001,"5min":163.49700000000001,"15min":163.49700000000001},"last":1.373},{"interface":"front","average":{"1min":8.5839999999999996,"5min":5.2400000000000002,"15min":4.351},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":163.512,"5min":163.512,"15min":163.512},"last":4.2960000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":12.541,"5min":6.2560000000000002,"15min":4.5679999999999996},"min":{"1min":0.33800000000000002,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":163.83500000000001,"5min":163.83500000000001,"15min":163.83500000000001},"last":1.2430000000000001},{"interface":"front","average":{"1min":11.702,"5min":6.319,"15min":4.4500000000000002},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":163.92099999999999,"5min":163.92099999999999,"15min":163.92099999999999},"last":1.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":2.4489999999999998,"15min":3.1349999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.3740000000000001,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.7669999999999999,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":5.4820000000000002,"5min":58.408000000000001,"15min":110.295},"last":3.7989999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":3.452,"15min":3.3799999999999999},"min":{"1min":0.316,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.5270000000000001,"5min":77.206000000000003,"15min":113.881},"last":3.7519999999999998},{"interface":"front","average":{"1min":0.80400000000000005,"5min":3.7250000000000001,"15min":3.2650000000000001},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":2.2930000000000001,"5min":77.968000000000004,"15min":114.598},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.629,"15min":3.8999999999999999},"min":{"1min":0.41599999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":2.2759999999999998,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.8439999999999999,"15min":4.0839999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":3.9279999999999999,"5min":58.701000000000001,"15min":82.233000000000004},"last":0.48199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.8210000000000002,"15min":3.4740000000000002},"min":{"1min":0.40000000000000002,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":2.2559999999999998,"5min":58.686,"15min":138.035},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.8679999999999999,"15min":3.4510000000000001},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.28699999999999998},"max":{"1min":3.4260000000000002,"5min":58.667999999999999,"15min":138.054},"last":0.54100000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019281,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1326468,"kb_used_data":992508,"kb_used_omap":0,"kb_used_meta":333888,"kb_avail":92418684,"statfs":{"total":95995035648,"available":94636732416,"internally_reserved":0,"allocated":1016328192,"data_stored":1015082529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":341901312},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.54,"15min":2.5699999999999998},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.189},"max":{"1min":3.2000000000000002,"5min":56.061999999999998,"15min":104.604},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.5409999999999999,"15min":2.5339999999999998},"min":{"1min":0.28899999999999998,"5min":0.218,"15min":0.218},"max":{"1min":3.1899999999999999,"5min":56.075000000000003,"15min":104.687},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":2.9489999999999998,"15min":5.5389999999999997},"min":{"1min":0.252,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":4.1669999999999998,"5min":119.892,"15min":161.613},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":3.2109999999999999,"15min":5.6139999999999999},"min":{"1min":0.28899999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.125,"5min":119.351,"15min":162.017},"last":0.45700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":6.6200000000000001,"15min":6.75},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.245},"max":{"1min":17.948,"5min":182.96700000000001,"15min":182.96700000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.349,"5min":6.2729999999999997,"15min":6.5039999999999996},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":17.978000000000002,"5min":183.244,"15min":183.244},"last":0.49399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":4.843,"15min":6.8440000000000003},"min":{"1min":0.30499999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":2.5259999999999998,"5min":184.37100000000001,"15min":266.54700000000003},"last":1.2270000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":4.9130000000000003,"15min":5.798},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":5.407,"5min":184.22300000000001,"15min":184.22300000000001},"last":0.46600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":2.1640000000000001,"15min":2.1160000000000001},"min":{"1min":0.25800000000000001,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":0.622,"5min":176.81899999999999,"15min":266.59899999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":2.2149999999999999,"15min":2.1880000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":0.56599999999999995,"5min":176.172,"15min":266.72899999999998},"last":0.378}]},{"osd":6,"last update":"Sat Sep 23 20:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.502,"15min":0.51000000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.804,"5min":2.5489999999999999,"15min":4.0410000000000004},"last":0.435},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60799999999999998,"15min":0.58899999999999997},"min":{"1min":0.38300000000000001,"5min":0.247,"15min":0.247},"max":{"1min":1.5880000000000001,"5min":4.2889999999999997,"15min":4.2889999999999997},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.337,"15min":1.446},"min":{"1min":0.36799999999999999,"5min":0.32000000000000001,"15min":0.26100000000000001},"max":{"1min":0.71699999999999997,"5min":74.040000000000006,"15min":76.507000000000005},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.5,"5min":1.3049999999999999,"15min":1.5629999999999999},"min":{"1min":0.36399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.61099999999999999,"5min":74.021000000000001,"15min":76.462000000000003},"last":0.41099999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757645,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1322560,"kb_used_data":920504,"kb_used_omap":0,"kb_used_meta":402048,"kb_avail":92422592,"statfs":{"total":95995035648,"available":94640734208,"internally_reserved":0,"allocated":942596096,"data_stored":941350802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":411697152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.88700000000000001,"15min":1.881},"min":{"1min":0.23000000000000001,"5min":0.224,"15min":0.216},"max":{"1min":0.625,"5min":8.6739999999999995,"15min":121.94799999999999},"last":0.316},{"interface":"front","average":{"1min":0.436,"5min":1.1399999999999999,"15min":2.0750000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":1.28,"5min":10.327,"15min":121.964},"last":1.6699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":2.093,"15min":5.8949999999999996},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":3.52,"5min":75.204999999999998,"15min":266.197},"last":0.629},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.1480000000000001,"15min":6.0700000000000003},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1399999999999997,"5min":75.231999999999999,"15min":266.14999999999998},"last":0.36899999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":5.6639999999999997,"15min":5.4379999999999997},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":4.1239999999999997,"5min":240.57300000000001,"15min":240.57300000000001},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.2509999999999999,"5min":5.7359999999999998,"15min":5.4969999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.313},"max":{"1min":4.1559999999999997,"5min":240.464,"15min":240.464},"last":0.95899999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.7030000000000001,"15min":4.8390000000000004},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.742,"5min":14.808,"15min":201.50899999999999},"last":6.9509999999999996},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.407,"15min":4.6959999999999997},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":3.4500000000000002,"5min":14.893000000000001,"15min":201.58099999999999},"last":0.76300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.71199999999999997,"15min":1.4490000000000001},"min":{"1min":0.27700000000000002,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":0.58799999999999997,"5min":9.7240000000000002,"15min":191.78899999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.59299999999999997,"15min":1.3360000000000001},"min":{"1min":0.311,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":1.3879999999999999,"5min":8.7360000000000007,"15min":191.821},"last":0.33300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.69299999999999995,"15min":0.64100000000000001},"min":{"1min":0.377,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":3.278,"5min":9.9009999999999998,"15min":9.9009999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.54600000000000004,"15min":0.51700000000000002},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.72799999999999998,"5min":5.0119999999999996,"15min":5.0119999999999996},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.56699999999999995,"15min":1.4930000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.3069999999999999,"5min":4.2080000000000002,"15min":191.84299999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65900000000000003,"15min":1.655},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":3.1440000000000001,"5min":9.6920000000000002,"15min":191.69},"last":0.42099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":741670912,"data_stored":741670912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":544604160,"data_stored":544604160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":916983808,"data_stored":916979736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1127677952,"data_stored":1127677952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1004470272,"data_stored":1004470272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":930549760,"data_stored":930545688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":378277888,"data_stored":378273816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:12:10.129 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:12:10.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:09 smithi144 ceph-mon[194833]: failed to encode map e391 with expected crc 2023-09-23T20:12:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:09 smithi144 ceph-mon[194833]: failed to encode map e391 with expected crc 2023-09-23T20:12:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:09 smithi144 ceph-mon[194833]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T20:12:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:09 smithi144 ceph-mon[194833]: failed to encode map e391 with expected crc 2023-09-23T20:12:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:09 smithi144 ceph-mon[194833]: failed to encode map e391 with expected crc 2023-09-23T20:12:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:09 smithi144 ceph-mon[194833]: pgmap v1100: 109 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 6519/53166 objects degraded (12.262%); 1086/53166 objects misplaced (2.043%); 4.7 MiB/s, 85 objects/s recovering 2023-09-23T20:12:10.180 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:10.294 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:10.398 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:12:10.616 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:10.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:10.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:10.666 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:12:10.666 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:12:10.730 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:10.758 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:10.804 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:10.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:10.911 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:10.952 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:10.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[221113]: failed to encode map e392 with expected crc 2023-09-23T20:12:10.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[221113]: failed to encode map e392 with expected crc 2023-09-23T20:12:10.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[221113]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T20:12:10.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[221113]: failed to encode map e392 with expected crc 2023-09-23T20:12:10.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[221113]: failed to encode map e392 with expected crc 2023-09-23T20:12:10.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[213871]: failed to encode map e392 with expected crc 2023-09-23T20:12:10.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[213871]: failed to encode map e392 with expected crc 2023-09-23T20:12:10.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[213871]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T20:12:10.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[213871]: failed to encode map e392 with expected crc 2023-09-23T20:12:10.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:10 smithi135 ceph-mon[213871]: failed to encode map e392 with expected crc 2023-09-23T20:12:11.012 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:11.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:11.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:11.021 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:10 smithi144 ceph-mon[194833]: failed to encode map e392 with expected crc 2023-09-23T20:12:11.021 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:10 smithi144 ceph-mon[194833]: failed to encode map e392 with expected crc 2023-09-23T20:12:11.021 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:10 smithi144 ceph-mon[194833]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T20:12:11.021 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:10 smithi144 ceph-mon[194833]: failed to encode map e392 with expected crc 2023-09-23T20:12:11.021 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:10 smithi144 ceph-mon[194833]: failed to encode map e392 with expected crc 2023-09-23T20:12:11.121 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:11.154 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:11.673 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:11.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:11.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:11.783 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:11.815 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:11.847 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:12:11.869 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:11.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:11.934 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:11 smithi135 ceph-mon[213871]: pgmap v1102: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 6519/51381 objects degraded (12.688%); 1075/51381 objects misplaced (2.092%); 3.5 MiB/s, 87 objects/s recovering 2023-09-23T20:12:11.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:11 smithi135 ceph-mon[221113]: pgmap v1102: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 6519/51381 objects degraded (12.688%); 1075/51381 objects misplaced (2.092%); 3.5 MiB/s, 87 objects/s recovering 2023-09-23T20:12:11.976 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:12.083 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:12.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:11 smithi144 ceph-mon[194833]: pgmap v1102: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 6519/51381 objects degraded (12.688%); 1075/51381 objects misplaced (2.092%); 3.5 MiB/s, 87 objects/s recovering 2023-09-23T20:12:12.276 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:12.305 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:12.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:12.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:12.339 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:12.433 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:12:12.515 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:12.755 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:12:12.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:12.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:12:12.798 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:12:12.868 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:12.900 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:12.942 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2392858548' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:12:12.943 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:12 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6519/51381 objects degraded (12.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:12.944 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2392858548' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:12:12.944 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:12 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6519/51381 objects degraded (12.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:12.947 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:12.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:13.052 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:13.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2392858548' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:12:13.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:12 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6519/51381 objects degraded (12.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:13.151 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:13.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:13.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2392858548' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:12:13.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[213871]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T20:12:13.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[213871]: failed to encode map e393 with expected crc 2023-09-23T20:12:13.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[213871]: failed to encode map e393 with expected crc 2023-09-23T20:12:13.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[213871]: failed to encode map e393 with expected crc 2023-09-23T20:12:13.977 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[213871]: pgmap v1104: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 6519/50496 objects degraded (12.910%); 1075/50496 objects misplaced (2.129%); 1.1 MiB/s, 58 objects/s recovering 2023-09-23T20:12:13.977 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:12:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:12:13] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-09-23T20:12:13.977 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:13.977 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2392858548' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:12:13.978 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[221113]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T20:12:13.978 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[221113]: failed to encode map e393 with expected crc 2023-09-23T20:12:13.978 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[221113]: failed to encode map e393 with expected crc 2023-09-23T20:12:13.978 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[221113]: failed to encode map e393 with expected crc 2023-09-23T20:12:13.978 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:13 smithi135 ceph-mon[221113]: pgmap v1104: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 6519/50496 objects degraded (12.910%); 1075/50496 objects misplaced (2.129%); 1.1 MiB/s, 58 objects/s recovering 2023-09-23T20:12:14.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:13 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:14.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2392858548' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:12:14.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:13 smithi144 ceph-mon[194833]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T20:12:14.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:13 smithi144 ceph-mon[194833]: failed to encode map e393 with expected crc 2023-09-23T20:12:14.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:13 smithi144 ceph-mon[194833]: failed to encode map e393 with expected crc 2023-09-23T20:12:14.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:13 smithi144 ceph-mon[194833]: failed to encode map e393 with expected crc 2023-09-23T20:12:14.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:13 smithi144 ceph-mon[194833]: pgmap v1104: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 6519/50496 objects degraded (12.910%); 1075/50496 objects misplaced (2.129%); 1.1 MiB/s, 58 objects/s recovering 2023-09-23T20:12:15.082 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:14 smithi135 ceph-mon[213871]: failed to encode map e393 with expected crc 2023-09-23T20:12:15.083 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:14 smithi135 ceph-mon[221113]: failed to encode map e393 with expected crc 2023-09-23T20:12:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:14 smithi144 ceph-mon[194833]: failed to encode map e393 with expected crc 2023-09-23T20:12:15.215 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:15.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:15.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:15.324 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:15.351 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:15.393 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:15.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:15.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:15.501 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:15.529 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:15.592 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:15.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:15.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:15.701 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:15.784 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:15.813 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:15.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:15 smithi144 ceph-mon[194833]: pgmap v1105: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 955 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 6519/45717 objects degraded (14.259%); 910/45717 objects misplaced (1.991%); 1.0 MiB/s, 138 objects/s recovering 2023-09-23T20:12:15.993 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:15.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:15.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:16.057 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:15 smithi135 ceph-mon[213871]: pgmap v1105: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 955 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 6519/45717 objects degraded (14.259%); 910/45717 objects misplaced (1.991%); 1.0 MiB/s, 138 objects/s recovering 2023-09-23T20:12:16.057 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:15 smithi135 ceph-mon[221113]: pgmap v1105: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 955 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 6519/45717 objects degraded (14.259%); 910/45717 objects misplaced (1.991%); 1.0 MiB/s, 138 objects/s recovering 2023-09-23T20:12:16.100 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:16.130 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:16.189 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:16.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:16.294 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:16.322 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:16.377 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:16.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:16.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:16.489 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:16.518 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:16.566 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:16.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:16.671 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:16.698 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:16.747 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:16.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:16.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:16.813 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:12:16.852 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:16.884 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:16.892 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:12:16.932 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:12:16.935 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:16.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:16.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:16.963 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:12:16.963 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6519/45717 objects degraded (14.259%), 4 pgs degraded, 4 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1506,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":393,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":7,"osd_in_since":1695499655,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":103},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":15239,"data_bytes":1001544770,"bytes_used":8509206528,"bytes_avail":663456043008,"bytes_total":671965249536,"degraded_objects":6519,"degraded_total":45717,"degraded_ratio":0.14259465844215499,"misplaced_objects":910,"misplaced_total":45717,"misplaced_ratio":0.019905068136579392,"recovering_objects_per_sec":138,"recovering_bytes_per_sec":1059811,"recovering_keys_per_sec":0,"num_objects_recovered":862,"num_bytes_recovered":6619136,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":481},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-23T20:06:56.817144+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"e76096c8-5249-4f94-9197-02047057b831":{"message":"Global Recovery Event (5m)\n [==========================..] (remaining: 18s)","progress":0.94495415687561035,"add_to_ceph_s":true}}} 2023-09-23T20:12:17.003 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/510854171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:17.003 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/510854171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:17.029 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:12:17.041 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:17.063 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:12:17.114 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:12:17.116 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:17.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/510854171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:17.161 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:12:17.200 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:12:17.233 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:12:17.248 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:12:17.248 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:12:17.249 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:12:17.249 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:12:17.249 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:12:17.249 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:12:17.249 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:12:17.249 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:12:18.178 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3276371268' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:12:18.178 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:17 smithi135 ceph-mon[221113]: pgmap v1106: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 886 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 6519/42411 objects degraded (15.371%); 910/42411 objects misplaced (2.146%); 88 KiB/s, 85 objects/s recovering 2023-09-23T20:12:18.178 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:17 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6519/45717 objects degraded (14.259%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:18.178 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3276371268' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:12:18.178 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:17 smithi135 ceph-mon[213871]: pgmap v1106: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 886 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 6519/42411 objects degraded (15.371%); 910/42411 objects misplaced (2.146%); 88 KiB/s, 85 objects/s recovering 2023-09-23T20:12:18.179 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:17 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6519/45717 objects degraded (14.259%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3276371268' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:12:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:17 smithi144 ceph-mon[194833]: pgmap v1106: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 886 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 6519/42411 objects degraded (15.371%); 910/42411 objects misplaced (2.146%); 88 KiB/s, 85 objects/s recovering 2023-09-23T20:12:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:17 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6519/45717 objects degraded (14.259%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:19.279 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:19.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:19.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:19.387 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:19.458 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:19.481 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:12:19.482 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:12:19.729 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:19.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:19.838 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:19.942 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:19.986 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:20.065 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:20.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:20.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:20.173 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:175181 7.7 7:eb856231:::benchmark_data_smithi135_404353_object78119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.719599+0000", 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 406.44405959300002, 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.34012610199999999, 2023-09-23T20:12:20.174 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 175181 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719599+0000", 2023-09-23T20:12:20.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719599+0000", 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719597+0000", 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719604+0000", 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9439999999999999e-06 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719617+0000", 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3339999999999999e-05 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719625+0000", 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5700000000000004e-06 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728365+0000", 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0087402499999999998 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728520+0000", 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000155014 2023-09-23T20:12:20.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728656+0000", 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013616600000000001 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.731769+0000", 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003113265 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.179 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038834+0000", 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30706430699999998 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038876+0000", 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2216999999999999e-05 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059552+0000", 2023-09-23T20:12:20.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.020676305999999998 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059607+0000", 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4753000000000002e-05 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059725+0000", 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117835 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.181 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:175178 7.4 e290/240 7:2901d72c:::benchmark_data_smithi135_404353_object78116:head v 290'10878, mlcod=290'10875)", 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.719930+0000", 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 406.443728475, 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31795096099999998, 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719930+0000", 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719930+0000", 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719929+0000", 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719935+0000", 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7559999999999996e-06 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719941+0000", 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9610000000000002e-06 2023-09-23T20:12:20.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719945+0000", 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7280000000000001e-06 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.723900+0000", 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0039551480000000003 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.723923+0000", 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3629e-05 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037838+0000", 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31391448599999999 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037881+0000", 2023-09-23T20:12:20.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2777000000000001e-05 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:175183 7.4 e290/240 7:299b4e88:::benchmark_data_smithi135_404353_object78121:head v 290'10879, mlcod=290'10875)", 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.724977+0000", 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 406.43868117699998, 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31294654799999999, 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.187 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724977+0000", 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724977+0000", 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724976+0000", 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724983+0000", 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7970000000000003e-06 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724994+0000", 2023-09-23T20:12:20.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1429e-05 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.725001+0000", 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7009999999999999e-06 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.730362+0000", 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053613630000000001 2023-09-23T20:12:20.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.730376+0000", 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3526000000000001e-05 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037888+0000", 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30751193100000002 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.191 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037924+0000", 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5802e-05 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:175182 7.5 e290/240 7:a10814f3:::benchmark_data_smithi135_404353_object78120:head v 290'11086, mlcod=290'11083)", 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.728954+0000", 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 406.43470420800003, 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30892793499999999, 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.192 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728954+0000", 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728954+0000", 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728953+0000", 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728959+0000", 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5990000000000001e-06 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.194 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728974+0000", 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4636e-05 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728980+0000", 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7680000000000003e-06 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.729049+0000", 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8345999999999999e-05 2023-09-23T20:12:20.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.729068+0000", 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9057000000000001e-05 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037848+0000", 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30878053300000002 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037882+0000", 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3633000000000001e-05 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:175190 7.7 7:eddb5a12:::benchmark_data_smithi135_404353_object78128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.732026+0000", 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 406.43163237599998, 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32787697900000001, 2023-09-23T20:12:20.197 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 175190 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732026+0000", 2023-09-23T20:12:20.198 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732026+0000", 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732024+0000", 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:20.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732031+0000", 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3180000000000001e-06 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037829+0000", 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30579754199999998 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037843+0000", 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4378e-05 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038899+0000", 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001056083 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.039026+0000", 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000126594 2023-09-23T20:12:20.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.039123+0000", 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6921000000000003e-05 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.042110+0000", 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029867309999999998 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.042985+0000", 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000875116 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.047677+0000", 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046925919999999998 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059761+0000", 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01208389 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059805+0000", 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3761999999999999e-05 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059903+0000", 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7800000000000006e-05 2023-09-23T20:12:20.204 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.35358.0:175181 7.7 e290/274 ondisk, result = 0)", 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.732356+0000", 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 406.43130182200002, 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30652289999999999, 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.205 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732356+0000", 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732356+0000", 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732355+0000", 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732361+0000", 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8690000000000001e-06 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037823+0000", 2023-09-23T20:12:20.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30546166600000002 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037835+0000", 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.168e-05 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038861+0000", 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010267570000000001 2023-09-23T20:12:20.208 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038869+0000", 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6520000000000006e-06 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038879+0000", 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0169e-05 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.209 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65675 8.1 8:8413818a:::benchmark_data_smithi135_460343_object65674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:19.118032+0000", 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 301.04562583299997, 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5792248369999999, 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.210 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65675 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118032+0000", 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118032+0000", 2023-09-23T20:12:20.211 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118031+0000", 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118053+0000", 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2157000000000001e-05 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.212 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118063+0000", 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0210000000000001e-05 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118071+0000", 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2199999999999992e-06 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118136+0000", 2023-09-23T20:12:20.213 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4697999999999996e-05 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118148+0000", 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1849e-05 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.676405+0000", 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5582569479999999 2023-09-23T20:12:20.214 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.678641+0000", 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022364920000000001 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.678889+0000", 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000247607 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.215 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.686930+0000", 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080408010000000002 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.688012+0000", 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001082118 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.693954+0000", 2023-09-23T20:12:20.216 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0059426150000000001 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.694023+0000", 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9093999999999999e-05 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.697257+0000", 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032336639999999998 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.217 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:91666 8.1 8:8cda5659:::benchmark_data_smithi135_460343_object7433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:14.988938+0000", 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 245.17472046, 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.349132473, 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.218 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 91666 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988938+0000", 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988938+0000", 2023-09-23T20:12:20.219 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988936+0000", 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988944+0000", 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6179999999999995e-06 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.220 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988958+0000", 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4174e-05 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988967+0000", 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4209999999999995e-06 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.221 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.993525+0000", 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045585570000000004 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.993654+0000", 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012858600000000001 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.993783+0000", 2023-09-23T20:12:20.222 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000129159 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.002953+0000", 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0091700099999999993 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.002991+0000", 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8149999999999999e-05 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.223 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.335673+0000", 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33268188599999998 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.335729+0000", 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5519999999999997e-05 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.224 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.338070+0000", 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023415509999999999 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:113327 8.1 8:8edd1613:::benchmark_data_smithi135_460343_object29094:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:58.101036+0000", 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 202.062622172, 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.23996662499999999, 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.225 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 113327 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101036+0000", 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.226 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101036+0000", 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101034+0000", 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.227 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101042+0000", 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3629999999999994e-06 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101056+0000", 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3973e-05 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101065+0000", 2023-09-23T20:12:20.228 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4689999999999997e-06 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.102649+0000", 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015837010000000001 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.102742+0000", 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3016999999999994e-05 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.229 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.102857+0000", 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000115259 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.109123+0000", 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0062665530000000002 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.114004+0000", 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048803420000000002 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.339586+0000", 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.225582702 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.339641+0000", 2023-09-23T20:12:20.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5003999999999997e-05 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.341003+0000", 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001361129 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:114810 8.1 8:890a0c99:::benchmark_data_smithi135_460343_object30577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:12:20.232 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:01.208771+0000", 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 198.954887063, 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24141304099999999, 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 114810 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208771+0000", 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208771+0000", 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208769+0000", 2023-09-23T20:12:20.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208777+0000", 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7319999999999998e-06 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208790+0000", 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3232e-05 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208799+0000", 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4120000000000007e-06 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.217359+0000", 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085597390000000002 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.236 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.217501+0000", 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014166100000000001 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.217635+0000", 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013428600000000001 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.224173+0000", 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065379990000000001 2023-09-23T20:12:20.237 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.227602+0000", 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034295369999999999 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.448962+0000", 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22135990799999999 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.449025+0000", 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2816000000000003e-05 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.450184+0000", 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0011589580000000001 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:114816 8.1 8:86e3ef4c:::benchmark_data_smithi135_460343_object30583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:01.212087+0000", 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 198.95157161700001, 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.248140421, 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.240 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 114816 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212087+0000", 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212087+0000", 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212085+0000", 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212093+0000", 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5459999999999992e-06 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.242 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212110+0000", 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7144000000000001e-05 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212120+0000", 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2590000000000006e-06 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.221012+0000", 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088927590000000001 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.221160+0000", 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014748699999999999 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.221300+0000", 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000140536 2023-09-23T20:12:20.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.224275+0000", 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029750200000000001 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.227642+0000", 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033664469999999998 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.458959+0000", 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.231317518 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.459016+0000", 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6428999999999998e-05 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.460227+0000", 2023-09-23T20:12:20.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001211061 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:137131 8.1 8:87785a6e:::benchmark_data_smithi135_460343_object52898:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:39.829778+0000", 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 40.333879750999998, 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16302156500000001, 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.247 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 137131 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.829778+0000", 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.248 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.829778+0000", 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.829777+0000", 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.829782+0000", 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.113e-06 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.829792+0000", 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9839999999999996e-06 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.829798+0000", 2023-09-23T20:12:20.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6210000000000001e-06 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.829834+0000", 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6338999999999999e-05 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.829942+0000", 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000108528 2023-09-23T20:12:20.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.830038+0000", 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5717999999999994e-05 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.832057+0000", 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020192809999999999 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.990152+0000", 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.158094552 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.991832+0000", 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001680267 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.991884+0000", 2023-09-23T20:12:20.253 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1940000000000001e-05 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:39.992800+0000", 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00091580999999999997 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.254 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:137726 8.1 8:8067c0eb:::benchmark_data_smithi135_460343_object53493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:41.383562+0000", 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 38.780095836000001, 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17975639500000001, 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 137726 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.255 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383562+0000", 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383562+0000", 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383561+0000", 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383567+0000", 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4130000000000003e-06 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383582+0000", 2023-09-23T20:12:20.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.438e-05 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383589+0000", 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3649999999999998e-06 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383630+0000", 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1159999999999999e-05 2023-09-23T20:12:20.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383742+0000", 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111358 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383836+0000", 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4891999999999999e-05 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.386813+0000", 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029765730000000002 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.387070+0000", 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000256514 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.562338+0000", 2023-09-23T20:12:20.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17526850099999999 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.562399+0000", 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0739000000000002e-05 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.563319+0000", 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00091990500000000005 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.261 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:137728 8.1 8:801befe5:::benchmark_data_smithi135_460343_object53495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:41.388353+0000", 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 38.775305541999998, 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17817260300000001, 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.262 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 137728 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388353+0000", 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388353+0000", 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388351+0000", 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388360+0000", 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6460000000000005e-06 2023-09-23T20:12:20.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388380+0000", 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9233999999999999e-05 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388388+0000", 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.789e-06 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388439+0000", 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0358999999999997e-05 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388610+0000", 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017112299999999999 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.266 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388733+0000", 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012366100000000001 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.391543+0000", 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028094040000000002 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.391953+0000", 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00041040100000000002 2023-09-23T20:12:20.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.565545+0000", 2023-09-23T20:12:20.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17359206199999999 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.565610+0000", 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4317000000000001e-05 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.566525+0000", 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00091557699999999999 2023-09-23T20:12:20.270 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:137735 8.1 8:8fa750a8:::benchmark_data_smithi135_460343_object53502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:41.401864+0000", 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 38.761793963999999, 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16520929200000001, 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.271 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 137735 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.401864+0000", 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.272 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.401864+0000", 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.401863+0000", 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.401870+0000", 2023-09-23T20:12:20.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7309999999999993e-06 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.401886+0000", 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5855e-05 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.401898+0000", 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2320999999999999e-05 2023-09-23T20:12:20.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.401961+0000", 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2995000000000002e-05 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.402096+0000", 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000134559 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.402231+0000", 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013460100000000001 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.404301+0000", 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002070798 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.404711+0000", 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00040927599999999998 2023-09-23T20:12:20.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.566574+0000", 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16186292199999999 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.566628+0000", 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4178999999999997e-05 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.567073+0000", 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000445714 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:144171 8.1 8:88e198c0:::benchmark_data_smithi135_460343_object59938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:55.556999+0000", 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 24.606658932999999, 2023-09-23T20:12:20.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24187935099999999, 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 144171 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.556999+0000", 2023-09-23T20:12:20.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.556999+0000", 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.556998+0000", 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557005+0000", 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5349999999999999e-06 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557020+0000", 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4883e-05 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557028+0000", 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3869999999999997e-06 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557086+0000", 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8470000000000001e-05 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557266+0000", 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000180199 2023-09-23T20:12:20.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557436+0000", 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000169603 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.560273+0000", 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00283665 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.789439+0000", 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.229166656 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.798080+0000", 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0086407440000000005 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.798142+0000", 2023-09-23T20:12:20.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2162999999999994e-05 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.798879+0000", 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00073647100000000004 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:156675 8.1 8:8c220d10:::benchmark_data_smithi135_460343_object72442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T20:12:20.285 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:17.553673+0000", 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.6099849559999999, 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.170080643, 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 156675 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.286 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.553673+0000", 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.553673+0000", 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.553671+0000", 2023-09-23T20:12:20.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.553679+0000", 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.469e-06 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.553693+0000", 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4168999999999999e-05 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.553702+0000", 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7029999999999995e-06 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.553751+0000", 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9345999999999998e-05 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.553886+0000", 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013487199999999999 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.554053+0000", 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000166725 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.556827+0000", 2023-09-23T20:12:20.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027740260000000002 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.561768+0000", 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049407899999999996 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.723148+0000", 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.161380942 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.723207+0000", 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8483999999999998e-05 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:17.723754+0000", 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00054686800000000005 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.292 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:157147 8.1 8:8e5789f8:::benchmark_data_smithi135_460343_object72914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:18.285993+0000", 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 1.8776647950000001, 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21334767800000001, 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.293 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 157147 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.285993+0000", 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.285993+0000", 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.285992+0000", 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.285998+0000", 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1319999999999999e-06 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286020+0000", 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2356000000000001e-05 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286030+0000", 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3470000000000005e-06 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286072+0000", 2023-09-23T20:12:20.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2633999999999999e-05 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286235+0000", 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000162917 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286388+0000", 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015320399999999999 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.289566+0000", 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031780200000000002 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.299076+0000", 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0095100389999999996 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.498675+0000", 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.199598674 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.498733+0000", 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8239999999999998e-05 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.499341+0000", 2023-09-23T20:12:20.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060763400000000002 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:157158 8.1 8:8cb317c3:::benchmark_data_smithi135_460343_object72925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:18.301097+0000", 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 1.862561328, 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.19862486200000001, 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.300 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 157158 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301097+0000", 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301097+0000", 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301095+0000", 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301103+0000", 2023-09-23T20:12:20.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8720000000000002e-06 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301116+0000", 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2967e-05 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301124+0000", 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3059999999999998e-06 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301981+0000", 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00085613599999999996 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.302131+0000", 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015003799999999999 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.304 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.302275+0000", 2023-09-23T20:12:20.305 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000143905 2023-09-23T20:12:20.305 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.305 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:20.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.304842+0000", 2023-09-23T20:12:20.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025672690000000001 2023-09-23T20:12:20.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.311933+0000", 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007091445 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.499390+0000", 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.18745663900000001 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.499438+0000", 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7888e-05 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.499722+0000", 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028388600000000001 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.308 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:157160 8.1 8:8e424eab:::benchmark_data_smithi135_460343_object72927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:18.305597+0000", 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 1.8580607929999999, 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.203750138, 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 157160 2023-09-23T20:12:20.309 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305597+0000", 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305597+0000", 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:20.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305596+0000", 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305604+0000", 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3690000000000007e-06 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305620+0000", 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6052000000000001e-05 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305627+0000", 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0509999999999999e-06 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305673+0000", 2023-09-23T20:12:20.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6063000000000003e-05 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305830+0000", 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000156362 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305947+0000", 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117244 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.311879+0000", 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005932629 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.313100+0000", 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001220715 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:20.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.508732+0000", 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.195631429 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.508787+0000", 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5819000000000002e-05 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.509347+0000", 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00056003599999999998 2023-09-23T20:12:20.315 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.316 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.316 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.316 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:20.316 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:20.316 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:12:20.316 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:20 smithi135 ceph-mon[221113]: pgmap v1107: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 776 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 6519/37116 objects degraded (17.564%); 910/37116 objects misplaced (2.452%); 76 KiB/s, 74 objects/s recovering 2023-09-23T20:12:20.316 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:20 smithi135 ceph-mon[213871]: pgmap v1107: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 776 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 6519/37116 objects degraded (17.564%); 910/37116 objects misplaced (2.452%); 76 KiB/s, 74 objects/s recovering 2023-09-23T20:12:20.334 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:20 smithi144 ceph-mon[194833]: pgmap v1107: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 776 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 6519/37116 objects degraded (17.564%); 910/37116 objects misplaced (2.452%); 76 KiB/s, 74 objects/s recovering 2023-09-23T20:12:20.335 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:20.393 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:20.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:20.500 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:20.529 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:20.592 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:20.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:20.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:20.698 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:20.752 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:21.221 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:21.258 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:21.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:21.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:21.363 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:21.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:21 smithi144 ceph-mon[194833]: pgmap v1108: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 685 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6519/32742 objects degraded (19.910%); 708/32742 objects misplaced (2.162%); 0 B/s, 86 objects/s recovering 2023-09-23T20:12:21.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:21 smithi135 ceph-mon[221113]: pgmap v1108: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 685 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6519/32742 objects degraded (19.910%); 708/32742 objects misplaced (2.162%); 0 B/s, 86 objects/s recovering 2023-09-23T20:12:21.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:21 smithi135 ceph-mon[213871]: pgmap v1108: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 685 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6519/32742 objects degraded (19.910%); 708/32742 objects misplaced (2.162%); 0 B/s, 86 objects/s recovering 2023-09-23T20:12:21.472 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:21.523 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:12:22.212 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:22.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:22.321 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:22.396 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:22.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:22.581 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:12:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:22.759 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:22.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:22.867 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:12:22.960 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6519/32742 objects degraded (19.910%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:23.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:23 smithi135 ceph-mon[213871]: pgmap v1109: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 659 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 6519/31494 objects degraded (20.699%); 708/31494 objects misplaced (2.248%); 0 B/s, 84 objects/s recovering 2023-09-23T20:12:23.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6519/32742 objects degraded (19.910%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:23.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:23 smithi135 ceph-mon[221113]: pgmap v1109: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 659 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 6519/31494 objects degraded (20.699%); 708/31494 objects misplaced (2.248%); 0 B/s, 84 objects/s recovering 2023-09-23T20:12:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6519/32742 objects degraded (19.910%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:23.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:23 smithi144 ceph-mon[194833]: pgmap v1109: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 659 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 6519/31494 objects degraded (20.699%); 708/31494 objects misplaced (2.248%); 0 B/s, 84 objects/s recovering 2023-09-23T20:12:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:12:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:12:23] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-09-23T20:12:24.365 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:12:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:24.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:12:24.479 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:24.491 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:24.522 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:24.575 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:24.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:24.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:24.684 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:24.722 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:24.775 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:25.019 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3818274592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:12:25.019 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:25 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:12:25.073 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:12:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:25.206 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:25.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:25.307 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:25.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3818274592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:12:25.379 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:25 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:12:25.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3818274592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:12:25.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:25 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:12:25.382 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:25.985 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:25.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:26.078 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:12:26.085 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:12:26.093 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:26.112 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[221113]: pgmap v1110: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 521 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 6519/24897 objects degraded (26.184%); 492/24897 objects misplaced (1.976%); 0 B/s, 106 objects/s recovering 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[221113]: failed to encode map e394 with expected crc 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[221113]: failed to encode map e394 with expected crc 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[221113]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[213871]: pgmap v1110: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 521 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 6519/24897 objects degraded (26.184%); 492/24897 objects misplaced (1.976%); 0 B/s, 106 objects/s recovering 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[213871]: failed to encode map e394 with expected crc 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[213871]: failed to encode map e394 with expected crc 2023-09-23T20:12:26.113 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:12:26.114 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:26 smithi135 ceph-mon[213871]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T20:12:26.120 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:26.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:26 smithi144 ceph-mon[194833]: pgmap v1110: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 521 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 6519/24897 objects degraded (26.184%); 492/24897 objects misplaced (1.976%); 0 B/s, 106 objects/s recovering 2023-09-23T20:12:26.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:26 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:26.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:26 smithi144 ceph-mon[194833]: failed to encode map e394 with expected crc 2023-09-23T20:12:26.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:26 smithi144 ceph-mon[194833]: failed to encode map e394 with expected crc 2023-09-23T20:12:26.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:12:26.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:26 smithi144 ceph-mon[194833]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T20:12:27.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:27 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:12:27.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:27 smithi135 ceph-mon[221113]: from='client.55873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:27.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:27 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:12:27.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:27 smithi135 ceph-mon[213871]: from='client.55873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:27.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:27 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:12:27.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:27 smithi144 ceph-mon[194833]: from='client.55873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:27.988 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:28.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:28.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:28.101 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:28.133 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:28.166 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1111,"stamp":"2023-09-23T20:12:25.053129+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":546724930,"num_objects":8299,"num_object_clones":0,"num_object_copies":24897,"num_objects_missing_on_primary":488,"num_objects_missing":488,"num_objects_degraded":6519,"num_objects_misplaced":492,"num_objects_unfound":0,"num_objects_dirty":8299,"num_whiteouts":0,"num_read":6158,"num_read_kb":13539,"num_write":161185,"num_write_kb":5404324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32723,"num_bytes_recovered":1628246271,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95425,"ondisk_log_size":95425,"up":327,"acting":323,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":376,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6986244,"kb_used_data":3283404,"kb_used_omap":32,"kb_used_meta":3702687,"kb_avail":649229820,"statfs":{"total":671965249536,"available":664811335680,"internally_reserved":0,"allocated":3362205696,"data_stored":3350729283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3791552006},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,4,3,8,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-454819840,"num_objects":-6940,"num_object_clones":0,"num_object_copies":-20820,"num_objects_missing_on_primary":-762,"num_objects_missing":-762,"num_objects_degraded":0,"num_objects_misplaced":-418,"num_objects_unfound":0,"num_objects_dirty":-6940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6940,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":762,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.079111"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":391,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728497+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:12:09.728497+0000","last_peered":"2023-09-23T20:12:09.728497+0000","last_clean":"2023-09-23T20:12:09.728497+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:12:09.728497+0000","last_undegraded":"2023-09-23T20:12:09.728497+0000","last_fullsized":"2023-09-23T20:12:09.728497+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":221,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759527+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:12:09.759527+0000","last_peered":"2023-09-23T20:12:09.759527+0000","last_clean":"2023-09-23T20:12:09.759527+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:12:09.759527+0000","last_undegraded":"2023-09-23T20:12:09.759527+0000","last_fullsized":"2023-09-23T20:12:09.759527+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":89,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.800517+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:12:12.800517+0000","last_peered":"2023-09-23T20:12:12.800517+0000","last_clean":"2023-09-23T20:12:12.800517+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:12:12.800517+0000","last_undegraded":"2023-09-23T20:12:12.800517+0000","last_fullsized":"2023-09-23T20:12:12.800517+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":101,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965894+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:12:09.965894+0000","last_peered":"2023-09-23T20:12:09.965894+0000","last_clean":"2023-09-23T20:12:09.965894+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:12:09.965894+0000","last_undegraded":"2023-09-23T20:12:09.965894+0000","last_fullsized":"2023-09-23T20:12:09.965894+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":221,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.746781+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:12:09.746781+0000","last_peered":"2023-09-23T20:12:09.746781+0000","last_clean":"2023-09-23T20:12:09.746781+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:12:09.746781+0000","last_undegraded":"2023-09-23T20:12:09.746781+0000","last_fullsized":"2023-09-23T20:12:09.746781+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":338,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785811+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:12:09.785811+0000","last_peered":"2023-09-23T20:12:09.785811+0000","last_clean":"2023-09-23T20:12:09.785811+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:12:09.785811+0000","last_undegraded":"2023-09-23T20:12:09.785811+0000","last_fullsized":"2023-09-23T20:12:09.785811+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":319,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785757+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:12:09.785757+0000","last_peered":"2023-09-23T20:12:09.785757+0000","last_clean":"2023-09-23T20:12:09.785757+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:12:09.785757+0000","last_undegraded":"2023-09-23T20:12:09.785757+0000","last_fullsized":"2023-09-23T20:12:09.785757+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":236,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.724341+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:12:09.724341+0000","last_peered":"2023-09-23T20:12:09.724341+0000","last_clean":"2023-09-23T20:12:09.724341+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:12:09.724341+0000","last_undegraded":"2023-09-23T20:12:09.724341+0000","last_fullsized":"2023-09-23T20:12:09.724341+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":703,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.786523+0000","last_change":"2023-09-23T20:09:56.042750+0000","last_active":"2023-09-23T20:12:12.786523+0000","last_peered":"2023-09-23T20:12:12.786523+0000","last_clean":"2023-09-23T20:12:12.786523+0000","last_became_active":"2023-09-23T20:09:56.042455+0000","last_became_peered":"2023-09-23T20:09:56.042455+0000","last_unstale":"2023-09-23T20:12:12.786523+0000","last_undegraded":"2023-09-23T20:12:12.786523+0000","last_fullsized":"2023-09-23T20:12:12.786523+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":392,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785647+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:12:09.785647+0000","last_peered":"2023-09-23T20:12:09.785647+0000","last_clean":"2023-09-23T20:12:09.785647+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:12:09.785647+0000","last_undegraded":"2023-09-23T20:12:09.785647+0000","last_fullsized":"2023-09-23T20:12:09.785647+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":255,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.724505+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:12:09.724505+0000","last_peered":"2023-09-23T20:12:09.724505+0000","last_clean":"2023-09-23T20:12:09.724505+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:12:09.724505+0000","last_undegraded":"2023-09-23T20:12:09.724505+0000","last_fullsized":"2023-09-23T20:12:09.724505+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":86,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965926+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:12:09.965926+0000","last_peered":"2023-09-23T20:12:09.965926+0000","last_clean":"2023-09-23T20:12:09.965926+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:12:09.965926+0000","last_undegraded":"2023-09-23T20:12:09.965926+0000","last_fullsized":"2023-09-23T20:12:09.965926+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759687+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:12:09.759687+0000","last_peered":"2023-09-23T20:12:09.759687+0000","last_clean":"2023-09-23T20:12:09.759687+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:12:09.759687+0000","last_undegraded":"2023-09-23T20:12:09.759687+0000","last_fullsized":"2023-09-23T20:12:09.759687+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":447,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728478+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:12:09.728478+0000","last_peered":"2023-09-23T20:12:09.728478+0000","last_clean":"2023-09-23T20:12:09.728478+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:12:09.728478+0000","last_undegraded":"2023-09-23T20:12:09.728478+0000","last_fullsized":"2023-09-23T20:12:09.728478+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":389,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728517+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:12:09.728517+0000","last_peered":"2023-09-23T20:12:09.728517+0000","last_clean":"2023-09-23T20:12:09.728517+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:12:09.728517+0000","last_undegraded":"2023-09-23T20:12:09.728517+0000","last_fullsized":"2023-09-23T20:12:09.728517+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":722,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.784953+0000","last_change":"2023-09-23T20:11:07.067995+0000","last_active":"2023-09-23T20:12:09.784953+0000","last_peered":"2023-09-23T20:12:09.784953+0000","last_clean":"2023-09-23T20:12:09.784953+0000","last_became_active":"2023-09-23T20:09:56.043288+0000","last_became_peered":"2023-09-23T20:09:56.043288+0000","last_unstale":"2023-09-23T20:12:09.784953+0000","last_undegraded":"2023-09-23T20:12:09.784953+0000","last_fullsized":"2023-09-23T20:12:09.784953+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":86,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965621+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:12:09.965621+0000","last_peered":"2023-09-23T20:12:09.965621+0000","last_clean":"2023-09-23T20:12:09.965621+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:12:09.965621+0000","last_undegraded":"2023-09-23T20:12:09.965621+0000","last_fullsized":"2023-09-23T20:12:09.965621+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":245,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759631+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:12:09.759631+0000","last_peered":"2023-09-23T20:12:09.759631+0000","last_clean":"2023-09-23T20:12:09.759631+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:12:09.759631+0000","last_undegraded":"2023-09-23T20:12:09.759631+0000","last_fullsized":"2023-09-23T20:12:09.759631+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":545,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965388+0000","last_change":"2023-09-23T20:12:00.290344+0000","last_active":"2023-09-23T20:12:09.965388+0000","last_peered":"2023-09-23T20:12:09.965388+0000","last_clean":"2023-09-23T20:12:09.965388+0000","last_became_active":"2023-09-23T20:09:56.050784+0000","last_became_peered":"2023-09-23T20:09:56.050784+0000","last_unstale":"2023-09-23T20:12:09.965388+0000","last_undegraded":"2023-09-23T20:12:09.965388+0000","last_fullsized":"2023-09-23T20:12:09.965388+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":101,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965540+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:12:09.965540+0000","last_peered":"2023-09-23T20:12:09.965540+0000","last_clean":"2023-09-23T20:12:09.965540+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:12:09.965540+0000","last_undegraded":"2023-09-23T20:12:09.965540+0000","last_fullsized":"2023-09-23T20:12:09.965540+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":391,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785608+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:12:09.785608+0000","last_peered":"2023-09-23T20:12:09.785608+0000","last_clean":"2023-09-23T20:12:09.785608+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:12:09.785608+0000","last_undegraded":"2023-09-23T20:12:09.785608+0000","last_fullsized":"2023-09-23T20:12:09.785608+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":519,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807349+0000","last_change":"2023-09-23T20:09:56.062364+0000","last_active":"2023-09-23T20:12:12.807349+0000","last_peered":"2023-09-23T20:12:12.807349+0000","last_clean":"2023-09-23T20:12:12.807349+0000","last_became_active":"2023-09-23T20:09:56.061865+0000","last_became_peered":"2023-09-23T20:09:56.061865+0000","last_unstale":"2023-09-23T20:12:12.807349+0000","last_undegraded":"2023-09-23T20:12:12.807349+0000","last_fullsized":"2023-09-23T20:12:12.807349+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":80,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965411+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:12:09.965411+0000","last_peered":"2023-09-23T20:12:09.965411+0000","last_clean":"2023-09-23T20:12:09.965411+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:12:09.965411+0000","last_undegraded":"2023-09-23T20:12:09.965411+0000","last_fullsized":"2023-09-23T20:12:09.965411+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":379,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.749882+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:12:09.749882+0000","last_peered":"2023-09-23T20:12:09.749882+0000","last_clean":"2023-09-23T20:12:09.749882+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:12:09.749882+0000","last_undegraded":"2023-09-23T20:12:09.749882+0000","last_fullsized":"2023-09-23T20:12:09.749882+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":620,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965366+0000","last_change":"2023-09-23T20:11:56.308383+0000","last_active":"2023-09-23T20:12:09.965366+0000","last_peered":"2023-09-23T20:12:09.965366+0000","last_clean":"2023-09-23T20:12:09.965366+0000","last_became_active":"2023-09-23T20:09:56.043157+0000","last_became_peered":"2023-09-23T20:09:56.043157+0000","last_unstale":"2023-09-23T20:12:09.965366+0000","last_undegraded":"2023-09-23T20:12:09.965366+0000","last_fullsized":"2023-09-23T20:12:09.965366+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":24,"reported_epoch":392,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:12:09.784946+0000","last_change":"2023-09-23T20:11:10.067603+0000","last_active":"2023-09-23T20:12:09.784946+0000","last_peered":"2023-09-23T20:12:09.784946+0000","last_clean":"2023-09-23T20:00:51.694506+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:12:09.784946+0000","last_undegraded":"2023-09-23T20:11:10.056843+0000","last_fullsized":"2023-09-23T20:12:09.784946+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":4}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.724670+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:12:09.724670+0000","last_peered":"2023-09-23T20:12:09.724670+0000","last_clean":"2023-09-23T20:12:09.724670+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:12:09.724670+0000","last_undegraded":"2023-09-23T20:12:09.724670+0000","last_fullsized":"2023-09-23T20:12:09.724670+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":816,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728521+0000","last_change":"2023-09-23T20:11:52.828453+0000","last_active":"2023-09-23T20:12:09.728521+0000","last_peered":"2023-09-23T20:12:09.728521+0000","last_clean":"2023-09-23T20:12:09.728521+0000","last_became_active":"2023-09-23T20:10:52.687280+0000","last_became_peered":"2023-09-23T20:10:52.687280+0000","last_unstale":"2023-09-23T20:12:09.728521+0000","last_undegraded":"2023-09-23T20:12:09.728521+0000","last_fullsized":"2023-09-23T20:12:09.728521+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.4","version":"393'10225","reported_seq":10327,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:23.269768+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:12:23.269768+0000","last_peered":"2023-09-23T20:12:23.269768+0000","last_clean":"2023-09-23T20:12:23.269768+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:12:23.269768+0000","last_undegraded":"2023-09-23T20:12:23.269768+0000","last_fullsized":"2023-09-23T20:12:23.269768+0000","mapping_epoch":299,"log_start":"315'3965","ondisk_log_start":"315'3965","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6260,"log_dups_size":0,"ondisk_log_size":6260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33095680,"num_objects":505,"num_object_clones":0,"num_object_copies":1515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10225,"num_write_kb":343360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":84,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.966084+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:12:09.966084+0000","last_peered":"2023-09-23T20:12:09.966084+0000","last_clean":"2023-09-23T20:12:09.966084+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:12:09.966084+0000","last_undegraded":"2023-09-23T20:12:09.966084+0000","last_fullsized":"2023-09-23T20:12:09.966084+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":401,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.784885+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:12:09.784885+0000","last_peered":"2023-09-23T20:12:09.784885+0000","last_clean":"2023-09-23T20:12:09.784885+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:12:09.784885+0000","last_undegraded":"2023-09-23T20:12:09.784885+0000","last_fullsized":"2023-09-23T20:12:09.784885+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":704,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:11.165645+0000","last_change":"2023-09-23T20:09:56.064782+0000","last_active":"2023-09-23T20:12:11.165645+0000","last_peered":"2023-09-23T20:12:11.165645+0000","last_clean":"2023-09-23T20:12:11.165645+0000","last_became_active":"2023-09-23T20:09:56.064411+0000","last_became_peered":"2023-09-23T20:09:56.064411+0000","last_unstale":"2023-09-23T20:12:11.165645+0000","last_undegraded":"2023-09-23T20:12:11.165645+0000","last_fullsized":"2023-09-23T20:12:11.165645+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218454,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.5","version":"393'9805","reported_seq":12501,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:23.585995+0000","last_change":"2023-09-23T20:07:45.697314+0000","last_active":"2023-09-23T20:12:23.585995+0000","last_peered":"2023-09-23T20:12:23.585995+0000","last_clean":"2023-09-23T20:12:23.585995+0000","last_became_active":"2023-09-23T20:07:45.690354+0000","last_became_peered":"2023-09-23T20:07:45.690354+0000","last_unstale":"2023-09-23T20:12:23.585995+0000","last_undegraded":"2023-09-23T20:12:23.585995+0000","last_fullsized":"2023-09-23T20:12:23.585995+0000","mapping_epoch":327,"log_start":"321'4118","ondisk_log_start":"321'4118","created":299,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.926631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33357824,"num_objects":509,"num_object_clones":0,"num_object_copies":1527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9805,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":85,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965934+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:12:09.965934+0000","last_peered":"2023-09-23T20:12:09.965934+0000","last_clean":"2023-09-23T20:12:09.965934+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:12:09.965934+0000","last_undegraded":"2023-09-23T20:12:09.965934+0000","last_fullsized":"2023-09-23T20:12:09.965934+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":449,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728283+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:12:09.728283+0000","last_peered":"2023-09-23T20:12:09.728283+0000","last_clean":"2023-09-23T20:12:09.728283+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:12:09.728283+0000","last_undegraded":"2023-09-23T20:12:09.728283+0000","last_fullsized":"2023-09-23T20:12:09.728283+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":661,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.724839+0000","last_change":"2023-09-23T20:11:20.856999+0000","last_active":"2023-09-23T20:12:09.724839+0000","last_peered":"2023-09-23T20:12:09.724839+0000","last_clean":"2023-09-23T20:12:09.724839+0000","last_became_active":"2023-09-23T20:11:20.856640+0000","last_became_peered":"2023-09-23T20:11:20.856640+0000","last_unstale":"2023-09-23T20:12:09.724839+0000","last_undegraded":"2023-09-23T20:12:09.724839+0000","last_fullsized":"2023-09-23T20:12:09.724839+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.6","version":"393'10211","reported_seq":11838,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:20.777210+0000","last_change":"2023-09-23T20:07:37.064417+0000","last_active":"2023-09-23T20:12:20.777210+0000","last_peered":"2023-09-23T20:12:20.777210+0000","last_clean":"2023-09-23T20:06:59.831272+0000","last_became_active":"2023-09-23T20:07:37.059682+0000","last_became_peered":"2023-09-23T20:07:37.059682+0000","last_unstale":"2023-09-23T20:12:20.777210+0000","last_undegraded":"2023-09-23T20:07:36.983730+0000","last_fullsized":"2023-09-23T20:07:36.974121+0000","mapping_epoch":322,"log_start":"329'5065","ondisk_log_start":"329'5065","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5146,"log_dups_size":0,"ondisk_log_size":5146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:33.575626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39124992,"num_objects":597,"num_object_clones":0,"num_object_copies":1791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10211,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-794}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":84,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965764+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:12:09.965764+0000","last_peered":"2023-09-23T20:12:09.965764+0000","last_clean":"2023-09-23T20:12:09.965764+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:12:09.965764+0000","last_undegraded":"2023-09-23T20:12:09.965764+0000","last_fullsized":"2023-09-23T20:12:09.965764+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759737+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:12:09.759737+0000","last_peered":"2023-09-23T20:12:09.759737+0000","last_clean":"2023-09-23T20:12:09.759737+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:12:09.759737+0000","last_undegraded":"2023-09-23T20:12:09.759737+0000","last_fullsized":"2023-09-23T20:12:09.759737+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":585,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728636+0000","last_change":"2023-09-23T20:11:43.909971+0000","last_active":"2023-09-23T20:12:09.728636+0000","last_peered":"2023-09-23T20:12:09.728636+0000","last_clean":"2023-09-23T20:12:09.728636+0000","last_became_active":"2023-09-23T20:09:56.043491+0000","last_became_peered":"2023-09-23T20:09:56.043491+0000","last_unstale":"2023-09-23T20:12:09.728636+0000","last_undegraded":"2023-09-23T20:12:09.728636+0000","last_fullsized":"2023-09-23T20:12:09.728636+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.7","version":"393'10072","reported_seq":17397,"reported_epoch":393,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:12:23.978931+0000","last_change":"2023-09-23T20:12:07.891624+0000","last_active":"2023-09-23T20:12:23.978931+0000","last_peered":"2023-09-23T20:12:23.978931+0000","last_clean":"2023-09-23T20:07:17.891274+0000","last_became_active":"2023-09-23T20:10:58.299550+0000","last_became_peered":"2023-09-23T20:10:58.299550+0000","last_unstale":"2023-09-23T20:12:23.978931+0000","last_undegraded":"2023-09-23T20:12:23.978931+0000","last_fullsized":"2023-09-23T20:10:53.692217+0000","mapping_epoch":369,"log_start":"321'4529","ondisk_log_start":"321'4529","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5543,"log_dups_size":0,"ondisk_log_size":5543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:48.194019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30539776,"num_objects":466,"num_object_clones":0,"num_object_copies":1398,"num_objects_missing_on_primary":484,"num_objects_missing":484,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10072,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3599,"num_bytes_recovered":146669568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2],"avail_no_missing":["3"],"object_location_counts":[{"shards":"1,2","objects":466}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":431,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785491+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:12:09.785491+0000","last_peered":"2023-09-23T20:12:09.785491+0000","last_clean":"2023-09-23T20:12:09.785491+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:12:09.785491+0000","last_undegraded":"2023-09-23T20:12:09.785491+0000","last_fullsized":"2023-09-23T20:12:09.785491+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.760034+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:12:09.760034+0000","last_peered":"2023-09-23T20:12:09.760034+0000","last_clean":"2023-09-23T20:12:09.760034+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:12:09.760034+0000","last_undegraded":"2023-09-23T20:12:09.760034+0000","last_fullsized":"2023-09-23T20:12:09.760034+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"393'9730","reported_seq":17864,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:21.129081+0000","last_change":"2023-09-23T20:10:10.729751+0000","last_active":"2023-09-23T20:12:21.129081+0000","last_peered":"2023-09-23T20:12:21.129081+0000","last_clean":"2023-09-23T20:12:21.129081+0000","last_became_active":"2023-09-23T20:10:10.708263+0000","last_became_peered":"2023-09-23T20:10:10.708263+0000","last_unstale":"2023-09-23T20:12:21.129081+0000","last_undegraded":"2023-09-23T20:12:21.129081+0000","last_fullsized":"2023-09-23T20:12:21.129081+0000","mapping_epoch":361,"log_start":"321'4210","ondisk_log_start":"321'4210","created":299,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5520,"log_dups_size":0,"ondisk_log_size":5520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:15.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37879808,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9730,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4003,"num_bytes_recovered":136118272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":775,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965377+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:12:09.965377+0000","last_peered":"2023-09-23T20:12:09.965377+0000","last_clean":"2023-09-23T20:12:09.965377+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:12:09.965377+0000","last_undegraded":"2023-09-23T20:12:09.965377+0000","last_fullsized":"2023-09-23T20:12:09.965377+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":83,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728493+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:12:09.728493+0000","last_peered":"2023-09-23T20:12:09.728493+0000","last_clean":"2023-09-23T20:12:09.728493+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:12:09.728493+0000","last_undegraded":"2023-09-23T20:12:09.728493+0000","last_fullsized":"2023-09-23T20:12:09.728493+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":419,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728583+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:12:09.728583+0000","last_peered":"2023-09-23T20:12:09.728583+0000","last_clean":"2023-09-23T20:12:09.728583+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:12:09.728583+0000","last_undegraded":"2023-09-23T20:12:09.728583+0000","last_fullsized":"2023-09-23T20:12:09.728583+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"393'10044","reported_seq":16449,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:22.803652+0000","last_change":"2023-09-23T20:10:58.556423+0000","last_active":"2023-09-23T20:12:22.803652+0000","last_peered":"2023-09-23T20:12:22.803652+0000","last_clean":"2023-09-23T20:07:17.847486+0000","last_became_active":"2023-09-23T20:07:20.666556+0000","last_became_peered":"2023-09-23T20:07:20.666556+0000","last_unstale":"2023-09-23T20:12:22.803652+0000","last_undegraded":"2023-09-23T20:10:58.555698+0000","last_fullsized":"2023-09-23T20:07:19.932561+0000","mapping_epoch":317,"log_start":"331'5146","ondisk_log_start":"331'5146","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4898,"log_dups_size":0,"ondisk_log_size":4898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:38.441814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29884416,"num_objects":456,"num_object_clones":0,"num_object_copies":1368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":996,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10044,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3145,"num_bytes_recovered":206110720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-540}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":727,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.784992+0000","last_change":"2023-09-23T20:11:11.229144+0000","last_active":"2023-09-23T20:12:09.784992+0000","last_peered":"2023-09-23T20:12:09.784992+0000","last_clean":"2023-09-23T20:12:09.784992+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:12:09.784992+0000","last_undegraded":"2023-09-23T20:12:09.784992+0000","last_fullsized":"2023-09-23T20:12:09.784992+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":359,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728423+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:12:09.728423+0000","last_peered":"2023-09-23T20:12:09.728423+0000","last_clean":"2023-09-23T20:12:09.728423+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:12:09.728423+0000","last_undegraded":"2023-09-23T20:12:09.728423+0000","last_fullsized":"2023-09-23T20:12:09.728423+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":389,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728979+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:12:09.728979+0000","last_peered":"2023-09-23T20:12:09.728979+0000","last_clean":"2023-09-23T20:12:09.728979+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:12:09.728979+0000","last_undegraded":"2023-09-23T20:12:09.728979+0000","last_fullsized":"2023-09-23T20:12:09.728979+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":519,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759508+0000","last_change":"2023-09-23T20:09:56.062162+0000","last_active":"2023-09-23T20:12:09.759508+0000","last_peered":"2023-09-23T20:12:09.759508+0000","last_clean":"2023-09-23T20:12:09.759508+0000","last_became_active":"2023-09-23T20:09:56.061518+0000","last_became_peered":"2023-09-23T20:09:56.061518+0000","last_unstale":"2023-09-23T20:12:09.759508+0000","last_undegraded":"2023-09-23T20:12:09.759508+0000","last_fullsized":"2023-09-23T20:12:09.759508+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"393'10002","reported_seq":10104,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:23.560710+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:12:23.560710+0000","last_peered":"2023-09-23T20:12:23.560710+0000","last_clean":"2023-09-23T20:12:23.560710+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:12:23.560710+0000","last_undegraded":"2023-09-23T20:12:23.560710+0000","last_fullsized":"2023-09-23T20:12:23.560710+0000","mapping_epoch":299,"log_start":"321'4285","ondisk_log_start":"321'4285","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5717,"log_dups_size":0,"ondisk_log_size":5717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30015488,"num_objects":458,"num_object_clones":0,"num_object_copies":1374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10002,"num_write_kb":334720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":383,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728150+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:12:09.728150+0000","last_peered":"2023-09-23T20:12:09.728150+0000","last_clean":"2023-09-23T20:12:09.728150+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:12:09.728150+0000","last_undegraded":"2023-09-23T20:12:09.728150+0000","last_fullsized":"2023-09-23T20:12:09.728150+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":419,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728733+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:12:09.728733+0000","last_peered":"2023-09-23T20:12:09.728733+0000","last_clean":"2023-09-23T20:12:09.728733+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:12:09.728733+0000","last_undegraded":"2023-09-23T20:12:09.728733+0000","last_fullsized":"2023-09-23T20:12:09.728733+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"393'9833","reported_seq":17995,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:23.792866+0000","last_change":"2023-09-23T20:09:37.763558+0000","last_active":"2023-09-23T20:12:23.792866+0000","last_peered":"2023-09-23T20:12:23.792866+0000","last_clean":"2023-09-23T20:12:23.792866+0000","last_became_active":"2023-09-23T20:09:37.732414+0000","last_became_peered":"2023-09-23T20:09:37.732414+0000","last_unstale":"2023-09-23T20:12:23.792866+0000","last_undegraded":"2023-09-23T20:12:23.792866+0000","last_fullsized":"2023-09-23T20:12:23.792866+0000","mapping_epoch":352,"log_start":"321'4281","ondisk_log_start":"321'4281","created":299,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:39.792571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30605312,"num_objects":467,"num_object_clones":0,"num_object_copies":1401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9833,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4021,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":731,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785457+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:12:09.785457+0000","last_peered":"2023-09-23T20:12:09.785457+0000","last_clean":"2023-09-23T20:12:09.785457+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:12:09.785457+0000","last_undegraded":"2023-09-23T20:12:09.785457+0000","last_fullsized":"2023-09-23T20:12:09.785457+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":23,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.724101+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:12:09.724101+0000","last_peered":"2023-09-23T20:12:09.724101+0000","last_clean":"2023-09-23T20:12:09.724101+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:12:09.724101+0000","last_undegraded":"2023-09-23T20:12:09.724101+0000","last_fullsized":"2023-09-23T20:12:09.724101+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.761706+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:12:09.761706+0000","last_peered":"2023-09-23T20:12:09.761706+0000","last_clean":"2023-09-23T20:12:09.761706+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:12:09.761706+0000","last_undegraded":"2023-09-23T20:12:09.761706+0000","last_fullsized":"2023-09-23T20:12:09.761706+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"393'9964","reported_seq":10069,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:20.831990+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:12:20.831990+0000","last_peered":"2023-09-23T20:12:20.831990+0000","last_clean":"2023-09-23T20:12:20.831990+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:12:20.831990+0000","last_undegraded":"2023-09-23T20:12:20.831990+0000","last_fullsized":"2023-09-23T20:12:20.831990+0000","mapping_epoch":299,"log_start":"326'4783","ondisk_log_start":"326'4783","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5181,"log_dups_size":0,"ondisk_log_size":5181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39190528,"num_objects":598,"num_object_clones":0,"num_object_copies":1794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9964,"num_write_kb":337984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":661,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.724756+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:12:09.724756+0000","last_peered":"2023-09-23T20:12:09.724756+0000","last_clean":"2023-09-23T20:12:09.724756+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:12:09.724756+0000","last_undegraded":"2023-09-23T20:12:09.724756+0000","last_fullsized":"2023-09-23T20:12:09.724756+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":526,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.760044+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:12:09.760044+0000","last_peered":"2023-09-23T20:12:09.760044+0000","last_clean":"2023-09-23T20:12:09.760044+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:12:09.760044+0000","last_undegraded":"2023-09-23T20:12:09.760044+0000","last_fullsized":"2023-09-23T20:12:09.760044+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":393,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785536+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:12:09.785536+0000","last_peered":"2023-09-23T20:12:09.785536+0000","last_clean":"2023-09-23T20:12:09.785536+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:12:09.785536+0000","last_undegraded":"2023-09-23T20:12:09.785536+0000","last_fullsized":"2023-09-23T20:12:09.785536+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759609+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:12:09.759609+0000","last_peered":"2023-09-23T20:12:09.759609+0000","last_clean":"2023-09-23T20:12:09.759609+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:12:09.759609+0000","last_undegraded":"2023-09-23T20:12:09.759609+0000","last_fullsized":"2023-09-23T20:12:09.759609+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"393'10038","reported_seq":14869,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:23.267996+0000","last_change":"2023-09-23T20:07:52.783215+0000","last_active":"2023-09-23T20:12:23.267996+0000","last_peered":"2023-09-23T20:12:23.267996+0000","last_clean":"2023-09-23T20:12:23.267996+0000","last_became_active":"2023-09-23T20:07:52.778874+0000","last_became_peered":"2023-09-23T20:07:52.778874+0000","last_unstale":"2023-09-23T20:12:23.267996+0000","last_undegraded":"2023-09-23T20:12:23.267996+0000","last_fullsized":"2023-09-23T20:12:23.267996+0000","mapping_epoch":330,"log_start":"315'3700","ondisk_log_start":"315'3700","created":299,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6338,"log_dups_size":0,"ondisk_log_size":6338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:31.036337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31850496,"num_objects":486,"num_object_clones":0,"num_object_copies":1458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10038,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2349,"num_bytes_recovered":153944064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":726,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785067+0000","last_change":"2023-09-23T20:11:06.026338+0000","last_active":"2023-09-23T20:12:09.785067+0000","last_peered":"2023-09-23T20:12:09.785067+0000","last_clean":"2023-09-23T20:12:09.785067+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:12:09.785067+0000","last_undegraded":"2023-09-23T20:12:09.785067+0000","last_fullsized":"2023-09-23T20:12:09.785067+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1013,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785249+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:12:09.785249+0000","last_peered":"2023-09-23T20:12:09.785249+0000","last_clean":"2023-09-23T20:12:09.785249+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:12:09.785249+0000","last_undegraded":"2023-09-23T20:12:09.785249+0000","last_fullsized":"2023-09-23T20:12:09.785249+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":412,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728615+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:12:09.728615+0000","last_peered":"2023-09-23T20:12:09.728615+0000","last_clean":"2023-09-23T20:12:09.728615+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:12:09.728615+0000","last_undegraded":"2023-09-23T20:12:09.728615+0000","last_fullsized":"2023-09-23T20:12:09.728615+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759988+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:12:09.759988+0000","last_peered":"2023-09-23T20:12:09.759988+0000","last_clean":"2023-09-23T20:12:09.759988+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:12:09.759988+0000","last_undegraded":"2023-09-23T20:12:09.759988+0000","last_fullsized":"2023-09-23T20:12:09.759988+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"393'9872","reported_seq":22241,"reported_epoch":393,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:12:23.769244+0000","last_change":"2023-09-23T20:11:18.839874+0000","last_active":"2023-09-23T20:12:23.769244+0000","last_peered":"2023-09-23T20:12:23.769244+0000","last_clean":"2023-09-23T20:11:16.510239+0000","last_became_active":"2023-09-23T20:11:18.809696+0000","last_became_peered":"2023-09-23T20:11:18.809696+0000","last_unstale":"2023-09-23T20:12:23.769244+0000","last_undegraded":"2023-09-23T20:12:23.769244+0000","last_fullsized":"2023-09-23T20:12:23.769244+0000","mapping_epoch":381,"log_start":"321'4400","ondisk_log_start":"321'4400","created":299,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:58.376473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32243712,"num_objects":492,"num_object_clones":0,"num_object_copies":1476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":492,"num_objects_unfound":0,"num_objects_dirty":492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9872,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":217251840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[1,4,2],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":492}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":618,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965533+0000","last_change":"2023-09-23T20:11:54.316411+0000","last_active":"2023-09-23T20:12:09.965533+0000","last_peered":"2023-09-23T20:12:09.965533+0000","last_clean":"2023-09-23T20:12:09.965533+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:12:09.965533+0000","last_undegraded":"2023-09-23T20:12:09.965533+0000","last_fullsized":"2023-09-23T20:12:09.965533+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1029,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728417+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:12:09.728417+0000","last_peered":"2023-09-23T20:12:09.728417+0000","last_clean":"2023-09-23T20:12:09.728417+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:12:09.728417+0000","last_undegraded":"2023-09-23T20:12:09.728417+0000","last_fullsized":"2023-09-23T20:12:09.728417+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759569+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:12:09.759569+0000","last_peered":"2023-09-23T20:12:09.759569+0000","last_clean":"2023-09-23T20:12:09.759569+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:12:09.759569+0000","last_undegraded":"2023-09-23T20:12:09.759569+0000","last_fullsized":"2023-09-23T20:12:09.759569+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":84,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.760314+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:12:09.760314+0000","last_peered":"2023-09-23T20:12:09.760314+0000","last_clean":"2023-09-23T20:12:09.760314+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:12:09.760314+0000","last_undegraded":"2023-09-23T20:12:09.760314+0000","last_fullsized":"2023-09-23T20:12:09.760314+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"393'10068","reported_seq":10170,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:23.277635+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:12:23.277635+0000","last_peered":"2023-09-23T20:12:23.277635+0000","last_clean":"2023-09-23T20:12:23.277635+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:12:23.277635+0000","last_undegraded":"2023-09-23T20:12:23.277635+0000","last_fullsized":"2023-09-23T20:12:23.277635+0000","mapping_epoch":299,"log_start":"315'3765","ondisk_log_start":"315'3765","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32768000,"num_objects":500,"num_object_clones":0,"num_object_copies":1500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10068,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1301,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785273+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:12:09.785273+0000","last_peered":"2023-09-23T20:12:09.785273+0000","last_clean":"2023-09-23T20:12:09.785273+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:12:09.785273+0000","last_undegraded":"2023-09-23T20:12:09.785273+0000","last_fullsized":"2023-09-23T20:12:09.785273+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":552,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965366+0000","last_change":"2023-09-23T20:11:53.362155+0000","last_active":"2023-09-23T20:12:09.965366+0000","last_peered":"2023-09-23T20:12:09.965366+0000","last_clean":"2023-09-23T20:12:09.965366+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:12:09.965366+0000","last_undegraded":"2023-09-23T20:12:09.965366+0000","last_fullsized":"2023-09-23T20:12:09.965366+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":258,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785343+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:12:09.785343+0000","last_peered":"2023-09-23T20:12:09.785343+0000","last_clean":"2023-09-23T20:12:09.785343+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:12:09.785343+0000","last_undegraded":"2023-09-23T20:12:09.785343+0000","last_fullsized":"2023-09-23T20:12:09.785343+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":288,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.729272+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:12:09.729272+0000","last_peered":"2023-09-23T20:12:09.729272+0000","last_clean":"2023-09-23T20:12:09.729272+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:12:09.729272+0000","last_undegraded":"2023-09-23T20:12:09.729272+0000","last_fullsized":"2023-09-23T20:12:09.729272+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"393'10057","reported_seq":10273,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:20.820086+0000","last_change":"2023-09-23T20:10:57.512712+0000","last_active":"2023-09-23T20:12:20.820086+0000","last_peered":"2023-09-23T20:12:20.820086+0000","last_clean":"2023-09-23T20:07:17.907438+0000","last_became_active":"2023-09-23T20:07:20.693608+0000","last_became_peered":"2023-09-23T20:07:20.693608+0000","last_unstale":"2023-09-23T20:12:20.820086+0000","last_undegraded":"2023-09-23T20:07:19.959595+0000","last_fullsized":"2023-09-23T20:07:19.924202+0000","mapping_epoch":317,"log_start":"326'4888","ondisk_log_start":"326'4888","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5169,"log_dups_size":0,"ondisk_log_size":5169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:21.022797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37945344,"num_objects":579,"num_object_clones":0,"num_object_copies":1737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10057,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":3407872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":-3549}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":519,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.760100+0000","last_change":"2023-09-23T20:09:56.051517+0000","last_active":"2023-09-23T20:12:09.760100+0000","last_peered":"2023-09-23T20:12:09.760100+0000","last_clean":"2023-09-23T20:12:09.760100+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:12:09.760100+0000","last_undegraded":"2023-09-23T20:12:09.760100+0000","last_fullsized":"2023-09-23T20:12:09.760100+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":829,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965209+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:12:09.965209+0000","last_peered":"2023-09-23T20:12:09.965209+0000","last_clean":"2023-09-23T20:12:09.965209+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:12:09.965209+0000","last_undegraded":"2023-09-23T20:12:09.965209+0000","last_fullsized":"2023-09-23T20:12:09.965209+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":379,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728353+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:12:09.728353+0000","last_peered":"2023-09-23T20:12:09.728353+0000","last_clean":"2023-09-23T20:12:09.728353+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:12:09.728353+0000","last_undegraded":"2023-09-23T20:12:09.728353+0000","last_fullsized":"2023-09-23T20:12:09.728353+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.760153+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:12:09.760153+0000","last_peered":"2023-09-23T20:12:09.760153+0000","last_clean":"2023-09-23T20:12:09.760153+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:12:09.760153+0000","last_undegraded":"2023-09-23T20:12:09.760153+0000","last_fullsized":"2023-09-23T20:12:09.760153+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"393'10058","reported_seq":17883,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:20.788140+0000","last_change":"2023-09-23T20:12:10.019046+0000","last_active":"2023-09-23T20:12:20.788140+0000","last_peered":"2023-09-23T20:12:20.788140+0000","last_clean":"2023-09-23T20:12:20.788140+0000","last_became_active":"2023-09-23T20:12:09.762053+0000","last_became_peered":"2023-09-23T20:12:09.762053+0000","last_unstale":"2023-09-23T20:12:20.788140+0000","last_undegraded":"2023-09-23T20:12:20.788140+0000","last_fullsized":"2023-09-23T20:12:20.788140+0000","mapping_epoch":391,"log_start":"328'4946","ondisk_log_start":"328'4946","created":299,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5112,"log_dups_size":0,"ondisk_log_size":5112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:41.902924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40763392,"num_objects":622,"num_object_clones":0,"num_object_copies":1866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10058,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4732,"num_bytes_recovered":261292032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":592,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728855+0000","last_change":"2023-09-23T20:11:55.924213+0000","last_active":"2023-09-23T20:12:09.728855+0000","last_peered":"2023-09-23T20:12:09.728855+0000","last_clean":"2023-09-23T20:12:09.728855+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:12:09.728855+0000","last_undegraded":"2023-09-23T20:12:09.728855+0000","last_fullsized":"2023-09-23T20:12:09.728855+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":93,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965802+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:12:09.965802+0000","last_peered":"2023-09-23T20:12:09.965802+0000","last_clean":"2023-09-23T20:12:09.965802+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:12:09.965802+0000","last_undegraded":"2023-09-23T20:12:09.965802+0000","last_fullsized":"2023-09-23T20:12:09.965802+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.759893+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:12:09.759893+0000","last_peered":"2023-09-23T20:12:09.759893+0000","last_clean":"2023-09-23T20:12:09.759893+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:12:09.759893+0000","last_undegraded":"2023-09-23T20:12:09.759893+0000","last_fullsized":"2023-09-23T20:12:09.759893+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":219,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785148+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:12:09.785148+0000","last_peered":"2023-09-23T20:12:09.785148+0000","last_clean":"2023-09-23T20:12:09.785148+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:12:09.785148+0000","last_undegraded":"2023-09-23T20:12:09.785148+0000","last_fullsized":"2023-09-23T20:12:09.785148+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"393'10148","reported_seq":14448,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:23.536579+0000","last_change":"2023-09-23T20:08:33.396812+0000","last_active":"2023-09-23T20:12:23.536579+0000","last_peered":"2023-09-23T20:12:23.536579+0000","last_clean":"2023-09-23T20:12:23.536579+0000","last_became_active":"2023-09-23T20:08:05.774750+0000","last_became_peered":"2023-09-23T20:08:05.774750+0000","last_unstale":"2023-09-23T20:12:23.536579+0000","last_undegraded":"2023-09-23T20:12:23.536579+0000","last_fullsized":"2023-09-23T20:12:23.536579+0000","mapping_epoch":334,"log_start":"312'3300","ondisk_log_start":"312'3300","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6848,"log_dups_size":0,"ondisk_log_size":6848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.633506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10148,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2086,"num_bytes_recovered":136708096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":817,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728655+0000","last_change":"2023-09-23T20:11:45.860623+0000","last_active":"2023-09-23T20:12:09.728655+0000","last_peered":"2023-09-23T20:12:09.728655+0000","last_clean":"2023-09-23T20:12:09.728655+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:12:09.728655+0000","last_undegraded":"2023-09-23T20:12:09.728655+0000","last_fullsized":"2023-09-23T20:12:09.728655+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":235,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.965968+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:12:09.965968+0000","last_peered":"2023-09-23T20:12:09.965968+0000","last_clean":"2023-09-23T20:12:09.965968+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:12:09.965968+0000","last_undegraded":"2023-09-23T20:12:09.965968+0000","last_fullsized":"2023-09-23T20:12:09.965968+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":431,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.760088+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:12:09.760088+0000","last_peered":"2023-09-23T20:12:09.760088+0000","last_clean":"2023-09-23T20:12:09.760088+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:12:09.760088+0000","last_undegraded":"2023-09-23T20:12:09.760088+0000","last_fullsized":"2023-09-23T20:12:09.760088+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":23,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.723967+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:12:09.723967+0000","last_peered":"2023-09-23T20:12:09.723967+0000","last_clean":"2023-09-23T20:12:09.723967+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:12:09.723967+0000","last_undegraded":"2023-09-23T20:12:09.723967+0000","last_fullsized":"2023-09-23T20:12:09.723967+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"393'10064","reported_seq":12835,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:23.196426+0000","last_change":"2023-09-23T20:09:08.683160+0000","last_active":"2023-09-23T20:12:23.196426+0000","last_peered":"2023-09-23T20:12:23.196426+0000","last_clean":"2023-09-23T20:12:23.196426+0000","last_became_active":"2023-09-23T20:09:08.670527+0000","last_became_peered":"2023-09-23T20:09:08.670527+0000","last_unstale":"2023-09-23T20:12:23.196426+0000","last_undegraded":"2023-09-23T20:12:23.196426+0000","last_fullsized":"2023-09-23T20:12:23.196426+0000","mapping_epoch":346,"log_start":"315'3800","ondisk_log_start":"315'3800","created":299,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:47.531381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30081048,"num_objects":460,"num_object_clones":0,"num_object_copies":1380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":460,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10064,"num_write_kb":336705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":704,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.786853+0000","last_change":"2023-09-23T20:09:56.036037+0000","last_active":"2023-09-23T20:12:12.786853+0000","last_peered":"2023-09-23T20:12:12.786853+0000","last_clean":"2023-09-23T20:12:12.786853+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:12:12.786853+0000","last_undegraded":"2023-09-23T20:12:12.786853+0000","last_fullsized":"2023-09-23T20:12:12.786853+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":94,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.966080+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:12:09.966080+0000","last_peered":"2023-09-23T20:12:09.966080+0000","last_clean":"2023-09-23T20:12:09.966080+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:12:09.966080+0000","last_undegraded":"2023-09-23T20:12:09.966080+0000","last_fullsized":"2023-09-23T20:12:09.966080+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":93,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.760012+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:12:09.760012+0000","last_peered":"2023-09-23T20:12:09.760012+0000","last_clean":"2023-09-23T20:12:09.760012+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:12:09.760012+0000","last_undegraded":"2023-09-23T20:12:09.760012+0000","last_fullsized":"2023-09-23T20:12:09.760012+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":352,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728150+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:12:09.728150+0000","last_peered":"2023-09-23T20:12:09.728150+0000","last_clean":"2023-09-23T20:12:09.728150+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:12:09.728150+0000","last_undegraded":"2023-09-23T20:12:09.728150+0000","last_fullsized":"2023-09-23T20:12:09.728150+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":168,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.745056+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:12:09.745056+0000","last_peered":"2023-09-23T20:12:09.745056+0000","last_clean":"2023-09-23T20:12:09.745056+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:12:09.745056+0000","last_undegraded":"2023-09-23T20:12:09.745056+0000","last_fullsized":"2023-09-23T20:12:09.745056+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":159,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.745878+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:12:09.745878+0000","last_peered":"2023-09-23T20:12:09.745878+0000","last_clean":"2023-09-23T20:12:09.745878+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:12:09.745878+0000","last_undegraded":"2023-09-23T20:12:09.745878+0000","last_fullsized":"2023-09-23T20:12:09.745878+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":396,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785441+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:12:09.785441+0000","last_peered":"2023-09-23T20:12:09.785441+0000","last_clean":"2023-09-23T20:12:09.785441+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:12:09.785441+0000","last_undegraded":"2023-09-23T20:12:09.785441+0000","last_fullsized":"2023-09-23T20:12:09.785441+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":409,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728686+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:12:09.728686+0000","last_peered":"2023-09-23T20:12:09.728686+0000","last_clean":"2023-09-23T20:12:09.728686+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:12:09.728686+0000","last_undegraded":"2023-09-23T20:12:09.728686+0000","last_fullsized":"2023-09-23T20:12:09.728686+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":386,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785520+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:12:09.785520+0000","last_peered":"2023-09-23T20:12:09.785520+0000","last_clean":"2023-09-23T20:12:09.785520+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:12:09.785520+0000","last_undegraded":"2023-09-23T20:12:09.785520+0000","last_fullsized":"2023-09-23T20:12:09.785520+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":389,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728352+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:12:09.728352+0000","last_peered":"2023-09-23T20:12:09.728352+0000","last_clean":"2023-09-23T20:12:09.728352+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:12:09.728352+0000","last_undegraded":"2023-09-23T20:12:09.728352+0000","last_fullsized":"2023-09-23T20:12:09.728352+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":94,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:10.831488+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:12:10.831488+0000","last_peered":"2023-09-23T20:12:10.831488+0000","last_clean":"2023-09-23T20:12:10.831488+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:12:10.831488+0000","last_undegraded":"2023-09-23T20:12:10.831488+0000","last_fullsized":"2023-09-23T20:12:10.831488+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":250,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728447+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:12:09.728447+0000","last_peered":"2023-09-23T20:12:09.728447+0000","last_clean":"2023-09-23T20:12:09.728447+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:12:09.728447+0000","last_undegraded":"2023-09-23T20:12:09.728447+0000","last_fullsized":"2023-09-23T20:12:09.728447+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":238,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.785739+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:12:09.785739+0000","last_peered":"2023-09-23T20:12:09.785739+0000","last_clean":"2023-09-23T20:12:09.785739+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:12:09.785739+0000","last_undegraded":"2023-09-23T20:12:09.785739+0000","last_fullsized":"2023-09-23T20:12:09.785739+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":439,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T20:12:09.728159+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:12:09.728159+0000","last_peered":"2023-09-23T20:12:09.728159+0000","last_clean":"2023-09-23T20:12:09.728159+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:12:09.728159+0000","last_undegraded":"2023-09-23T20:12:09.728159+0000","last_fullsized":"2023-09-23T20:12:09.728159+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":26174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":541982744,"num_objects":8271,"num_object_clones":0,"num_object_copies":24813,"num_objects_missing_on_primary":484,"num_objects_missing":484,"num_objects_degraded":6515,"num_objects_misplaced":492,"num_objects_unfound":0,"num_objects_dirty":8271,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":160191,"num_write_kb":5390721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32516,"num_bytes_recovered":1618542592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3221565440,"data_stored":3221553224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91010,"ondisk_log_size":91010,"up":48,"acting":44,"num_store_stats":7},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":542359,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":177,"num_read_kb":331,"num_write":87,"num_write_kb":878,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1299398,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234375,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":508336,"kb_used_data":168808,"kb_used_omap":0,"kb_used_meta":339520,"kb_avail":93236816,"statfs":{"total":95995035648,"available":95474499584,"internally_reserved":0,"allocated":172859392,"data_stored":171593375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":347668480},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.0449999999999999,"15min":2.2400000000000002},"min":{"1min":0.23499999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.5030000000000001,"5min":55.390000000000001,"15min":62.061999999999998},"last":4.3200000000000003},{"interface":"front","average":{"1min":1.034,"5min":1.9870000000000001,"15min":2.3780000000000001},"min":{"1min":0.248,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":3.726,"5min":56.664000000000001,"15min":62.389000000000003},"last":2.387}]},{"osd":1,"last update":"Sat Sep 23 20:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.907,"15min":3.4039999999999999},"min":{"1min":0.29299999999999998,"5min":0.28299999999999997,"15min":0.22700000000000001},"max":{"1min":3.7080000000000002,"5min":54.945999999999998,"15min":104.16800000000001},"last":2.2610000000000001},{"interface":"front","average":{"1min":0.84299999999999997,"5min":2,"15min":3.4929999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.223},"max":{"1min":2.9390000000000001,"5min":55.094000000000001,"15min":104.003},"last":5.2350000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":4.0800000000000001,"15min":3.6440000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":28.707000000000001,"5min":92.319000000000003,"15min":105.842},"last":5.3179999999999996},{"interface":"front","average":{"1min":2.8919999999999999,"5min":4.4699999999999998,"15min":4.1390000000000002},"min":{"1min":0.34300000000000003,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":28.596,"5min":92.049000000000007,"15min":105.66},"last":2.8879999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":3.1070000000000002,"15min":4.3840000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":4.0359999999999996,"5min":94.953000000000003,"15min":146.38800000000001},"last":1.724},{"interface":"front","average":{"1min":1.159,"5min":3.0390000000000001,"15min":4.4020000000000001},"min":{"1min":0.41399999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.6949999999999998,"5min":93.423000000000002,"15min":146.851},"last":3.3170000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.48999999999999999,"15min":0.71899999999999997},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.22500000000000001},"max":{"1min":3.2069999999999999,"5min":3.2069999999999999,"15min":19.215},"last":0.379},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.54600000000000004,"15min":0.83299999999999996},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.88400000000000001,"5min":2.7949999999999999,"15min":41.822000000000003},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.63700000000000001,"15min":0.97299999999999998},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":3.407,"5min":4.4080000000000004,"15min":35.488},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.628,"15min":0.85699999999999998},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":3.2549999999999999,"5min":4.2030000000000003,"15min":34.896000000000001},"last":0.51700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.625,"15min":0.80800000000000005},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":1.4650000000000001,"5min":4.3710000000000004,"15min":41.902999999999999},"last":3.2730000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.63200000000000001,"15min":0.64200000000000002},"min":{"1min":0.38400000000000001,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":0.85599999999999998,"5min":3.9889999999999999,"15min":6.3620000000000001},"last":0.54800000000000004}]}]},{"osd":6,"up_from":0,"seq":1078036791416,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762574,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":797192,"kb_used_data":221248,"kb_used_omap":7,"kb_used_meta":575928,"kb_avail":92947960,"statfs":{"total":95995035648,"available":95178711040,"internally_reserved":0,"allocated":226557952,"data_stored":224686813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":589751098},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.1339999999999999,"15min":3.698},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.151},"max":{"1min":9.9410000000000007,"5min":28.510000000000002,"15min":186.84},"last":1.4530000000000001},{"interface":"front","average":{"1min":1.776,"5min":2.4209999999999998,"15min":3.9929999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":8.6660000000000004,"5min":14.781000000000001,"15min":191.322},"last":4.173}]},{"osd":2,"last update":"Sat Sep 23 20:12:09 2023","interfaces":[{"interface":"back","average":{"1min":3.9369999999999998,"5min":4.1559999999999997,"15min":4.8419999999999996},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":38.040999999999997,"5min":58.499000000000002,"15min":134.72499999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":3.3700000000000001,"5min":3.6269999999999998,"15min":4.952},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":38.173999999999999,"5min":58.741,"15min":135.74600000000001},"last":6.1829999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.5459999999999998,"15min":3.7879999999999998},"min":{"1min":0.25700000000000001,"5min":0.255,"15min":0.22},"max":{"1min":3.9849999999999999,"5min":25.963999999999999,"15min":68.180999999999997},"last":6.0890000000000004},{"interface":"front","average":{"1min":1.4019999999999999,"5min":2.4430000000000001,"15min":3.6909999999999998},"min":{"1min":0.29299999999999998,"5min":0.28199999999999997,"15min":0.24299999999999999},"max":{"1min":6.4320000000000004,"5min":40.716000000000001,"15min":66.509},"last":4.101}]},{"osd":4,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.6140000000000001,"15min":1.9430000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23599999999999999},"max":{"1min":6.6029999999999998,"5min":18.087,"15min":60.951000000000001},"last":1.496},{"interface":"front","average":{"1min":1.573,"5min":1.764,"15min":2.1190000000000002},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":8.1669999999999998,"5min":14.890000000000001,"15min":60.841000000000001},"last":4.0579999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.8300000000000001,"15min":2.952},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.25900000000000001},"max":{"1min":6.5679999999999996,"5min":14.83,"15min":219.048},"last":4.1550000000000002},{"interface":"front","average":{"1min":1.3440000000000001,"5min":2.1819999999999999,"15min":3.0950000000000002},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28399999999999997},"max":{"1min":6.0190000000000001,"5min":25.893000000000001,"15min":218.69499999999999},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:11:43 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.7649999999999999,"15min":1.994},"min":{"1min":0.38800000000000001,"5min":0.38500000000000001,"15min":0.35899999999999999},"max":{"1min":6.6749999999999998,"5min":18.117999999999999,"15min":18.117999999999999},"last":1.5569999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.8089999999999999,"15min":2.6579999999999999},"min":{"1min":0.42299999999999999,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":6.5069999999999997,"5min":14.862,"15min":20.734999999999999},"last":4.1260000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":2.1869999999999998,"15min":2.3239999999999998},"min":{"1min":0.42999999999999999,"5min":0.33300000000000002,"15min":0.26800000000000002},"max":{"1min":4.5899999999999999,"5min":25.937000000000001,"15min":60.213000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.556,"5min":1.7749999999999999,"15min":2.0619999999999998},"min":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.30199999999999999},"max":{"1min":6.649,"5min":18.053999999999998,"15min":61.298999999999999},"last":1.5289999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926111,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1019988,"kb_used_data":389516,"kb_used_omap":8,"kb_used_meta":630455,"kb_avail":92725164,"statfs":{"total":95995035648,"available":94950567936,"internally_reserved":0,"allocated":398864384,"data_stored":396985154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":645586529},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5190000000000001,"5min":3.3839999999999999,"15min":3.8860000000000001},"min":{"1min":0.21299999999999999,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":72.534000000000006,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.33000000000000002},{"interface":"front","average":{"1min":5.5250000000000004,"5min":3.6579999999999999,"15min":4.0819999999999999},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":72.667000000000002,"5min":72.667000000000002,"15min":91.632999999999996},"last":1.214}]},{"osd":2,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":3.8780000000000001,"15min":5.0019999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19800000000000001},"max":{"1min":19.486999999999998,"5min":102.65300000000001,"15min":270.51100000000002},"last":5.0979999999999999},{"interface":"front","average":{"1min":3.0049999999999999,"5min":4.0129999999999999,"15min":4.8470000000000004},"min":{"1min":0.25,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.693999999999999,"5min":96.578999999999994,"15min":269.69299999999998},"last":1.3819999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":4.4909999999999997,"15min":5.5590000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":8.6489999999999991,"5min":126.95399999999999,"15min":145.72800000000001},"last":1.129},{"interface":"front","average":{"1min":1.4770000000000001,"5min":4.8780000000000001,"15min":5.7240000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.23999999999999999},"max":{"1min":5.4690000000000003,"5min":127.294,"15min":145.50700000000001},"last":0.80500000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.5629999999999999,"15min":2.2629999999999999},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":6.1369999999999996,"5min":15.688000000000001,"15min":148.20599999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.099,"5min":1.4039999999999999,"15min":2.0379999999999998},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":5.3940000000000001,"5min":13.428000000000001,"15min":148.11000000000001},"last":0.82899999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.298,"15min":2.464},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":1.1459999999999999,"5min":9.5719999999999992,"15min":131.34399999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.163,"5min":1.718,"15min":2.468},"min":{"1min":0.377,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.1059999999999999,"5min":15.717000000000001,"15min":131.386},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.7270000000000001,"15min":1.6439999999999999},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":4.9800000000000004,"5min":15.744999999999999,"15min":15.744999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.5549999999999999,"15min":1.841},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":5.4390000000000001,"5min":13.470000000000001,"15min":13.766999999999999},"last":0.85599999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.327,"15min":1.891},"min":{"1min":0.38900000000000001,"5min":0.38100000000000001,"15min":0.31900000000000001},"max":{"1min":1.0980000000000001,"5min":9.6069999999999993,"15min":65.081999999999994},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.724,"15min":2.0190000000000001},"min":{"1min":0.39200000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.1920000000000002,"5min":15.771000000000001,"15min":15.771000000000001},"last":0.499}]}]},{"osd":2,"up_from":104,"seq":446676599049,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1192604,"kb_used_data":453460,"kb_used_omap":12,"kb_used_meta":739059,"kb_avail":92552548,"statfs":{"total":95995035648,"available":94773809152,"internally_reserved":0,"allocated":464343040,"data_stored":462456413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":756797438},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":2.2160000000000002,"15min":3.0030000000000001},"min":{"1min":0.26200000000000001,"5min":0.19900000000000001,"15min":0.16900000000000001},"max":{"1min":14.794,"5min":17.876999999999999,"15min":76.251999999999995},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.339,"5min":2.4159999999999999,"15min":3.0339999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.192},"max":{"1min":5.077,"5min":18.561,"15min":76.590000000000003},"last":5.7690000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.8220000000000001,"15min":3.6280000000000001},"min":{"1min":0.25,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":32.677999999999997,"5min":32.677999999999997,"15min":94.768000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.7010000000000001,"5min":2.4820000000000002,"15min":3.577},"min":{"1min":0.30599999999999999,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":32.582000000000001,"5min":32.582000000000001,"15min":96.808000000000007},"last":0.501}]},{"osd":3,"last update":"Sat Sep 23 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.5169999999999999,"15min":4.4029999999999996},"min":{"1min":0.32800000000000001,"5min":0.25,"15min":0.214},"max":{"1min":8.5850000000000009,"5min":13.513,"15min":107.381},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.0900000000000001,"5min":2.097,"15min":3.7610000000000001},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.23699999999999999},"max":{"1min":4.7919999999999998,"5min":20.308,"15min":106.301},"last":1.954}]},{"osd":4,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.698,"15min":2.0790000000000002},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.26900000000000002},"max":{"1min":14.757999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.7090000000000001,"15min":2.1259999999999999},"min":{"1min":0.34899999999999998,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":13.387,"5min":13.387,"15min":76.161000000000001},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":4.258,"5min":2.1099999999999999,"15min":2.6899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":29.433,"5min":29.433,"15min":172.505},"last":0.66400000000000003},{"interface":"front","average":{"1min":3.6040000000000001,"5min":2.1920000000000002,"15min":2.8500000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.29099999999999998},"max":{"1min":29.196999999999999,"5min":29.196999999999999,"15min":172.375},"last":5.7400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.5900000000000001,"15min":1.601},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":4.718,"5min":10.209,"15min":10.209},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.8220000000000001,"15min":1.909},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.776,"5min":14.279999999999999,"15min":14.279999999999999},"last":5.6929999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.837,"15min":1.829},"min":{"1min":0.44,"5min":0.377,"15min":0.30199999999999999},"max":{"1min":13.481999999999999,"5min":13.481999999999999,"15min":23.079999999999998},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.4159999999999999,"5min":1.869,"15min":1.9139999999999999},"min":{"1min":0.39900000000000002,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":14.698,"5min":14.698,"15min":20.614999999999998},"last":0.63600000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435523,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514320,"kb_used_data":831880,"kb_used_omap":4,"kb_used_meta":682427,"kb_avail":92230832,"statfs":{"total":95995035648,"available":94444371968,"internally_reserved":0,"allocated":851845120,"data_stored":849787073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":698805357},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":4.2430000000000003,"15min":3.4049999999999998},"min":{"1min":0.19700000000000001,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":7.6440000000000001,"5min":74.323999999999998,"15min":74.323999999999998},"last":0.996},{"interface":"front","average":{"1min":0.94599999999999995,"5min":4.0170000000000003,"15min":3.758},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":4.9000000000000004,"5min":74.447000000000003,"15min":74.447000000000003},"last":0.95399999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":8.7829999999999995,"5min":4.7240000000000002,"15min":4.0599999999999996},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":163.49700000000001,"5min":163.49700000000001,"15min":163.49700000000001},"last":3.3519999999999999},{"interface":"front","average":{"1min":8.5839999999999996,"5min":5.2400000000000002,"15min":4.351},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":163.512,"5min":163.512,"15min":163.512},"last":3.5299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":12.541,"5min":6.2560000000000002,"15min":4.5679999999999996},"min":{"1min":0.33800000000000002,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":163.83500000000001,"5min":163.83500000000001,"15min":163.83500000000001},"last":3.3820000000000001},{"interface":"front","average":{"1min":11.702,"5min":6.319,"15min":4.4500000000000002},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":163.92099999999999,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.80700000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":2.4489999999999998,"15min":3.1349999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.3740000000000001,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.7669999999999999,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":5.4820000000000002,"5min":58.408000000000001,"15min":110.295},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":3.452,"15min":3.3799999999999999},"min":{"1min":0.316,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.5270000000000001,"5min":77.206000000000003,"15min":113.881},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.80400000000000005,"5min":3.7250000000000001,"15min":3.2650000000000001},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":2.2930000000000001,"5min":77.968000000000004,"15min":114.598},"last":0.97099999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.629,"15min":3.8999999999999999},"min":{"1min":0.41599999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":2.2759999999999998,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.8439999999999999,"15min":4.0839999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":3.9279999999999999,"5min":58.701000000000001,"15min":82.233000000000004},"last":0.65700000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.8210000000000002,"15min":3.4740000000000002},"min":{"1min":0.40000000000000002,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":2.2559999999999998,"5min":58.686,"15min":138.035},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.8679999999999999,"15min":3.4510000000000001},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.28699999999999998},"max":{"1min":3.4260000000000002,"5min":58.667999999999999,"15min":138.054},"last":0.72999999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019285,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1028712,"kb_used_data":691616,"kb_used_omap":0,"kb_used_meta":337088,"kb_avail":92716440,"statfs":{"total":95995035648,"available":94941634560,"internally_reserved":0,"allocated":708214784,"data_stored":706956923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":345178112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:21 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.0509999999999999,"15min":2.202},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.189},"max":{"1min":3.6150000000000002,"5min":11.743,"15min":89.004999999999995},"last":2.1400000000000001},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.085,"15min":2.1659999999999999},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.218},"max":{"1min":5.3250000000000002,"5min":11.776,"15min":89.024000000000001},"last":0.40300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":2.9489999999999998,"15min":5.5389999999999997},"min":{"1min":0.252,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":4.1669999999999998,"5min":119.892,"15min":161.613},"last":3.6600000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":3.2109999999999999,"15min":5.6139999999999999},"min":{"1min":0.28899999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.125,"5min":119.351,"15min":162.017},"last":0.25800000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":8.6579999999999995,"15min":6.8689999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":144.916,"5min":182.96700000000001,"15min":182.96700000000001},"last":4.3360000000000003},{"interface":"front","average":{"1min":11.116,"5min":8.2240000000000002,"15min":6.6360000000000001},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":144.83099999999999,"5min":183.244,"15min":183.244},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":4.843,"15min":6.8440000000000003},"min":{"1min":0.30499999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":2.5259999999999998,"5min":184.37100000000001,"15min":266.54700000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":4.9130000000000003,"15min":5.798},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":5.407,"5min":184.22300000000001,"15min":184.22300000000001},"last":1.8380000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:12:21 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48799999999999999,"15min":2.117},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":2.8919999999999999,"5min":3.3290000000000002,"15min":266.59899999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.45800000000000002,"15min":2.1419999999999999},"min":{"1min":0.29999999999999999,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":1.2230000000000001,"5min":1.722,"15min":266.72899999999998},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.502,"15min":0.51000000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.804,"5min":2.5489999999999999,"15min":4.0410000000000004},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60799999999999998,"15min":0.58899999999999997},"min":{"1min":0.38300000000000001,"5min":0.247,"15min":0.247},"max":{"1min":1.5880000000000001,"5min":4.2889999999999997,"15min":4.2889999999999997},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":1.244,"15min":1.411},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":0.59699999999999998,"5min":74.040000000000006,"15min":76.507000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":1.25,"15min":1.536},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":3.2890000000000001,"5min":74.021000000000001,"15min":76.462000000000003},"last":1.5429999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757649,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925092,"kb_used_data":526876,"kb_used_omap":0,"kb_used_meta":398208,"kb_avail":92820060,"statfs":{"total":95995035648,"available":95047741440,"internally_reserved":0,"allocated":539521024,"data_stored":538263532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":407764992},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.82099999999999995,"15min":1.7290000000000001},"min":{"1min":0.26100000000000001,"5min":0.224,"15min":0.216},"max":{"1min":2.173,"5min":8.6739999999999995,"15min":121.94799999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.103,"15min":1.9970000000000001},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":9.4250000000000007,"5min":10.327,"15min":121.964},"last":4.2750000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":2.093,"15min":5.8949999999999996},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":3.52,"5min":75.204999999999998,"15min":266.197},"last":0.222},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.1480000000000001,"15min":6.0700000000000003},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1399999999999997,"5min":75.231999999999999,"15min":266.14999999999998},"last":2.6099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":5.6639999999999997,"15min":5.4379999999999997},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":4.1239999999999997,"5min":240.57300000000001,"15min":240.57300000000001},"last":4.3639999999999999},{"interface":"front","average":{"1min":1.2509999999999999,"5min":5.7359999999999998,"15min":5.4969999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.313},"max":{"1min":4.1559999999999997,"5min":240.464,"15min":240.464},"last":4.407}]},{"osd":3,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.7030000000000001,"15min":4.8390000000000004},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.742,"5min":14.808,"15min":201.50899999999999},"last":4.4249999999999998},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.407,"15min":4.6959999999999997},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":3.4500000000000002,"5min":14.893000000000001,"15min":201.58099999999999},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:12:16 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.48599999999999999,"15min":1.4119999999999999},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":2.5790000000000002,"5min":4.0510000000000002,"15min":191.78899999999999},"last":0.255},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.436,"15min":1.3129999999999999},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":0.59499999999999997,"5min":1.4359999999999999,"15min":191.821},"last":0.29999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.69299999999999995,"15min":0.64100000000000001},"min":{"1min":0.377,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":3.278,"5min":9.9009999999999998,"15min":9.9009999999999998},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.54600000000000004,"15min":0.51700000000000002},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.72799999999999998,"5min":5.0119999999999996,"15min":5.0119999999999996},"last":0.35599999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.56699999999999995,"15min":1.4930000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.3069999999999999,"5min":4.2080000000000002,"15min":191.84299999999999},"last":0.372},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65900000000000003,"15min":1.655},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":3.1440000000000001,"5min":9.6920000000000002,"15min":191.69},"last":0.39100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":387448832,"data_stored":387448832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":210632704,"data_stored":210632704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":428544000,"data_stored":428539928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":827260928,"data_stored":827260928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":685572096,"data_stored":685572096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":524947456,"data_stored":524943384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":157159424,"data_stored":157155352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:12:28.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:28.202 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:28.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:28.275 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:28 smithi135 ceph-mon[221113]: 1.e scrub starts 2023-09-23T20:12:28.275 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:28 smithi135 ceph-mon[221113]: pgmap v1112: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 455 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 6519/21714 objects degraded (30.022%); 492/21714 objects misplaced (2.266%); 0 B/s, 76 objects/s recovering 2023-09-23T20:12:28.275 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6519/21714 objects degraded (30.022%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:28.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:28 smithi135 ceph-mon[213871]: 1.e scrub starts 2023-09-23T20:12:28.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:28 smithi135 ceph-mon[213871]: pgmap v1112: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 455 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 6519/21714 objects degraded (30.022%); 492/21714 objects misplaced (2.266%); 0 B/s, 76 objects/s recovering 2023-09-23T20:12:28.276 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6519/21714 objects degraded (30.022%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:28.303 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:28.375 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:28 smithi144 ceph-mon[194833]: 1.e scrub starts 2023-09-23T20:12:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:28 smithi144 ceph-mon[194833]: pgmap v1112: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 455 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 6519/21714 objects degraded (30.022%); 492/21714 objects misplaced (2.266%); 0 B/s, 76 objects/s recovering 2023-09-23T20:12:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6519/21714 objects degraded (30.022%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:28.500 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:28.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:28.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:28.611 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:28.639 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:28.680 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:28.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:28.786 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:28.812 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:28.861 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:28.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:28.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:28.969 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:29.004 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:29.063 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:29.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:29.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:29.126 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:29 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:12:29.126 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:29 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:12:29.127 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:29 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:12:29.127 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:29 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:12:29.168 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:29.202 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:29.252 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:29.259 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:29 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:12:29.259 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:29 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:12:29.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:29.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:29.361 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:29.394 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:29.452 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:29.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:29.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:29.560 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:29.591 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:29.608 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:12:29.644 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:29.645 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:12:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:29.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:29.696 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:12:29.746 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:12:29.751 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:29.782 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:29.783 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:12:29.823 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:12:29.839 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:29.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:29.860 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:12:29.907 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:12:29.931 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:12:29.931 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:12:29.932 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:12:29.932 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:12:29.932 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:12:29.932 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:12:29.932 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:12:29.932 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:12:29.944 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:29.971 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:30.027 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:30.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:30.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:30.131 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:30.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:30 smithi144 ceph-mon[194833]: pgmap v1113: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 354 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 6519/16842 objects degraded (38.707%); 492/16842 objects misplaced (2.921%); 0 B/s, 76 objects/s recovering 2023-09-23T20:12:30.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2592283143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:30.141 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:12:30.160 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:30.215 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:30.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:30 smithi135 ceph-mon[213871]: pgmap v1113: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 354 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 6519/16842 objects degraded (38.707%); 492/16842 objects misplaced (2.921%); 0 B/s, 76 objects/s recovering 2023-09-23T20:12:30.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2592283143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:30.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:30 smithi135 ceph-mon[221113]: pgmap v1113: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 354 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 6519/16842 objects degraded (38.707%); 492/16842 objects misplaced (2.921%); 0 B/s, 76 objects/s recovering 2023-09-23T20:12:30.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2592283143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:30.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:30.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:30.324 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:12:30.379 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:12:30.379 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:12:30.431 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:30.550 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:12:30.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:30.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:12:30.661 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:30.688 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:31.202 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:12:31.520 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:31.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:31.627 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:31.656 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:31.709 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:31.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:31.817 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:31.909 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:32.066 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:32 smithi144 ceph-mon[194833]: pgmap v1114: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 267 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 6519/12699 objects degraded (51.335%); 277/12699 objects misplaced (2.181%); 0 B/s, 82 objects/s recovering 2023-09-23T20:12:32.067 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:32 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:12:32.071 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:32 smithi135 ceph-mon[213871]: pgmap v1114: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 267 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 6519/12699 objects degraded (51.335%); 277/12699 objects misplaced (2.181%); 0 B/s, 82 objects/s recovering 2023-09-23T20:12:32.071 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:32 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:12:32.161 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:12:32.213 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:32.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:32.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:32.286 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:32.320 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:32.353 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:32.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:32 smithi135 ceph-mon[221113]: pgmap v1114: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 267 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 6519/12699 objects degraded (51.335%); 277/12699 objects misplaced (2.181%); 0 B/s, 82 objects/s recovering 2023-09-23T20:12:32.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:32 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:12:32.435 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:32.617 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:32.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:32.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:32.725 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:32.760 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:32.811 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:32.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:32.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:32.921 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:33.013 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:33.068 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:33 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:12:33.068 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:33 smithi144 ceph-mon[194833]: 1.12 scrub starts 2023-09-23T20:12:33.068 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6519/12699 objects degraded (51.335%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:33.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:33 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:12:33.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:33 smithi135 ceph-mon[221113]: 1.12 scrub starts 2023-09-23T20:12:33.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6519/12699 objects degraded (51.335%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:33.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:33 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:12:33.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:33 smithi135 ceph-mon[213871]: 1.12 scrub starts 2023-09-23T20:12:33.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6519/12699 objects degraded (51.335%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:33.849 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:12:33.894 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:12:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:12:33] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-09-23T20:12:34.140 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:34.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:34.148 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:34 smithi135 ceph-mon[213871]: pgmap v1115: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 245 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 6519/11607 objects degraded (56.164%); 277/11607 objects misplaced (2.386%); 0 B/s, 82 objects/s recovering 2023-09-23T20:12:34.148 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:34 smithi135 ceph-mon[221113]: pgmap v1115: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 245 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 6519/11607 objects degraded (56.164%); 277/11607 objects misplaced (2.386%); 0 B/s, 82 objects/s recovering 2023-09-23T20:12:34.248 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:34.278 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:34.333 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:34.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:34.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:34.339 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:34 smithi144 ceph-mon[194833]: pgmap v1115: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 245 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 6519/11607 objects degraded (56.164%); 277/11607 objects misplaced (2.386%); 0 B/s, 82 objects/s recovering 2023-09-23T20:12:34.440 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:34.551 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:34.995 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:35.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:35.065 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:35.108 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:35.135 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:35.183 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:35.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:35.190 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Removed 84230 objects 2023-09-23T20:12:35.190 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Clean up completed and total clean up time :276.178 2023-09-23T20:12:35.217 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:35 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:12:35.217 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:35 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:12:35.289 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:35.315 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:35.356 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:35.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:35 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:12:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:35.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:35.444 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:35.465 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:12:35.499 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:35.879 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:12:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:35.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:12:35.986 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:36.017 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:36.065 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:36.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:36.170 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:12:36.172 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:12:36.173 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:36.199 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:36.204 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:36 smithi135 ceph-mon[221113]: 1.e scrub starts 2023-09-23T20:12:36.204 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:36 smithi135 ceph-mon[221113]: pgmap v1116: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 111 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 6519/5193 objects degraded (125.534%); 53/5193 objects misplaced (1.021%); 0 B/s, 69 objects/s recovering 2023-09-23T20:12:36.205 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:36 smithi135 ceph-mon[213871]: 1.e scrub starts 2023-09-23T20:12:36.205 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:36 smithi135 ceph-mon[213871]: pgmap v1116: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 111 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 6519/5193 objects degraded (125.534%); 53/5193 objects misplaced (1.021%); 0 B/s, 69 objects/s recovering 2023-09-23T20:12:36.241 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:36.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:36.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:36.347 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:36 smithi144 ceph-mon[194833]: 1.e scrub starts 2023-09-23T20:12:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:36 smithi144 ceph-mon[194833]: pgmap v1116: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 111 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 6519/5193 objects degraded (125.534%); 53/5193 objects misplaced (1.021%); 0 B/s, 69 objects/s recovering 2023-09-23T20:12:36.393 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:37.050 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:37.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:37.115 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:12:37.154 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:37.180 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:37.222 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:37.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:37.328 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:37.354 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:12:37.355 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[221113]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.355 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[221113]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T20:12:37.355 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[221113]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.355 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[221113]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.355 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[221113]: from='client.36108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:37.355 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/455122957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:12:37.355 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[221113]: pgmap v1118: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 52 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 6519/2382 objects degraded (273.678%); 53/2382 objects misplaced (2.225%); 0 B/s, 69 objects/s recovering 2023-09-23T20:12:37.355 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:12:37.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[213871]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[213871]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T20:12:37.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[213871]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[213871]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[213871]: from='client.36108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:37.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/455122957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:12:37.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:37 smithi135 ceph-mon[213871]: pgmap v1118: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 52 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 6519/2382 objects degraded (273.678%); 53/2382 objects misplaced (2.225%); 0 B/s, 69 objects/s recovering 2023-09-23T20:12:37.357 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:37 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:12:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:37 smithi144 ceph-mon[194833]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:37 smithi144 ceph-mon[194833]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T20:12:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:37 smithi144 ceph-mon[194833]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:37 smithi144 ceph-mon[194833]: failed to encode map e395 with expected crc 2023-09-23T20:12:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:37 smithi144 ceph-mon[194833]: from='client.36108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:37 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/455122957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:12:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:37 smithi144 ceph-mon[194833]: pgmap v1118: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 52 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 6519/2382 objects degraded (273.678%); 53/2382 objects misplaced (2.225%); 0 B/s, 69 objects/s recovering 2023-09-23T20:12:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:12:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: 1.8 scrub starts 2023-09-23T20:12:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/455122957' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6519/2382 objects degraded (273.678%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: 1.8 scrub starts 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/455122957' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6519/2382 objects degraded (273.678%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: 1.8 scrub starts 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/455122957' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: failed to encode map e396 with expected crc 2023-09-23T20:12:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:38.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:38.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6519/2382 objects degraded (273.678%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: failed to encode map e396 with expected crc 2023-09-23T20:12:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T20:12:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: 1.4 scrub starts 2023-09-23T20:12:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: 1.3 scrub starts 2023-09-23T20:12:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: 1.3 scrub ok 2023-09-23T20:12:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:39 smithi144 ceph-mon[194833]: pgmap v1121: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 15 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 5927/591 objects degraded (1002.876%); 53/591 objects misplaced (8.968%); 0 B/s, 145 objects/s recovering 2023-09-23T20:12:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: failed to encode map e396 with expected crc 2023-09-23T20:12:39.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: 1.4 scrub starts 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: 1.3 scrub starts 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: 1.3 scrub ok 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[221113]: pgmap v1121: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 15 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 5927/591 objects degraded (1002.876%); 53/591 objects misplaced (8.968%); 0 B/s, 145 objects/s recovering 2023-09-23T20:12:39.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: failed to encode map e396 with expected crc 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: 1.4 scrub starts 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: failed to encode map e397 with expected crc 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: 1.3 scrub starts 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: 1.3 scrub ok 2023-09-23T20:12:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:39 smithi135 ceph-mon[213871]: pgmap v1121: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 15 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 5927/591 objects degraded (1002.876%); 53/591 objects misplaced (8.968%); 0 B/s, 145 objects/s recovering 2023-09-23T20:12:40.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:40 smithi144 ceph-mon[194833]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:40 smithi144 ceph-mon[194833]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:40 smithi144 ceph-mon[194833]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T20:12:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:40 smithi144 ceph-mon[194833]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:40 smithi144 ceph-mon[194833]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[213871]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[213871]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[213871]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T20:12:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[213871]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[213871]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[221113]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[221113]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[221113]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T20:12:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[221113]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:40 smithi135 ceph-mon[221113]: failed to encode map e398 with expected crc 2023-09-23T20:12:40.897 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T20:12:40.897 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T20:12:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:12:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[213871]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[213871]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[213871]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[213871]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[213871]: pgmap v1124: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 7.8 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 5745/243 objects degraded (2364.198%); 53/243 objects misplaced (21.811%); 0 B/s, 288 objects/s recovering 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[221113]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[221113]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[221113]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[221113]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:41 smithi135 ceph-mon[221113]: pgmap v1124: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 7.8 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 5745/243 objects degraded (2364.198%); 53/243 objects misplaced (21.811%); 0 B/s, 288 objects/s recovering 2023-09-23T20:12:41.335 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1117,"stamp":"2023-09-23T20:12:36.092625+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":65952834,"num_objects":963,"num_object_clones":0,"num_object_copies":2889,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":6519,"num_objects_misplaced":53,"num_objects_unfound":0,"num_objects_dirty":963,"num_whiteouts":0,"num_read":6158,"num_read_kb":13539,"num_write":168521,"num_write_kb":5404324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33420,"num_bytes_recovered":1628246271,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95561,"ondisk_log_size":95561,"up":327,"acting":323,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":376,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5531296,"kb_used_data":1818664,"kb_used_omap":32,"kb_used_meta":3712223,"kb_avail":650684768,"statfs":{"total":671965249536,"available":666301202432,"internally_reserved":0,"allocated":1862311936,"data_stored":1850799751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3801316870},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,0,4,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-480772096,"num_objects":-7336,"num_object_clones":0,"num_object_copies":-22008,"num_objects_missing_on_primary":-484,"num_objects_missing":-484,"num_objects_degraded":0,"num_objects_misplaced":-439,"num_objects_unfound":0,"num_objects_dirty":-7336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7336,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":697,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.039496"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":392,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801014+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:12:12.801014+0000","last_peered":"2023-09-23T20:12:12.801014+0000","last_clean":"2023-09-23T20:12:12.801014+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:12:12.801014+0000","last_undegraded":"2023-09-23T20:12:12.801014+0000","last_fullsized":"2023-09-23T20:12:12.801014+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":222,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807461+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:12:12.807461+0000","last_peered":"2023-09-23T20:12:12.807461+0000","last_clean":"2023-09-23T20:12:12.807461+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:12:12.807461+0000","last_undegraded":"2023-09-23T20:12:12.807461+0000","last_fullsized":"2023-09-23T20:12:12.807461+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:13.929658+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:12:13.929658+0000","last_peered":"2023-09-23T20:12:13.929658+0000","last_clean":"2023-09-23T20:12:13.929658+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:12:13.929658+0000","last_undegraded":"2023-09-23T20:12:13.929658+0000","last_fullsized":"2023-09-23T20:12:13.929658+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":102,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798258+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:12:12.798258+0000","last_peered":"2023-09-23T20:12:12.798258+0000","last_clean":"2023-09-23T20:12:12.798258+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:12:12.798258+0000","last_undegraded":"2023-09-23T20:12:12.798258+0000","last_fullsized":"2023-09-23T20:12:12.798258+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":222,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.806861+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:12:12.806861+0000","last_peered":"2023-09-23T20:12:12.806861+0000","last_clean":"2023-09-23T20:12:12.806861+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:12:12.806861+0000","last_undegraded":"2023-09-23T20:12:12.806861+0000","last_fullsized":"2023-09-23T20:12:12.806861+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":339,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799693+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:12:12.799693+0000","last_peered":"2023-09-23T20:12:12.799693+0000","last_clean":"2023-09-23T20:12:12.799693+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:12:12.799693+0000","last_undegraded":"2023-09-23T20:12:12.799693+0000","last_fullsized":"2023-09-23T20:12:12.799693+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":320,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799616+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:12:12.799616+0000","last_peered":"2023-09-23T20:12:12.799616+0000","last_clean":"2023-09-23T20:12:12.799616+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:12:12.799616+0000","last_undegraded":"2023-09-23T20:12:12.799616+0000","last_fullsized":"2023-09-23T20:12:12.799616+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":237,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.787383+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:12:12.787383+0000","last_peered":"2023-09-23T20:12:12.787383+0000","last_clean":"2023-09-23T20:12:12.787383+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:12:12.787383+0000","last_undegraded":"2023-09-23T20:12:12.787383+0000","last_fullsized":"2023-09-23T20:12:12.787383+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":704,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:16.166267+0000","last_change":"2023-09-23T20:09:56.042750+0000","last_active":"2023-09-23T20:12:16.166267+0000","last_peered":"2023-09-23T20:12:16.166267+0000","last_clean":"2023-09-23T20:12:16.166267+0000","last_became_active":"2023-09-23T20:09:56.042455+0000","last_became_peered":"2023-09-23T20:09:56.042455+0000","last_unstale":"2023-09-23T20:12:16.166267+0000","last_undegraded":"2023-09-23T20:12:16.166267+0000","last_fullsized":"2023-09-23T20:12:16.166267+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":393,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799473+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:12:12.799473+0000","last_peered":"2023-09-23T20:12:12.799473+0000","last_clean":"2023-09-23T20:12:12.799473+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:12:12.799473+0000","last_undegraded":"2023-09-23T20:12:12.799473+0000","last_fullsized":"2023-09-23T20:12:12.799473+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":256,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.783532+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:12:12.783532+0000","last_peered":"2023-09-23T20:12:12.783532+0000","last_clean":"2023-09-23T20:12:12.783532+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:12:12.783532+0000","last_undegraded":"2023-09-23T20:12:12.783532+0000","last_fullsized":"2023-09-23T20:12:12.783532+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":87,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799224+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:12:12.799224+0000","last_peered":"2023-09-23T20:12:12.799224+0000","last_clean":"2023-09-23T20:12:12.799224+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:12:12.799224+0000","last_undegraded":"2023-09-23T20:12:12.799224+0000","last_fullsized":"2023-09-23T20:12:12.799224+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807499+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:12:12.807499+0000","last_peered":"2023-09-23T20:12:12.807499+0000","last_clean":"2023-09-23T20:12:12.807499+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:12:12.807499+0000","last_undegraded":"2023-09-23T20:12:12.807499+0000","last_fullsized":"2023-09-23T20:12:12.807499+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":448,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801008+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:12:12.801008+0000","last_peered":"2023-09-23T20:12:12.801008+0000","last_clean":"2023-09-23T20:12:12.801008+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:12:12.801008+0000","last_undegraded":"2023-09-23T20:12:12.801008+0000","last_fullsized":"2023-09-23T20:12:12.801008+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":390,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801203+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:12:12.801203+0000","last_peered":"2023-09-23T20:12:12.801203+0000","last_clean":"2023-09-23T20:12:12.801203+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:12:12.801203+0000","last_undegraded":"2023-09-23T20:12:12.801203+0000","last_fullsized":"2023-09-23T20:12:12.801203+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":723,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.795998+0000","last_change":"2023-09-23T20:11:07.067995+0000","last_active":"2023-09-23T20:12:12.795998+0000","last_peered":"2023-09-23T20:12:12.795998+0000","last_clean":"2023-09-23T20:12:12.795998+0000","last_became_active":"2023-09-23T20:09:56.043288+0000","last_became_peered":"2023-09-23T20:09:56.043288+0000","last_unstale":"2023-09-23T20:12:12.795998+0000","last_undegraded":"2023-09-23T20:12:12.795998+0000","last_fullsized":"2023-09-23T20:12:12.795998+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":87,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.797793+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:12:12.797793+0000","last_peered":"2023-09-23T20:12:12.797793+0000","last_clean":"2023-09-23T20:12:12.797793+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:12:12.797793+0000","last_undegraded":"2023-09-23T20:12:12.797793+0000","last_fullsized":"2023-09-23T20:12:12.797793+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":246,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807382+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:12:12.807382+0000","last_peered":"2023-09-23T20:12:12.807382+0000","last_clean":"2023-09-23T20:12:12.807382+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:12:12.807382+0000","last_undegraded":"2023-09-23T20:12:12.807382+0000","last_fullsized":"2023-09-23T20:12:12.807382+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":546,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.797397+0000","last_change":"2023-09-23T20:12:00.290344+0000","last_active":"2023-09-23T20:12:12.797397+0000","last_peered":"2023-09-23T20:12:12.797397+0000","last_clean":"2023-09-23T20:12:12.797397+0000","last_became_active":"2023-09-23T20:09:56.050784+0000","last_became_peered":"2023-09-23T20:09:56.050784+0000","last_unstale":"2023-09-23T20:12:12.797397+0000","last_undegraded":"2023-09-23T20:12:12.797397+0000","last_fullsized":"2023-09-23T20:12:12.797397+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":102,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.797605+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:12:12.797605+0000","last_peered":"2023-09-23T20:12:12.797605+0000","last_clean":"2023-09-23T20:12:12.797605+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:12:12.797605+0000","last_undegraded":"2023-09-23T20:12:12.797605+0000","last_fullsized":"2023-09-23T20:12:12.797605+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":392,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799574+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:12:12.799574+0000","last_peered":"2023-09-23T20:12:12.799574+0000","last_clean":"2023-09-23T20:12:12.799574+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:12:12.799574+0000","last_undegraded":"2023-09-23T20:12:12.799574+0000","last_fullsized":"2023-09-23T20:12:12.799574+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":520,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:18.976789+0000","last_change":"2023-09-23T20:09:56.062364+0000","last_active":"2023-09-23T20:12:18.976789+0000","last_peered":"2023-09-23T20:12:18.976789+0000","last_clean":"2023-09-23T20:12:18.976789+0000","last_became_active":"2023-09-23T20:09:56.061865+0000","last_became_peered":"2023-09-23T20:09:56.061865+0000","last_unstale":"2023-09-23T20:12:18.976789+0000","last_undegraded":"2023-09-23T20:12:18.976789+0000","last_fullsized":"2023-09-23T20:12:18.976789+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.797471+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:12:12.797471+0000","last_peered":"2023-09-23T20:12:12.797471+0000","last_clean":"2023-09-23T20:12:12.797471+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:12:12.797471+0000","last_undegraded":"2023-09-23T20:12:12.797471+0000","last_fullsized":"2023-09-23T20:12:12.797471+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":380,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.783937+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:12:12.783937+0000","last_peered":"2023-09-23T20:12:12.783937+0000","last_clean":"2023-09-23T20:12:12.783937+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:12:12.783937+0000","last_undegraded":"2023-09-23T20:12:12.783937+0000","last_fullsized":"2023-09-23T20:12:12.783937+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":624,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:32.086658+0000","last_change":"2023-09-23T20:12:32.086658+0000","last_active":"2023-09-23T20:12:32.086658+0000","last_peered":"2023-09-23T20:12:32.086658+0000","last_clean":"2023-09-23T20:12:32.086658+0000","last_became_active":"2023-09-23T20:09:56.043157+0000","last_became_peered":"2023-09-23T20:09:56.043157+0000","last_unstale":"2023-09-23T20:12:32.086658+0000","last_undegraded":"2023-09-23T20:12:32.086658+0000","last_fullsized":"2023-09-23T20:12:32.086658+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":25,"reported_epoch":393,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:12:12.798412+0000","last_change":"2023-09-23T20:11:10.067603+0000","last_active":"2023-09-23T20:12:12.798412+0000","last_peered":"2023-09-23T20:12:12.798412+0000","last_clean":"2023-09-23T20:00:51.694506+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:12:12.798412+0000","last_undegraded":"2023-09-23T20:11:10.056843+0000","last_fullsized":"2023-09-23T20:12:12.798412+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":4}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.784454+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:12:12.784454+0000","last_peered":"2023-09-23T20:12:12.784454+0000","last_clean":"2023-09-23T20:12:12.784454+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:12:12.784454+0000","last_undegraded":"2023-09-23T20:12:12.784454+0000","last_fullsized":"2023-09-23T20:12:12.784454+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":820,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:31.731737+0000","last_change":"2023-09-23T20:12:31.731737+0000","last_active":"2023-09-23T20:12:31.731737+0000","last_peered":"2023-09-23T20:12:31.731737+0000","last_clean":"2023-09-23T20:12:31.731737+0000","last_became_active":"2023-09-23T20:10:52.687280+0000","last_became_peered":"2023-09-23T20:10:52.687280+0000","last_unstale":"2023-09-23T20:12:31.731737+0000","last_undegraded":"2023-09-23T20:12:31.731737+0000","last_fullsized":"2023-09-23T20:12:31.731737+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.4","version":"394'10653","reported_seq":10756,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:33.275771+0000","last_change":"2023-09-23T20:06:19.579196+0000","last_active":"2023-09-23T20:12:33.275771+0000","last_peered":"2023-09-23T20:12:33.275771+0000","last_clean":"2023-09-23T20:12:33.275771+0000","last_became_active":"2023-09-23T20:06:19.578932+0000","last_became_peered":"2023-09-23T20:06:19.578932+0000","last_unstale":"2023-09-23T20:12:33.275771+0000","last_undegraded":"2023-09-23T20:12:33.275771+0000","last_fullsized":"2023-09-23T20:12:33.275771+0000","mapping_epoch":299,"log_start":"321'4365","ondisk_log_start":"321'4365","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6288,"log_dups_size":0,"ondisk_log_size":6288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.600475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5046272,"num_objects":77,"num_object_clones":0,"num_object_copies":231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10653,"num_write_kb":343360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":85,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798145+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:12:12.798145+0000","last_peered":"2023-09-23T20:12:12.798145+0000","last_clean":"2023-09-23T20:12:12.798145+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:12:12.798145+0000","last_undegraded":"2023-09-23T20:12:12.798145+0000","last_fullsized":"2023-09-23T20:12:12.798145+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":402,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798576+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:12:12.798576+0000","last_peered":"2023-09-23T20:12:12.798576+0000","last_clean":"2023-09-23T20:12:12.798576+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:12:12.798576+0000","last_undegraded":"2023-09-23T20:12:12.798576+0000","last_fullsized":"2023-09-23T20:12:12.798576+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":705,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.786438+0000","last_change":"2023-09-23T20:09:56.064782+0000","last_active":"2023-09-23T20:12:12.786438+0000","last_peered":"2023-09-23T20:12:12.786438+0000","last_clean":"2023-09-23T20:12:12.786438+0000","last_became_active":"2023-09-23T20:09:56.064411+0000","last_became_peered":"2023-09-23T20:09:56.064411+0000","last_unstale":"2023-09-23T20:12:12.786438+0000","last_undegraded":"2023-09-23T20:12:12.786438+0000","last_fullsized":"2023-09-23T20:12:12.786438+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218454,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.5","version":"394'10251","reported_seq":12948,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:33.581369+0000","last_change":"2023-09-23T20:07:45.697314+0000","last_active":"2023-09-23T20:12:33.581369+0000","last_peered":"2023-09-23T20:12:33.581369+0000","last_clean":"2023-09-23T20:12:33.581369+0000","last_became_active":"2023-09-23T20:07:45.690354+0000","last_became_peered":"2023-09-23T20:07:45.690354+0000","last_unstale":"2023-09-23T20:12:33.581369+0000","last_undegraded":"2023-09-23T20:12:33.581369+0000","last_fullsized":"2023-09-23T20:12:33.581369+0000","mapping_epoch":327,"log_start":"325'4518","ondisk_log_start":"325'4518","created":299,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.926631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10251,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":86,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798320+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:12:12.798320+0000","last_peered":"2023-09-23T20:12:12.798320+0000","last_clean":"2023-09-23T20:12:12.798320+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:12:12.798320+0000","last_undegraded":"2023-09-23T20:12:12.798320+0000","last_fullsized":"2023-09-23T20:12:12.798320+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":450,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.800839+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:12:12.800839+0000","last_peered":"2023-09-23T20:12:12.800839+0000","last_clean":"2023-09-23T20:12:12.800839+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:12:12.800839+0000","last_undegraded":"2023-09-23T20:12:12.800839+0000","last_fullsized":"2023-09-23T20:12:12.800839+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":662,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.784612+0000","last_change":"2023-09-23T20:11:20.856999+0000","last_active":"2023-09-23T20:12:12.784612+0000","last_peered":"2023-09-23T20:12:12.784612+0000","last_clean":"2023-09-23T20:12:12.784612+0000","last_became_active":"2023-09-23T20:11:20.856640+0000","last_became_peered":"2023-09-23T20:11:20.856640+0000","last_unstale":"2023-09-23T20:12:12.784612+0000","last_undegraded":"2023-09-23T20:12:12.784612+0000","last_fullsized":"2023-09-23T20:12:12.784612+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.6","version":"394'10808","reported_seq":12436,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:35.107633+0000","last_change":"2023-09-23T20:07:37.064417+0000","last_active":"2023-09-23T20:12:35.107633+0000","last_peered":"2023-09-23T20:12:35.107633+0000","last_clean":"2023-09-23T20:06:59.831272+0000","last_became_active":"2023-09-23T20:07:37.059682+0000","last_became_peered":"2023-09-23T20:07:37.059682+0000","last_unstale":"2023-09-23T20:12:35.107633+0000","last_undegraded":"2023-09-23T20:07:36.983730+0000","last_fullsized":"2023-09-23T20:07:36.974121+0000","mapping_epoch":322,"log_start":"335'5665","ondisk_log_start":"335'5665","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5143,"log_dups_size":0,"ondisk_log_size":5143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:33.575626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10808,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49348608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-1391}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":85,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799125+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:12:12.799125+0000","last_peered":"2023-09-23T20:12:12.799125+0000","last_clean":"2023-09-23T20:12:12.799125+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:12:12.799125+0000","last_undegraded":"2023-09-23T20:12:12.799125+0000","last_fullsized":"2023-09-23T20:12:12.799125+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":99,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807746+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:12:12.807746+0000","last_peered":"2023-09-23T20:12:12.807746+0000","last_clean":"2023-09-23T20:12:12.807746+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:12:12.807746+0000","last_undegraded":"2023-09-23T20:12:12.807746+0000","last_fullsized":"2023-09-23T20:12:12.807746+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":589,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:26.821542+0000","last_change":"2023-09-23T20:12:26.821542+0000","last_active":"2023-09-23T20:12:26.821542+0000","last_peered":"2023-09-23T20:12:26.821542+0000","last_clean":"2023-09-23T20:12:26.821542+0000","last_became_active":"2023-09-23T20:09:56.043491+0000","last_became_peered":"2023-09-23T20:09:56.043491+0000","last_unstale":"2023-09-23T20:12:26.821542+0000","last_undegraded":"2023-09-23T20:12:26.821542+0000","last_fullsized":"2023-09-23T20:12:26.821542+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.7","version":"394'10475","reported_seq":19195,"reported_epoch":394,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:12:33.965513+0000","last_change":"2023-09-23T20:12:07.891624+0000","last_active":"2023-09-23T20:12:33.965513+0000","last_peered":"2023-09-23T20:12:33.965513+0000","last_clean":"2023-09-23T20:07:17.891274+0000","last_became_active":"2023-09-23T20:10:58.299550+0000","last_became_peered":"2023-09-23T20:10:58.299550+0000","last_unstale":"2023-09-23T20:12:33.965513+0000","last_undegraded":"2023-09-23T20:12:33.965513+0000","last_fullsized":"2023-09-23T20:10:53.692217+0000","mapping_epoch":369,"log_start":"329'4929","ondisk_log_start":"329'4929","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5546,"log_dups_size":0,"ondisk_log_size":5546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:48.194019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10475,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4296,"num_bytes_recovered":146669568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2","objects":63}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":432,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798938+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:12:12.798938+0000","last_peered":"2023-09-23T20:12:12.798938+0000","last_clean":"2023-09-23T20:12:12.798938+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:12:12.798938+0000","last_undegraded":"2023-09-23T20:12:12.798938+0000","last_fullsized":"2023-09-23T20:12:12.798938+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807721+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:12:12.807721+0000","last_peered":"2023-09-23T20:12:12.807721+0000","last_clean":"2023-09-23T20:12:12.807721+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:12:12.807721+0000","last_undegraded":"2023-09-23T20:12:12.807721+0000","last_fullsized":"2023-09-23T20:12:12.807721+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"394'10139","reported_seq":18274,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:31.160239+0000","last_change":"2023-09-23T20:10:10.729751+0000","last_active":"2023-09-23T20:12:31.160239+0000","last_peered":"2023-09-23T20:12:31.160239+0000","last_clean":"2023-09-23T20:12:31.160239+0000","last_became_active":"2023-09-23T20:10:10.708263+0000","last_became_peered":"2023-09-23T20:10:10.708263+0000","last_unstale":"2023-09-23T20:12:31.160239+0000","last_undegraded":"2023-09-23T20:12:31.160239+0000","last_fullsized":"2023-09-23T20:12:31.160239+0000","mapping_epoch":361,"log_start":"326'4610","ondisk_log_start":"326'4610","created":299,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:15.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11075584,"num_objects":169,"num_object_clones":0,"num_object_copies":507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10139,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4003,"num_bytes_recovered":136118272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":776,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.797394+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:12:12.797394+0000","last_peered":"2023-09-23T20:12:12.797394+0000","last_clean":"2023-09-23T20:12:12.797394+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:12:12.797394+0000","last_undegraded":"2023-09-23T20:12:12.797394+0000","last_fullsized":"2023-09-23T20:12:12.797394+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":84,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801121+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:12:12.801121+0000","last_peered":"2023-09-23T20:12:12.801121+0000","last_clean":"2023-09-23T20:12:12.801121+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:12:12.801121+0000","last_undegraded":"2023-09-23T20:12:12.801121+0000","last_fullsized":"2023-09-23T20:12:12.801121+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":420,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801093+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:12:12.801093+0000","last_peered":"2023-09-23T20:12:12.801093+0000","last_clean":"2023-09-23T20:12:12.801093+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:12:12.801093+0000","last_undegraded":"2023-09-23T20:12:12.801093+0000","last_fullsized":"2023-09-23T20:12:12.801093+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"394'10418","reported_seq":16824,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:32.874434+0000","last_change":"2023-09-23T20:10:58.556423+0000","last_active":"2023-09-23T20:12:32.874434+0000","last_peered":"2023-09-23T20:12:32.874434+0000","last_clean":"2023-09-23T20:07:17.847486+0000","last_became_active":"2023-09-23T20:07:20.666556+0000","last_became_peered":"2023-09-23T20:07:20.666556+0000","last_unstale":"2023-09-23T20:12:32.874434+0000","last_undegraded":"2023-09-23T20:10:58.555698+0000","last_fullsized":"2023-09-23T20:07:19.932561+0000","mapping_epoch":317,"log_start":"335'5546","ondisk_log_start":"335'5546","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":4872,"log_dups_size":0,"ondisk_log_size":4872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:38.441814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5373952,"num_objects":82,"num_object_clones":0,"num_object_copies":246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":996,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10418,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3145,"num_bytes_recovered":206110720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-914}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":728,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798652+0000","last_change":"2023-09-23T20:11:11.229144+0000","last_active":"2023-09-23T20:12:12.798652+0000","last_peered":"2023-09-23T20:12:12.798652+0000","last_clean":"2023-09-23T20:12:12.798652+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:12:12.798652+0000","last_undegraded":"2023-09-23T20:12:12.798652+0000","last_fullsized":"2023-09-23T20:12:12.798652+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:58.802951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:58.802951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":360,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.800974+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:12:12.800974+0000","last_peered":"2023-09-23T20:12:12.800974+0000","last_clean":"2023-09-23T20:12:12.800974+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:12:12.800974+0000","last_undegraded":"2023-09-23T20:12:12.800974+0000","last_fullsized":"2023-09-23T20:12:12.800974+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":390,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801288+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:12:12.801288+0000","last_peered":"2023-09-23T20:12:12.801288+0000","last_clean":"2023-09-23T20:12:12.801288+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:12:12.801288+0000","last_undegraded":"2023-09-23T20:12:12.801288+0000","last_fullsized":"2023-09-23T20:12:12.801288+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":520,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807383+0000","last_change":"2023-09-23T20:09:56.062162+0000","last_active":"2023-09-23T20:12:12.807383+0000","last_peered":"2023-09-23T20:12:12.807383+0000","last_clean":"2023-09-23T20:12:12.807383+0000","last_became_active":"2023-09-23T20:09:56.061518+0000","last_became_peered":"2023-09-23T20:09:56.061518+0000","last_unstale":"2023-09-23T20:12:12.807383+0000","last_undegraded":"2023-09-23T20:12:12.807383+0000","last_fullsized":"2023-09-23T20:12:12.807383+0000","mapping_epoch":357,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"394'10404","reported_seq":10507,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:33.569934+0000","last_change":"2023-09-23T20:06:19.576421+0000","last_active":"2023-09-23T20:12:33.569934+0000","last_peered":"2023-09-23T20:12:33.569934+0000","last_clean":"2023-09-23T20:12:33.569934+0000","last_became_active":"2023-09-23T20:06:19.573955+0000","last_became_peered":"2023-09-23T20:06:19.573955+0000","last_unstale":"2023-09-23T20:12:33.569934+0000","last_undegraded":"2023-09-23T20:12:33.569934+0000","last_fullsized":"2023-09-23T20:12:33.569934+0000","mapping_epoch":299,"log_start":"326'4685","ondisk_log_start":"326'4685","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.422647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3670016,"num_objects":56,"num_object_clones":0,"num_object_copies":168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":56,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10404,"num_write_kb":334720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":384,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.800688+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:12:12.800688+0000","last_peered":"2023-09-23T20:12:12.800688+0000","last_clean":"2023-09-23T20:12:12.800688+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:12:12.800688+0000","last_undegraded":"2023-09-23T20:12:12.800688+0000","last_fullsized":"2023-09-23T20:12:12.800688+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":420,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801212+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:12:12.801212+0000","last_peered":"2023-09-23T20:12:12.801212+0000","last_clean":"2023-09-23T20:12:12.801212+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:12:12.801212+0000","last_undegraded":"2023-09-23T20:12:12.801212+0000","last_fullsized":"2023-09-23T20:12:12.801212+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"394'10247","reported_seq":18409,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:33.964741+0000","last_change":"2023-09-23T20:09:37.763558+0000","last_active":"2023-09-23T20:12:33.964741+0000","last_peered":"2023-09-23T20:12:33.964741+0000","last_clean":"2023-09-23T20:12:33.964741+0000","last_became_active":"2023-09-23T20:09:37.732414+0000","last_became_peered":"2023-09-23T20:09:37.732414+0000","last_unstale":"2023-09-23T20:12:33.964741+0000","last_undegraded":"2023-09-23T20:12:33.964741+0000","last_fullsized":"2023-09-23T20:12:33.964741+0000","mapping_epoch":352,"log_start":"328'4781","ondisk_log_start":"328'4781","created":299,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:39.792571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3473408,"num_objects":53,"num_object_clones":0,"num_object_copies":159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10247,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4021,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":732,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799298+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:12:12.799298+0000","last_peered":"2023-09-23T20:12:12.799298+0000","last_clean":"2023-09-23T20:12:12.799298+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:12:12.799298+0000","last_undegraded":"2023-09-23T20:12:12.799298+0000","last_fullsized":"2023-09-23T20:12:12.799298+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.787070+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:12:12.787070+0000","last_peered":"2023-09-23T20:12:12.787070+0000","last_clean":"2023-09-23T20:12:12.787070+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:12:12.787070+0000","last_undegraded":"2023-09-23T20:12:12.787070+0000","last_fullsized":"2023-09-23T20:12:12.787070+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":85,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.809101+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:12:12.809101+0000","last_peered":"2023-09-23T20:12:12.809101+0000","last_clean":"2023-09-23T20:12:12.809101+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:12:12.809101+0000","last_undegraded":"2023-09-23T20:12:12.809101+0000","last_fullsized":"2023-09-23T20:12:12.809101+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"394'10562","reported_seq":10668,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:35.147868+0000","last_change":"2023-09-23T20:06:19.581545+0000","last_active":"2023-09-23T20:12:35.147868+0000","last_peered":"2023-09-23T20:12:35.147868+0000","last_clean":"2023-09-23T20:12:35.147868+0000","last_became_active":"2023-09-23T20:06:19.581254+0000","last_became_peered":"2023-09-23T20:06:19.581254+0000","last_unstale":"2023-09-23T20:12:35.147868+0000","last_undegraded":"2023-09-23T20:12:35.147868+0000","last_fullsized":"2023-09-23T20:12:35.147868+0000","mapping_epoch":299,"log_start":"332'5383","ondisk_log_start":"332'5383","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5179,"log_dups_size":0,"ondisk_log_size":5179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:19.400919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10562,"num_write_kb":337984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":662,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.784533+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:12:12.784533+0000","last_peered":"2023-09-23T20:12:12.784533+0000","last_clean":"2023-09-23T20:12:12.784533+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:12:12.784533+0000","last_undegraded":"2023-09-23T20:12:12.784533+0000","last_fullsized":"2023-09-23T20:12:12.784533+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":527,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.808132+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:12:12.808132+0000","last_peered":"2023-09-23T20:12:12.808132+0000","last_clean":"2023-09-23T20:12:12.808132+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:12:12.808132+0000","last_undegraded":"2023-09-23T20:12:12.808132+0000","last_fullsized":"2023-09-23T20:12:12.808132+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":394,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799539+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:12:12.799539+0000","last_peered":"2023-09-23T20:12:12.799539+0000","last_clean":"2023-09-23T20:12:12.799539+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:12:12.799539+0000","last_undegraded":"2023-09-23T20:12:12.799539+0000","last_fullsized":"2023-09-23T20:12:12.799539+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801295+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:12:12.801295+0000","last_peered":"2023-09-23T20:12:12.801295+0000","last_clean":"2023-09-23T20:12:12.801295+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:12:12.801295+0000","last_undegraded":"2023-09-23T20:12:12.801295+0000","last_fullsized":"2023-09-23T20:12:12.801295+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"394'10442","reported_seq":15274,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:33.275635+0000","last_change":"2023-09-23T20:07:52.783215+0000","last_active":"2023-09-23T20:12:33.275635+0000","last_peered":"2023-09-23T20:12:33.275635+0000","last_clean":"2023-09-23T20:12:33.275635+0000","last_became_active":"2023-09-23T20:07:52.778874+0000","last_became_peered":"2023-09-23T20:07:52.778874+0000","last_unstale":"2023-09-23T20:12:33.275635+0000","last_undegraded":"2023-09-23T20:12:33.275635+0000","last_fullsized":"2023-09-23T20:12:33.275635+0000","mapping_epoch":330,"log_start":"319'4100","ondisk_log_start":"319'4100","created":299,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6342,"log_dups_size":0,"ondisk_log_size":6342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:31.036337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5373952,"num_objects":82,"num_object_clones":0,"num_object_copies":246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10442,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2349,"num_bytes_recovered":153944064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":727,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.796185+0000","last_change":"2023-09-23T20:11:06.026338+0000","last_active":"2023-09-23T20:12:12.796185+0000","last_peered":"2023-09-23T20:12:12.796185+0000","last_clean":"2023-09-23T20:12:12.796185+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:12:12.796185+0000","last_undegraded":"2023-09-23T20:12:12.796185+0000","last_fullsized":"2023-09-23T20:12:12.796185+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1014,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799054+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:12:12.799054+0000","last_peered":"2023-09-23T20:12:12.799054+0000","last_clean":"2023-09-23T20:12:12.799054+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:12:12.799054+0000","last_undegraded":"2023-09-23T20:12:12.799054+0000","last_fullsized":"2023-09-23T20:12:12.799054+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":413,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801369+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:12:12.801369+0000","last_peered":"2023-09-23T20:12:12.801369+0000","last_clean":"2023-09-23T20:12:12.801369+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:12:12.801369+0000","last_undegraded":"2023-09-23T20:12:12.801369+0000","last_fullsized":"2023-09-23T20:12:12.801369+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.808074+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:12:12.808074+0000","last_peered":"2023-09-23T20:12:12.808074+0000","last_clean":"2023-09-23T20:12:12.808074+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:12:12.808074+0000","last_undegraded":"2023-09-23T20:12:12.808074+0000","last_fullsized":"2023-09-23T20:12:12.808074+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"394'10311","reported_seq":22681,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:12:33.961552+0000","last_change":"2023-09-23T20:11:18.839874+0000","last_active":"2023-09-23T20:12:33.961552+0000","last_peered":"2023-09-23T20:12:33.961552+0000","last_clean":"2023-09-23T20:11:16.510239+0000","last_became_active":"2023-09-23T20:11:18.809696+0000","last_became_peered":"2023-09-23T20:11:18.809696+0000","last_unstale":"2023-09-23T20:12:33.961552+0000","last_undegraded":"2023-09-23T20:12:33.961552+0000","last_fullsized":"2023-09-23T20:12:33.961552+0000","mapping_epoch":381,"log_start":"328'4800","ondisk_log_start":"328'4800","created":299,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5511,"log_dups_size":0,"ondisk_log_size":5511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:58.376473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3473408,"num_objects":53,"num_object_clones":0,"num_object_copies":159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":53,"num_objects_unfound":0,"num_objects_dirty":53,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10311,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":217251840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[1,4,2],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":53}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":622,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:31.122075+0000","last_change":"2023-09-23T20:12:31.122075+0000","last_active":"2023-09-23T20:12:31.122075+0000","last_peered":"2023-09-23T20:12:31.122075+0000","last_clean":"2023-09-23T20:12:31.122075+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:12:31.122075+0000","last_undegraded":"2023-09-23T20:12:31.122075+0000","last_fullsized":"2023-09-23T20:12:31.122075+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:03:50.811793+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1030,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.812630+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:12:12.812630+0000","last_peered":"2023-09-23T20:12:12.812630+0000","last_clean":"2023-09-23T20:12:12.812630+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:12:12.812630+0000","last_undegraded":"2023-09-23T20:12:12.812630+0000","last_fullsized":"2023-09-23T20:12:12.812630+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807050+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:12:12.807050+0000","last_peered":"2023-09-23T20:12:12.807050+0000","last_clean":"2023-09-23T20:12:12.807050+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:12:12.807050+0000","last_undegraded":"2023-09-23T20:12:12.807050+0000","last_fullsized":"2023-09-23T20:12:12.807050+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.809003+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:12:12.809003+0000","last_peered":"2023-09-23T20:12:12.809003+0000","last_clean":"2023-09-23T20:12:12.809003+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:12:12.809003+0000","last_undegraded":"2023-09-23T20:12:12.809003+0000","last_fullsized":"2023-09-23T20:12:12.809003+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"394'10480","reported_seq":10583,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:33.276392+0000","last_change":"2023-09-23T20:06:19.574677+0000","last_active":"2023-09-23T20:12:33.276392+0000","last_peered":"2023-09-23T20:12:33.276392+0000","last_clean":"2023-09-23T20:12:33.276392+0000","last_became_active":"2023-09-23T20:06:19.574349+0000","last_became_peered":"2023-09-23T20:06:19.574349+0000","last_unstale":"2023-09-23T20:12:33.276392+0000","last_undegraded":"2023-09-23T20:12:33.276392+0000","last_fullsized":"2023-09-23T20:12:33.276392+0000","mapping_epoch":299,"log_start":"320'4165","ondisk_log_start":"320'4165","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6315,"log_dups_size":0,"ondisk_log_size":6315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:56.737869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5767168,"num_objects":88,"num_object_clones":0,"num_object_copies":264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10480,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1302,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798938+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:12:12.798938+0000","last_peered":"2023-09-23T20:12:12.798938+0000","last_clean":"2023-09-23T20:12:12.798938+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:12:12.798938+0000","last_undegraded":"2023-09-23T20:12:12.798938+0000","last_fullsized":"2023-09-23T20:12:12.798938+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":556,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:28.097772+0000","last_change":"2023-09-23T20:12:28.097772+0000","last_active":"2023-09-23T20:12:28.097772+0000","last_peered":"2023-09-23T20:12:28.097772+0000","last_clean":"2023-09-23T20:12:28.097772+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:12:28.097772+0000","last_undegraded":"2023-09-23T20:12:28.097772+0000","last_fullsized":"2023-09-23T20:12:28.097772+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:50.668369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:50.668369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":259,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799175+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:12:12.799175+0000","last_peered":"2023-09-23T20:12:12.799175+0000","last_clean":"2023-09-23T20:12:12.799175+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:12:12.799175+0000","last_undegraded":"2023-09-23T20:12:12.799175+0000","last_fullsized":"2023-09-23T20:12:12.799175+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":289,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801503+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:12:12.801503+0000","last_peered":"2023-09-23T20:12:12.801503+0000","last_clean":"2023-09-23T20:12:12.801503+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:12:12.801503+0000","last_undegraded":"2023-09-23T20:12:12.801503+0000","last_fullsized":"2023-09-23T20:12:12.801503+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"394'10636","reported_seq":10853,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:12:35.186319+0000","last_change":"2023-09-23T20:10:57.512712+0000","last_active":"2023-09-23T20:12:35.186319+0000","last_peered":"2023-09-23T20:12:35.186319+0000","last_clean":"2023-09-23T20:07:17.907438+0000","last_became_active":"2023-09-23T20:07:20.693608+0000","last_became_peered":"2023-09-23T20:07:20.693608+0000","last_unstale":"2023-09-23T20:12:35.186319+0000","last_undegraded":"2023-09-23T20:07:19.959595+0000","last_fullsized":"2023-09-23T20:07:19.924202+0000","mapping_epoch":317,"log_start":"335'5488","ondisk_log_start":"335'5488","created":299,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5148,"log_dups_size":0,"ondisk_log_size":5148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:21.022797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10636,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":3407872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":-4128}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":520,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807799+0000","last_change":"2023-09-23T20:09:56.051517+0000","last_active":"2023-09-23T20:12:12.807799+0000","last_peered":"2023-09-23T20:12:12.807799+0000","last_clean":"2023-09-23T20:12:12.807799+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:12:12.807799+0000","last_undegraded":"2023-09-23T20:12:12.807799+0000","last_fullsized":"2023-09-23T20:12:12.807799+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":830,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.797072+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:12:12.797072+0000","last_peered":"2023-09-23T20:12:12.797072+0000","last_clean":"2023-09-23T20:12:12.797072+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:12:12.797072+0000","last_undegraded":"2023-09-23T20:12:12.797072+0000","last_fullsized":"2023-09-23T20:12:12.797072+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":380,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.800850+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:12:12.800850+0000","last_peered":"2023-09-23T20:12:12.800850+0000","last_clean":"2023-09-23T20:12:12.800850+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:12:12.800850+0000","last_undegraded":"2023-09-23T20:12:12.800850+0000","last_fullsized":"2023-09-23T20:12:12.800850+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.808195+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:12:12.808195+0000","last_peered":"2023-09-23T20:12:12.808195+0000","last_clean":"2023-09-23T20:12:12.808195+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:12:12.808195+0000","last_undegraded":"2023-09-23T20:12:12.808195+0000","last_fullsized":"2023-09-23T20:12:12.808195+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"394'10680","reported_seq":18506,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:35.166508+0000","last_change":"2023-09-23T20:12:10.019046+0000","last_active":"2023-09-23T20:12:35.166508+0000","last_peered":"2023-09-23T20:12:35.166508+0000","last_clean":"2023-09-23T20:12:35.166508+0000","last_became_active":"2023-09-23T20:12:09.762053+0000","last_became_peered":"2023-09-23T20:12:09.762053+0000","last_unstale":"2023-09-23T20:12:35.166508+0000","last_undegraded":"2023-09-23T20:12:35.166508+0000","last_fullsized":"2023-09-23T20:12:35.166508+0000","mapping_epoch":391,"log_start":"335'5546","ondisk_log_start":"335'5546","created":299,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":5134,"log_dups_size":0,"ondisk_log_size":5134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:41.902924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10680,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4732,"num_bytes_recovered":261292032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":596,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:25.781028+0000","last_change":"2023-09-23T20:12:25.781028+0000","last_active":"2023-09-23T20:12:25.781028+0000","last_peered":"2023-09-23T20:12:25.781028+0000","last_clean":"2023-09-23T20:12:25.781028+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:12:25.781028+0000","last_undegraded":"2023-09-23T20:12:25.781028+0000","last_fullsized":"2023-09-23T20:12:25.781028+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:47.077830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:01:47.077830+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00139573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":94,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799108+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:12:12.799108+0000","last_peered":"2023-09-23T20:12:12.799108+0000","last_clean":"2023-09-23T20:12:12.799108+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:12:12.799108+0000","last_undegraded":"2023-09-23T20:12:12.799108+0000","last_fullsized":"2023-09-23T20:12:12.799108+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807992+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:12:12.807992+0000","last_peered":"2023-09-23T20:12:12.807992+0000","last_clean":"2023-09-23T20:12:12.807992+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:12:12.807992+0000","last_undegraded":"2023-09-23T20:12:12.807992+0000","last_fullsized":"2023-09-23T20:12:12.807992+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":220,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798785+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:12:12.798785+0000","last_peered":"2023-09-23T20:12:12.798785+0000","last_clean":"2023-09-23T20:12:12.798785+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:12:12.798785+0000","last_undegraded":"2023-09-23T20:12:12.798785+0000","last_fullsized":"2023-09-23T20:12:12.798785+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"394'10571","reported_seq":14872,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:33.569786+0000","last_change":"2023-09-23T20:08:33.396812+0000","last_active":"2023-09-23T20:12:33.569786+0000","last_peered":"2023-09-23T20:12:33.569786+0000","last_clean":"2023-09-23T20:12:33.569786+0000","last_became_active":"2023-09-23T20:08:05.774750+0000","last_became_peered":"2023-09-23T20:08:05.774750+0000","last_unstale":"2023-09-23T20:12:33.569786+0000","last_undegraded":"2023-09-23T20:12:33.569786+0000","last_fullsized":"2023-09-23T20:12:33.569786+0000","mapping_epoch":334,"log_start":"315'3700","ondisk_log_start":"315'3700","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6871,"log_dups_size":0,"ondisk_log_size":6871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.633506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4915200,"num_objects":75,"num_object_clones":0,"num_object_copies":225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10571,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2086,"num_bytes_recovered":136708096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":821,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:27.782431+0000","last_change":"2023-09-23T20:12:27.782431+0000","last_active":"2023-09-23T20:12:27.782431+0000","last_peered":"2023-09-23T20:12:27.782431+0000","last_clean":"2023-09-23T20:12:27.782431+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:12:27.782431+0000","last_undegraded":"2023-09-23T20:12:27.782431+0000","last_fullsized":"2023-09-23T20:12:27.782431+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.792373+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.792373+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":236,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.797934+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:12:12.797934+0000","last_peered":"2023-09-23T20:12:12.797934+0000","last_clean":"2023-09-23T20:12:12.797934+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:12:12.797934+0000","last_undegraded":"2023-09-23T20:12:12.797934+0000","last_fullsized":"2023-09-23T20:12:12.797934+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":432,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.808123+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:12:12.808123+0000","last_peered":"2023-09-23T20:12:12.808123+0000","last_clean":"2023-09-23T20:12:12.808123+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:12:12.808123+0000","last_undegraded":"2023-09-23T20:12:12.808123+0000","last_fullsized":"2023-09-23T20:12:12.808123+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":24,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.783996+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:12:12.783996+0000","last_peered":"2023-09-23T20:12:12.783996+0000","last_clean":"2023-09-23T20:12:12.783996+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:12:12.783996+0000","last_undegraded":"2023-09-23T20:12:12.783996+0000","last_fullsized":"2023-09-23T20:12:12.783996+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"394'10450","reported_seq":13222,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T20:12:33.287921+0000","last_change":"2023-09-23T20:09:08.683160+0000","last_active":"2023-09-23T20:12:33.287921+0000","last_peered":"2023-09-23T20:12:33.287921+0000","last_clean":"2023-09-23T20:12:33.287921+0000","last_became_active":"2023-09-23T20:09:08.670527+0000","last_became_peered":"2023-09-23T20:09:08.670527+0000","last_unstale":"2023-09-23T20:12:33.287921+0000","last_undegraded":"2023-09-23T20:12:33.287921+0000","last_fullsized":"2023-09-23T20:12:33.287921+0000","mapping_epoch":346,"log_start":"315'4100","ondisk_log_start":"315'4100","created":299,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:06:18.539488+0000","last_clean_scrub_stamp":"2023-09-23T20:06:18.539488+0000","objects_scrubbed":0,"log_size":6350,"log_dups_size":0,"ondisk_log_size":6350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:47.531381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4784152,"num_objects":74,"num_object_clones":0,"num_object_copies":222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10450,"num_write_kb":336705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":705,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:21.166821+0000","last_change":"2023-09-23T20:09:56.036037+0000","last_active":"2023-09-23T20:12:21.166821+0000","last_peered":"2023-09-23T20:12:21.166821+0000","last_clean":"2023-09-23T20:12:21.166821+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:12:21.166821+0000","last_undegraded":"2023-09-23T20:12:21.166821+0000","last_fullsized":"2023-09-23T20:12:21.166821+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":95,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798094+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:12:12.798094+0000","last_peered":"2023-09-23T20:12:12.798094+0000","last_clean":"2023-09-23T20:12:12.798094+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:12:12.798094+0000","last_undegraded":"2023-09-23T20:12:12.798094+0000","last_fullsized":"2023-09-23T20:12:12.798094+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":94,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.807993+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:12:12.807993+0000","last_peered":"2023-09-23T20:12:12.807993+0000","last_clean":"2023-09-23T20:12:12.807993+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:12:12.807993+0000","last_undegraded":"2023-09-23T20:12:12.807993+0000","last_fullsized":"2023-09-23T20:12:12.807993+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":353,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.800699+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:12:12.800699+0000","last_peered":"2023-09-23T20:12:12.800699+0000","last_clean":"2023-09-23T20:12:12.800699+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:12:12.800699+0000","last_undegraded":"2023-09-23T20:12:12.800699+0000","last_fullsized":"2023-09-23T20:12:12.800699+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":169,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.815667+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:12:12.815667+0000","last_peered":"2023-09-23T20:12:12.815667+0000","last_clean":"2023-09-23T20:12:12.815667+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:12:12.815667+0000","last_undegraded":"2023-09-23T20:12:12.815667+0000","last_fullsized":"2023-09-23T20:12:12.815667+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":160,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.817527+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:12:12.817527+0000","last_peered":"2023-09-23T20:12:12.817527+0000","last_clean":"2023-09-23T20:12:12.817527+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:12:12.817527+0000","last_undegraded":"2023-09-23T20:12:12.817527+0000","last_fullsized":"2023-09-23T20:12:12.817527+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":397,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.798874+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:12:12.798874+0000","last_peered":"2023-09-23T20:12:12.798874+0000","last_clean":"2023-09-23T20:12:12.798874+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:12:12.798874+0000","last_undegraded":"2023-09-23T20:12:12.798874+0000","last_fullsized":"2023-09-23T20:12:12.798874+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":410,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.801402+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:12:12.801402+0000","last_peered":"2023-09-23T20:12:12.801402+0000","last_clean":"2023-09-23T20:12:12.801402+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:12:12.801402+0000","last_undegraded":"2023-09-23T20:12:12.801402+0000","last_fullsized":"2023-09-23T20:12:12.801402+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":387,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799493+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:12:12.799493+0000","last_peered":"2023-09-23T20:12:12.799493+0000","last_clean":"2023-09-23T20:12:12.799493+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:12:12.799493+0000","last_undegraded":"2023-09-23T20:12:12.799493+0000","last_fullsized":"2023-09-23T20:12:12.799493+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":390,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.800925+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:12:12.800925+0000","last_peered":"2023-09-23T20:12:12.800925+0000","last_clean":"2023-09-23T20:12:12.800925+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:12:12.800925+0000","last_undegraded":"2023-09-23T20:12:12.800925+0000","last_fullsized":"2023-09-23T20:12:12.800925+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":95,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.817589+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:12:12.817589+0000","last_peered":"2023-09-23T20:12:12.817589+0000","last_clean":"2023-09-23T20:12:12.817589+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:12:12.817589+0000","last_undegraded":"2023-09-23T20:12:12.817589+0000","last_fullsized":"2023-09-23T20:12:12.817589+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":251,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.812703+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:12:12.812703+0000","last_peered":"2023-09-23T20:12:12.812703+0000","last_clean":"2023-09-23T20:12:12.812703+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:12:12.812703+0000","last_undegraded":"2023-09-23T20:12:12.812703+0000","last_fullsized":"2023-09-23T20:12:12.812703+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":239,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.799583+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:12:12.799583+0000","last_peered":"2023-09-23T20:12:12.799583+0000","last_clean":"2023-09-23T20:12:12.799583+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:12:12.799583+0000","last_undegraded":"2023-09-23T20:12:12.799583+0000","last_fullsized":"2023-09-23T20:12:12.799583+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":440,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T20:12:12.800706+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:12:12.800706+0000","last_peered":"2023-09-23T20:12:12.800706+0000","last_clean":"2023-09-23T20:12:12.800706+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:12:12.800706+0000","last_undegraded":"2023-09-23T20:12:12.800706+0000","last_fullsized":"2023-09-23T20:12:12.800706+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":26174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":61210648,"num_objects":935,"num_object_clones":0,"num_object_copies":2805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6515,"num_objects_misplaced":53,"num_objects_unfound":0,"num_objects_dirty":935,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167527,"num_write_kb":5390721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33213,"num_bytes_recovered":1618542592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1735204864,"data_stored":1735196720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91146,"ondisk_log_size":91146,"up":48,"acting":44,"num_store_stats":7},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":542359,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":177,"num_read_kb":331,"num_write":87,"num_write_kb":878,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1299398,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234377,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":372028,"kb_used_data":39156,"kb_used_omap":0,"kb_used_meta":332864,"kb_avail":93373124,"statfs":{"total":95995035648,"available":95614078976,"internally_reserved":0,"allocated":40095744,"data_stored":38825661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340852736},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.0449999999999999,"15min":2.2400000000000002},"min":{"1min":0.23499999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.5030000000000001,"5min":55.390000000000001,"15min":62.061999999999998},"last":3.8239999999999998},{"interface":"front","average":{"1min":1.034,"5min":1.9870000000000001,"15min":2.3780000000000001},"min":{"1min":0.248,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":3.726,"5min":56.664000000000001,"15min":62.389000000000003},"last":2.5499999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.907,"15min":3.4039999999999999},"min":{"1min":0.29299999999999998,"5min":0.28299999999999997,"15min":0.22700000000000001},"max":{"1min":3.7080000000000002,"5min":54.945999999999998,"15min":104.16800000000001},"last":0.874},{"interface":"front","average":{"1min":0.84299999999999997,"5min":2,"15min":3.4929999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.223},"max":{"1min":2.9390000000000001,"5min":55.094000000000001,"15min":104.003},"last":1.0109999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":4.0800000000000001,"15min":3.6440000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":28.707000000000001,"5min":92.319000000000003,"15min":105.842},"last":3.794},{"interface":"front","average":{"1min":2.8919999999999999,"5min":4.4699999999999998,"15min":4.1390000000000002},"min":{"1min":0.34300000000000003,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":28.596,"5min":92.049000000000007,"15min":105.66},"last":6.2800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":3.1070000000000002,"15min":4.3840000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":4.0359999999999996,"5min":94.953000000000003,"15min":146.38800000000001},"last":3.754},{"interface":"front","average":{"1min":1.159,"5min":3.0390000000000001,"15min":4.4020000000000001},"min":{"1min":0.41399999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.6949999999999998,"5min":93.423000000000002,"15min":146.851},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.48999999999999999,"15min":0.71899999999999997},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.22500000000000001},"max":{"1min":3.2069999999999999,"5min":3.2069999999999999,"15min":19.215},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.54600000000000004,"15min":0.83299999999999996},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.88400000000000001,"5min":2.7949999999999999,"15min":41.822000000000003},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.63700000000000001,"15min":0.97299999999999998},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":3.407,"5min":4.4080000000000004,"15min":35.488},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.628,"15min":0.85699999999999998},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":3.2549999999999999,"5min":4.2030000000000003,"15min":34.896000000000001},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.625,"15min":0.80800000000000005},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":1.4650000000000001,"5min":4.3710000000000004,"15min":41.902999999999999},"last":3.2309999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.63200000000000001,"15min":0.64200000000000002},"min":{"1min":0.38400000000000001,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":0.85599999999999998,"5min":3.9889999999999999,"15min":6.3620000000000001},"last":0.45300000000000001}]}]},{"osd":6,"up_from":0,"seq":1078036791418,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762576,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":608856,"kb_used_data":37008,"kb_used_omap":7,"kb_used_meta":571832,"kb_avail":93136296,"statfs":{"total":95995035648,"available":95371567104,"internally_reserved":0,"allocated":37896192,"data_stored":36017193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":585556794},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8660000000000001,"5min":2.5230000000000001,"15min":3.2349999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.16900000000000001},"max":{"1min":11.930999999999999,"5min":28.510000000000002,"15min":144.273},"last":3.8690000000000002},{"interface":"front","average":{"1min":2.6230000000000002,"5min":2.5409999999999999,"15min":3.468},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.19400000000000001},"max":{"1min":16.870999999999999,"5min":16.870999999999999,"15min":144.321},"last":8.9830000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:12:09 2023","interfaces":[{"interface":"back","average":{"1min":3.9369999999999998,"5min":4.1559999999999997,"15min":4.8419999999999996},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":38.040999999999997,"5min":58.499000000000002,"15min":134.72499999999999},"last":12.343999999999999},{"interface":"front","average":{"1min":3.3700000000000001,"5min":3.6269999999999998,"15min":4.952},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":38.173999999999999,"5min":58.741,"15min":135.74600000000001},"last":9.3599999999999994}]},{"osd":3,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.8159999999999998,"15min":3.786},"min":{"1min":0.255,"5min":0.255,"15min":0.22},"max":{"1min":17.212,"5min":25.963999999999999,"15min":68.180999999999997},"last":12.384},{"interface":"front","average":{"1min":2.5720000000000001,"5min":2.7000000000000002,"15min":3.7450000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.247},"max":{"1min":12.723000000000001,"5min":40.716000000000001,"15min":66.509},"last":9.1760000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7170000000000001,"15min":1.998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":11.698,"5min":18.087,"15min":60.951000000000001},"last":1.095},{"interface":"front","average":{"1min":1.3839999999999999,"5min":1.837,"15min":2.1429999999999998},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":11.08,"5min":14.890000000000001,"15min":60.841000000000001},"last":9.0280000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":5.2670000000000003,"5min":2.621,"15min":2.5459999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":83.510999999999996,"5min":83.510999999999996,"15min":83.510999999999996},"last":9.0540000000000003},{"interface":"front","average":{"1min":5.6289999999999996,"5min":3.121,"15min":2.6360000000000001},"min":{"1min":0.40200000000000002,"5min":0.28899999999999998,"15min":0.28399999999999997},"max":{"1min":83.353999999999999,"5min":83.353999999999999,"15min":83.353999999999999},"last":9.4290000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:11:43 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.7649999999999999,"15min":1.994},"min":{"1min":0.38800000000000001,"5min":0.38500000000000001,"15min":0.35899999999999999},"max":{"1min":6.6749999999999998,"5min":18.117999999999999,"15min":18.117999999999999},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.8089999999999999,"15min":2.6579999999999999},"min":{"1min":0.42299999999999999,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":6.5069999999999997,"5min":14.862,"15min":20.734999999999999},"last":9.0779999999999994}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":2.359,"15min":2.2869999999999999},"min":{"1min":0.434,"5min":0.33300000000000002,"15min":0.30399999999999999},"max":{"1min":12.823,"5min":25.937000000000001,"15min":60.213000000000001},"last":9.4730000000000008},{"interface":"front","average":{"1min":2.0070000000000001,"5min":1.8979999999999999,"15min":2.1030000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.30199999999999999},"max":{"1min":11.747,"5min":18.053999999999998,"15min":61.298999999999999},"last":1.2290000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926113,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":833312,"kb_used_data":206808,"kb_used_omap":8,"kb_used_meta":626359,"kb_avail":92911840,"statfs":{"total":95995035648,"available":95141724160,"internally_reserved":0,"allocated":211771392,"data_stored":209888096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":641392225},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5190000000000001,"5min":3.3839999999999999,"15min":3.8860000000000001},"min":{"1min":0.21299999999999999,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":72.534000000000006,"5min":72.534000000000006,"15min":94.055999999999997},"last":5.5419999999999998},{"interface":"front","average":{"1min":5.5250000000000004,"5min":3.6579999999999999,"15min":4.0819999999999999},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":72.667000000000002,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.222}]},{"osd":2,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":3.8780000000000001,"15min":5.0019999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19800000000000001},"max":{"1min":19.486999999999998,"5min":102.65300000000001,"15min":270.51100000000002},"last":0.20499999999999999},{"interface":"front","average":{"1min":3.0049999999999999,"5min":4.0129999999999999,"15min":4.8470000000000004},"min":{"1min":0.25,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.693999999999999,"5min":96.578999999999994,"15min":269.69299999999998},"last":0.90200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":4.4909999999999997,"15min":5.5590000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":8.6489999999999991,"5min":126.95399999999999,"15min":145.72800000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":4.8780000000000001,"15min":5.7240000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.23999999999999999},"max":{"1min":5.4690000000000003,"5min":127.294,"15min":145.50700000000001},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.631,"15min":2.2469999999999999},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":9.6959999999999997,"5min":15.688000000000001,"15min":148.20599999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.998,"5min":1.629,"15min":2.0270000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27600000000000002},"max":{"1min":8.4429999999999996,"5min":13.428000000000001,"15min":148.11000000000001},"last":0.432}]},{"osd":5,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.341,"15min":1.9410000000000001},"min":{"1min":0.374,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":3.7480000000000002,"5min":9.5719999999999992,"15min":64.725999999999999},"last":0.307},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.742,"15min":1.988},"min":{"1min":0.36499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.7479999999999993,"5min":15.717000000000001,"15min":27.609999999999999},"last":0.86599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.7270000000000001,"15min":1.6439999999999999},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":4.9800000000000004,"5min":15.744999999999999,"15min":15.744999999999999},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.5549999999999999,"15min":1.841},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":5.4390000000000001,"5min":13.470000000000001,"15min":13.766999999999999},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.429,"15min":1.891},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":5.1929999999999996,"5min":9.6069999999999993,"15min":65.081999999999994},"last":0.58399999999999996},{"interface":"front","average":{"1min":2.2589999999999999,"5min":1.867,"15min":2.0089999999999999},"min":{"1min":0.41199999999999998,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.0510000000000002,"5min":15.771000000000001,"15min":15.771000000000001},"last":0.97799999999999998}]}]},{"osd":2,"up_from":104,"seq":446676599052,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":773096,"kb_used_data":29920,"kb_used_omap":12,"kb_used_meta":743155,"kb_avail":92972056,"statfs":{"total":95995035648,"available":95203385344,"internally_reserved":0,"allocated":30638080,"data_stored":28747665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":760991742},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":2.2160000000000002,"15min":3.0030000000000001},"min":{"1min":0.26200000000000001,"5min":0.19900000000000001,"15min":0.16900000000000001},"max":{"1min":14.794,"5min":17.876999999999999,"15min":76.251999999999995},"last":1.9059999999999999},{"interface":"front","average":{"1min":1.339,"5min":2.4159999999999999,"15min":3.0339999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.192},"max":{"1min":5.077,"5min":18.561,"15min":76.590000000000003},"last":2.1909999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.8220000000000001,"15min":3.6280000000000001},"min":{"1min":0.25,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":32.677999999999997,"5min":32.677999999999997,"15min":94.768000000000001},"last":12.756},{"interface":"front","average":{"1min":3.7010000000000001,"5min":2.4820000000000002,"15min":3.577},"min":{"1min":0.30599999999999999,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":32.582000000000001,"5min":32.582000000000001,"15min":96.808000000000007},"last":1.744}]},{"osd":3,"last update":"Sat Sep 23 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.5169999999999999,"15min":4.4029999999999996},"min":{"1min":0.32800000000000001,"5min":0.25,"15min":0.214},"max":{"1min":8.5850000000000009,"5min":13.513,"15min":107.381},"last":12.606},{"interface":"front","average":{"1min":1.0900000000000001,"5min":2.097,"15min":3.7610000000000001},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.23699999999999999},"max":{"1min":4.7919999999999998,"5min":20.308,"15min":106.301},"last":1.9319999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.698,"15min":2.0790000000000002},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.26900000000000002},"max":{"1min":14.757999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":1.8200000000000001},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.7090000000000001,"15min":2.1259999999999999},"min":{"1min":0.34899999999999998,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":13.387,"5min":13.387,"15min":76.161000000000001},"last":12.728999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":4.258,"5min":2.1099999999999999,"15min":2.6899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":29.433,"5min":29.433,"15min":172.505},"last":1.7889999999999999},{"interface":"front","average":{"1min":3.6040000000000001,"5min":2.1920000000000002,"15min":2.8500000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.29099999999999998},"max":{"1min":29.196999999999999,"5min":29.196999999999999,"15min":172.375},"last":2.0880000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.5900000000000001,"15min":1.601},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":4.718,"5min":10.209,"15min":10.209},"last":1.8480000000000001},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.8220000000000001,"15min":1.909},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.776,"5min":14.279999999999999,"15min":14.279999999999999},"last":2.0449999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.837,"15min":1.829},"min":{"1min":0.44,"5min":0.377,"15min":0.30199999999999999},"max":{"1min":13.481999999999999,"5min":13.481999999999999,"15min":23.079999999999998},"last":12.645},{"interface":"front","average":{"1min":2.4159999999999999,"5min":1.869,"15min":1.9139999999999999},"min":{"1min":0.39900000000000002,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":14.698,"5min":14.698,"15min":20.614999999999998},"last":1.8799999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435525,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1346848,"kb_used_data":668312,"kb_used_omap":4,"kb_used_meta":678331,"kb_avail":92398304,"statfs":{"total":95995035648,"available":94615863296,"internally_reserved":0,"allocated":684351488,"data_stored":682285581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":694611053},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,0,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":4.2430000000000003,"15min":3.4049999999999998},"min":{"1min":0.19700000000000001,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":7.6440000000000001,"5min":74.323999999999998,"15min":74.323999999999998},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.94599999999999995,"5min":4.0170000000000003,"15min":3.758},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":4.9000000000000004,"5min":74.447000000000003,"15min":74.447000000000003},"last":1.976}]},{"osd":1,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":8.7829999999999995,"5min":4.7240000000000002,"15min":4.0599999999999996},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":163.49700000000001,"5min":163.49700000000001,"15min":163.49700000000001},"last":2.3490000000000002},{"interface":"front","average":{"1min":8.5839999999999996,"5min":5.2400000000000002,"15min":4.351},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":163.512,"5min":163.512,"15min":163.512},"last":1.073}]},{"osd":2,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":12.541,"5min":6.2560000000000002,"15min":4.5679999999999996},"min":{"1min":0.33800000000000002,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":163.83500000000001,"5min":163.83500000000001,"15min":163.83500000000001},"last":2.2480000000000002},{"interface":"front","average":{"1min":11.702,"5min":6.319,"15min":4.4500000000000002},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":163.92099999999999,"5min":163.92099999999999,"15min":163.92099999999999},"last":1.482}]},{"osd":4,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":2.4489999999999998,"15min":3.1349999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.3740000000000001,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.7669999999999999,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":5.4820000000000002,"5min":58.408000000000001,"15min":110.295},"last":1.907}]},{"osd":5,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":3.452,"15min":3.3799999999999999},"min":{"1min":0.316,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.5270000000000001,"5min":77.206000000000003,"15min":113.881},"last":1.9450000000000001},{"interface":"front","average":{"1min":0.80400000000000005,"5min":3.7250000000000001,"15min":3.2650000000000001},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":2.2930000000000001,"5min":77.968000000000004,"15min":114.598},"last":0.91300000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.629,"15min":3.8999999999999999},"min":{"1min":0.41599999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":2.2759999999999998,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.8439999999999999,"15min":4.0839999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":3.9279999999999999,"5min":58.701000000000001,"15min":82.233000000000004},"last":0.66700000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.8210000000000002,"15min":3.4740000000000002},"min":{"1min":0.40000000000000002,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":2.2559999999999998,"5min":58.686,"15min":138.035},"last":2.2879999999999998},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.8679999999999999,"15min":3.4510000000000001},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.28699999999999998},"max":{"1min":3.4260000000000002,"5min":58.667999999999999,"15min":138.054},"last":1.099}]}]},{"osd":4,"up_from":30,"seq":128849019287,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":865268,"kb_used_data":518380,"kb_used_omap":0,"kb_used_meta":346880,"kb_avail":92879884,"statfs":{"total":95995035648,"available":95109001216,"internally_reserved":0,"allocated":530821120,"data_stored":529559193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":355205120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:21 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.0509999999999999,"15min":2.202},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.189},"max":{"1min":3.6150000000000002,"5min":11.743,"15min":89.004999999999995},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.085,"15min":2.1659999999999999},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.218},"max":{"1min":5.3250000000000002,"5min":11.776,"15min":89.024000000000001},"last":0.68500000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":7.7149999999999999,"5min":4.2800000000000002,"15min":5.9630000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":138.47999999999999,"5min":138.47999999999999,"15min":161.613},"last":0.49299999999999999},{"interface":"front","average":{"1min":7.806,"5min":4.548,"15min":6.0780000000000003},"min":{"1min":0.247,"5min":0.217,"15min":0.217},"max":{"1min":138.429,"5min":138.429,"15min":162.017},"last":2.0179999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":8.6579999999999995,"15min":6.8689999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":144.916,"5min":182.96700000000001,"15min":182.96700000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":11.116,"5min":8.2240000000000002,"15min":6.6360000000000001},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":144.83099999999999,"5min":183.244,"15min":183.244},"last":2.8479999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":3.4319999999999999,"15min":6.5419999999999998},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":4.4340000000000002,"5min":184.37100000000001,"15min":266.54700000000003},"last":2.6779999999999999},{"interface":"front","average":{"1min":2.0329999999999999,"5min":3.6309999999999998,"15min":5.4809999999999999},"min":{"1min":0.36199999999999999,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":11.144,"5min":184.22300000000001,"15min":184.22300000000001},"last":2.7850000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:12:21 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48799999999999999,"15min":2.117},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":2.8919999999999999,"5min":3.3290000000000002,"15min":266.59899999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.45800000000000002,"15min":2.1419999999999999},"min":{"1min":0.29999999999999999,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":1.2230000000000001,"5min":1.722,"15min":266.72899999999998},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.502,"15min":0.51000000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.804,"5min":2.5489999999999999,"15min":4.0410000000000004},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60799999999999998,"15min":0.58899999999999997},"min":{"1min":0.38300000000000001,"5min":0.247,"15min":0.247},"max":{"1min":1.5880000000000001,"5min":4.2889999999999997,"15min":4.2889999999999997},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":1.244,"15min":1.411},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":0.59699999999999998,"5min":74.040000000000006,"15min":76.507000000000005},"last":0.375},{"interface":"front","average":{"1min":0.56200000000000006,"5min":1.25,"15min":1.536},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":3.2890000000000001,"5min":74.021000000000001,"15min":76.462000000000003},"last":0.55900000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757651,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":731888,"kb_used_data":319080,"kb_used_omap":0,"kb_used_meta":412800,"kb_avail":93013264,"statfs":{"total":95995035648,"available":95245582336,"internally_reserved":0,"allocated":326737920,"data_stored":325476362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":422707200},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.82099999999999995,"15min":1.7290000000000001},"min":{"1min":0.26100000000000001,"5min":0.224,"15min":0.216},"max":{"1min":2.173,"5min":8.6739999999999995,"15min":121.94799999999999},"last":6.1029999999999998},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.103,"15min":1.9970000000000001},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":9.4250000000000007,"5min":10.327,"15min":121.964},"last":1.9099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":2.093,"15min":5.8949999999999996},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":3.52,"5min":75.204999999999998,"15min":266.197},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.1480000000000001,"15min":6.0700000000000003},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1399999999999997,"5min":75.231999999999999,"15min":266.14999999999998},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":5.6639999999999997,"15min":5.4379999999999997},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":4.1239999999999997,"5min":240.57300000000001,"15min":240.57300000000001},"last":5.0890000000000004},{"interface":"front","average":{"1min":1.2509999999999999,"5min":5.7359999999999998,"15min":5.4969999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.313},"max":{"1min":4.1559999999999997,"5min":240.464,"15min":240.464},"last":5.1189999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.7030000000000001,"15min":4.8390000000000004},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.742,"5min":14.808,"15min":201.50899999999999},"last":0.625},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.407,"15min":4.6959999999999997},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":3.4500000000000002,"5min":14.893000000000001,"15min":201.58099999999999},"last":0.68999999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:12:16 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.48599999999999999,"15min":1.4119999999999999},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":2.5790000000000002,"5min":4.0510000000000002,"15min":191.78899999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.436,"15min":1.3129999999999999},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":0.59499999999999997,"5min":1.4359999999999999,"15min":191.821},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.69299999999999995,"15min":0.64100000000000001},"min":{"1min":0.377,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":3.278,"5min":9.9009999999999998,"15min":9.9009999999999998},"last":0.5},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.54600000000000004,"15min":0.51700000000000002},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.72799999999999998,"5min":5.0119999999999996,"15min":5.0119999999999996},"last":1.841}]},{"osd":7,"last update":"Sat Sep 23 20:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.56699999999999995,"15min":1.4930000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.3069999999999999,"5min":4.2080000000000002,"15min":191.84299999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65900000000000003,"15min":1.655},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":3.1440000000000001,"5min":9.6920000000000002,"15min":191.69},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":199950336,"data_stored":199950336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":25231360,"data_stored":25231360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3407872,"data_stored":3407872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":665124864,"data_stored":665124864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":506986496,"data_stored":506986496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":312610816,"data_stored":312606744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":21893120,"data_stored":21889048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:12:41.345 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:12:41.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:41.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:41.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:41.527 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:41.555 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:41 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:12:41.555 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:41 smithi144 ceph-mon[194833]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.555 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:41 smithi144 ceph-mon[194833]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T20:12:41.556 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:41 smithi144 ceph-mon[194833]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.556 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:41 smithi144 ceph-mon[194833]: failed to encode map e399 with expected crc 2023-09-23T20:12:41.556 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:41 smithi144 ceph-mon[194833]: pgmap v1124: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 7.8 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 5745/243 objects degraded (2364.198%); 53/243 objects misplaced (21.811%); 0 B/s, 288 objects/s recovering 2023-09-23T20:12:41.558 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:41.606 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:41.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:41.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:41.713 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:41.740 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:41.783 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:41.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:41.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:41.890 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:41.918 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:42.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[221113]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[221113]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[221113]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[221113]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[221113]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[221113]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[221113]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[213871]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[213871]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[213871]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T20:12:42.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[213871]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[213871]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[213871]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:42 smithi135 ceph-mon[213871]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:42 smithi144 ceph-mon[194833]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:42 smithi144 ceph-mon[194833]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:42 smithi144 ceph-mon[194833]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T20:12:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:42 smithi144 ceph-mon[194833]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:42 smithi144 ceph-mon[194833]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:42 smithi144 ceph-mon[194833]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:42 smithi144 ceph-mon[194833]: failed to encode map e400 with expected crc 2023-09-23T20:12:42.871 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:42.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:42.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:42.976 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:43.007 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:43.055 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:43.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:43.123 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:43.164 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:43.180 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:43.182 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:12:43.203 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:43.212 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:12:43.245 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:12:43.254 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:12:43.254 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:12:43.254 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:12:43.254 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:12:43.254 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:12:43.277 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:12:43.280 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:43 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5745/243 objects degraded (2364.198%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:43.280 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:43 smithi135 ceph-mon[213871]: pgmap v1126: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 5745/84 objects degraded (6839.286%); 0 B/s, 116 objects/s recovering 2023-09-23T20:12:43.280 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:43 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5745/243 objects degraded (2364.198%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:43.280 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:43 smithi135 ceph-mon[221113]: pgmap v1126: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 5745/84 objects degraded (6839.286%); 0 B/s, 116 objects/s recovering 2023-09-23T20:12:43.304 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:12:43.325 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:12:43.371 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:12:43.376 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:12:43.378 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:12:43.378 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:12:43.378 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:12:43.378 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:12:43.378 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:12:43.378 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:12:43.378 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:12:43.379 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:12:43.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:43 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5745/243 objects degraded (2364.198%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:12:43.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:43 smithi144 ceph-mon[194833]: pgmap v1126: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 5745/84 objects degraded (6839.286%); 0 B/s, 116 objects/s recovering 2023-09-23T20:12:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:12:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:12:43] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-09-23T20:12:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:44 smithi135 ceph-mon[213871]: 1.e deep-scrub starts 2023-09-23T20:12:44.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:44 smithi135 ceph-mon[213871]: 1.e deep-scrub ok 2023-09-23T20:12:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:44 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4217494746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:44 smithi135 ceph-mon[213871]: 1.14 deep-scrub starts 2023-09-23T20:12:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:44 smithi135 ceph-mon[221113]: 1.e deep-scrub starts 2023-09-23T20:12:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:44 smithi135 ceph-mon[221113]: 1.e deep-scrub ok 2023-09-23T20:12:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:44 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4217494746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:44 smithi135 ceph-mon[221113]: 1.14 deep-scrub starts 2023-09-23T20:12:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:44 smithi144 ceph-mon[194833]: 1.e deep-scrub starts 2023-09-23T20:12:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:44 smithi144 ceph-mon[194833]: 1.e deep-scrub ok 2023-09-23T20:12:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:44 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4217494746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:44 smithi144 ceph-mon[194833]: 1.14 deep-scrub starts 2023-09-23T20:12:44.768 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:12:44.818 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:44.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:44.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:44.923 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:44.951 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:45.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:45 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:12:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:45 smithi135 ceph-mon[221113]: pgmap v1127: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 3946/84 objects degraded (4697.619%); 0 B/s, 161 objects/s recovering 2023-09-23T20:12:45.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:45 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:12:45.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:45 smithi135 ceph-mon[213871]: pgmap v1127: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 3946/84 objects degraded (4697.619%); 0 B/s, 161 objects/s recovering 2023-09-23T20:12:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:45 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:12:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:45 smithi144 ceph-mon[194833]: pgmap v1127: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 3946/84 objects degraded (4697.619%); 0 B/s, 161 objects/s recovering 2023-09-23T20:12:46.148 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:46.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:46.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:46.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:46 smithi135 ceph-mon[221113]: 1.10 scrub starts 2023-09-23T20:12:46.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:46 smithi135 ceph-mon[213871]: 1.10 scrub starts 2023-09-23T20:12:46.254 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:46.286 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:46.333 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:46.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:46.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:46.439 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:46.468 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:12:46.476 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:46 smithi144 ceph-mon[194833]: 1.10 scrub starts 2023-09-23T20:12:46.733 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:46.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:46.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:46.837 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:46.869 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:46.917 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:46.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:47.022 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:12:47.052 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:47.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:47 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:12:47.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:47 smithi135 ceph-mon[213871]: pgmap v1128: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 3334/84 objects degraded (3969.048%); 0 B/s, 152 objects/s recovering 2023-09-23T20:12:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:47 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:12:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:47 smithi135 ceph-mon[221113]: pgmap v1128: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 3334/84 objects degraded (3969.048%); 0 B/s, 152 objects/s recovering 2023-09-23T20:12:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:47 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:12:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:47 smithi144 ceph-mon[194833]: pgmap v1128: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 3334/84 objects degraded (3969.048%); 0 B/s, 152 objects/s recovering 2023-09-23T20:12:48.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:48 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:12:48.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:48 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 3334/84 objects degraded (3969.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:12:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:48 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:12:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:48 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 3334/84 objects degraded (3969.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:12:48.564 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:12:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:48 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:12:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:48 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 3334/84 objects degraded (3969.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:12:49.186 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:12:49.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:49.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:12:49.281 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:49.295 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:49.327 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:49.338 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:49.368 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:49.373 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:49.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:49.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:49 smithi135 ceph-mon[213871]: 1.c scrub starts 2023-09-23T20:12:49.451 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:49 smithi135 ceph-mon[213871]: 1.2 scrub starts 2023-09-23T20:12:49.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:49 smithi135 ceph-mon[213871]: 1.2 scrub ok 2023-09-23T20:12:49.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:49 smithi135 ceph-mon[213871]: pgmap v1129: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 3334/84 objects degraded (3969.048%); 0 B/s, 135 objects/s recovering 2023-09-23T20:12:49.452 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:49 smithi135 ceph-mon[221113]: 1.c scrub starts 2023-09-23T20:12:49.452 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:49 smithi135 ceph-mon[221113]: 1.2 scrub starts 2023-09-23T20:12:49.452 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:49 smithi135 ceph-mon[221113]: 1.2 scrub ok 2023-09-23T20:12:49.452 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:49 smithi135 ceph-mon[221113]: pgmap v1129: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 3334/84 objects degraded (3969.048%); 0 B/s, 135 objects/s recovering 2023-09-23T20:12:49.480 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:49.510 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:49 smithi144 ceph-mon[194833]: 1.c scrub starts 2023-09-23T20:12:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:49 smithi144 ceph-mon[194833]: 1.2 scrub starts 2023-09-23T20:12:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:49 smithi144 ceph-mon[194833]: 1.2 scrub ok 2023-09-23T20:12:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:49 smithi144 ceph-mon[194833]: pgmap v1129: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 3334/84 objects degraded (3969.048%); 0 B/s, 135 objects/s recovering 2023-09-23T20:12:50.159 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:50 smithi135 ceph-mon[213871]: 1.12 scrub starts 2023-09-23T20:12:50.159 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:50 smithi135 ceph-mon[213871]: 1.12 scrub ok 2023-09-23T20:12:50.460 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:50.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:50 smithi135 ceph-mon[221113]: 1.12 scrub starts 2023-09-23T20:12:50.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:50 smithi135 ceph-mon[221113]: 1.12 scrub ok 2023-09-23T20:12:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:50.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:50.567 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:50.596 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:50 smithi144 ceph-mon[194833]: 1.12 scrub starts 2023-09-23T20:12:50.596 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:50 smithi144 ceph-mon[194833]: 1.12 scrub ok 2023-09-23T20:12:50.599 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:50.647 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:50.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:50.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:50.752 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:12:50.783 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:50.914 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:12:50.914 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 3334/84 objects degraded (3969.048%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1540,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":400,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":7,"osd_in_since":1695499655,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":107},{"state_name":"active+clean+scrubbing+deep","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":28,"data_bytes":4697461,"bytes_used":5060501504,"bytes_avail":666904748032,"bytes_total":671965249536,"degraded_objects":3334,"degraded_total":84,"degraded_ratio":39.69047619047619,"recovering_objects_per_sec":135,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":1202,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":4},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-23T20:06:56.817144+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"e76096c8-5249-4f94-9197-02047057b831":{"message":"Global Recovery Event (5m)\n [===========================.] (remaining: 3s)","progress":0.99082571268081665,"add_to_ceph_s":true}}} 2023-09-23T20:12:51.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:12:51.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[213871]: 1.6 scrub ok 2023-09-23T20:12:51.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3398039253' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:12:51.166 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1094331996' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:12:51.166 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[213871]: pgmap v1130: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 2630/84 objects degraded (3130.952%); 0 B/s, 172 objects/s recovering 2023-09-23T20:12:51.171 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_3' removed 2023-09-23T20:12:51.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:12:51.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[221113]: 1.6 scrub ok 2023-09-23T20:12:51.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3398039253' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:12:51.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1094331996' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:12:51.475 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:51 smithi135 ceph-mon[221113]: pgmap v1130: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 2630/84 objects degraded (3130.952%); 0 B/s, 172 objects/s recovering 2023-09-23T20:12:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:51 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:12:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:51 smithi144 ceph-mon[194833]: 1.6 scrub ok 2023-09-23T20:12:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3398039253' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:12:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1094331996' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:12:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:51 smithi144 ceph-mon[194833]: pgmap v1130: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 2630/84 objects degraded (3130.952%); 0 B/s, 172 objects/s recovering 2023-09-23T20:12:52.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[221113]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[221113]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[221113]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3398039253' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:12:52.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[221113]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T20:12:52.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:52.421 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[213871]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.421 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[213871]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.421 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[213871]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3398039253' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:12:52.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[213871]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T20:12:52.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:52 smithi144 ceph-mon[194833]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:52 smithi144 ceph-mon[194833]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:52 smithi144 ceph-mon[194833]: failed to encode map e401 with expected crc 2023-09-23T20:12:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3398039253' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:12:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:52 smithi144 ceph-mon[194833]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T20:12:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:53.019 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:12:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:53.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:12:53.124 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:53.152 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:53.337 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:53 smithi135 ceph-mon[221113]: 1.5 deep-scrub starts 2023-09-23T20:12:53.338 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 2630/84 objects degraded (3130.952%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:12:53.338 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:53 smithi135 ceph-mon[221113]: pgmap v1132: 93 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 91 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:12:53.338 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:12:53.338 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:53 smithi135 ceph-mon[213871]: 1.5 deep-scrub starts 2023-09-23T20:12:53.338 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 2630/84 objects degraded (3130.952%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:12:53.338 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:53 smithi135 ceph-mon[213871]: pgmap v1132: 93 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 91 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:12:53.338 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:12:53.351 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:53.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:53.456 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:12:53.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:53 smithi144 ceph-mon[194833]: 1.5 deep-scrub starts 2023-09-23T20:12:53.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 2630/84 objects degraded (3130.952%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:12:53.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:53 smithi144 ceph-mon[194833]: pgmap v1132: 93 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 91 active+clean; 4.5 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:12:53.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:12:53.488 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:53.537 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:12:53.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:53.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:12:53.610 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:12:53.610 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:12:53.642 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:53.670 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:53.715 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:53.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:53.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:53.826 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:53.853 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:53.914 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:12:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:12:53] "GET /metrics HTTP/1.1" 200 34118 "" "Prometheus/2.43.0" 2023-09-23T20:12:54.093 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:12:54.093 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:12:54.093 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T20:12:54.094 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T20:12:54.170 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: 1.d deep-scrub starts 2023-09-23T20:12:54.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: 1.d deep-scrub ok 2023-09-23T20:12:54.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: 1.10 scrub starts 2023-09-23T20:12:54.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: 1.10 scrub ok 2023-09-23T20:12:54.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2630/84 objects degraded (3130.952%), 1 pg degraded, 1 pg undersized) 2023-09-23T20:12:54.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:12:54.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: failed to encode map e402 with expected crc 2023-09-23T20:12:54.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:12:54.171 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[213871]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T20:12:54.337 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:12:54.337 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:12:54.337 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:12:54.337 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:12:54.337 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116287 8.7 8:e86b327e:::benchmark_data_smithi135_460343_object32054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:30.976436+0000", 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 203.35755639800001, 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6110225419999999, 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116287 2023-09-23T20:12:54.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976436+0000", 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976436+0000", 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976434+0000", 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976442+0000", 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3610000000000003e-06 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976458+0000", 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6008999999999999e-05 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976468+0000", 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0068000000000001e-05 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.047679+0000", 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.071211117000000004 2023-09-23T20:12:54.341 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.057547+0000", 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0098680380000000009 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.085877+0000", 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.028330208999999999 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.091492+0000", 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056143829999999997 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.418689+0000", 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.327197631 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.343 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.432118+0000", 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013428544000000001 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.456884+0000", 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024766474 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.498418+0000", 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.041533931000000003 2023-09-23T20:12:54.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.545058+0000", 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.046639997000000002 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.548122+0000", 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003063891 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.345 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.553892+0000", 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0057700349999999997 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.587458+0000", 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.033566209999999999 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.346 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116291 8.7 8:e6b5a36d:::benchmark_data_smithi135_460343_object32058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.051182+0000", 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 203.28281071200001, 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6927078039999999, 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116291 2023-09-23T20:12:54.347 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051182+0000", 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051182+0000", 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051180+0000", 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051189+0000", 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9240000000000002e-06 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051207+0000", 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7159e-05 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051216+0000", 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3489999999999998e-06 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.058998+0000", 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007781769 2023-09-23T20:12:54.350 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.065129+0000", 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061313779999999998 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.111839+0000", 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.046710086999999997 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.351 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.117249+0000", 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0054098159999999996 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.630036+0000", 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.512786883 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.637705+0000", 2023-09-23T20:12:54.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0076694720000000001 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.653502+0000", 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015796528000000001 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.718486+0000", 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.064984564999999994 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.353 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.719651+0000", 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001165031 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.722210+0000", 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025589889999999998 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.726730+0000", 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004519866 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.743889+0000", 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017158965000000002 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116311 8.f 8:f8d2a2d2:::benchmark_data_smithi135_460343_object32078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.295114+0000", 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 203.03887781899999, 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6782180090000001, 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116311 2023-09-23T20:12:54.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295114+0000", 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295114+0000", 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295113+0000", 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295121+0000", 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0660000000000004e-06 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295136+0000", 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4980000000000001e-05 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295144+0000", 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7789999999999996e-06 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.336228+0000", 2023-09-23T20:12:54.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.041084794000000001 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.345483+0000", 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0092550489999999996 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.401553+0000", 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.056069359999999999 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.411366+0000", 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0098127839999999997 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.914818+0000", 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5034523229999999 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.921608+0000", 2023-09-23T20:12:54.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067902980000000002 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.936336+0000", 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014727888999999999 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.950371+0000", 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014034744 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.953642+0000", 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003270911 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.955053+0000", 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014112020000000001 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.959874+0000", 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048215749999999998 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.973332+0000", 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013457920999999999 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.364 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116312 8.7 8:e7a522c9:::benchmark_data_smithi135_460343_object32079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.297531+0000", 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 203.03646088400001, 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.711426772, 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116312 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.365 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297531+0000", 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297531+0000", 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297531+0000", 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297535+0000", 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4750000000000004e-06 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297545+0000", 2023-09-23T20:12:54.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7459999999999995e-06 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297550+0000", 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7189999999999998e-06 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.336588+0000", 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.039037534999999998 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.345661+0000", 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0090730800000000007 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.401924+0000", 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.056263201999999998 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.412620+0000", 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010695377000000001 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.917825+0000", 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5052050750000001 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.926015+0000", 2023-09-23T20:12:54.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081900619999999997 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.939379+0000", 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013364467 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.954191+0000", 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014811892 2023-09-23T20:12:54.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.957357+0000", 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031655889999999999 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.974900+0000", 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017543361 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.980566+0000", 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056655630000000002 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.008958+0000", 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.028392463999999999 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116323 8.f 8:f7cc7664:::benchmark_data_smithi135_460343_object32090:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.357259+0000", 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 202.97673372, 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7112210409999999, 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116323 2023-09-23T20:12:54.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357259+0000", 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357259+0000", 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357257+0000", 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357266+0000", 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0759999999999992e-06 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357278+0000", 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2388999999999999e-05 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357287+0000", 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4870000000000002e-06 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.451249+0000", 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.093962813000000006 2023-09-23T20:12:54.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.462359+0000", 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01111006 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.521611+0000", 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.059251486999999999 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.536134+0000", 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014522765 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.004103+0000", 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.467969313 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.015518+0000", 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011414565 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.031900+0000", 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016382906999999999 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.051050+0000", 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019149817 2023-09-23T20:12:54.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.051647+0000", 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00059682099999999998 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.053566+0000", 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00191895 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.056171+0000", 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002605357 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.068480+0000", 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012308127 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116336 8.7 8:ee959f5b:::benchmark_data_smithi135_460343_object32103:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.535286+0000", 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 202.798706646, 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6772203409999999, 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116336 2023-09-23T20:12:54.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535286+0000", 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535286+0000", 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535284+0000", 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535297+0000", 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2571e-05 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535308+0000", 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0910999999999999e-05 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535314+0000", 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0209999999999996e-06 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.541877+0000", 2023-09-23T20:12:54.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065639729999999999 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.547255+0000", 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053776240000000001 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.639621+0000", 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.092366243000000001 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.645009+0000", 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053872670000000003 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.150126+0000", 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5051173279999999 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.155252+0000", 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051262410000000001 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.163894+0000", 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0086421810000000005 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.174763+0000", 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010868681 2023-09-23T20:12:54.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.176261+0000", 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014980390000000001 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.197173+0000", 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.020911762 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.200121+0000", 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029481490000000002 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.212506+0000", 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012384904 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116353 8.7 8:ecc5e610:::benchmark_data_smithi135_460343_object32120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.633770+0000", 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 202.70022197500001, 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6240762310000001, 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116353 2023-09-23T20:12:54.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633770+0000", 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633770+0000", 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633769+0000", 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633776+0000", 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5170000000000002e-06 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633790+0000", 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3749e-05 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633797+0000", 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9399999999999996e-06 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.663793+0000", 2023-09-23T20:12:54.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029996364000000001 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.670817+0000", 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007023334 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.751707+0000", 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.080890225999999996 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.758423+0000", 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067167019999999997 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.213064+0000", 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.454640854 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.217981+0000", 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049167359999999997 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.225569+0000", 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0075880000000000001 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.240287+0000", 2023-09-23T20:12:54.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014717701999999999 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.240780+0000", 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00049317100000000004 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.242363+0000", 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015826379999999999 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.245271+0000", 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029087150000000001 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.257846+0000", 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012575197999999999 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116354 8.f 8:f582891d:::benchmark_data_smithi135_460343_object32121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.651719+0000", 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 202.68227309, 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.688988763, 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116354 2023-09-23T20:12:54.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651719+0000", 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651719+0000", 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651718+0000", 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651726+0000", 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6769999999999995e-06 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651740+0000", 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4953999999999999e-05 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651749+0000", 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9050000000000003e-06 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.664771+0000", 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013021807999999999 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.676017+0000", 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011246143 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.759969+0000", 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.083951210999999998 2023-09-23T20:12:54.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.771322+0000", 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011353439 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.283281+0000", 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5119590860000001 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.406 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.289058+0000", 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0057770820000000002 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.299557+0000", 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010498664 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.310230+0000", 2023-09-23T20:12:54.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010673498 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.312280+0000", 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002049309 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.313009+0000", 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00072948999999999996 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.316874+0000", 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038645979999999999 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.340708+0000", 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.023834191000000001 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116396 8.f 8:f2fcea18:::benchmark_data_smithi135_460343_object32163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:32.088587+0000", 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 202.24540502900001, 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6114060879999998, 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.410 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116396 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088587+0000", 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088587+0000", 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088586+0000", 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088594+0000", 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9899999999999997e-06 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088606+0000", 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2315e-05 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088613+0000", 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6320000000000002e-06 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.227428+0000", 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13881549100000001 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.236699+0000", 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0092712660000000002 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.288248+0000", 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.051549141 2023-09-23T20:12:54.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.295587+0000", 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073383440000000001 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.658757+0000", 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3631700179999999 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.663837+0000", 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050801659999999997 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.672583+0000", 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0087457950000000007 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.680624+0000", 2023-09-23T20:12:54.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080414230000000007 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.684408+0000", 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003783818 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.685105+0000", 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00069654199999999997 2023-09-23T20:12:54.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.687598+0000", 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024939239999999998 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.699993+0000", 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012394789 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.418 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116398 8.f 8:f8620ac9:::benchmark_data_smithi135_460343_object32165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:32.098708+0000", 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 202.23528448299999, 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6899823980000002, 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.419 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116398 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098708+0000", 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098708+0000", 2023-09-23T20:12:54.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098706+0000", 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098715+0000", 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0599999999999997e-06 2023-09-23T20:12:54.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098731+0000", 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5438e-05 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098739+0000", 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2470000000000008e-06 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.240960+0000", 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14222137600000001 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.249002+0000", 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080412609999999992 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.319967+0000", 2023-09-23T20:12:54.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.070965500000000001 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.326060+0000", 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0060927430000000003 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.713791+0000", 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3877313149999999 2023-09-23T20:12:54.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.718352+0000", 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045612430000000004 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.726422+0000", 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080694430000000008 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.769597+0000", 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.043174953000000002 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.771434+0000", 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018374450000000001 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.773270+0000", 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018357479999999999 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.776449+0000", 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003178934 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.788690+0000", 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01224129 2023-09-23T20:12:54.427 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:117890 8.f 8:f3976cab:::benchmark_data_smithi135_460343_object33657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:48.672824+0000", 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 185.66116808300001, 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.58877402, 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.428 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117890 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672824+0000", 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.429 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672824+0000", 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672823+0000", 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672830+0000", 2023-09-23T20:12:54.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8759999999999999e-06 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672841+0000", 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0763e-05 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672847+0000", 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.737e-06 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.674677+0000", 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001829458 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.683189+0000", 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.008512759 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.738282+0000", 2023-09-23T20:12:54.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.055092231999999998 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.745725+0000", 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0074430210000000002 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.088007+0000", 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3422824659999999 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.093539+0000", 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0055319289999999997 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.103612+0000", 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010072561000000001 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.220680+0000", 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.117068696 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.222650+0000", 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001969518 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.247834+0000", 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.025184372 2023-09-23T20:12:54.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.250538+0000", 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027036679999999998 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.261598+0000", 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011060199 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.436 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:117909 8.f 8:fab9314c:::benchmark_data_smithi135_460343_object33676:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:49.049290+0000", 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 185.284702033, 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5971690810000001, 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.437 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117909 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049290+0000", 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049290+0000", 2023-09-23T20:12:54.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049288+0000", 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049300+0000", 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2026e-05 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049320+0000", 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0078e-05 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049330+0000", 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9110000000000005e-06 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.440 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.170287+0000", 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.120956735 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.177100+0000", 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0068132369999999998 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.220172+0000", 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.043072023000000001 2023-09-23T20:12:54.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.232953+0000", 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012781073 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.576010+0000", 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3430568809999999 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.580714+0000", 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047042589999999997 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.590160+0000", 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094457099999999995 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.630047+0000", 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.039886963999999997 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.630551+0000", 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050392899999999997 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.632109+0000", 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015582059999999999 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.634635+0000", 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025259560000000002 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.646459+0000", 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011824328 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:117927 8.f 8:f5e97549:::benchmark_data_smithi135_460343_object33694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:49.254223+0000", 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 185.07976911899999, 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5708444940000001, 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117927 2023-09-23T20:12:54.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254223+0000", 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254223+0000", 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254220+0000", 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999971 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254233+0000", 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2813000000000001e-05 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.448 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254253+0000", 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0253000000000001e-05 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254265+0000", 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1114999999999999e-05 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.288521+0000", 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.034256712000000002 2023-09-23T20:12:54.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.301312+0000", 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012790920000000001 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.417009+0000", 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.11569644599999999 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.424745+0000", 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0077367269999999997 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.726652+0000", 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.301907047 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.735070+0000", 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084176070000000006 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.755181+0000", 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.020110801000000001 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.810068+0000", 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.054886783000000001 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.810571+0000", 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050297199999999997 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.812309+0000", 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001738327 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.814876+0000", 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025667760000000002 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.825068+0000", 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010191891999999999 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123384 8.f 8:fe9c53da:::benchmark_data_smithi135_460343_object39151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 132.42797361199999, 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4434314349999999, 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123384 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.455 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906017+0000", 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906025+0000", 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6260000000000003e-06 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.457 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906041+0000", 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6183000000000001e-05 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906048+0000", 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7000000000000008e-06 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.969587+0000", 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.063539010000000007 2023-09-23T20:12:54.458 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.974774+0000", 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051870179999999998 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.005518+0000", 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030743594999999999 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.459 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.011676+0000", 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061576579999999999 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.259997+0000", 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2483216180000001 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.274504+0000", 2023-09-23T20:12:54.460 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014506939999999999 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.290807+0000", 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016302543999999999 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.312744+0000", 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021937076999999999 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.461 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.314532+0000", 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017877699999999999 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.336416+0000", 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021884023999999998 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.462 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.338670+0000", 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022543989999999998 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.349450+0000", 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010779924 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.463 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123388 8.f 8:f5171e3d:::benchmark_data_smithi135_460343_object39155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 132.25537622100001, 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5028154900000001, 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123388 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.464 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.465 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078615+0000", 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078622+0000", 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.345e-06 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078636+0000", 2023-09-23T20:12:54.466 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3645000000000001e-05 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078642+0000", 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5810000000000002e-06 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.092915+0000", 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014273083000000001 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.467 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.098087+0000", 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051715119999999996 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.137553+0000", 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.039466342000000001 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.468 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.146411+0000", 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088579100000000001 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.513388+0000", 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.366976599 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.518018+0000", 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046298229999999999 2023-09-23T20:12:54.469 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.527773+0000", 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097548610000000001 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.566237+0000", 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.038464921999999999 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.470 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.567647+0000", 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001409684 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.568272+0000", 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00062472500000000002 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.570874+0000", 2023-09-23T20:12:54.471 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026022689999999999 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.581431+0000", 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010557383 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123394 8.f 8:ff01962c:::benchmark_data_smithi135_460343_object39161:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:12:54.472 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 132.133613852, 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.413050616, 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123394 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.473 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200376+0000", 2023-09-23T20:12:54.474 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200388+0000", 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2403e-05 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200413+0000", 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4596999999999998e-05 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.475 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200430+0000", 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6908000000000001e-05 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.221890+0000", 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021460679999999999 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.476 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.238665+0000", 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0167742 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.309438+0000", 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.070773321 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.319840+0000", 2023-09-23T20:12:54.477 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010402428 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.545580+0000", 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.225739777 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.550259+0000", 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046789290000000001 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.478 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.558399+0000", 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081401330000000008 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.583234+0000", 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024835052999999999 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.479 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.585402+0000", 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0021673310000000002 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.586049+0000", 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00064698200000000005 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.591270+0000", 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052211779999999999 2023-09-23T20:12:54.480 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.613429+0000", 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.022159160000000001 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123402 8.7 8:ec120430:::benchmark_data_smithi135_460343_object39169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:12:54.481 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 132.080863924, 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4055715360000001, 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123402 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.482 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253126+0000", 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:12:54.483 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253138+0000", 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1725e-05 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253158+0000", 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0772999999999999e-05 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.484 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253169+0000", 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.022e-05 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.321269+0000", 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.068100777000000001 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.485 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.331911+0000", 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010641866999999999 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.411508+0000", 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.079596497000000002 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.422363+0000", 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010854744 2023-09-23T20:12:54.486 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.612810+0000", 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1904476230000001 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.617814+0000", 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050038449999999998 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.487 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.627404+0000", 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0095902439999999995 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.643653+0000", 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016248987999999999 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.645238+0000", 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00158438 2023-09-23T20:12:54.488 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.645689+0000", 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045135999999999999 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.647776+0000", 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020871879999999998 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.489 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.658700+0000", 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010923696 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123442 8.f 8:f6660517:::benchmark_data_smithi135_460343_object39209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 131.79780713299999, 2023-09-23T20:12:54.490 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4673699600000001, 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123442 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:12:54.491 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536183+0000", 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.492 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536192+0000", 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1039999999999996e-06 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536207+0000", 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4834000000000001e-05 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.493 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536219+0000", 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1691999999999999e-05 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.615218+0000", 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.078999571000000005 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.620140+0000", 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049211439999999997 2023-09-23T20:12:54.494 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.675848+0000", 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.055708879000000003 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.680973+0000", 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051250449999999999 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.495 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.828876+0000", 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1479029780000001 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.834877+0000", 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0060005409999999999 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.496 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.844577+0000", 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097000209999999996 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.858053+0000", 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013476228 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.859480+0000", 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001426407 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.497 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.991498+0000", 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13201818500000001 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.993799+0000", 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023010399999999999 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.498 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.003555+0000", 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097562010000000008 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123449 8.f 8:f357d011:::benchmark_data_smithi135_460343_object39216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 131.76312262100001, 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5039241049999998, 2023-09-23T20:12:54.499 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123449 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.500 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570867+0000", 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999971 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.501 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570879+0000", 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2435e-05 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570900+0000", 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0509999999999998e-05 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570919+0000", 2023-09-23T20:12:54.502 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8947999999999999e-05 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.620845+0000", 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.049925803999999997 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.626417+0000", 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0055726200000000004 2023-09-23T20:12:54.503 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.505 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.505 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.698693+0000", 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.072275866999999994 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.703708+0000", 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050152590000000002 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.506 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.016373+0000", 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.312664507 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.020480+0000", 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041066089999999998 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.507 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.028636+0000", 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081566039999999996 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.041139+0000", 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012503377 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.042755+0000", 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016158360000000001 2023-09-23T20:12:54.508 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.061415+0000", 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018660039 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.063733+0000", 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023174549999999999 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.509 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.074794+0000", 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011060876000000001 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123630 8.7 8:e7d430c0:::benchmark_data_smithi135_460343_object39397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 130.01061647099999, 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4124915579999999, 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:12:54.510 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123630 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.511 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323374+0000", 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.512 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323381+0000", 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8360000000000003e-06 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323393+0000", 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.226e-05 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323402+0000", 2023-09-23T20:12:54.513 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9330000000000007e-06 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.394929+0000", 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.071526432000000001 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.399736+0000", 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048073309999999998 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.514 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.434854+0000", 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.035117795 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.439795+0000", 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049409620000000001 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:12:54.515 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.537860+0000", 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0980649309999999 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.542831+0000", 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004970833 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.551732+0000", 2023-09-23T20:12:54.516 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0089008879999999992 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.719631+0000", 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16789942299999999 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.721555+0000", 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019236259999999999 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.517 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.723874+0000", 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023192669999999999 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.726072+0000", 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0021980229999999999 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.735867+0000", 2023-09-23T20:12:54.518 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097953720000000001 2023-09-23T20:12:54.519 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.519 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.519 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.519 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:12:54.519 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:12:54.519 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: 1.d deep-scrub starts 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: 1.d deep-scrub ok 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: 1.10 scrub starts 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: 1.10 scrub ok 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2630/84 objects degraded (3130.952%), 1 pg degraded, 1 pg undersized) 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: failed to encode map e402 with expected crc 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:12:54.520 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:54 smithi135 ceph-mon[221113]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T20:12:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: 1.d deep-scrub starts 2023-09-23T20:12:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: 1.d deep-scrub ok 2023-09-23T20:12:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: 1.10 scrub starts 2023-09-23T20:12:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: 1.10 scrub ok 2023-09-23T20:12:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2630/84 objects degraded (3130.952%), 1 pg degraded, 1 pg undersized) 2023-09-23T20:12:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:12:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: failed to encode map e402 with expected crc 2023-09-23T20:12:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:12:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:54 smithi144 ceph-mon[194833]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T20:12:55.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: 1.5 scrub ok 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: 1.1 scrub starts 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: 1.1 scrub ok 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: 1.0 scrub starts 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: 1.0 scrub ok 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: failed to encode map e403 with expected crc 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[213871]: pgmap v1135: 93 pgs: 2 peering, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 89 active+clean; 4.4 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:12:55.439 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: 1.5 scrub ok 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: 1.1 scrub starts 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: 1.1 scrub ok 2023-09-23T20:12:55.440 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: 1.0 scrub starts 2023-09-23T20:12:55.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: 1.0 scrub ok 2023-09-23T20:12:55.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: failed to encode map e403 with expected crc 2023-09-23T20:12:55.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T20:12:55.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:55 smithi135 ceph-mon[221113]: pgmap v1135: 93 pgs: 2 peering, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 89 active+clean; 4.4 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:12:55.475 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:12:55.550 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:55.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: failed to encode map e402 with expected crc 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:12:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: 1.5 scrub ok 2023-09-23T20:12:55.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: 1.1 scrub starts 2023-09-23T20:12:55.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: 1.1 scrub ok 2023-09-23T20:12:55.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: 1.0 scrub starts 2023-09-23T20:12:55.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: 1.0 scrub ok 2023-09-23T20:12:55.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: failed to encode map e403 with expected crc 2023-09-23T20:12:55.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T20:12:55.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:55 smithi144 ceph-mon[194833]: pgmap v1135: 93 pgs: 2 peering, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 89 active+clean; 4.4 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:12:55.655 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:55.665 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:55.692 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:55.761 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:55.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:55.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:55.865 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:55.897 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:55.944 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:55.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:56.052 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:56.083 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:56.090 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:56.100 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:12:56.122 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:12:56.130 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:56.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:56.143 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:12:56.163 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:12:56.184 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:12:56.205 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:12:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: 1.8 scrub starts 2023-09-23T20:12:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: 1.8 scrub ok 2023-09-23T20:12:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: failed to encode map e404 with expected crc 2023-09-23T20:12:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T20:12:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3699508571' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:12:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3792784256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: 1.8 scrub starts 2023-09-23T20:12:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: 1.8 scrub ok 2023-09-23T20:12:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: failed to encode map e404 with expected crc 2023-09-23T20:12:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T20:12:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3699508571' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:12:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3792784256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:56.230 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:12:56.235 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:12:56.266 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:56.279 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:12:56.279 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:12:56.279 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:12:56.279 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:12:56.279 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:12:56.280 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:12:56.280 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:12:56.280 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:12:56.280 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:12:56.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: failed to encode map e403 with expected crc 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: 1.8 scrub starts 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: 1.8 scrub ok 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: failed to encode map e404 with expected crc 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3699508571' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:12:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3792784256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:12:56.731 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:12:57.164 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:12:57.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:57.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:12:57.270 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:57.302 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:57.350 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:57.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:57.457 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: failed to encode map e405 with expected crc 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3699508571' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[213871]: pgmap v1138: 93 pgs: 2 peering, 3 active+undersized+remapped, 88 active+clean; 4.4 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: failed to encode map e405 with expected crc 2023-09-23T20:12:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3699508571' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:12:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T20:12:57.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:57 smithi135 ceph-mon[221113]: pgmap v1138: 93 pgs: 2 peering, 3 active+undersized+remapped, 88 active+clean; 4.4 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-09-23T20:12:57.486 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: failed to encode map e404 with expected crc 2023-09-23T20:12:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:12:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: failed to encode map e405 with expected crc 2023-09-23T20:12:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3699508571' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:12:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T20:12:57.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:57 smithi144 ceph-mon[194833]: pgmap v1138: 93 pgs: 2 peering, 3 active+undersized+remapped, 88 active+clean; 4.4 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-09-23T20:12:57.885 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:57.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:57.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:57.989 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:12:58.017 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:58.414 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:12:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:58.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:12:58.465 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.465 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[221113]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.466 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:58.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:58.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:58 smithi135 ceph-mon[213871]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T20:12:58.518 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:12:58.547 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.547 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.548 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.548 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.548 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.548 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: failed to encode map e405 with expected crc 2023-09-23T20:12:58.548 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:12:58.548 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:58.548 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:58 smithi144 ceph-mon[194833]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T20:12:58.550 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:58.599 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:12:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:58.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:12:58.704 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:12:58.733 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:59.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:59.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[213871]: pgmap v1140: 93 pgs: 2 active+undersized+remapped, 91 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 239 B/s rd, 478 B/s wr, 0 op/s; 75 KiB/s, 0 objects/s recovering 2023-09-23T20:12:59.234 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:12:59.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:59.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:12:59.277 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:12:59.279 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:12:59.289 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:59.338 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:12:59.366 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:59.459 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:12:59.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.463 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.463 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:59.463 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:12:59 smithi144 ceph-mon[194833]: pgmap v1140: 93 pgs: 2 active+undersized+remapped, 91 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 239 B/s rd, 478 B/s wr, 0 op/s; 75 KiB/s, 0 objects/s recovering 2023-09-23T20:12:59.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:59.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: failed to encode map e406 with expected crc 2023-09-23T20:12:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:59.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:12:59 smithi135 ceph-mon[221113]: pgmap v1140: 93 pgs: 2 active+undersized+remapped, 91 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 239 B/s rd, 478 B/s wr, 0 op/s; 75 KiB/s, 0 objects/s recovering 2023-09-23T20:12:59.565 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:12:59.592 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:59.636 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:12:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:59.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:12:59.739 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:12:59.769 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:12:59.936 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:12:59.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:12:59.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:00.040 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:00.066 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:00.110 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:00.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:00.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:00.139 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1140,"stamp":"2023-09-23T20:12:58.994493+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4906480,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":6077,"num_read_kb":13388,"num_write":955,"num_write_kb":13202,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":212,"num_bytes_recovered":9736176,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4415,"ondisk_log_size":4415,"up":279,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":296,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4875060,"kb_used_data":1103996,"kb_used_omap":32,"kb_used_meta":3770975,"kb_avail":651341004,"statfs":{"total":671965249536,"available":666973188096,"internally_reserved":0,"allocated":1130491904,"data_stored":1118807294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3861478918},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":36,"apply_latency_ms":36,"commit_latency_ns":36000000,"apply_latency_ns":36000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":262689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":327682,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.278868"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":404,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728639+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:12:56.728639+0000","last_peered":"2023-09-23T20:12:56.728639+0000","last_clean":"2023-09-23T20:12:56.728639+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:12:56.728639+0000","last_undegraded":"2023-09-23T20:12:56.728639+0000","last_fullsized":"2023-09-23T20:12:56.728639+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":234,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.726916+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:12:56.726916+0000","last_peered":"2023-09-23T20:12:56.726916+0000","last_clean":"2023-09-23T20:12:56.726916+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:12:56.726916+0000","last_undegraded":"2023-09-23T20:12:56.726916+0000","last_fullsized":"2023-09-23T20:12:56.726916+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.726664+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:12:56.726664+0000","last_peered":"2023-09-23T20:12:56.726664+0000","last_clean":"2023-09-23T20:12:56.726664+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:12:56.726664+0000","last_undegraded":"2023-09-23T20:12:56.726664+0000","last_fullsized":"2023-09-23T20:12:56.726664+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":114,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.733776+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:12:56.733776+0000","last_peered":"2023-09-23T20:12:56.733776+0000","last_clean":"2023-09-23T20:12:56.733776+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:12:56.733776+0000","last_undegraded":"2023-09-23T20:12:56.733776+0000","last_fullsized":"2023-09-23T20:12:56.733776+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":235,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.731212+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:12:57.731212+0000","last_peered":"2023-09-23T20:12:57.731212+0000","last_clean":"2023-09-23T20:12:57.731212+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:12:57.731212+0000","last_undegraded":"2023-09-23T20:12:57.731212+0000","last_fullsized":"2023-09-23T20:12:57.731212+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":351,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.730994+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:12:56.730994+0000","last_peered":"2023-09-23T20:12:56.730994+0000","last_clean":"2023-09-23T20:12:56.730994+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:12:56.730994+0000","last_undegraded":"2023-09-23T20:12:56.730994+0000","last_fullsized":"2023-09-23T20:12:56.730994+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":332,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.730852+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:12:56.730852+0000","last_peered":"2023-09-23T20:12:56.730852+0000","last_clean":"2023-09-23T20:12:56.730852+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:12:56.730852+0000","last_undegraded":"2023-09-23T20:12:56.730852+0000","last_fullsized":"2023-09-23T20:12:56.730852+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":247,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.719580+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:12:54.719580+0000","last_peered":"2023-09-23T20:12:54.719580+0000","last_clean":"2023-09-23T20:12:54.719580+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:12:54.719580+0000","last_undegraded":"2023-09-23T20:12:54.719580+0000","last_fullsized":"2023-09-23T20:12:54.719580+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":721,"reported_epoch":404,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:12:55.736226+0000","last_change":"2023-09-23T20:12:55.736142+0000","last_active":"2023-09-23T20:12:55.736226+0000","last_peered":"2023-09-23T20:12:55.736226+0000","last_clean":"2023-09-23T20:12:51.474750+0000","last_became_active":"2023-09-23T20:12:55.736142+0000","last_became_peered":"2023-09-23T20:12:55.736142+0000","last_unstale":"2023-09-23T20:12:55.736226+0000","last_undegraded":"2023-09-23T20:12:55.736226+0000","last_fullsized":"2023-09-23T20:12:55.730325+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[3,7],"avail_no_missing":["3","2","4","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":405,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.726928+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:12:56.726928+0000","last_peered":"2023-09-23T20:12:56.726928+0000","last_clean":"2023-09-23T20:12:56.726928+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:12:56.726928+0000","last_undegraded":"2023-09-23T20:12:56.726928+0000","last_fullsized":"2023-09-23T20:12:56.726928+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":266,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.719766+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:12:54.719766+0000","last_peered":"2023-09-23T20:12:54.719766+0000","last_clean":"2023-09-23T20:12:54.719766+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:12:54.719766+0000","last_undegraded":"2023-09-23T20:12:54.719766+0000","last_fullsized":"2023-09-23T20:12:54.719766+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":99,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.733651+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:12:56.733651+0000","last_peered":"2023-09-23T20:12:56.733651+0000","last_clean":"2023-09-23T20:12:56.733651+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:12:56.733651+0000","last_undegraded":"2023-09-23T20:12:56.733651+0000","last_fullsized":"2023-09-23T20:12:56.733651+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727072+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:12:56.727072+0000","last_peered":"2023-09-23T20:12:56.727072+0000","last_clean":"2023-09-23T20:12:56.727072+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:12:56.727072+0000","last_undegraded":"2023-09-23T20:12:56.727072+0000","last_fullsized":"2023-09-23T20:12:56.727072+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":460,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729113+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:12:56.729113+0000","last_peered":"2023-09-23T20:12:56.729113+0000","last_clean":"2023-09-23T20:12:56.729113+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:12:56.729113+0000","last_undegraded":"2023-09-23T20:12:56.729113+0000","last_fullsized":"2023-09-23T20:12:56.729113+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":402,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.733949+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:12:56.733949+0000","last_peered":"2023-09-23T20:12:56.733949+0000","last_clean":"2023-09-23T20:12:56.733949+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:12:56.733949+0000","last_undegraded":"2023-09-23T20:12:56.733949+0000","last_fullsized":"2023-09-23T20:12:56.733949+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":755,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.745067+0000","last_change":"2023-09-23T20:12:57.745067+0000","last_active":"2023-09-23T20:12:57.745067+0000","last_peered":"2023-09-23T20:12:57.745067+0000","last_clean":"2023-09-23T20:12:57.745067+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:12:57.745067+0000","last_undegraded":"2023-09-23T20:12:57.745067+0000","last_fullsized":"2023-09-23T20:12:57.745067+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":99,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.732835+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:12:56.732835+0000","last_peered":"2023-09-23T20:12:56.732835+0000","last_clean":"2023-09-23T20:12:56.732835+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:12:56.732835+0000","last_undegraded":"2023-09-23T20:12:56.732835+0000","last_fullsized":"2023-09-23T20:12:56.732835+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":259,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.726850+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:12:56.726850+0000","last_peered":"2023-09-23T20:12:56.726850+0000","last_clean":"2023-09-23T20:12:56.726850+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:12:56.726850+0000","last_undegraded":"2023-09-23T20:12:56.726850+0000","last_fullsized":"2023-09-23T20:12:56.726850+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":444,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.746858+0000","last_change":"2023-09-23T20:12:57.746858+0000","last_active":"2023-09-23T20:12:57.746858+0000","last_peered":"2023-09-23T20:12:57.746858+0000","last_clean":"2023-09-23T20:12:57.746858+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:12:57.746858+0000","last_undegraded":"2023-09-23T20:12:57.746858+0000","last_fullsized":"2023-09-23T20:12:57.746858+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.079134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.079134+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":114,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.732811+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:12:56.732811+0000","last_peered":"2023-09-23T20:12:56.732811+0000","last_clean":"2023-09-23T20:12:56.732811+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:12:56.732811+0000","last_undegraded":"2023-09-23T20:12:56.732811+0000","last_fullsized":"2023-09-23T20:12:56.732811+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":404,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729256+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:12:56.729256+0000","last_peered":"2023-09-23T20:12:56.729256+0000","last_clean":"2023-09-23T20:12:56.729256+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:12:56.729256+0000","last_undegraded":"2023-09-23T20:12:56.729256+0000","last_fullsized":"2023-09-23T20:12:56.729256+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":549,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.739426+0000","last_change":"2023-09-23T20:12:57.739426+0000","last_active":"2023-09-23T20:12:57.739426+0000","last_peered":"2023-09-23T20:12:57.739426+0000","last_clean":"2023-09-23T20:12:57.739426+0000","last_became_active":"2023-09-23T20:12:57.738891+0000","last_became_peered":"2023-09-23T20:12:57.738891+0000","last_unstale":"2023-09-23T20:12:57.739426+0000","last_undegraded":"2023-09-23T20:12:57.739426+0000","last_fullsized":"2023-09-23T20:12:57.739426+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.732663+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:12:56.732663+0000","last_peered":"2023-09-23T20:12:56.732663+0000","last_clean":"2023-09-23T20:12:56.732663+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:12:56.732663+0000","last_undegraded":"2023-09-23T20:12:56.732663+0000","last_fullsized":"2023-09-23T20:12:56.732663+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":390,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.720975+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:12:54.720975+0000","last_peered":"2023-09-23T20:12:54.720975+0000","last_clean":"2023-09-23T20:12:54.720975+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:12:54.720975+0000","last_undegraded":"2023-09-23T20:12:54.720975+0000","last_fullsized":"2023-09-23T20:12:54.720975+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":513,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.745248+0000","last_change":"2023-09-23T20:12:57.745248+0000","last_active":"2023-09-23T20:12:57.745248+0000","last_peered":"2023-09-23T20:12:57.745248+0000","last_clean":"2023-09-23T20:12:57.745248+0000","last_became_active":"2023-09-23T20:12:57.744840+0000","last_became_peered":"2023-09-23T20:12:57.744840+0000","last_unstale":"2023-09-23T20:12:57.745248+0000","last_undegraded":"2023-09-23T20:12:57.745248+0000","last_fullsized":"2023-09-23T20:12:57.745248+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.075018+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.075018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":49,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.726645+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:12:56.726645+0000","last_peered":"2023-09-23T20:12:56.726645+0000","last_clean":"2023-09-23T20:12:56.726645+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:12:56.726645+0000","last_undegraded":"2023-09-23T20:12:56.726645+0000","last_fullsized":"2023-09-23T20:12:56.726645+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":34,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.719928+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:12:54.719928+0000","last_peered":"2023-09-23T20:12:54.719928+0000","last_clean":"2023-09-23T20:12:54.719928+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:12:54.719928+0000","last_undegraded":"2023-09-23T20:12:54.719928+0000","last_fullsized":"2023-09-23T20:12:54.719928+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":636,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.739839+0000","last_change":"2023-09-23T20:12:57.739839+0000","last_active":"2023-09-23T20:12:57.739839+0000","last_peered":"2023-09-23T20:12:57.739839+0000","last_clean":"2023-09-23T20:12:57.739839+0000","last_became_active":"2023-09-23T20:12:57.739435+0000","last_became_peered":"2023-09-23T20:12:57.739435+0000","last_unstale":"2023-09-23T20:12:57.739839+0000","last_undegraded":"2023-09-23T20:12:57.739839+0000","last_fullsized":"2023-09-23T20:12:57.739839+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:12:52.744296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":97,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.734393+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:12:56.734393+0000","last_peered":"2023-09-23T20:12:56.734393+0000","last_clean":"2023-09-23T20:12:56.734393+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:12:56.734393+0000","last_undegraded":"2023-09-23T20:12:56.734393+0000","last_fullsized":"2023-09-23T20:12:56.734393+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":414,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.726951+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:12:56.726951+0000","last_peered":"2023-09-23T20:12:56.726951+0000","last_clean":"2023-09-23T20:12:56.726951+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:12:56.726951+0000","last_undegraded":"2023-09-23T20:12:56.726951+0000","last_fullsized":"2023-09-23T20:12:56.726951+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":729,"reported_epoch":404,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:12:55.746203+0000","last_change":"2023-09-23T20:12:55.746496+0000","last_active":"2023-09-23T20:12:55.746203+0000","last_peered":"2023-09-23T20:12:55.746203+0000","last_clean":"2023-09-23T20:12:51.474658+0000","last_became_active":"2023-09-23T20:12:55.737220+0000","last_became_peered":"2023-09-23T20:12:55.737220+0000","last_unstale":"2023-09-23T20:12:55.746203+0000","last_undegraded":"2023-09-23T20:12:55.746203+0000","last_fullsized":"2023-09-23T20:12:55.730854+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":546134,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0],"avail_no_missing":["3","0","4","5","7"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":98,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.733882+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:12:56.733882+0000","last_peered":"2023-09-23T20:12:56.733882+0000","last_clean":"2023-09-23T20:12:56.733882+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:12:56.733882+0000","last_undegraded":"2023-09-23T20:12:56.733882+0000","last_fullsized":"2023-09-23T20:12:56.733882+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":462,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727609+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:12:56.727609+0000","last_peered":"2023-09-23T20:12:56.727609+0000","last_clean":"2023-09-23T20:12:56.727609+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:12:56.727609+0000","last_undegraded":"2023-09-23T20:12:56.727609+0000","last_fullsized":"2023-09-23T20:12:56.727609+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":468,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.745239+0000","last_change":"2023-09-23T20:12:57.745239+0000","last_active":"2023-09-23T20:12:57.745239+0000","last_peered":"2023-09-23T20:12:57.745239+0000","last_clean":"2023-09-23T20:12:57.745239+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:12:57.745239+0000","last_undegraded":"2023-09-23T20:12:57.745239+0000","last_fullsized":"2023-09-23T20:12:57.745239+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":97,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.733114+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:12:56.733114+0000","last_peered":"2023-09-23T20:12:56.733114+0000","last_clean":"2023-09-23T20:12:56.733114+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:12:56.733114+0000","last_undegraded":"2023-09-23T20:12:56.733114+0000","last_fullsized":"2023-09-23T20:12:56.733114+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":111,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727151+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:12:56.727151+0000","last_peered":"2023-09-23T20:12:56.727151+0000","last_clean":"2023-09-23T20:12:56.727151+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:12:56.727151+0000","last_undegraded":"2023-09-23T20:12:56.727151+0000","last_fullsized":"2023-09-23T20:12:56.727151+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":634,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.740764+0000","last_change":"2023-09-23T20:12:57.740764+0000","last_active":"2023-09-23T20:12:57.740764+0000","last_peered":"2023-09-23T20:12:57.740764+0000","last_clean":"2023-09-23T20:12:57.740764+0000","last_became_active":"2023-09-23T20:12:57.740404+0000","last_became_peered":"2023-09-23T20:12:57.740404+0000","last_unstale":"2023-09-23T20:12:57.740764+0000","last_undegraded":"2023-09-23T20:12:57.740764+0000","last_fullsized":"2023-09-23T20:12:57.740764+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:05.339351+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":444,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.731814+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:12:56.731814+0000","last_peered":"2023-09-23T20:12:56.731814+0000","last_clean":"2023-09-23T20:12:56.731814+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:12:56.731814+0000","last_undegraded":"2023-09-23T20:12:56.731814+0000","last_fullsized":"2023-09-23T20:12:56.731814+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728430+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:12:56.728430+0000","last_peered":"2023-09-23T20:12:56.728430+0000","last_clean":"2023-09-23T20:12:56.728430+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:12:56.728430+0000","last_undegraded":"2023-09-23T20:12:56.728430+0000","last_fullsized":"2023-09-23T20:12:56.728430+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":788,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.732581+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:12:56.732581+0000","last_peered":"2023-09-23T20:12:56.732581+0000","last_clean":"2023-09-23T20:12:56.732581+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:12:56.732581+0000","last_undegraded":"2023-09-23T20:12:56.732581+0000","last_fullsized":"2023-09-23T20:12:56.732581+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":96,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.733859+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:12:56.733859+0000","last_peered":"2023-09-23T20:12:56.733859+0000","last_clean":"2023-09-23T20:12:56.733859+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:12:56.733859+0000","last_undegraded":"2023-09-23T20:12:56.733859+0000","last_fullsized":"2023-09-23T20:12:56.733859+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":432,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729252+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:12:56.729252+0000","last_peered":"2023-09-23T20:12:56.729252+0000","last_clean":"2023-09-23T20:12:56.729252+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:12:56.729252+0000","last_undegraded":"2023-09-23T20:12:56.729252+0000","last_fullsized":"2023-09-23T20:12:56.729252+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":753,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727105+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:12:56.727105+0000","last_peered":"2023-09-23T20:12:56.727105+0000","last_clean":"2023-09-23T20:12:56.727105+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:12:56.727105+0000","last_undegraded":"2023-09-23T20:12:56.727105+0000","last_fullsized":"2023-09-23T20:12:56.727105+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":372,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728453+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:12:56.728453+0000","last_peered":"2023-09-23T20:12:56.728453+0000","last_clean":"2023-09-23T20:12:56.728453+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:12:56.728453+0000","last_undegraded":"2023-09-23T20:12:56.728453+0000","last_fullsized":"2023-09-23T20:12:56.728453+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":402,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729698+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:12:56.729698+0000","last_peered":"2023-09-23T20:12:56.729698+0000","last_clean":"2023-09-23T20:12:56.729698+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:12:56.729698+0000","last_undegraded":"2023-09-23T20:12:56.729698+0000","last_fullsized":"2023-09-23T20:12:56.729698+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":549,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:12:57.745940+0000","last_change":"2023-09-23T20:12:57.745940+0000","last_active":"2023-09-23T20:12:57.745940+0000","last_peered":"2023-09-23T20:12:57.745940+0000","last_clean":"2023-09-23T20:12:57.745940+0000","last_became_active":"2023-09-23T20:12:57.745568+0000","last_became_peered":"2023-09-23T20:12:57.745568+0000","last_unstale":"2023-09-23T20:12:57.745940+0000","last_undegraded":"2023-09-23T20:12:57.745940+0000","last_fullsized":"2023-09-23T20:12:57.745940+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":396,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727262+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:12:56.727262+0000","last_peered":"2023-09-23T20:12:56.727262+0000","last_clean":"2023-09-23T20:12:56.727262+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:12:56.727262+0000","last_undegraded":"2023-09-23T20:12:56.727262+0000","last_fullsized":"2023-09-23T20:12:56.727262+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":432,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.734833+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:12:56.734833+0000","last_peered":"2023-09-23T20:12:56.734833+0000","last_clean":"2023-09-23T20:12:56.734833+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:12:56.734833+0000","last_undegraded":"2023-09-23T20:12:56.734833+0000","last_fullsized":"2023-09-23T20:12:56.734833+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":744,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729587+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:12:56.729587+0000","last_peered":"2023-09-23T20:12:56.729587+0000","last_clean":"2023-09-23T20:12:56.729587+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:12:56.729587+0000","last_undegraded":"2023-09-23T20:12:56.729587+0000","last_fullsized":"2023-09-23T20:12:56.729587+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":34,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.719406+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:12:54.719406+0000","last_peered":"2023-09-23T20:12:54.719406+0000","last_clean":"2023-09-23T20:12:54.719406+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:12:54.719406+0000","last_undegraded":"2023-09-23T20:12:54.719406+0000","last_fullsized":"2023-09-23T20:12:54.719406+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":97,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728641+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:12:56.728641+0000","last_peered":"2023-09-23T20:12:56.728641+0000","last_clean":"2023-09-23T20:12:56.728641+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:12:56.728641+0000","last_undegraded":"2023-09-23T20:12:56.728641+0000","last_fullsized":"2023-09-23T20:12:56.728641+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":672,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.720014+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:12:54.720014+0000","last_peered":"2023-09-23T20:12:54.720014+0000","last_clean":"2023-09-23T20:12:54.720014+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:12:54.720014+0000","last_undegraded":"2023-09-23T20:12:54.720014+0000","last_fullsized":"2023-09-23T20:12:54.720014+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":539,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727416+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:12:56.727416+0000","last_peered":"2023-09-23T20:12:56.727416+0000","last_clean":"2023-09-23T20:12:56.727416+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:12:56.727416+0000","last_undegraded":"2023-09-23T20:12:56.727416+0000","last_fullsized":"2023-09-23T20:12:56.727416+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":406,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728347+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:12:56.728347+0000","last_peered":"2023-09-23T20:12:56.728347+0000","last_clean":"2023-09-23T20:12:56.728347+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:12:56.728347+0000","last_undegraded":"2023-09-23T20:12:56.728347+0000","last_fullsized":"2023-09-23T20:12:56.728347+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.726988+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:12:56.726988+0000","last_peered":"2023-09-23T20:12:56.726988+0000","last_clean":"2023-09-23T20:12:56.726988+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:12:56.726988+0000","last_undegraded":"2023-09-23T20:12:56.726988+0000","last_fullsized":"2023-09-23T20:12:56.726988+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":749,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728242+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:12:56.728242+0000","last_peered":"2023-09-23T20:12:56.728242+0000","last_clean":"2023-09-23T20:12:56.728242+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:12:56.728242+0000","last_undegraded":"2023-09-23T20:12:56.728242+0000","last_fullsized":"2023-09-23T20:12:56.728242+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1026,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729363+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:12:56.729363+0000","last_peered":"2023-09-23T20:12:56.729363+0000","last_clean":"2023-09-23T20:12:56.729363+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:12:56.729363+0000","last_undegraded":"2023-09-23T20:12:56.729363+0000","last_fullsized":"2023-09-23T20:12:56.729363+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":425,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.734142+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:12:56.734142+0000","last_peered":"2023-09-23T20:12:56.734142+0000","last_clean":"2023-09-23T20:12:56.734142+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:12:56.734142+0000","last_undegraded":"2023-09-23T20:12:56.734142+0000","last_fullsized":"2023-09-23T20:12:56.734142+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727320+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:12:56.727320+0000","last_peered":"2023-09-23T20:12:56.727320+0000","last_clean":"2023-09-23T20:12:56.727320+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:12:56.727320+0000","last_undegraded":"2023-09-23T20:12:56.727320+0000","last_fullsized":"2023-09-23T20:12:56.727320+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":644,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.732928+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:12:56.732928+0000","last_peered":"2023-09-23T20:12:56.732928+0000","last_clean":"2023-09-23T20:12:56.732928+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:12:56.732928+0000","last_undegraded":"2023-09-23T20:12:56.732928+0000","last_fullsized":"2023-09-23T20:12:56.732928+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1040,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.721328+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:12:54.721328+0000","last_peered":"2023-09-23T20:12:54.721328+0000","last_clean":"2023-09-23T20:12:54.721328+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:12:54.721328+0000","last_undegraded":"2023-09-23T20:12:54.721328+0000","last_fullsized":"2023-09-23T20:12:54.721328+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727301+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:12:56.727301+0000","last_peered":"2023-09-23T20:12:56.727301+0000","last_clean":"2023-09-23T20:12:56.727301+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:12:56.727301+0000","last_undegraded":"2023-09-23T20:12:56.727301+0000","last_fullsized":"2023-09-23T20:12:56.727301+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":97,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727932+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:12:56.727932+0000","last_peered":"2023-09-23T20:12:56.727932+0000","last_clean":"2023-09-23T20:12:56.727932+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:12:56.727932+0000","last_undegraded":"2023-09-23T20:12:56.727932+0000","last_fullsized":"2023-09-23T20:12:56.727932+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1314,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727537+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:12:56.727537+0000","last_peered":"2023-09-23T20:12:56.727537+0000","last_clean":"2023-09-23T20:12:56.727537+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:12:56.727537+0000","last_undegraded":"2023-09-23T20:12:56.727537+0000","last_fullsized":"2023-09-23T20:12:56.727537+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":581,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.731619+0000","last_change":"2023-09-23T20:12:54.043697+0000","last_active":"2023-09-23T20:12:56.731619+0000","last_peered":"2023-09-23T20:12:56.731619+0000","last_clean":"2023-09-23T20:12:56.731619+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:12:56.731619+0000","last_undegraded":"2023-09-23T20:12:56.731619+0000","last_fullsized":"2023-09-23T20:12:56.731619+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:01.598813+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":271,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729501+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:12:56.729501+0000","last_peered":"2023-09-23T20:12:56.729501+0000","last_clean":"2023-09-23T20:12:56.729501+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:12:56.729501+0000","last_undegraded":"2023-09-23T20:12:56.729501+0000","last_fullsized":"2023-09-23T20:12:56.729501+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":301,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729996+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:12:56.729996+0000","last_peered":"2023-09-23T20:12:56.729996+0000","last_clean":"2023-09-23T20:12:56.729996+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:12:56.729996+0000","last_undegraded":"2023-09-23T20:12:56.729996+0000","last_fullsized":"2023-09-23T20:12:56.729996+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":544,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728514+0000","last_change":"2023-09-23T20:12:38.758800+0000","last_active":"2023-09-23T20:12:56.728514+0000","last_peered":"2023-09-23T20:12:56.728514+0000","last_clean":"2023-09-23T20:12:56.728514+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:12:56.728514+0000","last_undegraded":"2023-09-23T20:12:56.728514+0000","last_fullsized":"2023-09-23T20:12:56.728514+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:56.065108+0000","scrub_duration":0.001864159,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":842,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.731281+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:12:56.731281+0000","last_peered":"2023-09-23T20:12:56.731281+0000","last_clean":"2023-09-23T20:12:56.731281+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:12:56.731281+0000","last_undegraded":"2023-09-23T20:12:56.731281+0000","last_fullsized":"2023-09-23T20:12:56.731281+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":392,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727709+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:12:56.727709+0000","last_peered":"2023-09-23T20:12:56.727709+0000","last_clean":"2023-09-23T20:12:56.727709+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:12:56.727709+0000","last_undegraded":"2023-09-23T20:12:56.727709+0000","last_fullsized":"2023-09-23T20:12:56.727709+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727405+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:12:56.727405+0000","last_peered":"2023-09-23T20:12:56.727405+0000","last_clean":"2023-09-23T20:12:56.727405+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:12:56.727405+0000","last_undegraded":"2023-09-23T20:12:56.727405+0000","last_fullsized":"2023-09-23T20:12:56.727405+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":624,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.735108+0000","last_change":"2023-09-23T20:12:49.763399+0000","last_active":"2023-09-23T20:12:56.735108+0000","last_peered":"2023-09-23T20:12:56.735108+0000","last_clean":"2023-09-23T20:12:56.735108+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:12:56.735108+0000","last_undegraded":"2023-09-23T20:12:56.735108+0000","last_fullsized":"2023-09-23T20:12:56.735108+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:23.024903+0000","scrub_duration":0.001589575,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":106,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.734004+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:12:56.734004+0000","last_peered":"2023-09-23T20:12:56.734004+0000","last_clean":"2023-09-23T20:12:56.734004+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:12:56.734004+0000","last_undegraded":"2023-09-23T20:12:56.734004+0000","last_fullsized":"2023-09-23T20:12:56.734004+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727240+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:12:56.727240+0000","last_peered":"2023-09-23T20:12:56.727240+0000","last_clean":"2023-09-23T20:12:56.727240+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:12:56.727240+0000","last_undegraded":"2023-09-23T20:12:56.727240+0000","last_fullsized":"2023-09-23T20:12:56.727240+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":232,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.731052+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:12:56.731052+0000","last_peered":"2023-09-23T20:12:56.731052+0000","last_clean":"2023-09-23T20:12:56.731052+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:12:56.731052+0000","last_undegraded":"2023-09-23T20:12:56.731052+0000","last_fullsized":"2023-09-23T20:12:56.731052+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":852,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.729582+0000","last_change":"2023-09-23T20:12:53.737343+0000","last_active":"2023-09-23T20:12:56.729582+0000","last_peered":"2023-09-23T20:12:56.729582+0000","last_clean":"2023-09-23T20:12:56.729582+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:12:56.729582+0000","last_undegraded":"2023-09-23T20:12:56.729582+0000","last_fullsized":"2023-09-23T20:12:56.729582+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.611543+0000","scrub_duration":0.0015738379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":248,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.734111+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:12:56.734111+0000","last_peered":"2023-09-23T20:12:56.734111+0000","last_clean":"2023-09-23T20:12:56.734111+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:12:56.734111+0000","last_undegraded":"2023-09-23T20:12:56.734111+0000","last_fullsized":"2023-09-23T20:12:56.734111+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":444,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727384+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:12:56.727384+0000","last_peered":"2023-09-23T20:12:56.727384+0000","last_clean":"2023-09-23T20:12:56.727384+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:12:56.727384+0000","last_undegraded":"2023-09-23T20:12:56.727384+0000","last_fullsized":"2023-09-23T20:12:56.727384+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":34,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.719359+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:12:54.719359+0000","last_peered":"2023-09-23T20:12:54.719359+0000","last_clean":"2023-09-23T20:12:54.719359+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:12:54.719359+0000","last_undegraded":"2023-09-23T20:12:54.719359+0000","last_fullsized":"2023-09-23T20:12:54.719359+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":716,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.721948+0000","last_change":"2023-09-23T20:12:39.118887+0000","last_active":"2023-09-23T20:12:54.721948+0000","last_peered":"2023-09-23T20:12:54.721948+0000","last_clean":"2023-09-23T20:12:54.721948+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:12:54.721948+0000","last_undegraded":"2023-09-23T20:12:54.721948+0000","last_fullsized":"2023-09-23T20:12:54.721948+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":107,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.734327+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:12:56.734327+0000","last_peered":"2023-09-23T20:12:56.734327+0000","last_clean":"2023-09-23T20:12:56.734327+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:12:56.734327+0000","last_undegraded":"2023-09-23T20:12:56.734327+0000","last_fullsized":"2023-09-23T20:12:56.734327+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":106,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727321+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:12:56.727321+0000","last_peered":"2023-09-23T20:12:56.727321+0000","last_clean":"2023-09-23T20:12:56.727321+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:12:56.727321+0000","last_undegraded":"2023-09-23T20:12:56.727321+0000","last_fullsized":"2023-09-23T20:12:56.727321+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":365,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.727436+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:12:56.727436+0000","last_peered":"2023-09-23T20:12:56.727436+0000","last_clean":"2023-09-23T20:12:56.727436+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:12:56.727436+0000","last_undegraded":"2023-09-23T20:12:56.727436+0000","last_fullsized":"2023-09-23T20:12:56.727436+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":179,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.721013+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:12:54.721013+0000","last_peered":"2023-09-23T20:12:54.721013+0000","last_clean":"2023-09-23T20:12:54.721013+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:12:54.721013+0000","last_undegraded":"2023-09-23T20:12:54.721013+0000","last_fullsized":"2023-09-23T20:12:54.721013+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":170,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.722301+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:12:54.722301+0000","last_peered":"2023-09-23T20:12:54.722301+0000","last_clean":"2023-09-23T20:12:54.722301+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:12:54.722301+0000","last_undegraded":"2023-09-23T20:12:54.722301+0000","last_fullsized":"2023-09-23T20:12:54.722301+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":409,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.731751+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:12:56.731751+0000","last_peered":"2023-09-23T20:12:56.731751+0000","last_clean":"2023-09-23T20:12:56.731751+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:12:56.731751+0000","last_undegraded":"2023-09-23T20:12:56.731751+0000","last_fullsized":"2023-09-23T20:12:56.731751+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":422,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.734220+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:12:56.734220+0000","last_peered":"2023-09-23T20:12:56.734220+0000","last_clean":"2023-09-23T20:12:56.734220+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:12:56.734220+0000","last_undegraded":"2023-09-23T20:12:56.734220+0000","last_fullsized":"2023-09-23T20:12:56.734220+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":399,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728267+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:12:56.728267+0000","last_peered":"2023-09-23T20:12:56.728267+0000","last_clean":"2023-09-23T20:12:56.728267+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:12:56.728267+0000","last_undegraded":"2023-09-23T20:12:56.728267+0000","last_fullsized":"2023-09-23T20:12:56.728267+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":402,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728356+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:12:56.728356+0000","last_peered":"2023-09-23T20:12:56.728356+0000","last_clean":"2023-09-23T20:12:56.728356+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:12:56.728356+0000","last_undegraded":"2023-09-23T20:12:56.728356+0000","last_fullsized":"2023-09-23T20:12:56.728356+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":105,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.720593+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:12:54.720593+0000","last_peered":"2023-09-23T20:12:54.720593+0000","last_clean":"2023-09-23T20:12:54.720593+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:12:54.720593+0000","last_undegraded":"2023-09-23T20:12:54.720593+0000","last_fullsized":"2023-09-23T20:12:54.720593+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":261,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T20:12:54.721407+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:12:54.721407+0000","last_peered":"2023-09-23T20:12:54.721407+0000","last_clean":"2023-09-23T20:12:54.721407+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:12:54.721407+0000","last_undegraded":"2023-09-23T20:12:54.721407+0000","last_fullsized":"2023-09-23T20:12:54.721407+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":251,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.730795+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:12:56.730795+0000","last_peered":"2023-09-23T20:12:56.730795+0000","last_clean":"2023-09-23T20:12:56.730795+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:12:56.730795+0000","last_undegraded":"2023-09-23T20:12:56.730795+0000","last_fullsized":"2023-09-23T20:12:56.730795+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":452,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T20:12:56.728955+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:12:56.728955+0000","last_peered":"2023-09-23T20:12:56.728955+0000","last_clean":"2023-09-23T20:12:56.728955+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:12:56.728955+0000","last_undegraded":"2023-09-23T20:12:56.728955+0000","last_fullsized":"2023-09-23T20:12:56.728955+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":706653,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":97,"num_read_kb":181,"num_write":48,"num_write_kb":477,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1331895,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":61,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234382,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":375200,"kb_used_data":8344,"kb_used_omap":0,"kb_used_meta":366848,"kb_avail":93369952,"statfs":{"total":95995035648,"available":95610830848,"internally_reserved":0,"allocated":8544256,"data_stored":7267790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":375652352},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.0019999999999998,"15min":2.2599999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":6.3630000000000004,"5min":55.390000000000001,"15min":62.061999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.0960000000000001,"15min":2.3090000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.19900000000000001},"max":{"1min":6.0049999999999999,"5min":56.664000000000001,"15min":62.389000000000003},"last":0.71299999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":8.8119999999999994,"5min":3.4590000000000001,"15min":3.8889999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":91.335999999999999,"5min":91.335999999999999,"15min":104.16800000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":11.324,"5min":4.0149999999999997,"15min":4.1639999999999997},"min":{"1min":0.29399999999999998,"5min":0.249,"15min":0.223},"max":{"1min":91.516999999999996,"5min":91.516999999999996,"15min":104.003},"last":0.56799999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":9.4710000000000001,"5min":5.3049999999999997,"15min":4.2329999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":67.760000000000005,"5min":92.319000000000003,"15min":105.842},"last":0.78200000000000003},{"interface":"front","average":{"1min":10.063000000000001,"5min":5.7400000000000002,"15min":4.7290000000000001},"min":{"1min":0.27700000000000002,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":67.488,"5min":92.049000000000007,"15min":105.66},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.5950000000000002,"15min":4.4379999999999997},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":8.2720000000000002,"5min":94.953000000000003,"15min":146.38800000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.4370000000000001,"5min":2.444,"15min":4.375},"min":{"1min":0.29699999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.5369999999999999,"5min":93.423000000000002,"15min":146.851},"last":0.503}]},{"osd":4,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.48999999999999999,"15min":0.71899999999999997},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.22500000000000001},"max":{"1min":3.2069999999999999,"5min":3.2069999999999999,"15min":19.215},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.54600000000000004,"15min":0.83299999999999996},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.88400000000000001,"5min":2.7949999999999999,"15min":41.822000000000003},"last":0.73599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.63700000000000001,"15min":0.97299999999999998},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":3.407,"5min":4.4080000000000004,"15min":35.488},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.628,"15min":0.85699999999999998},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":3.2549999999999999,"5min":4.2030000000000003,"15min":34.896000000000001},"last":0.69699999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.70299999999999996,"15min":0.83399999999999996},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":3.4849999999999999,"5min":3.4849999999999999,"15min":41.902999999999999},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.55800000000000005,"15min":0.64000000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30399999999999999},"max":{"1min":2.3500000000000001,"5min":2.4590000000000001,"15min":6.3620000000000001},"last":0.751}]}]},{"osd":6,"up_from":0,"seq":1078036791422,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762581,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603048,"kb_used_data":9056,"kb_used_omap":7,"kb_used_meta":593976,"kb_avail":93142104,"statfs":{"total":95995035648,"available":95377514496,"internally_reserved":0,"allocated":9273344,"data_stored":7346782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":608232250},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8660000000000001,"5min":2.5230000000000001,"15min":3.2349999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.16900000000000001},"max":{"1min":11.930999999999999,"5min":28.510000000000002,"15min":144.273},"last":0.5},{"interface":"front","average":{"1min":2.6230000000000002,"5min":2.5409999999999999,"15min":3.468},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.19400000000000001},"max":{"1min":16.870999999999999,"5min":16.870999999999999,"15min":144.321},"last":0.317}]},{"osd":2,"last update":"Sat Sep 23 20:12:09 2023","interfaces":[{"interface":"back","average":{"1min":3.9369999999999998,"5min":4.1559999999999997,"15min":4.8419999999999996},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":38.040999999999997,"5min":58.499000000000002,"15min":134.72499999999999},"last":0.376},{"interface":"front","average":{"1min":3.3700000000000001,"5min":3.6269999999999998,"15min":4.952},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":38.173999999999999,"5min":58.741,"15min":135.74600000000001},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.8159999999999998,"15min":3.786},"min":{"1min":0.255,"5min":0.255,"15min":0.22},"max":{"1min":17.212,"5min":25.963999999999999,"15min":68.180999999999997},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.5720000000000001,"5min":2.7000000000000002,"15min":3.7450000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.247},"max":{"1min":12.723000000000001,"5min":40.716000000000001,"15min":66.509},"last":0.67200000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7170000000000001,"15min":1.998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":11.698,"5min":18.087,"15min":60.951000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.3839999999999999,"5min":1.837,"15min":2.1429999999999998},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":11.08,"5min":14.890000000000001,"15min":60.841000000000001},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":5.2670000000000003,"5min":2.621,"15min":2.5459999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":83.510999999999996,"5min":83.510999999999996,"15min":83.510999999999996},"last":0.627},{"interface":"front","average":{"1min":5.6289999999999996,"5min":3.121,"15min":2.6360000000000001},"min":{"1min":0.40200000000000002,"5min":0.28899999999999998,"15min":0.28399999999999997},"max":{"1min":83.353999999999999,"5min":83.353999999999999,"15min":83.353999999999999},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":1.8959999999999999,"15min":2.0009999999999999},"min":{"1min":0.42299999999999999,"5min":0.38500000000000001,"15min":0.35899999999999999},"max":{"1min":11.868,"5min":18.117999999999999,"15min":18.117999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.8260000000000001,"5min":1.6799999999999999,"15min":2.5499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":11.109,"5min":11.109,"15min":20.734999999999999},"last":0.70699999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":2.359,"15min":2.2869999999999999},"min":{"1min":0.434,"5min":0.33300000000000002,"15min":0.30399999999999999},"max":{"1min":12.823,"5min":25.937000000000001,"15min":60.213000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":2.0070000000000001,"5min":1.8979999999999999,"15min":2.1030000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.30199999999999999},"max":{"1min":11.747,"5min":18.053999999999998,"15min":61.298999999999999},"last":0.56100000000000005}]}]},{"osd":0,"up_from":94,"seq":403726926118,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":749772,"kb_used_data":123396,"kb_used_omap":8,"kb_used_meta":626359,"kb_avail":92995380,"statfs":{"total":95995035648,"available":95227269120,"internally_reserved":0,"allocated":126357504,"data_stored":124450315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":641392225},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":3.0899999999999999,"15min":3.8639999999999999},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":6.5199999999999996,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.314},{"interface":"front","average":{"1min":2.4540000000000002,"5min":3.6890000000000001,"15min":4.1440000000000001},"min":{"1min":0.218,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":14.513999999999999,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.24399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":3.8780000000000001,"15min":5.0019999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19800000000000001},"max":{"1min":19.486999999999998,"5min":102.65300000000001,"15min":270.51100000000002},"last":0.35299999999999998},{"interface":"front","average":{"1min":3.0049999999999999,"5min":4.0129999999999999,"15min":4.8470000000000004},"min":{"1min":0.25,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.693999999999999,"5min":96.578999999999994,"15min":269.69299999999998},"last":0.28699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":2.8220000000000001,"15min":5.6210000000000004},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":10.082000000000001,"5min":32.204999999999998,"15min":145.72800000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.8680000000000001,"5min":3.1120000000000001,"15min":5.6890000000000001},"min":{"1min":0.29999999999999999,"5min":0.27800000000000002,"15min":0.23999999999999999},"max":{"1min":9.3970000000000002,"5min":33.457000000000001,"15min":145.50700000000001},"last":0.29999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.631,"15min":2.2469999999999999},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":9.6959999999999997,"5min":15.688000000000001,"15min":148.20599999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.998,"5min":1.629,"15min":2.0270000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27600000000000002},"max":{"1min":8.4429999999999996,"5min":13.428000000000001,"15min":148.11000000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.341,"15min":1.9410000000000001},"min":{"1min":0.374,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":3.7480000000000002,"5min":9.5719999999999992,"15min":64.725999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.742,"15min":1.988},"min":{"1min":0.36499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.7479999999999993,"5min":15.717000000000001,"15min":27.609999999999999},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.5589999999999999,"15min":1.6319999999999999},"min":{"1min":0.51400000000000001,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":6.0839999999999996,"5min":15.744999999999999,"15min":15.744999999999999},"last":0.628},{"interface":"front","average":{"1min":1.675,"5min":1.633,"15min":1.8260000000000001},"min":{"1min":0.437,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":8.3919999999999995,"5min":13.470000000000001,"15min":13.766999999999999},"last":0.53600000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.429,"15min":1.891},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":5.1929999999999996,"5min":9.6069999999999993,"15min":65.081999999999994},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.2589999999999999,"5min":1.867,"15min":2.0089999999999999},"min":{"1min":0.41199999999999998,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.0510000000000002,"5min":15.771000000000001,"15min":15.771000000000001},"last":0.57299999999999995}]}]},{"osd":2,"up_from":104,"seq":446676599056,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":750036,"kb_used_data":5196,"kb_used_omap":12,"kb_used_meta":744819,"kb_avail":92995116,"statfs":{"total":95995035648,"available":95226998784,"internally_reserved":0,"allocated":5320704,"data_stored":3395278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":762695678},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":25,"apply_latency_ms":25,"commit_latency_ns":25000000,"apply_latency_ns":25000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":2.2160000000000002,"15min":3.0030000000000001},"min":{"1min":0.26200000000000001,"5min":0.19900000000000001,"15min":0.16900000000000001},"max":{"1min":14.794,"5min":17.876999999999999,"15min":76.251999999999995},"last":0.252},{"interface":"front","average":{"1min":1.339,"5min":2.4159999999999999,"15min":3.0339999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.192},"max":{"1min":5.077,"5min":18.561,"15min":76.590000000000003},"last":0.33300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.8220000000000001,"15min":3.6280000000000001},"min":{"1min":0.25,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":32.677999999999997,"5min":32.677999999999997,"15min":94.768000000000001},"last":0.27400000000000002},{"interface":"front","average":{"1min":3.7010000000000001,"5min":2.4820000000000002,"15min":3.577},"min":{"1min":0.30599999999999999,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":32.582000000000001,"5min":32.582000000000001,"15min":96.808000000000007},"last":0.315}]},{"osd":3,"last update":"Sat Sep 23 20:12:36 2023","interfaces":[{"interface":"back","average":{"1min":3.395,"5min":2.5600000000000001,"15min":4.2320000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":14.131,"5min":14.131,"15min":107.381},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.5259999999999998,"5min":2.25,"15min":3.508},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23699999999999999},"max":{"1min":14.778,"5min":20.308,"15min":106.301},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.698,"15min":2.0790000000000002},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.26900000000000002},"max":{"1min":14.757999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.7090000000000001,"15min":2.1259999999999999},"min":{"1min":0.34899999999999998,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":13.387,"5min":13.387,"15min":76.161000000000001},"last":0.53600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":4.258,"5min":2.1099999999999999,"15min":2.6899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":29.433,"5min":29.433,"15min":172.505},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.6040000000000001,"5min":2.1920000000000002,"15min":2.8500000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.29099999999999998},"max":{"1min":29.196999999999999,"5min":29.196999999999999,"15min":172.375},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:12:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.6799999999999999,"15min":1.641},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":14.664,"5min":14.664,"15min":14.664},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.214,"5min":1.8640000000000001,"15min":1.9259999999999999},"min":{"1min":0.42099999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":11.081,"5min":14.279999999999999,"15min":14.279999999999999},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.837,"15min":1.829},"min":{"1min":0.44,"5min":0.377,"15min":0.30199999999999999},"max":{"1min":13.481999999999999,"5min":13.481999999999999,"15min":23.079999999999998},"last":0.60099999999999998},{"interface":"front","average":{"1min":2.4159999999999999,"5min":1.869,"15min":1.9139999999999999},"min":{"1min":0.39900000000000002,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":14.698,"5min":14.698,"15min":20.614999999999998},"last":0.57299999999999995}]}]},{"osd":3,"up_from":109,"seq":468151435530,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1054684,"kb_used_data":372244,"kb_used_omap":4,"kb_used_meta":682427,"kb_avail":92690468,"statfs":{"total":95995035648,"available":94915039232,"internally_reserved":0,"allocated":381177856,"data_stored":379069212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":698805357},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":4.1029999999999998,"15min":3.4489999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":4.9340000000000002,"5min":74.323999999999998,"15min":74.323999999999998},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.9690000000000001,"5min":4.0149999999999997,"15min":3.7639999999999998},"min":{"1min":0.26500000000000001,"5min":0.20100000000000001,"15min":0.19600000000000001},"max":{"1min":10.484,"5min":74.447000000000003,"15min":74.447000000000003},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":4.7510000000000003,"15min":4.1319999999999997},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":5.5949999999999998,"5min":163.49700000000001,"15min":163.49700000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":2.6779999999999999,"5min":5.258,"15min":4.4859999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":9.9760000000000009,"5min":163.512,"15min":163.512},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1629999999999998,"5min":6.6520000000000001,"15min":4.351},"min":{"1min":0.27400000000000002,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":14.9,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.252,"5min":6.0380000000000003,"15min":4.0140000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":4.2439999999999998,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.35499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.202,"15min":3.1429999999999998},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.6259999999999999,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.714,"5min":2.8420000000000001,"15min":3.5510000000000002},"min":{"1min":0.308,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":7.5759999999999996,"5min":58.408000000000001,"15min":110.295},"last":0.66000000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.6160000000000001,"15min":3.395},"min":{"1min":0.40300000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":7.6200000000000001,"5min":58.506,"15min":113.881},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.321,"5min":2.8919999999999999,"15min":3.2959999999999998},"min":{"1min":0.36299999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":4.274,"5min":58.643000000000001,"15min":114.598},"last":0.67100000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":2.625,"15min":3.6059999999999999},"min":{"1min":0.41399999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":4.3120000000000003,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.88700000000000001,"5min":2.2389999999999999,"15min":3.7599999999999998},"min":{"1min":0.376,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":3.6720000000000002,"5min":58.701000000000001,"15min":82.233000000000004},"last":0.69199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":2.9359999999999999,"15min":3.516},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.29599999999999999},"max":{"1min":4.335,"5min":58.686,"15min":138.035},"last":0.70399999999999996},{"interface":"front","average":{"1min":1.52,"5min":2.6989999999999998,"15min":3.5070000000000001},"min":{"1min":0.41899999999999998,"5min":0.38400000000000001,"15min":0.32600000000000001},"max":{"1min":4.125,"5min":58.667999999999999,"15min":138.054},"last":0.57899999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019292,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":702044,"kb_used_data":348756,"kb_used_omap":0,"kb_used_meta":353280,"kb_avail":93043108,"statfs":{"total":95995035648,"available":95276142592,"internally_reserved":0,"allocated":357126144,"data_stored":355853762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":361758720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:21 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.0509999999999999,"15min":2.202},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.189},"max":{"1min":3.6150000000000002,"5min":11.743,"15min":89.004999999999995},"last":0.436},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.085,"15min":2.1659999999999999},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.218},"max":{"1min":5.3250000000000002,"5min":11.776,"15min":89.024000000000001},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":7.7149999999999999,"5min":4.2800000000000002,"15min":5.9630000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":138.47999999999999,"5min":138.47999999999999,"15min":161.613},"last":0.48799999999999999},{"interface":"front","average":{"1min":7.806,"5min":4.548,"15min":6.0780000000000003},"min":{"1min":0.247,"5min":0.217,"15min":0.217},"max":{"1min":138.429,"5min":138.429,"15min":162.017},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":8.6579999999999995,"15min":6.8689999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":144.916,"5min":182.96700000000001,"15min":182.96700000000001},"last":0.378},{"interface":"front","average":{"1min":11.116,"5min":8.2240000000000002,"15min":6.6360000000000001},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":144.83099999999999,"5min":183.244,"15min":183.244},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":3.4319999999999999,"15min":6.5419999999999998},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":4.4340000000000002,"5min":184.37100000000001,"15min":266.54700000000003},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.0329999999999999,"5min":3.6309999999999998,"15min":5.4809999999999999},"min":{"1min":0.36199999999999999,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":11.144,"5min":184.22300000000001,"15min":184.22300000000001},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:12:21 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48799999999999999,"15min":2.117},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":2.8919999999999999,"5min":3.3290000000000002,"15min":266.59899999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.45800000000000002,"15min":2.1419999999999999},"min":{"1min":0.29999999999999999,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":1.2230000000000001,"5min":1.722,"15min":266.72899999999998},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.50900000000000001,"15min":0.51400000000000001},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.23499999999999999},"max":{"1min":2.9350000000000001,"5min":2.9350000000000001,"15min":4.0410000000000004},"last":0.501},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.51100000000000001,"15min":0.58899999999999997},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.247},"max":{"1min":0.81100000000000005,"5min":1.6240000000000001,"15min":4.2889999999999997},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":1.244,"15min":1.411},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":0.59699999999999998,"5min":74.040000000000006,"15min":76.507000000000005},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":1.25,"15min":1.536},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":3.2890000000000001,"5min":74.021000000000001,"15min":76.462000000000003},"last":0.52100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757656,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640276,"kb_used_data":237004,"kb_used_omap":0,"kb_used_meta":403264,"kb_avail":93104876,"statfs":{"total":95995035648,"available":95339393024,"internally_reserved":0,"allocated":242692096,"data_stored":241424155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":412942336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.82099999999999995,"15min":1.7290000000000001},"min":{"1min":0.26100000000000001,"5min":0.224,"15min":0.216},"max":{"1min":2.173,"5min":8.6739999999999995,"15min":121.94799999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.103,"15min":1.9970000000000001},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":9.4250000000000007,"5min":10.327,"15min":121.964},"last":0.63700000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.2480000000000002,"15min":5.9630000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":20.337,"5min":75.204999999999998,"15min":266.197},"last":0.497},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.2330000000000001,"15min":6.1399999999999997},"min":{"1min":0.28199999999999997,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":23.497,"5min":75.231999999999999,"15min":266.14999999999998},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":5.6639999999999997,"15min":5.4379999999999997},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":4.1239999999999997,"5min":240.57300000000001,"15min":240.57300000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.2509999999999999,"5min":5.7359999999999998,"15min":5.4969999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.313},"max":{"1min":4.1559999999999997,"5min":240.464,"15min":240.464},"last":0.65900000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.631,"15min":4.8849999999999998},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.608000000000001,"5min":12.608000000000001,"15min":201.50899999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.0820000000000001,"15min":4.6719999999999997},"min":{"1min":0.313,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":1.9299999999999999,"5min":7.3739999999999997,"15min":201.58099999999999},"last":0.69499999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:12:16 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.48599999999999999,"15min":1.4119999999999999},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":2.5790000000000002,"5min":4.0510000000000002,"15min":191.78899999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.436,"15min":1.3129999999999999},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":0.59499999999999997,"5min":1.4359999999999999,"15min":191.821},"last":0.78900000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.52200000000000002,"15min":0.63100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":1.7410000000000001,"5min":3.278,"15min":9.9009999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.47299999999999998,"15min":0.51900000000000002},"min":{"1min":0.315,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.841,"5min":1.841,"15min":5.0119999999999996},"last":0.74299999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.56699999999999995,"15min":1.4930000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.3069999999999999,"5min":4.2080000000000002,"15min":191.84299999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65900000000000003,"15min":1.655},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":3.1440000000000001,"5min":9.6920000000000002,"15min":191.69},"last":0.77100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:13:00.198 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:13:00.214 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:00.268 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:00 smithi144 ceph-mon[194833]: from='client.36147 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:00.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:00.425 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:00 smithi135 ceph-mon[221113]: from='client.36147 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:00.425 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:00 smithi135 ceph-mon[213871]: from='client.36147 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:00.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:00.526 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:13:00.526 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:00.562 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:00.622 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:00.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:00.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:00.727 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:00.759 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:01.205 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_4' created 2023-09-23T20:13:01.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:01 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4288062397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:13:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:01 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:13:01.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:01 smithi135 ceph-mon[221113]: pgmap v1141: 93 pgs: 1 active+undersized+remapped, 92 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:13:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:01 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4288062397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:13:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:01 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:13:01.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:01 smithi135 ceph-mon[213871]: pgmap v1141: 93 pgs: 1 active+undersized+remapped, 92 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:13:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:01 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4288062397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:13:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:01 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:13:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:01 smithi144 ceph-mon[194833]: pgmap v1141: 93 pgs: 1 active+undersized+remapped, 92 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:13:02.308 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T20:13:02.317 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:02.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:02.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[221113]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:13:02.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[221113]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T20:13:02.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[221113]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[221113]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[213871]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:13:02.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[213871]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T20:13:02.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[213871]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.346 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:02 smithi135 ceph-mon[213871]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.394 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:02.421 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:02.446 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:02 smithi144 ceph-mon[194833]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.446 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:02 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:13:02.446 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:02 smithi144 ceph-mon[194833]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T20:13:02.447 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:02 smithi144 ceph-mon[194833]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.447 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:02 smithi144 ceph-mon[194833]: failed to encode map e407 with expected crc 2023-09-23T20:13:02.447 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:02.493 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:02.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:02.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:02.601 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:02.630 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:02.667 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:02.679 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:02.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:02.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:02.744 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:13:02.744 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:13:02.785 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:02.822 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:03.147 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:13:03.453 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[221113]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[221113]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[221113]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[221113]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[221113]: pgmap v1144: 109 pgs: 16 unknown, 93 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[213871]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[213871]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.454 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[213871]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[213871]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:03 smithi135 ceph-mon[213871]: pgmap v1144: 109 pgs: 16 unknown, 93 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:03 smithi144 ceph-mon[194833]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T20:13:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:03 smithi144 ceph-mon[194833]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:03 smithi144 ceph-mon[194833]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:03 smithi144 ceph-mon[194833]: failed to encode map e408 with expected crc 2023-09-23T20:13:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:03 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:03 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:03 smithi144 ceph-mon[194833]: pgmap v1144: 109 pgs: 16 unknown, 93 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:03.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:13:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:13:03] "GET /metrics HTTP/1.1" 200 34118 "" "Prometheus/2.43.0" 2023-09-23T20:13:03.999 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:13:04.278 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:04 smithi135 ceph-mon[221113]: failed to encode map e408 with expected crc 2023-09-23T20:13:04.279 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:04 smithi135 ceph-mon[221113]: failed to encode map e409 with expected crc 2023-09-23T20:13:04.279 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:04 smithi135 ceph-mon[221113]: failed to encode map e409 with expected crc 2023-09-23T20:13:04.279 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:04 smithi135 ceph-mon[221113]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T20:13:04.280 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:04 smithi135 ceph-mon[213871]: failed to encode map e408 with expected crc 2023-09-23T20:13:04.280 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:04 smithi135 ceph-mon[213871]: failed to encode map e409 with expected crc 2023-09-23T20:13:04.280 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:04 smithi135 ceph-mon[213871]: failed to encode map e409 with expected crc 2023-09-23T20:13:04.280 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:04 smithi135 ceph-mon[213871]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T20:13:04.332 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:04.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:04.436 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:04.460 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:04.465 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:04 smithi144 ceph-mon[194833]: failed to encode map e408 with expected crc 2023-09-23T20:13:04.466 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:04 smithi144 ceph-mon[194833]: failed to encode map e409 with expected crc 2023-09-23T20:13:04.466 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:04 smithi144 ceph-mon[194833]: failed to encode map e409 with expected crc 2023-09-23T20:13:04.466 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:04 smithi144 ceph-mon[194833]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T20:13:04.470 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:04.520 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:04.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:04.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:04.625 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:04.654 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:04.699 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:04.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:04.805 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:04.834 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:05.230 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:05.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:05.331 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:05.334 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:05.368 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:05.443 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:13:05.462 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:13:05.464 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:05.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:05 smithi135 ceph-mon[221113]: failed to encode map e409 with expected crc 2023-09-23T20:13:05.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:05 smithi135 ceph-mon[221113]: failed to encode map e409 with expected crc 2023-09-23T20:13:05.466 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:05 smithi135 ceph-mon[221113]: pgmap v1146: 109 pgs: 2 unknown, 107 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:05.466 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:05 smithi135 ceph-mon[213871]: failed to encode map e409 with expected crc 2023-09-23T20:13:05.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:05 smithi135 ceph-mon[213871]: failed to encode map e409 with expected crc 2023-09-23T20:13:05.467 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:05 smithi135 ceph-mon[213871]: pgmap v1146: 109 pgs: 2 unknown, 107 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:05.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:05.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:05.481 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:13:05.500 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:13:05.520 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:13:05.539 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:13:05.558 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:13:05.569 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:05.577 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:13:05.592 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:13:05.592 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:13:05.592 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:13:05.592 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:13:05.592 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:13:05.592 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:13:05.593 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:13:05.593 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:13:05.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:05 smithi144 ceph-mon[194833]: failed to encode map e409 with expected crc 2023-09-23T20:13:05.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:05 smithi144 ceph-mon[194833]: failed to encode map e409 with expected crc 2023-09-23T20:13:05.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:05 smithi144 ceph-mon[194833]: pgmap v1146: 109 pgs: 2 unknown, 107 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:05.604 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:05.655 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:05.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:05.761 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:05.790 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:05.833 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:05.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:05.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:05.876 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:05.938 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:05.966 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:06.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/607230915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:06.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/607230915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:06.519 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:06.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:06.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:06.624 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/607230915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:06.651 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:07.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:07 smithi144 ceph-mon[194833]: pgmap v1147: 109 pgs: 109 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:07.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:07.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:07.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:07 smithi135 ceph-mon[221113]: pgmap v1147: 109 pgs: 109 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:07.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:07.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:07 smithi135 ceph-mon[213871]: pgmap v1147: 109 pgs: 109 active+clean; 4.7 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:07.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:07.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:07.968 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:07.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:07.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:08.072 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:08.103 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:08.471 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:08.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:08.575 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:08.607 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:08.655 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:08.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:08.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:08.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:08 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2647801144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:08.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:08 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:08.712 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:08 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2647801144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:08.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:08 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:08.720 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:13:08.763 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:08.794 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:08 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2647801144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:08.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:08 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:09.261 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:13:09.262 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:13:09.262 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:13:09.262 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:13:09.262 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:13:09.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:09.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: failed to encode map e410 with expected crc 2023-09-23T20:13:09.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:09.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: failed to encode map e410 with expected crc 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: failed to encode map e410 with expected crc 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: failed to encode map e410 with expected crc 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: pgmap v1149: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: failed to encode map e410 with expected crc 2023-09-23T20:13:09.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: failed to encode map e410 with expected crc 2023-09-23T20:13:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T20:13:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: pgmap v1149: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:09.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:09 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: failed to encode map e410 with expected crc 2023-09-23T20:13:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: failed to encode map e410 with expected crc 2023-09-23T20:13:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: failed to encode map e410 with expected crc 2023-09-23T20:13:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T20:13:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: pgmap v1149: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:10.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:10.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:09 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:10.361 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:10.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:10.466 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:10.499 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:10.548 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:10.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:10.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:10.593 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:13:10.657 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:10.666 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:13:10.695 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:10.817 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:10.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:10.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:10.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:10 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:10.922 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:10.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:10 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:10.949 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:10 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:10.952 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:11.004 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:11.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:11.108 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:11.135 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:11.159 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:11.181 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:11.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:11.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:11.209 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:11.281 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:13:11.283 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:13:11.286 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:11.318 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:11.366 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:11.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:11.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:11.471 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:11.499 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:11.548 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:11.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:11.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:11.653 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:11.682 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:11.738 INFO:teuthology.orchestra.run.smithi135.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T20:13:11.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:11.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:11.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:11 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3729086551' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:13:11.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:11 smithi135 ceph-mon[221113]: pgmap v1150: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:11.938 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:11.938 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:11.938 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:11 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3729086551' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:13:11.939 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:11 smithi135 ceph-mon[213871]: pgmap v1150: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:12.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:12.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:12.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:11 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3729086551' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:13:12.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:11 smithi144 ceph-mon[194833]: pgmap v1150: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:12.620 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:13:12.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[221113]: from='client.46052 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:12.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[221113]: failed to encode map e411 with expected crc 2023-09-23T20:13:12.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[221113]: failed to encode map e411 with expected crc 2023-09-23T20:13:12.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3729086551' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:13:12.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[221113]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T20:13:12.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:12.929 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[213871]: from='client.46052 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:12.929 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[213871]: failed to encode map e411 with expected crc 2023-09-23T20:13:12.929 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[213871]: failed to encode map e411 with expected crc 2023-09-23T20:13:12.929 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3729086551' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:13:12.929 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[213871]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T20:13:12.929 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:12 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:12.979 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:12.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:12.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:13.084 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:13.111 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:13.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:12 smithi144 ceph-mon[194833]: from='client.46052 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:13.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:12 smithi144 ceph-mon[194833]: failed to encode map e411 with expected crc 2023-09-23T20:13:13.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:12 smithi144 ceph-mon[194833]: failed to encode map e411 with expected crc 2023-09-23T20:13:13.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3729086551' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:13:13.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:12 smithi144 ceph-mon[194833]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T20:13:13.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:12 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:13.157 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:13.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:13.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:13.170 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1150,"stamp":"2023-09-23T20:13:10.997355+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4906480,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":6077,"num_read_kb":13388,"num_write":955,"num_write_kb":13202,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":212,"num_bytes_recovered":9736176,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4415,"ondisk_log_size":4415,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4745436,"kb_used_data":969892,"kb_used_omap":32,"kb_used_meta":3775455,"kb_avail":651470628,"statfs":{"total":671965249536,"available":667105923072,"internally_reserved":0,"allocated":993169408,"data_stored":981414680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":3866066438},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002041"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":407,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735870+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:13:02.735870+0000","last_peered":"2023-09-23T20:13:02.735870+0000","last_clean":"2023-09-23T20:13:02.735870+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:13:02.735870+0000","last_undegraded":"2023-09-23T20:13:02.735870+0000","last_fullsized":"2023-09-23T20:13:02.735870+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":238,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:03.768250+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:13:03.768250+0000","last_peered":"2023-09-23T20:13:03.768250+0000","last_clean":"2023-09-23T20:13:03.768250+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:13:03.768250+0000","last_undegraded":"2023-09-23T20:13:03.768250+0000","last_fullsized":"2023-09-23T20:13:03.768250+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":105,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.206836+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:13:02.206836+0000","last_peered":"2023-09-23T20:13:02.206836+0000","last_clean":"2023-09-23T20:13:02.206836+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:13:02.206836+0000","last_undegraded":"2023-09-23T20:13:02.206836+0000","last_fullsized":"2023-09-23T20:13:02.206836+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":117,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210408+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:13:02.210408+0000","last_peered":"2023-09-23T20:13:02.210408+0000","last_clean":"2023-09-23T20:13:02.210408+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:13:02.210408+0000","last_undegraded":"2023-09-23T20:13:02.210408+0000","last_fullsized":"2023-09-23T20:13:02.210408+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":238,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.206666+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:13:02.206666+0000","last_peered":"2023-09-23T20:13:02.206666+0000","last_clean":"2023-09-23T20:13:02.206666+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:13:02.206666+0000","last_undegraded":"2023-09-23T20:13:02.206666+0000","last_fullsized":"2023-09-23T20:13:02.206666+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":354,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208667+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:13:02.208667+0000","last_peered":"2023-09-23T20:13:02.208667+0000","last_clean":"2023-09-23T20:13:02.208667+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:13:02.208667+0000","last_undegraded":"2023-09-23T20:13:02.208667+0000","last_fullsized":"2023-09-23T20:13:02.208667+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":335,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208556+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:13:02.208556+0000","last_peered":"2023-09-23T20:13:02.208556+0000","last_clean":"2023-09-23T20:13:02.208556+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:13:02.208556+0000","last_undegraded":"2023-09-23T20:13:02.208556+0000","last_fullsized":"2023-09-23T20:13:02.208556+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":252,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.203274+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:13:02.203274+0000","last_peered":"2023-09-23T20:13:02.203274+0000","last_clean":"2023-09-23T20:13:02.203274+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:13:02.203274+0000","last_undegraded":"2023-09-23T20:13:02.203274+0000","last_fullsized":"2023-09-23T20:13:02.203274+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":730,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:04.094773+0000","last_change":"2023-09-23T20:12:57.740154+0000","last_active":"2023-09-23T20:13:04.094773+0000","last_peered":"2023-09-23T20:13:04.094773+0000","last_clean":"2023-09-23T20:13:04.094773+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:13:04.094773+0000","last_undegraded":"2023-09-23T20:13:04.094773+0000","last_fullsized":"2023-09-23T20:13:04.094773+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":408,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208202+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:13:02.208202+0000","last_peered":"2023-09-23T20:13:02.208202+0000","last_clean":"2023-09-23T20:13:02.208202+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:13:02.208202+0000","last_undegraded":"2023-09-23T20:13:02.208202+0000","last_fullsized":"2023-09-23T20:13:02.208202+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":271,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.204857+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:13:02.204857+0000","last_peered":"2023-09-23T20:13:02.204857+0000","last_clean":"2023-09-23T20:13:02.204857+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:13:02.204857+0000","last_undegraded":"2023-09-23T20:13:02.204857+0000","last_fullsized":"2023-09-23T20:13:02.204857+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":102,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210319+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:13:02.210319+0000","last_peered":"2023-09-23T20:13:02.210319+0000","last_clean":"2023-09-23T20:13:02.210319+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:13:02.210319+0000","last_undegraded":"2023-09-23T20:13:02.210319+0000","last_fullsized":"2023-09-23T20:13:02.210319+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207129+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:13:02.207129+0000","last_peered":"2023-09-23T20:13:02.207129+0000","last_clean":"2023-09-23T20:13:02.207129+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:13:02.207129+0000","last_undegraded":"2023-09-23T20:13:02.207129+0000","last_fullsized":"2023-09-23T20:13:02.207129+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":463,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736147+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:13:02.736147+0000","last_peered":"2023-09-23T20:13:02.736147+0000","last_clean":"2023-09-23T20:13:02.736147+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:13:02.736147+0000","last_undegraded":"2023-09-23T20:13:02.736147+0000","last_fullsized":"2023-09-23T20:13:02.736147+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":405,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736102+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:13:02.736102+0000","last_peered":"2023-09-23T20:13:02.736102+0000","last_clean":"2023-09-23T20:13:02.736102+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:13:02.736102+0000","last_undegraded":"2023-09-23T20:13:02.736102+0000","last_fullsized":"2023-09-23T20:13:02.736102+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":758,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207650+0000","last_change":"2023-09-23T20:12:57.745067+0000","last_active":"2023-09-23T20:13:02.207650+0000","last_peered":"2023-09-23T20:13:02.207650+0000","last_clean":"2023-09-23T20:13:02.207650+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:13:02.207650+0000","last_undegraded":"2023-09-23T20:13:02.207650+0000","last_fullsized":"2023-09-23T20:13:02.207650+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:57.794739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:57.794739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051716449999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":102,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210174+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:13:02.210174+0000","last_peered":"2023-09-23T20:13:02.210174+0000","last_clean":"2023-09-23T20:13:02.210174+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:13:02.210174+0000","last_undegraded":"2023-09-23T20:13:02.210174+0000","last_fullsized":"2023-09-23T20:13:02.210174+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":262,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207122+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:13:02.207122+0000","last_peered":"2023-09-23T20:13:02.207122+0000","last_clean":"2023-09-23T20:13:02.207122+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:13:02.207122+0000","last_undegraded":"2023-09-23T20:13:02.207122+0000","last_fullsized":"2023-09-23T20:13:02.207122+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":447,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208546+0000","last_change":"2023-09-23T20:12:57.746858+0000","last_active":"2023-09-23T20:13:02.208546+0000","last_peered":"2023-09-23T20:13:02.208546+0000","last_clean":"2023-09-23T20:13:02.208546+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:13:02.208546+0000","last_undegraded":"2023-09-23T20:13:02.208546+0000","last_fullsized":"2023-09-23T20:13:02.208546+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.079134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.079134+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":117,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.209733+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:13:02.209733+0000","last_peered":"2023-09-23T20:13:02.209733+0000","last_clean":"2023-09-23T20:13:02.209733+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:13:02.209733+0000","last_undegraded":"2023-09-23T20:13:02.209733+0000","last_fullsized":"2023-09-23T20:13:02.209733+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":407,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208454+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:13:02.208454+0000","last_peered":"2023-09-23T20:13:02.208454+0000","last_clean":"2023-09-23T20:13:02.208454+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:13:02.208454+0000","last_undegraded":"2023-09-23T20:13:02.208454+0000","last_fullsized":"2023-09-23T20:13:02.208454+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":552,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.206935+0000","last_change":"2023-09-23T20:12:57.739426+0000","last_active":"2023-09-23T20:13:02.206935+0000","last_peered":"2023-09-23T20:13:02.206935+0000","last_clean":"2023-09-23T20:13:02.206935+0000","last_became_active":"2023-09-23T20:12:57.738891+0000","last_became_peered":"2023-09-23T20:12:57.738891+0000","last_unstale":"2023-09-23T20:13:02.206935+0000","last_undegraded":"2023-09-23T20:13:02.206935+0000","last_fullsized":"2023-09-23T20:13:02.206935+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":96,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.209882+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:13:02.209882+0000","last_peered":"2023-09-23T20:13:02.209882+0000","last_clean":"2023-09-23T20:13:02.209882+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:13:02.209882+0000","last_undegraded":"2023-09-23T20:13:02.209882+0000","last_fullsized":"2023-09-23T20:13:02.209882+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":395,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.205215+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:13:02.205215+0000","last_peered":"2023-09-23T20:13:02.205215+0000","last_clean":"2023-09-23T20:13:02.205215+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:13:02.205215+0000","last_undegraded":"2023-09-23T20:13:02.205215+0000","last_fullsized":"2023-09-23T20:13:02.205215+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":516,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207956+0000","last_change":"2023-09-23T20:12:57.745248+0000","last_active":"2023-09-23T20:13:02.207956+0000","last_peered":"2023-09-23T20:13:02.207956+0000","last_clean":"2023-09-23T20:13:02.207956+0000","last_became_active":"2023-09-23T20:12:57.744840+0000","last_became_peered":"2023-09-23T20:12:57.744840+0000","last_unstale":"2023-09-23T20:13:02.207956+0000","last_undegraded":"2023-09-23T20:13:02.207956+0000","last_fullsized":"2023-09-23T20:13:02.207956+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.075018+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.075018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":52,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208258+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:13:02.208258+0000","last_peered":"2023-09-23T20:13:02.208258+0000","last_clean":"2023-09-23T20:13:02.208258+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:13:02.208258+0000","last_undegraded":"2023-09-23T20:13:02.208258+0000","last_fullsized":"2023-09-23T20:13:02.208258+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":39,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.203949+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:13:02.203949+0000","last_peered":"2023-09-23T20:13:02.203949+0000","last_clean":"2023-09-23T20:13:02.203949+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:13:02.203949+0000","last_undegraded":"2023-09-23T20:13:02.203949+0000","last_fullsized":"2023-09-23T20:13:02.203949+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.795467+0000","last_change":"2023-09-23T20:13:02.794951+0000","last_active":"2023-09-23T20:13:02.795467+0000","last_peered":"2023-09-23T20:13:02.795467+0000","last_clean":"2023-09-23T20:13:02.795467+0000","last_became_active":"2023-09-23T20:13:02.794601+0000","last_became_peered":"2023-09-23T20:13:02.794601+0000","last_unstale":"2023-09-23T20:13:02.795467+0000","last_undegraded":"2023-09-23T20:13:02.795467+0000","last_fullsized":"2023-09-23T20:13:02.795467+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:25.583752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":639,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210040+0000","last_change":"2023-09-23T20:12:57.739839+0000","last_active":"2023-09-23T20:13:02.210040+0000","last_peered":"2023-09-23T20:13:02.210040+0000","last_clean":"2023-09-23T20:13:02.210040+0000","last_became_active":"2023-09-23T20:12:57.739435+0000","last_became_peered":"2023-09-23T20:12:57.739435+0000","last_unstale":"2023-09-23T20:13:02.210040+0000","last_undegraded":"2023-09-23T20:13:02.210040+0000","last_fullsized":"2023-09-23T20:13:02.210040+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:12:52.744296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":100,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210029+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:13:02.210029+0000","last_peered":"2023-09-23T20:13:02.210029+0000","last_clean":"2023-09-23T20:13:02.210029+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:13:02.210029+0000","last_undegraded":"2023-09-23T20:13:02.210029+0000","last_fullsized":"2023-09-23T20:13:02.210029+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":417,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208339+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:13:02.208339+0000","last_peered":"2023-09-23T20:13:02.208339+0000","last_clean":"2023-09-23T20:13:02.208339+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:13:02.208339+0000","last_undegraded":"2023-09-23T20:13:02.208339+0000","last_fullsized":"2023-09-23T20:13:02.208339+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":11,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:03.768359+0000","last_change":"2023-09-23T20:13:02.737552+0000","last_active":"2023-09-23T20:13:03.768359+0000","last_peered":"2023-09-23T20:13:03.768359+0000","last_clean":"2023-09-23T20:13:03.768359+0000","last_became_active":"2023-09-23T20:13:02.736984+0000","last_became_peered":"2023-09-23T20:13:02.736984+0000","last_unstale":"2023-09-23T20:13:03.768359+0000","last_undegraded":"2023-09-23T20:13:03.768359+0000","last_fullsized":"2023-09-23T20:13:03.768359+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:21.010081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":743,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.204254+0000","last_change":"2023-09-23T20:12:57.740105+0000","last_active":"2023-09-23T20:13:02.204254+0000","last_peered":"2023-09-23T20:13:02.204254+0000","last_clean":"2023-09-23T20:13:02.204254+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:13:02.204254+0000","last_undegraded":"2023-09-23T20:13:02.204254+0000","last_fullsized":"2023-09-23T20:13:02.204254+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":546134,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":101,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210456+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:13:02.210456+0000","last_peered":"2023-09-23T20:13:02.210456+0000","last_clean":"2023-09-23T20:13:02.210456+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:13:02.210456+0000","last_undegraded":"2023-09-23T20:13:02.210456+0000","last_fullsized":"2023-09-23T20:13:02.210456+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":465,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735754+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:13:02.735754+0000","last_peered":"2023-09-23T20:13:02.735754+0000","last_clean":"2023-09-23T20:13:02.735754+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:13:02.735754+0000","last_undegraded":"2023-09-23T20:13:02.735754+0000","last_fullsized":"2023-09-23T20:13:02.735754+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.213080+0000","last_change":"2023-09-23T20:13:02.212695+0000","last_active":"2023-09-23T20:13:02.213080+0000","last_peered":"2023-09-23T20:13:02.213080+0000","last_clean":"2023-09-23T20:13:02.213080+0000","last_became_active":"2023-09-23T20:13:02.212400+0000","last_became_peered":"2023-09-23T20:13:02.212400+0000","last_unstale":"2023-09-23T20:13:02.213080+0000","last_undegraded":"2023-09-23T20:13:02.213080+0000","last_fullsized":"2023-09-23T20:13:02.213080+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:13.554058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":471,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208448+0000","last_change":"2023-09-23T20:12:57.745239+0000","last_active":"2023-09-23T20:13:02.208448+0000","last_peered":"2023-09-23T20:13:02.208448+0000","last_clean":"2023-09-23T20:13:02.208448+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:13:02.208448+0000","last_undegraded":"2023-09-23T20:13:02.208448+0000","last_fullsized":"2023-09-23T20:13:02.208448+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":100,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210189+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:13:02.210189+0000","last_peered":"2023-09-23T20:13:02.210189+0000","last_clean":"2023-09-23T20:13:02.210189+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:13:02.210189+0000","last_undegraded":"2023-09-23T20:13:02.210189+0000","last_fullsized":"2023-09-23T20:13:02.210189+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":114,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207207+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:13:02.207207+0000","last_peered":"2023-09-23T20:13:02.207207+0000","last_clean":"2023-09-23T20:13:02.207207+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:13:02.207207+0000","last_undegraded":"2023-09-23T20:13:02.207207+0000","last_fullsized":"2023-09-23T20:13:02.207207+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.212571+0000","last_change":"2023-09-23T20:13:02.211796+0000","last_active":"2023-09-23T20:13:02.212571+0000","last_peered":"2023-09-23T20:13:02.212571+0000","last_clean":"2023-09-23T20:13:02.212571+0000","last_became_active":"2023-09-23T20:13:02.211407+0000","last_became_peered":"2023-09-23T20:13:02.211407+0000","last_unstale":"2023-09-23T20:13:02.212571+0000","last_undegraded":"2023-09-23T20:13:02.212571+0000","last_fullsized":"2023-09-23T20:13:02.212571+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:11.120676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":637,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735991+0000","last_change":"2023-09-23T20:12:57.740764+0000","last_active":"2023-09-23T20:13:02.735991+0000","last_peered":"2023-09-23T20:13:02.735991+0000","last_clean":"2023-09-23T20:13:02.735991+0000","last_became_active":"2023-09-23T20:12:57.740404+0000","last_became_peered":"2023-09-23T20:12:57.740404+0000","last_unstale":"2023-09-23T20:13:02.735991+0000","last_undegraded":"2023-09-23T20:13:02.735991+0000","last_fullsized":"2023-09-23T20:13:02.735991+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:05.339351+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":447,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208121+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:13:02.208121+0000","last_peered":"2023-09-23T20:13:02.208121+0000","last_clean":"2023-09-23T20:13:02.208121+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:13:02.208121+0000","last_undegraded":"2023-09-23T20:13:02.208121+0000","last_fullsized":"2023-09-23T20:13:02.208121+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":105,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207131+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:13:02.207131+0000","last_peered":"2023-09-23T20:13:02.207131+0000","last_clean":"2023-09-23T20:13:02.207131+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:13:02.207131+0000","last_undegraded":"2023-09-23T20:13:02.207131+0000","last_fullsized":"2023-09-23T20:13:02.207131+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.737378+0000","last_change":"2023-09-23T20:13:02.737123+0000","last_active":"2023-09-23T20:13:02.737378+0000","last_peered":"2023-09-23T20:13:02.737378+0000","last_clean":"2023-09-23T20:13:02.737378+0000","last_became_active":"2023-09-23T20:13:02.736852+0000","last_became_peered":"2023-09-23T20:13:02.736852+0000","last_unstale":"2023-09-23T20:13:02.737378+0000","last_undegraded":"2023-09-23T20:13:02.737378+0000","last_fullsized":"2023-09-23T20:13:02.737378+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:02.273532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":791,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.209753+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:13:02.209753+0000","last_peered":"2023-09-23T20:13:02.209753+0000","last_clean":"2023-09-23T20:13:02.209753+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:13:02.209753+0000","last_undegraded":"2023-09-23T20:13:02.209753+0000","last_fullsized":"2023-09-23T20:13:02.209753+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":99,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736019+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:13:02.736019+0000","last_peered":"2023-09-23T20:13:02.736019+0000","last_clean":"2023-09-23T20:13:02.736019+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:13:02.736019+0000","last_undegraded":"2023-09-23T20:13:02.736019+0000","last_fullsized":"2023-09-23T20:13:02.736019+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":435,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736228+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:13:02.736228+0000","last_peered":"2023-09-23T20:13:02.736228+0000","last_clean":"2023-09-23T20:13:02.736228+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:13:02.736228+0000","last_undegraded":"2023-09-23T20:13:02.736228+0000","last_fullsized":"2023-09-23T20:13:02.736228+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":11,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:03.767817+0000","last_change":"2023-09-23T20:13:02.794915+0000","last_active":"2023-09-23T20:13:03.767817+0000","last_peered":"2023-09-23T20:13:03.767817+0000","last_clean":"2023-09-23T20:13:03.767817+0000","last_became_active":"2023-09-23T20:13:02.794591+0000","last_became_peered":"2023-09-23T20:13:02.794591+0000","last_unstale":"2023-09-23T20:13:03.767817+0000","last_undegraded":"2023-09-23T20:13:03.767817+0000","last_fullsized":"2023-09-23T20:13:03.767817+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:28.806237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":756,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208444+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:13:02.208444+0000","last_peered":"2023-09-23T20:13:02.208444+0000","last_clean":"2023-09-23T20:13:02.208444+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:13:02.208444+0000","last_undegraded":"2023-09-23T20:13:02.208444+0000","last_fullsized":"2023-09-23T20:13:02.208444+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":375,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735777+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:13:02.735777+0000","last_peered":"2023-09-23T20:13:02.735777+0000","last_clean":"2023-09-23T20:13:02.735777+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:13:02.735777+0000","last_undegraded":"2023-09-23T20:13:02.735777+0000","last_fullsized":"2023-09-23T20:13:02.735777+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":405,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736491+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:13:02.736491+0000","last_peered":"2023-09-23T20:13:02.736491+0000","last_clean":"2023-09-23T20:13:02.736491+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:13:02.736491+0000","last_undegraded":"2023-09-23T20:13:02.736491+0000","last_fullsized":"2023-09-23T20:13:02.736491+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.212778+0000","last_change":"2023-09-23T20:13:02.212538+0000","last_active":"2023-09-23T20:13:02.212778+0000","last_peered":"2023-09-23T20:13:02.212778+0000","last_clean":"2023-09-23T20:13:02.212778+0000","last_became_active":"2023-09-23T20:13:02.212305+0000","last_became_peered":"2023-09-23T20:13:02.212305+0000","last_unstale":"2023-09-23T20:13:02.212778+0000","last_undegraded":"2023-09-23T20:13:02.212778+0000","last_fullsized":"2023-09-23T20:13:02.212778+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:10.419377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":552,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.206940+0000","last_change":"2023-09-23T20:12:57.745940+0000","last_active":"2023-09-23T20:13:02.206940+0000","last_peered":"2023-09-23T20:13:02.206940+0000","last_clean":"2023-09-23T20:13:02.206940+0000","last_became_active":"2023-09-23T20:12:57.745568+0000","last_became_peered":"2023-09-23T20:12:57.745568+0000","last_unstale":"2023-09-23T20:13:02.206940+0000","last_undegraded":"2023-09-23T20:13:02.206940+0000","last_fullsized":"2023-09-23T20:13:02.206940+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:06:43.228148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:06:43.228148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":399,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735603+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:13:02.735603+0000","last_peered":"2023-09-23T20:13:02.735603+0000","last_clean":"2023-09-23T20:13:02.735603+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:13:02.735603+0000","last_undegraded":"2023-09-23T20:13:02.735603+0000","last_fullsized":"2023-09-23T20:13:02.735603+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":435,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736492+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:13:02.736492+0000","last_peered":"2023-09-23T20:13:02.736492+0000","last_clean":"2023-09-23T20:13:02.736492+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:13:02.736492+0000","last_undegraded":"2023-09-23T20:13:02.736492+0000","last_fullsized":"2023-09-23T20:13:02.736492+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":11,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:03.215407+0000","last_change":"2023-09-23T20:13:02.209574+0000","last_active":"2023-09-23T20:13:03.215407+0000","last_peered":"2023-09-23T20:13:03.215407+0000","last_clean":"2023-09-23T20:13:03.215407+0000","last_became_active":"2023-09-23T20:13:02.209307+0000","last_became_peered":"2023-09-23T20:13:02.209307+0000","last_unstale":"2023-09-23T20:13:03.215407+0000","last_undegraded":"2023-09-23T20:13:03.215407+0000","last_fullsized":"2023-09-23T20:13:03.215407+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:18.271180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":747,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208253+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:13:02.208253+0000","last_peered":"2023-09-23T20:13:02.208253+0000","last_clean":"2023-09-23T20:13:02.208253+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:13:02.208253+0000","last_undegraded":"2023-09-23T20:13:02.208253+0000","last_fullsized":"2023-09-23T20:13:02.208253+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":39,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.203029+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:13:02.203029+0000","last_peered":"2023-09-23T20:13:02.203029+0000","last_clean":"2023-09-23T20:13:02.203029+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:13:02.203029+0000","last_undegraded":"2023-09-23T20:13:02.203029+0000","last_fullsized":"2023-09-23T20:13:02.203029+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207602+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:13:02.207602+0000","last_peered":"2023-09-23T20:13:02.207602+0000","last_clean":"2023-09-23T20:13:02.207602+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:13:02.207602+0000","last_undegraded":"2023-09-23T20:13:02.207602+0000","last_fullsized":"2023-09-23T20:13:02.207602+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.212189+0000","last_change":"2023-09-23T20:13:02.211701+0000","last_active":"2023-09-23T20:13:02.212189+0000","last_peered":"2023-09-23T20:13:02.212189+0000","last_clean":"2023-09-23T20:13:02.212189+0000","last_became_active":"2023-09-23T20:13:02.211349+0000","last_became_peered":"2023-09-23T20:13:02.211349+0000","last_unstale":"2023-09-23T20:13:02.212189+0000","last_undegraded":"2023-09-23T20:13:02.212189+0000","last_fullsized":"2023-09-23T20:13:02.212189+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:35.001450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":677,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.204039+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:13:02.204039+0000","last_peered":"2023-09-23T20:13:02.204039+0000","last_clean":"2023-09-23T20:13:02.204039+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:13:02.204039+0000","last_undegraded":"2023-09-23T20:13:02.204039+0000","last_fullsized":"2023-09-23T20:13:02.204039+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":542,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207464+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:13:02.207464+0000","last_peered":"2023-09-23T20:13:02.207464+0000","last_clean":"2023-09-23T20:13:02.207464+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:13:02.207464+0000","last_undegraded":"2023-09-23T20:13:02.207464+0000","last_fullsized":"2023-09-23T20:13:02.207464+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":409,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208055+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:13:02.208055+0000","last_peered":"2023-09-23T20:13:02.208055+0000","last_clean":"2023-09-23T20:13:02.208055+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:13:02.208055+0000","last_undegraded":"2023-09-23T20:13:02.208055+0000","last_fullsized":"2023-09-23T20:13:02.208055+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207042+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:13:02.207042+0000","last_peered":"2023-09-23T20:13:02.207042+0000","last_clean":"2023-09-23T20:13:02.207042+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:13:02.207042+0000","last_undegraded":"2023-09-23T20:13:02.207042+0000","last_fullsized":"2023-09-23T20:13:02.207042+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.796453+0000","last_change":"2023-09-23T20:13:02.796250+0000","last_active":"2023-09-23T20:13:02.796453+0000","last_peered":"2023-09-23T20:13:02.796453+0000","last_clean":"2023-09-23T20:13:02.796453+0000","last_became_active":"2023-09-23T20:13:02.795989+0000","last_became_peered":"2023-09-23T20:13:02.795989+0000","last_unstale":"2023-09-23T20:13:02.796453+0000","last_undegraded":"2023-09-23T20:13:02.796453+0000","last_fullsized":"2023-09-23T20:13:02.796453+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:49.452140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":752,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207776+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:13:02.207776+0000","last_peered":"2023-09-23T20:13:02.207776+0000","last_clean":"2023-09-23T20:13:02.207776+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:13:02.207776+0000","last_undegraded":"2023-09-23T20:13:02.207776+0000","last_fullsized":"2023-09-23T20:13:02.207776+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1029,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208156+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:13:02.208156+0000","last_peered":"2023-09-23T20:13:02.208156+0000","last_clean":"2023-09-23T20:13:02.208156+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:13:02.208156+0000","last_undegraded":"2023-09-23T20:13:02.208156+0000","last_fullsized":"2023-09-23T20:13:02.208156+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":428,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736261+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:13:02.736261+0000","last_peered":"2023-09-23T20:13:02.736261+0000","last_clean":"2023-09-23T20:13:02.736261+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:13:02.736261+0000","last_undegraded":"2023-09-23T20:13:02.736261+0000","last_fullsized":"2023-09-23T20:13:02.736261+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":105,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207483+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:13:02.207483+0000","last_peered":"2023-09-23T20:13:02.207483+0000","last_clean":"2023-09-23T20:13:02.207483+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:13:02.207483+0000","last_undegraded":"2023-09-23T20:13:02.207483+0000","last_fullsized":"2023-09-23T20:13:02.207483+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.213071+0000","last_change":"2023-09-23T20:13:02.212686+0000","last_active":"2023-09-23T20:13:02.213071+0000","last_peered":"2023-09-23T20:13:02.213071+0000","last_clean":"2023-09-23T20:13:02.213071+0000","last_became_active":"2023-09-23T20:13:02.212438+0000","last_became_peered":"2023-09-23T20:13:02.212438+0000","last_unstale":"2023-09-23T20:13:02.213071+0000","last_undegraded":"2023-09-23T20:13:02.213071+0000","last_fullsized":"2023-09-23T20:13:02.213071+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:09.919399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":647,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210718+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:13:02.210718+0000","last_peered":"2023-09-23T20:13:02.210718+0000","last_clean":"2023-09-23T20:13:02.210718+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:13:02.210718+0000","last_undegraded":"2023-09-23T20:13:02.210718+0000","last_fullsized":"2023-09-23T20:13:02.210718+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1046,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:04.093947+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:13:04.093947+0000","last_peered":"2023-09-23T20:13:04.093947+0000","last_clean":"2023-09-23T20:13:04.093947+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:13:04.093947+0000","last_undegraded":"2023-09-23T20:13:04.093947+0000","last_fullsized":"2023-09-23T20:13:04.093947+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.206823+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:13:02.206823+0000","last_peered":"2023-09-23T20:13:02.206823+0000","last_clean":"2023-09-23T20:13:02.206823+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:13:02.206823+0000","last_undegraded":"2023-09-23T20:13:02.206823+0000","last_fullsized":"2023-09-23T20:13:02.206823+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":100,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207549+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:13:02.207549+0000","last_peered":"2023-09-23T20:13:02.207549+0000","last_clean":"2023-09-23T20:13:02.207549+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:13:02.207549+0000","last_undegraded":"2023-09-23T20:13:02.207549+0000","last_fullsized":"2023-09-23T20:13:02.207549+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.795487+0000","last_change":"2023-09-23T20:13:02.794985+0000","last_active":"2023-09-23T20:13:02.795487+0000","last_peered":"2023-09-23T20:13:02.795487+0000","last_clean":"2023-09-23T20:13:02.795487+0000","last_became_active":"2023-09-23T20:13:02.794652+0000","last_became_peered":"2023-09-23T20:13:02.794652+0000","last_unstale":"2023-09-23T20:13:02.795487+0000","last_undegraded":"2023-09-23T20:13:02.795487+0000","last_fullsized":"2023-09-23T20:13:02.795487+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:32.459920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1317,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208293+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:13:02.208293+0000","last_peered":"2023-09-23T20:13:02.208293+0000","last_clean":"2023-09-23T20:13:02.208293+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:13:02.208293+0000","last_undegraded":"2023-09-23T20:13:02.208293+0000","last_fullsized":"2023-09-23T20:13:02.208293+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":584,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.209482+0000","last_change":"2023-09-23T20:12:54.043697+0000","last_active":"2023-09-23T20:13:02.209482+0000","last_peered":"2023-09-23T20:13:02.209482+0000","last_clean":"2023-09-23T20:13:02.209482+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:13:02.209482+0000","last_undegraded":"2023-09-23T20:13:02.209482+0000","last_fullsized":"2023-09-23T20:13:02.209482+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:01.598813+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":274,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208151+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:13:02.208151+0000","last_peered":"2023-09-23T20:13:02.208151+0000","last_clean":"2023-09-23T20:13:02.208151+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:13:02.208151+0000","last_undegraded":"2023-09-23T20:13:02.208151+0000","last_fullsized":"2023-09-23T20:13:02.208151+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":304,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736655+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:13:02.736655+0000","last_peered":"2023-09-23T20:13:02.736655+0000","last_clean":"2023-09-23T20:13:02.736655+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:13:02.736655+0000","last_undegraded":"2023-09-23T20:13:02.736655+0000","last_fullsized":"2023-09-23T20:13:02.736655+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.795474+0000","last_change":"2023-09-23T20:13:02.795014+0000","last_active":"2023-09-23T20:13:02.795474+0000","last_peered":"2023-09-23T20:13:02.795474+0000","last_clean":"2023-09-23T20:13:02.795474+0000","last_became_active":"2023-09-23T20:13:02.794587+0000","last_became_peered":"2023-09-23T20:13:02.794587+0000","last_unstale":"2023-09-23T20:13:02.795474+0000","last_undegraded":"2023-09-23T20:13:02.795474+0000","last_fullsized":"2023-09-23T20:13:02.795474+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:08.535861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":547,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207294+0000","last_change":"2023-09-23T20:12:38.758800+0000","last_active":"2023-09-23T20:13:02.207294+0000","last_peered":"2023-09-23T20:13:02.207294+0000","last_clean":"2023-09-23T20:13:02.207294+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:13:02.207294+0000","last_undegraded":"2023-09-23T20:13:02.207294+0000","last_fullsized":"2023-09-23T20:13:02.207294+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:56.065108+0000","scrub_duration":0.001864159,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":845,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.209339+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:13:02.209339+0000","last_peered":"2023-09-23T20:13:02.209339+0000","last_clean":"2023-09-23T20:13:02.209339+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:13:02.209339+0000","last_undegraded":"2023-09-23T20:13:02.209339+0000","last_fullsized":"2023-09-23T20:13:02.209339+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":395,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735776+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:13:02.735776+0000","last_peered":"2023-09-23T20:13:02.735776+0000","last_clean":"2023-09-23T20:13:02.735776+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:13:02.735776+0000","last_undegraded":"2023-09-23T20:13:02.735776+0000","last_fullsized":"2023-09-23T20:13:02.735776+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":105,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207475+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:13:02.207475+0000","last_peered":"2023-09-23T20:13:02.207475+0000","last_clean":"2023-09-23T20:13:02.207475+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:13:02.207475+0000","last_undegraded":"2023-09-23T20:13:02.207475+0000","last_fullsized":"2023-09-23T20:13:02.207475+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736745+0000","last_change":"2023-09-23T20:13:02.736405+0000","last_active":"2023-09-23T20:13:02.736745+0000","last_peered":"2023-09-23T20:13:02.736745+0000","last_clean":"2023-09-23T20:13:02.736745+0000","last_became_active":"2023-09-23T20:13:02.736146+0000","last_became_peered":"2023-09-23T20:13:02.736146+0000","last_unstale":"2023-09-23T20:13:02.736745+0000","last_undegraded":"2023-09-23T20:13:02.736745+0000","last_fullsized":"2023-09-23T20:13:02.736745+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:41.788802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":627,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736705+0000","last_change":"2023-09-23T20:12:49.763399+0000","last_active":"2023-09-23T20:13:02.736705+0000","last_peered":"2023-09-23T20:13:02.736705+0000","last_clean":"2023-09-23T20:13:02.736705+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:13:02.736705+0000","last_undegraded":"2023-09-23T20:13:02.736705+0000","last_fullsized":"2023-09-23T20:13:02.736705+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:23.024903+0000","scrub_duration":0.001589575,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":109,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.210240+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:13:02.210240+0000","last_peered":"2023-09-23T20:13:02.210240+0000","last_clean":"2023-09-23T20:13:02.210240+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:13:02.210240+0000","last_undegraded":"2023-09-23T20:13:02.210240+0000","last_fullsized":"2023-09-23T20:13:02.210240+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":105,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207385+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:13:02.207385+0000","last_peered":"2023-09-23T20:13:02.207385+0000","last_clean":"2023-09-23T20:13:02.207385+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:13:02.207385+0000","last_undegraded":"2023-09-23T20:13:02.207385+0000","last_fullsized":"2023-09-23T20:13:02.207385+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":235,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208031+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:13:02.208031+0000","last_peered":"2023-09-23T20:13:02.208031+0000","last_clean":"2023-09-23T20:13:02.208031+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:13:02.208031+0000","last_undegraded":"2023-09-23T20:13:02.208031+0000","last_fullsized":"2023-09-23T20:13:02.208031+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":10,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.214020+0000","last_change":"2023-09-23T20:13:02.213807+0000","last_active":"2023-09-23T20:13:02.214020+0000","last_peered":"2023-09-23T20:13:02.214020+0000","last_clean":"2023-09-23T20:13:02.214020+0000","last_became_active":"2023-09-23T20:13:02.213521+0000","last_became_peered":"2023-09-23T20:13:02.213521+0000","last_unstale":"2023-09-23T20:13:02.214020+0000","last_undegraded":"2023-09-23T20:13:02.214020+0000","last_fullsized":"2023-09-23T20:13:02.214020+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:09.558945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":855,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736359+0000","last_change":"2023-09-23T20:12:53.737343+0000","last_active":"2023-09-23T20:13:02.736359+0000","last_peered":"2023-09-23T20:13:02.736359+0000","last_clean":"2023-09-23T20:13:02.736359+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:13:02.736359+0000","last_undegraded":"2023-09-23T20:13:02.736359+0000","last_fullsized":"2023-09-23T20:13:02.736359+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.611543+0000","scrub_duration":0.0015738379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":251,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.209788+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:13:02.209788+0000","last_peered":"2023-09-23T20:13:02.209788+0000","last_clean":"2023-09-23T20:13:02.209788+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:13:02.209788+0000","last_undegraded":"2023-09-23T20:13:02.209788+0000","last_fullsized":"2023-09-23T20:13:02.209788+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":447,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207472+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:13:02.207472+0000","last_peered":"2023-09-23T20:13:02.207472+0000","last_clean":"2023-09-23T20:13:02.207472+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:13:02.207472+0000","last_undegraded":"2023-09-23T20:13:02.207472+0000","last_fullsized":"2023-09-23T20:13:02.207472+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":39,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.202869+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:13:02.202869+0000","last_peered":"2023-09-23T20:13:02.202869+0000","last_clean":"2023-09-23T20:13:02.202869+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:13:02.202869+0000","last_undegraded":"2023-09-23T20:13:02.202869+0000","last_fullsized":"2023-09-23T20:13:02.202869+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":11,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:03.216172+0000","last_change":"2023-09-23T20:13:02.737701+0000","last_active":"2023-09-23T20:13:03.216172+0000","last_peered":"2023-09-23T20:13:03.216172+0000","last_clean":"2023-09-23T20:13:03.216172+0000","last_became_active":"2023-09-23T20:13:02.737397+0000","last_became_peered":"2023-09-23T20:13:02.737397+0000","last_unstale":"2023-09-23T20:13:03.216172+0000","last_undegraded":"2023-09-23T20:13:03.216172+0000","last_fullsized":"2023-09-23T20:13:03.216172+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.355044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":721,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.204928+0000","last_change":"2023-09-23T20:12:39.118887+0000","last_active":"2023-09-23T20:13:02.204928+0000","last_peered":"2023-09-23T20:13:02.204928+0000","last_clean":"2023-09-23T20:13:02.204928+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:13:02.204928+0000","last_undegraded":"2023-09-23T20:13:02.204928+0000","last_fullsized":"2023-09-23T20:13:02.204928+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":110,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.209976+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:13:02.209976+0000","last_peered":"2023-09-23T20:13:02.209976+0000","last_clean":"2023-09-23T20:13:02.209976+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:13:02.209976+0000","last_undegraded":"2023-09-23T20:13:02.209976+0000","last_fullsized":"2023-09-23T20:13:02.209976+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":109,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207382+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:13:02.207382+0000","last_peered":"2023-09-23T20:13:02.207382+0000","last_clean":"2023-09-23T20:13:02.207382+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:13:02.207382+0000","last_undegraded":"2023-09-23T20:13:02.207382+0000","last_fullsized":"2023-09-23T20:13:02.207382+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":368,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735605+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:13:02.735605+0000","last_peered":"2023-09-23T20:13:02.735605+0000","last_clean":"2023-09-23T20:13:02.735605+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:13:02.735605+0000","last_undegraded":"2023-09-23T20:13:02.735605+0000","last_fullsized":"2023-09-23T20:13:02.735605+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":185,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:04.095477+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:13:04.095477+0000","last_peered":"2023-09-23T20:13:04.095477+0000","last_clean":"2023-09-23T20:13:04.095477+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:13:04.095477+0000","last_undegraded":"2023-09-23T20:13:04.095477+0000","last_fullsized":"2023-09-23T20:13:04.095477+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":176,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:04.095034+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:13:04.095034+0000","last_peered":"2023-09-23T20:13:04.095034+0000","last_clean":"2023-09-23T20:13:04.095034+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:13:04.095034+0000","last_undegraded":"2023-09-23T20:13:04.095034+0000","last_fullsized":"2023-09-23T20:13:04.095034+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":412,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.207962+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:13:02.207962+0000","last_peered":"2023-09-23T20:13:02.207962+0000","last_clean":"2023-09-23T20:13:02.207962+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:13:02.207962+0000","last_undegraded":"2023-09-23T20:13:02.207962+0000","last_fullsized":"2023-09-23T20:13:02.207962+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":425,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.736253+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:13:02.736253+0000","last_peered":"2023-09-23T20:13:02.736253+0000","last_clean":"2023-09-23T20:13:02.736253+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:13:02.736253+0000","last_undegraded":"2023-09-23T20:13:02.736253+0000","last_fullsized":"2023-09-23T20:13:02.736253+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":402,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208064+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:13:02.208064+0000","last_peered":"2023-09-23T20:13:02.208064+0000","last_clean":"2023-09-23T20:13:02.208064+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:13:02.208064+0000","last_undegraded":"2023-09-23T20:13:02.208064+0000","last_fullsized":"2023-09-23T20:13:02.208064+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":405,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735773+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:13:02.735773+0000","last_peered":"2023-09-23T20:13:02.735773+0000","last_clean":"2023-09-23T20:13:02.735773+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:13:02.735773+0000","last_undegraded":"2023-09-23T20:13:02.735773+0000","last_fullsized":"2023-09-23T20:13:02.735773+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":111,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:04.094497+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:13:04.094497+0000","last_peered":"2023-09-23T20:13:04.094497+0000","last_clean":"2023-09-23T20:13:04.094497+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:13:04.094497+0000","last_undegraded":"2023-09-23T20:13:04.094497+0000","last_fullsized":"2023-09-23T20:13:04.094497+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":268,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T20:13:04.093945+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:13:04.093945+0000","last_peered":"2023-09-23T20:13:04.093945+0000","last_clean":"2023-09-23T20:13:04.093945+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:13:04.093945+0000","last_undegraded":"2023-09-23T20:13:04.093945+0000","last_fullsized":"2023-09-23T20:13:04.093945+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":254,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.208562+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:13:02.208562+0000","last_peered":"2023-09-23T20:13:02.208562+0000","last_clean":"2023-09-23T20:13:02.208562+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:13:02.208562+0000","last_undegraded":"2023-09-23T20:13:02.208562+0000","last_fullsized":"2023-09-23T20:13:02.208562+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":455,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:13:02.735587+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:13:02.735587+0000","last_peered":"2023-09-23T20:13:02.735587+0000","last_clean":"2023-09-23T20:13:02.735587+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:13:02.735587+0000","last_undegraded":"2023-09-23T20:13:02.735587+0000","last_fullsized":"2023-09-23T20:13:02.735587+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":706653,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":97,"num_read_kb":181,"num_write":48,"num_write_kb":477,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1331895,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234384,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":375352,"kb_used_data":8368,"kb_used_omap":0,"kb_used_meta":366976,"kb_avail":93369800,"statfs":{"total":95995035648,"available":95610675200,"internally_reserved":0,"allocated":8568832,"data_stored":7291990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":375783424},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.0019999999999998,"15min":2.2599999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":6.3630000000000004,"5min":55.390000000000001,"15min":62.061999999999998},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.0960000000000001,"15min":2.3090000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.19900000000000001},"max":{"1min":6.0049999999999999,"5min":56.664000000000001,"15min":62.389000000000003},"last":0.68400000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":8.8119999999999994,"5min":3.4590000000000001,"15min":3.8889999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":91.335999999999999,"5min":91.335999999999999,"15min":104.16800000000001},"last":0.82399999999999995},{"interface":"front","average":{"1min":11.324,"5min":4.0149999999999997,"15min":4.1639999999999997},"min":{"1min":0.29399999999999998,"5min":0.249,"15min":0.223},"max":{"1min":91.516999999999996,"5min":91.516999999999996,"15min":104.003},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":9.4710000000000001,"5min":5.3049999999999997,"15min":4.2329999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":67.760000000000005,"5min":92.319000000000003,"15min":105.842},"last":0.80100000000000005},{"interface":"front","average":{"1min":10.063000000000001,"5min":5.7400000000000002,"15min":4.7290000000000001},"min":{"1min":0.27700000000000002,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":67.488,"5min":92.049000000000007,"15min":105.66},"last":0.64700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.5950000000000002,"15min":4.4379999999999997},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":8.2720000000000002,"5min":94.953000000000003,"15min":146.38800000000001},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.4370000000000001,"5min":2.444,"15min":4.375},"min":{"1min":0.29699999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.5369999999999999,"5min":93.423000000000002,"15min":146.851},"last":0.55900000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.48999999999999999,"15min":0.71899999999999997},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.22500000000000001},"max":{"1min":3.2069999999999999,"5min":3.2069999999999999,"15min":19.215},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.54600000000000004,"15min":0.83299999999999996},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.88400000000000001,"5min":2.7949999999999999,"15min":41.822000000000003},"last":0.73599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.63700000000000001,"15min":0.97299999999999998},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":3.407,"5min":4.4080000000000004,"15min":35.488},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.628,"15min":0.85699999999999998},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":3.2549999999999999,"5min":4.2030000000000003,"15min":34.896000000000001},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.70299999999999996,"15min":0.83399999999999996},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":3.4849999999999999,"5min":3.4849999999999999,"15min":41.902999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.55800000000000005,"15min":0.64000000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30399999999999999},"max":{"1min":2.3500000000000001,"5min":2.4590000000000001,"15min":6.3620000000000001},"last":0.59899999999999998}]}]},{"osd":6,"up_from":0,"seq":1078036791425,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762583,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603084,"kb_used_data":9092,"kb_used_omap":7,"kb_used_meta":593976,"kb_avail":93142068,"statfs":{"total":95995035648,"available":95377477632,"internally_reserved":0,"allocated":9310208,"data_stored":7372599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":608232250},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8660000000000001,"5min":2.5230000000000001,"15min":3.2349999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.16900000000000001},"max":{"1min":11.930999999999999,"5min":28.510000000000002,"15min":144.273},"last":0.316},{"interface":"front","average":{"1min":2.6230000000000002,"5min":2.5409999999999999,"15min":3.468},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.19400000000000001},"max":{"1min":16.870999999999999,"5min":16.870999999999999,"15min":144.321},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:12:09 2023","interfaces":[{"interface":"back","average":{"1min":3.9369999999999998,"5min":4.1559999999999997,"15min":4.8419999999999996},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":38.040999999999997,"5min":58.499000000000002,"15min":134.72499999999999},"last":0.373},{"interface":"front","average":{"1min":3.3700000000000001,"5min":3.6269999999999998,"15min":4.952},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":38.173999999999999,"5min":58.741,"15min":135.74600000000001},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.8159999999999998,"15min":3.786},"min":{"1min":0.255,"5min":0.255,"15min":0.22},"max":{"1min":17.212,"5min":25.963999999999999,"15min":68.180999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.5720000000000001,"5min":2.7000000000000002,"15min":3.7450000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.247},"max":{"1min":12.723000000000001,"5min":40.716000000000001,"15min":66.509},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7170000000000001,"15min":1.998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":11.698,"5min":18.087,"15min":60.951000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.3839999999999999,"5min":1.837,"15min":2.1429999999999998},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":11.08,"5min":14.890000000000001,"15min":60.841000000000001},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":5.2670000000000003,"5min":2.621,"15min":2.5459999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":83.510999999999996,"5min":83.510999999999996,"15min":83.510999999999996},"last":0.73699999999999999},{"interface":"front","average":{"1min":5.6289999999999996,"5min":3.121,"15min":2.6360000000000001},"min":{"1min":0.40200000000000002,"5min":0.28899999999999998,"15min":0.28399999999999997},"max":{"1min":83.353999999999999,"5min":83.353999999999999,"15min":83.353999999999999},"last":0.623}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":1.8959999999999999,"15min":2.0009999999999999},"min":{"1min":0.42299999999999999,"5min":0.38500000000000001,"15min":0.35899999999999999},"max":{"1min":11.868,"5min":18.117999999999999,"15min":18.117999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.8260000000000001,"5min":1.6799999999999999,"15min":2.5499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":11.109,"5min":11.109,"15min":20.734999999999999},"last":0.76800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":2.359,"15min":2.2869999999999999},"min":{"1min":0.434,"5min":0.33300000000000002,"15min":0.30399999999999999},"max":{"1min":12.823,"5min":25.937000000000001,"15min":60.213000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":2.0070000000000001,"5min":1.8979999999999999,"15min":2.1030000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.30199999999999999},"max":{"1min":11.747,"5min":18.053999999999998,"15min":61.298999999999999},"last":0.64900000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926120,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":734716,"kb_used_data":104244,"kb_used_omap":8,"kb_used_meta":630455,"kb_avail":93010436,"statfs":{"total":95995035648,"available":95242686464,"internally_reserved":0,"allocated":106745856,"data_stored":104823390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":645586529},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":3.0899999999999999,"15min":3.8639999999999999},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":6.5199999999999996,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.4540000000000002,"5min":3.6890000000000001,"15min":4.1440000000000001},"min":{"1min":0.218,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":14.513999999999999,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.48399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":3.8780000000000001,"15min":5.0019999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19800000000000001},"max":{"1min":19.486999999999998,"5min":102.65300000000001,"15min":270.51100000000002},"last":0.621},{"interface":"front","average":{"1min":3.0049999999999999,"5min":4.0129999999999999,"15min":4.8470000000000004},"min":{"1min":0.25,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.693999999999999,"5min":96.578999999999994,"15min":269.69299999999998},"last":0.34300000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":2.8220000000000001,"15min":5.6210000000000004},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":10.082000000000001,"5min":32.204999999999998,"15min":145.72800000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.8680000000000001,"5min":3.1120000000000001,"15min":5.6890000000000001},"min":{"1min":0.29999999999999999,"5min":0.27800000000000002,"15min":0.23999999999999999},"max":{"1min":9.3970000000000002,"5min":33.457000000000001,"15min":145.50700000000001},"last":2.3650000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.631,"15min":2.2469999999999999},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":9.6959999999999997,"5min":15.688000000000001,"15min":148.20599999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.998,"5min":1.629,"15min":2.0270000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27600000000000002},"max":{"1min":8.4429999999999996,"5min":13.428000000000001,"15min":148.11000000000001},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.341,"15min":1.9410000000000001},"min":{"1min":0.374,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":3.7480000000000002,"5min":9.5719999999999992,"15min":64.725999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.742,"15min":1.988},"min":{"1min":0.36499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.7479999999999993,"5min":15.717000000000001,"15min":27.609999999999999},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.5589999999999999,"15min":1.6319999999999999},"min":{"1min":0.51400000000000001,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":6.0839999999999996,"5min":15.744999999999999,"15min":15.744999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.675,"5min":1.633,"15min":1.8260000000000001},"min":{"1min":0.437,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":8.3919999999999995,"5min":13.470000000000001,"15min":13.766999999999999},"last":0.67000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:12:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.429,"15min":1.891},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":5.1929999999999996,"5min":9.6069999999999993,"15min":65.081999999999994},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.2589999999999999,"5min":1.867,"15min":2.0089999999999999},"min":{"1min":0.41199999999999998,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.0510000000000002,"5min":15.771000000000001,"15min":15.771000000000001},"last":0.58199999999999996}]}]},{"osd":2,"up_from":104,"seq":446676599059,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":750116,"kb_used_data":5276,"kb_used_omap":12,"kb_used_meta":744819,"kb_avail":92995036,"statfs":{"total":95995035648,"available":95226916864,"internally_reserved":0,"allocated":5402624,"data_stored":3453829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":762695678},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":2.2160000000000002,"15min":3.0030000000000001},"min":{"1min":0.26200000000000001,"5min":0.19900000000000001,"15min":0.16900000000000001},"max":{"1min":14.794,"5min":17.876999999999999,"15min":76.251999999999995},"last":0.26700000000000002},{"interface":"front","average":{"1min":1.339,"5min":2.4159999999999999,"15min":3.0339999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.192},"max":{"1min":5.077,"5min":18.561,"15min":76.590000000000003},"last":0.316}]},{"osd":1,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.8220000000000001,"15min":3.6280000000000001},"min":{"1min":0.25,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":32.677999999999997,"5min":32.677999999999997,"15min":94.768000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":3.7010000000000001,"5min":2.4820000000000002,"15min":3.577},"min":{"1min":0.30599999999999999,"5min":0.24299999999999999,"15min":0.21099999999999999},"max":{"1min":32.582000000000001,"5min":32.582000000000001,"15min":96.808000000000007},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:12:36 2023","interfaces":[{"interface":"back","average":{"1min":3.395,"5min":2.5600000000000001,"15min":4.2320000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":14.131,"5min":14.131,"15min":107.381},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.5259999999999998,"5min":2.25,"15min":3.508},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23699999999999999},"max":{"1min":14.778,"5min":20.308,"15min":106.301},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.698,"15min":2.0790000000000002},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.26900000000000002},"max":{"1min":14.757999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.7090000000000001,"15min":2.1259999999999999},"min":{"1min":0.34899999999999998,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":13.387,"5min":13.387,"15min":76.161000000000001},"last":0.67000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":4.258,"5min":2.1099999999999999,"15min":2.6899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":29.433,"5min":29.433,"15min":172.505},"last":0.504},{"interface":"front","average":{"1min":3.6040000000000001,"5min":2.1920000000000002,"15min":2.8500000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.29099999999999998},"max":{"1min":29.196999999999999,"5min":29.196999999999999,"15min":172.375},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:12:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.6799999999999999,"15min":1.641},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":14.664,"5min":14.664,"15min":14.664},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.214,"5min":1.8640000000000001,"15min":1.9259999999999999},"min":{"1min":0.42099999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":11.081,"5min":14.279999999999999,"15min":14.279999999999999},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:12:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.837,"15min":1.829},"min":{"1min":0.44,"5min":0.377,"15min":0.30199999999999999},"max":{"1min":13.481999999999999,"5min":13.481999999999999,"15min":23.079999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.4159999999999999,"5min":1.869,"15min":1.9139999999999999},"min":{"1min":0.39900000000000002,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":14.698,"5min":14.698,"15min":20.614999999999998},"last":0.59599999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435532,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1016344,"kb_used_data":333904,"kb_used_omap":4,"kb_used_meta":682427,"kb_avail":92728808,"statfs":{"total":95995035648,"available":94954299392,"internally_reserved":0,"allocated":341917696,"data_stored":339789755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":698805357},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":4.1029999999999998,"15min":3.4489999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":4.9340000000000002,"5min":74.323999999999998,"15min":74.323999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.9690000000000001,"5min":4.0149999999999997,"15min":3.7639999999999998},"min":{"1min":0.26500000000000001,"5min":0.20100000000000001,"15min":0.19600000000000001},"max":{"1min":10.484,"5min":74.447000000000003,"15min":74.447000000000003},"last":0.33600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":4.7510000000000003,"15min":4.1319999999999997},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":5.5949999999999998,"5min":163.49700000000001,"15min":163.49700000000001},"last":0.308},{"interface":"front","average":{"1min":2.6779999999999999,"5min":5.258,"15min":4.4859999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":9.9760000000000009,"5min":163.512,"15min":163.512},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1629999999999998,"5min":6.6520000000000001,"15min":4.351},"min":{"1min":0.27400000000000002,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":14.9,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.378},{"interface":"front","average":{"1min":1.252,"5min":6.0380000000000003,"15min":4.0140000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":4.2439999999999998,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.42899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.202,"15min":3.1429999999999998},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.6259999999999999,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.714,"5min":2.8420000000000001,"15min":3.5510000000000002},"min":{"1min":0.308,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":7.5759999999999996,"5min":58.408000000000001,"15min":110.295},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.6160000000000001,"15min":3.395},"min":{"1min":0.40300000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":7.6200000000000001,"5min":58.506,"15min":113.881},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.321,"5min":2.8919999999999999,"15min":3.2959999999999998},"min":{"1min":0.36299999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":4.274,"5min":58.643000000000001,"15min":114.598},"last":0.53800000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":2.625,"15min":3.6059999999999999},"min":{"1min":0.41399999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":4.3120000000000003,"5min":58.588000000000001,"15min":82.281999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.88700000000000001,"5min":2.2389999999999999,"15min":3.7599999999999998},"min":{"1min":0.376,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":3.6720000000000002,"5min":58.701000000000001,"15min":82.233000000000004},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":2.9359999999999999,"15min":3.516},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.29599999999999999},"max":{"1min":4.335,"5min":58.686,"15min":138.035},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.52,"5min":2.6989999999999998,"15min":3.5070000000000001},"min":{"1min":0.41899999999999998,"5min":0.38400000000000001,"15min":0.32600000000000001},"max":{"1min":4.125,"5min":58.667999999999999,"15min":138.054},"last":0.48599999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019294,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":644596,"kb_used_data":291180,"kb_used_omap":0,"kb_used_meta":353408,"kb_avail":93100556,"statfs":{"total":95995035648,"available":95334969344,"internally_reserved":0,"allocated":298168320,"data_stored":296895562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":361889792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:21 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.0509999999999999,"15min":2.202},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.189},"max":{"1min":3.6150000000000002,"5min":11.743,"15min":89.004999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.085,"15min":2.1659999999999999},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.218},"max":{"1min":5.3250000000000002,"5min":11.776,"15min":89.024000000000001},"last":0.54500000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":7.7149999999999999,"5min":4.2800000000000002,"15min":5.9630000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":138.47999999999999,"5min":138.47999999999999,"15min":161.613},"last":0.46500000000000002},{"interface":"front","average":{"1min":7.806,"5min":4.548,"15min":6.0780000000000003},"min":{"1min":0.247,"5min":0.217,"15min":0.217},"max":{"1min":138.429,"5min":138.429,"15min":162.017},"last":0.50800000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":8.6579999999999995,"15min":6.8689999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":144.916,"5min":182.96700000000001,"15min":182.96700000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":11.116,"5min":8.2240000000000002,"15min":6.6360000000000001},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":144.83099999999999,"5min":183.244,"15min":183.244},"last":0.42899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":3.4319999999999999,"15min":6.5419999999999998},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":4.4340000000000002,"5min":184.37100000000001,"15min":266.54700000000003},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.0329999999999999,"5min":3.6309999999999998,"15min":5.4809999999999999},"min":{"1min":0.36199999999999999,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":11.144,"5min":184.22300000000001,"15min":184.22300000000001},"last":0.441}]},{"osd":5,"last update":"Sat Sep 23 20:12:21 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48799999999999999,"15min":2.117},"min":{"1min":0.246,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":2.8919999999999999,"5min":3.3290000000000002,"15min":266.59899999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.45800000000000002,"15min":2.1419999999999999},"min":{"1min":0.29999999999999999,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":1.2230000000000001,"5min":1.722,"15min":266.72899999999998},"last":0.33500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.50900000000000001,"15min":0.51400000000000001},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.23499999999999999},"max":{"1min":2.9350000000000001,"5min":2.9350000000000001,"15min":4.0410000000000004},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.51100000000000001,"15min":0.58899999999999997},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.247},"max":{"1min":0.81100000000000005,"5min":1.6240000000000001,"15min":4.2889999999999997},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:12:08 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":1.244,"15min":1.411},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":0.59699999999999998,"5min":74.040000000000006,"15min":76.507000000000005},"last":0.37},{"interface":"front","average":{"1min":0.56200000000000006,"5min":1.25,"15min":1.536},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":3.2890000000000001,"5min":74.021000000000001,"15min":76.462000000000003},"last":0.41699999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757658,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":621228,"kb_used_data":217828,"kb_used_omap":0,"kb_used_meta":403392,"kb_avail":93123924,"statfs":{"total":95995035648,"available":95358898176,"internally_reserved":0,"allocated":223055872,"data_stored":221787555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":413073408},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.82099999999999995,"15min":1.7290000000000001},"min":{"1min":0.26100000000000001,"5min":0.224,"15min":0.216},"max":{"1min":2.173,"5min":8.6739999999999995,"15min":121.94799999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.103,"15min":1.9970000000000001},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":9.4250000000000007,"5min":10.327,"15min":121.964},"last":0.60699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.2480000000000002,"15min":5.9630000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":20.337,"5min":75.204999999999998,"15min":266.197},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.2330000000000001,"15min":6.1399999999999997},"min":{"1min":0.28199999999999997,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":23.497,"5min":75.231999999999999,"15min":266.14999999999998},"last":0.67100000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:13:01 2023","interfaces":[{"interface":"back","average":{"1min":11.291,"5min":6.4000000000000004,"15min":5.8250000000000002},"min":{"1min":0.36799999999999999,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":120.297,"5min":240.57300000000001,"15min":240.57300000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":11.327,"5min":6.4740000000000002,"15min":5.8840000000000003},"min":{"1min":0.41499999999999998,"5min":0.374,"15min":0.313},"max":{"1min":120.325,"5min":240.464,"15min":240.464},"last":0.629}]},{"osd":3,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.631,"15min":4.8849999999999998},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.608000000000001,"5min":12.608000000000001,"15min":201.50899999999999},"last":1.593},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.0820000000000001,"15min":4.6719999999999997},"min":{"1min":0.313,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":1.9299999999999999,"5min":7.3739999999999997,"15min":201.58099999999999},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:16 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.48599999999999999,"15min":1.4119999999999999},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":2.5790000000000002,"5min":4.0510000000000002,"15min":191.78899999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.436,"15min":1.3129999999999999},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":0.59499999999999997,"5min":1.4359999999999999,"15min":191.821},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.52200000000000002,"15min":0.63100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":1.7410000000000001,"5min":3.278,"15min":9.9009999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.47299999999999998,"15min":0.51900000000000002},"min":{"1min":0.315,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.841,"5min":1.841,"15min":5.0119999999999996},"last":0.53900000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:13:01 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.51000000000000001,"15min":1.486},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26200000000000001},"max":{"1min":0.70999999999999996,"5min":2.3069999999999999,"15min":191.84299999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.54400000000000004,"15min":1.645},"min":{"1min":0.32800000000000001,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":0.77100000000000002,"5min":3.1440000000000001,"15min":191.69},"last":0.64300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:13:13.178 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:13:13.178 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:13:13.178 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T20:13:13.178 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:13:13.178 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:13:13.179 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T20:13:13.262 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:13.294 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:13.343 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:13.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:13.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:13.450 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:13.481 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:13.581 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:13.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:13.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:13.684 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:13.711 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:13:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:13:13] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.43.0" 2023-09-23T20:13:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:13 smithi135 ceph-mon[221113]: pgmap v1152: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:13.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:13 smithi135 ceph-mon[213871]: pgmap v1152: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:13.983 DEBUG:teuthology.orchestra.run.smithi135:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T20:13:13.986 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:13:13.986 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:13:14.046 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:hints = 1 2023-09-23T20:13:14.046 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:13:14.047 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object prefix: benchmark_data_smithi135_549536 2023-09-23T20:13:14.047 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:13:14.047 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:13:14.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:13 smithi144 ceph-mon[194833]: pgmap v1152: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:14.227 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:14.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:14.334 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:14.402 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:14.523 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:13:14.523 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:13:14.523 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:13:14.523 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52207 8.6 8:6fd8f8f3:::benchmark_data_smithi135_460343_object52206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 374.60812167500001, 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1234665740000001, 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.524 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52207 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:13:14.525 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909076+0000", 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.526 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909102+0000", 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.605e-05 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909495+0000", 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00039314200000000002 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909506+0000", 2023-09-23T20:13:14.527 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0699999999999999e-05 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910549+0000", 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001043034 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910557+0000", 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.816e-06 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.528 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914406+0000", 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0038485909999999 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914418+0000", 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2016e-05 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.529 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.031329+0000", 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1169108190000001 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.032544+0000", 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001214705 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.530 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52216 8.6 8:60b8198d:::benchmark_data_smithi135_460343_object52215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 374.60768062199998, 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1446887769999998, 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.531 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52216 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.532 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909517+0000", 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909548+0000", 2023-09-23T20:13:14.533 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0108999999999999e-05 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909557+0000", 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.842e-06 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909563+0000", 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0759999999999999e-06 2023-09-23T20:13:14.534 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910589+0000", 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010254730000000001 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910596+0000", 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4970000000000004e-06 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.535 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914445+0000", 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.003848268 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914452+0000", 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7100000000000007e-06 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.536 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.033864+0000", 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1194116679999999 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.034445+0000", 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00058121900000000001 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.034641+0000", 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019598099999999999 2023-09-23T20:13:14.537 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050209+0000", 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015567925 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050294+0000", 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5099000000000006e-05 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.538 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.052593+0000", 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022989210000000002 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.052660+0000", 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6950000000000001e-05 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.539 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054207+0000", 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001546658 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52226 8.6 8:64c4735f:::benchmark_data_smithi135_460343_object52225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 374.60210253700001, 2023-09-23T20:13:14.540 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1418229260000001, 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52226 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:13:14.541 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915095+0000", 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.542 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915130+0000", 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.536e-05 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.918885+0000", 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0037552050000000002 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.543 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.918902+0000", 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6691e-05 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.919727+0000", 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00082474700000000002 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.544 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.919738+0000", 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1235e-05 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914477+0000", 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.99473878900000001 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914486+0000", 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0350000000000007e-06 2023-09-23T20:13:14.545 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036650+0000", 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.122164451 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036775+0000", 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012457700000000001 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.546 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036925+0000", 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000150464 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050301+0000", 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013375705999999999 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.547 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054287+0000", 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0039856179999999998 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055326+0000", 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001039744 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.548 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055373+0000", 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6264999999999998e-05 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.056919+0000", 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001546551 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.549 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52231 8.6 8:64b25628:::benchmark_data_smithi135_460343_object52230:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 374.58894466200002, 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.130363241, 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.550 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52231 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.551 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.552 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928273+0000", 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9911e-05 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928281+0000", 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2910000000000002e-06 2023-09-23T20:13:14.553 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928286+0000", 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3159999999999996e-06 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928328+0000", 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1780000000000003e-05 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.554 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928337+0000", 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5899999999999997e-06 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914511+0000", 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98617375299999999 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914519+0000", 2023-09-23T20:13:14.555 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5680000000000002e-06 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.038973+0000", 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.12445419 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039130+0000", 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000156861 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.556 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039293+0000", 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000163535 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050307+0000", 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011013761 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.557 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054370+0000", 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0040626919999999997 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.057016+0000", 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026464700000000002 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.057055+0000", 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8605000000000001e-05 2023-09-23T20:13:14.558 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058617+0000", 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015624199999999999 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52236 8.6 8:693d9ca7:::benchmark_data_smithi135_460343_object52235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:13:14.559 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 374.57897823799999, 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1218329040000001, 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52236 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.560 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:13:14.561 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938241+0000", 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1342999999999999e-05 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938249+0000", 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7800000000000001e-06 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.562 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938253+0000", 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.777e-06 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.939183+0000", 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00093006800000000004 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.563 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.939192+0000", 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2129999999999997e-06 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914544+0000", 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97535231300000003 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914552+0000", 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5190000000000003e-06 2023-09-23T20:13:14.564 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041404+0000", 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.126851673 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041553+0000", 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000148877 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.565 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041722+0000", 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016971799999999999 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050312+0000", 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085901160000000001 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.566 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054415+0000", 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041022569999999998 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058671+0000", 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0042560890000000002 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058717+0000", 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6471000000000003e-05 2023-09-23T20:13:14.567 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060053+0000", 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001336222 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52242 8.6 8:663215dc:::benchmark_data_smithi135_460343_object52241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:13:14.568 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 374.57217544100001, 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.116376196, 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52242 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.569 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945022+0000", 2023-09-23T20:13:14.570 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945047+0000", 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5188e-05 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945066+0000", 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8719000000000002e-05 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.571 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945074+0000", 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0460000000000006e-06 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945123+0000", 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9542000000000001e-05 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.572 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945133+0000", 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6490000000000003e-06 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914577+0000", 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96944391900000004 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914584+0000", 2023-09-23T20:13:14.573 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4880000000000001e-06 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043641+0000", 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.129056426 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043776+0000", 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0001348 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.574 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043935+0000", 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015974000000000001 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050322+0000", 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0063863770000000004 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.575 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054682+0000", 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00436075 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060087+0000", 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005404252 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.576 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060126+0000", 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9467e-05 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061399+0000", 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001273281 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.577 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52249 8.6 8:6b568471:::benchmark_data_smithi135_460343_object52248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 374.567626201, 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1131886460000002, 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52249 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.578 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.579 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949571+0000", 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949593+0000", 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1430999999999999e-05 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.580 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949607+0000", 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4381e-05 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949614+0000", 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0330000000000003e-06 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949662+0000", 2023-09-23T20:13:14.581 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7685000000000002e-05 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949675+0000", 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2575000000000001e-05 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.915769+0000", 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96609494600000001 2023-09-23T20:13:14.582 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.915781+0000", 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1364e-05 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.047967+0000", 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1321858119999999 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.583 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.048151+0000", 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000184138 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.048355+0000", 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020468600000000001 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050329+0000", 2023-09-23T20:13:14.584 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019734399999999999 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055290+0000", 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049610740000000002 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061446+0000", 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061564150000000002 2023-09-23T20:13:14.585 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061507+0000", 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0739999999999998e-05 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.062761+0000", 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012539999999999999 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.586 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65642 8.7 8:e801740d:::benchmark_data_smithi135_460343_object65641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 355.59109313200003, 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.760197558, 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.587 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65642 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:13:14.588 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926104+0000", 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926155+0000", 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1158e-05 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.589 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926172+0000", 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.664e-05 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926183+0000", 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0896999999999999e-05 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926311+0000", 2023-09-23T20:13:14.590 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012874499999999999 2023-09-23T20:13:14.591 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926319+0000", 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.294e-06 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927738+0000", 2023-09-23T20:13:14.593 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001418891 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927746+0000", 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5960000000000004e-06 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.684466+0000", 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.756720061 2023-09-23T20:13:14.594 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.686303+0000", 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001836935 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65643 8.3 8:c7d581fe:::benchmark_data_smithi135_460343_object65642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:13:14.595 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 355.59099626300002, 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7775143610000002, 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65643 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.596 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.597 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926223+0000", 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0891999999999999e-05 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.598 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926230+0000", 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3799999999999996e-06 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926236+0000", 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2179999999999996e-06 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926302+0000", 2023-09-23T20:13:14.599 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5423e-05 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926309+0000", 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3889999999999999e-06 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927982+0000", 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016733589999999999 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.600 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927990+0000", 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8369999999999997e-06 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.701891+0000", 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.773900823 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.601 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.703717+0000", 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018257550000000001 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65645 8.3 8:cf69bd2e:::benchmark_data_smithi135_460343_object65644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 355.59095238700002, 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7804623400000001, 2023-09-23T20:13:14.602 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65645 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.603 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.604 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926264+0000", 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8465e-05 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926271+0000", 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.849e-06 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.605 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926334+0000", 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2448999999999998e-05 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926375+0000", 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1554999999999998e-05 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926381+0000", 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.083e-06 2023-09-23T20:13:14.606 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.928014+0000", 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001632362 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.928020+0000", 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4629999999999998e-06 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.607 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.706613+0000", 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7785930159999999 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.706709+0000", 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5585e-05 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.608 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65667 8.a 8:5e718162:::benchmark_data_smithi135_460343_object65666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 355.590857245, 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.820842632, 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65667 2023-09-23T20:13:14.609 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.610 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926366+0000", 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4946000000000001e-05 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.611 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926374+0000", 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0150000000000003e-06 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926380+0000", 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.981e-06 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.612 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.933090+0000", 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067101180000000002 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.933100+0000", 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0087e-05 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704577+0000", 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7714773450000001 2023-09-23T20:13:14.613 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704719+0000", 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000141182 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704870+0000", 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015157699999999999 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.614 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713627+0000", 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0087563789999999999 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713675+0000", 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7994000000000002e-05 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.615 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.743266+0000", 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029591681000000002 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.743323+0000", 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7122e-05 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747184+0000", 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038607680000000001 2023-09-23T20:13:14.616 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65682 8.a 8:51fe7b50:::benchmark_data_smithi135_460343_object65681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 355.383415275, 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.61738943, 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.617 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65682 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.618 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133782+0000", 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133803+0000", 2023-09-23T20:13:14.619 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0800000000000001e-05 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133815+0000", 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2177999999999999e-05 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133821+0000", 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0170000000000002e-06 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.620 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133873+0000", 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2039000000000001e-05 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133885+0000", 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1158000000000001e-05 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.621 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710505+0000", 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.576620546 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710683+0000", 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017801700000000001 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710887+0000", 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020342900000000001 2023-09-23T20:13:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713666+0000", 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027793280000000002 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713740+0000", 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4468000000000006e-05 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.623 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747234+0000", 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.033493756 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747286+0000", 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.185e-05 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.751173+0000", 2023-09-23T20:13:14.624 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038869239999999999 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86869 8.f 8:fd41b8fd:::benchmark_data_smithi135_460343_object2636:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 309.775068729, 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0711724490000001, 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.625 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86869 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.626 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742129+0000", 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.627 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742133+0000", 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5920000000000002e-06 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742144+0000", 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0358e-05 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742153+0000", 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1579999999999994e-06 2023-09-23T20:13:14.628 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742207+0000", 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4622e-05 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742216+0000", 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3820000000000004e-06 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.629 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744303+0000", 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020874449999999998 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744310+0000", 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9330000000000004e-06 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.767416+0000", 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.023105787 2023-09-23T20:13:14.630 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.768079+0000", 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00066287699999999998 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.799565+0000", 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.031486449999999999 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.631 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.800979+0000", 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001414092 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.801070+0000", 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0818999999999999e-05 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803811+0000", 2023-09-23T20:13:14.632 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002740592 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804527+0000", 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00071653999999999999 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.811597+0000", 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0070696429999999996 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.633 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.811634+0000", 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7245e-05 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.813302+0000", 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016681619999999999 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.634 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86884 8.f 8:f6bbdc58:::benchmark_data_smithi135_460343_object2651:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 309.77503119800002, 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1026705059999999, 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.635 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86884 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:13:14.636 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742167+0000", 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742170+0000", 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5610000000000001e-06 2023-09-23T20:13:14.637 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742176+0000", 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0070000000000003e-06 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742180+0000", 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.651e-06 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.638 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742295+0000", 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011517499999999999 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742308+0000", 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2274e-05 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744328+0000", 2023-09-23T20:13:14.639 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002019982 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744334+0000", 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2779999999999999e-06 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803248+0000", 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0589140269999999 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.640 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803789+0000", 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00054099499999999995 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817686+0000", 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013896531 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.641 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817787+0000", 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101207 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817881+0000", 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4238000000000001e-05 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.832086+0000", 2023-09-23T20:13:14.642 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014205493 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843262+0000", 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011175433 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843318+0000", 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6292000000000002e-05 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.643 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843373+0000", 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4737000000000001e-05 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844838+0000", 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001465084 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.644 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86888 8.f 8:ff8ee601:::benchmark_data_smithi135_460343_object2655:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 309.774989178, 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1038876470000001, 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.645 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86888 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.646 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742209+0000", 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742214+0000", 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8230000000000002e-06 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.647 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742260+0000", 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5964000000000002e-05 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742266+0000", 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6209999999999998e-06 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.648 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742385+0000", 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011896900000000001 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742393+0000", 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4590000000000001e-06 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744351+0000", 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019579710000000002 2023-09-23T20:13:14.649 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744358+0000", 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803866+0000", 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0595081719999999 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.650 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804470+0000", 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060409500000000002 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823610+0000", 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019139976999999999 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.651 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823691+0000", 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0525999999999994e-05 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823816+0000", 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000125029 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.837821+0000", 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014005521999999999 2023-09-23T20:13:14.652 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844894+0000", 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007073368 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844944+0000", 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9799999999999998e-05 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.653 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.845000+0000", 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5933999999999997e-05 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846097+0000", 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001096999 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.654 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86947 8.f 8:fc570a5a:::benchmark_data_smithi135_460343_object2714:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 309.77491020899998, 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.10541584, 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86947 2023-09-23T20:13:14.655 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.656 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742288+0000", 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742292+0000", 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4179999999999999e-06 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.657 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742299+0000", 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0110000000000003e-06 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742304+0000", 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4710000000000001e-06 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742429+0000", 2023-09-23T20:13:14.658 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012482199999999999 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742435+0000", 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5150000000000003e-06 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744369+0000", 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001934203 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.659 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744375+0000", 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9190000000000004e-06 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804569+0000", 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0601939 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.660 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805072+0000", 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050289400000000004 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829232+0000", 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024160198000000001 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.661 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829317+0000", 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4447e-05 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829401+0000", 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4468999999999994e-05 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843205+0000", 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013804293 2023-09-23T20:13:14.662 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846168+0000", 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029630279999999999 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846218+0000", 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9574999999999999e-05 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.663 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846273+0000", 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5058e-05 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847704+0000", 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014312109999999999 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.664 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86957 8.f 8:fdc750af:::benchmark_data_smithi135_460343_object2724:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 309.77489020799999, 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.106922379, 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.665 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86957 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.666 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742308+0000", 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742311+0000", 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5840000000000002e-06 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.667 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742316+0000", 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.746e-06 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742320+0000", 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1649999999999999e-06 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742500+0000", 2023-09-23T20:13:14.668 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000180003 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742506+0000", 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.263e-06 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744392+0000", 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001886284 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.669 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744398+0000", 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1149999999999999e-06 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805115+0000", 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0607168629999999 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.670 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805787+0000", 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00067139499999999998 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.834880+0000", 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029093853999999999 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.835004+0000", 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123545 2023-09-23T20:13:14.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.835169+0000", 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000164506 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844858+0000", 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0096890859999999995 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847759+0000", 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029018820000000002 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847800+0000", 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0590999999999998e-05 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847847+0000", 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6615e-05 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849231+0000", 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0013841960000000001 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:87007 8.f 8:ff8f150f:::benchmark_data_smithi135_460343_object2774:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 309.774875678, 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.108736019, 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 87007 2023-09-23T20:13:14.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742325+0000", 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.824e-06 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742329+0000", 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7099999999999998e-06 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742333+0000", 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8589999999999998e-06 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742551+0000", 2023-09-23T20:13:14.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00021761999999999999 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742559+0000", 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2020000000000004e-06 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744418+0000", 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001859468 2023-09-23T20:13:14.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744426+0000", 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0770000000000002e-06 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805831+0000", 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0614059330000001 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.806364+0000", 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00053292300000000003 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840788+0000", 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.034423435000000002 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840892+0000", 2023-09-23T20:13:14.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010409299999999999 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840986+0000", 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3930000000000004e-05 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846116+0000", 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005129761 2023-09-23T20:13:14.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849292+0000", 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031762359999999998 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849356+0000", 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4146000000000006e-05 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.683 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849403+0000", 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7151999999999998e-05 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.851059+0000", 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001655903 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:115182 8.7 8.c781a1a7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:02.044539+0000", 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 252.47266006300001, 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.60009978200000003, 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 115182 2023-09-23T20:13:14.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.044539+0000", 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.044539+0000", 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.044537+0000", 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.044546+0000", 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4130000000000006e-06 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.687 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.044570+0000", 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4068999999999999e-05 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.044582+0000", 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1999e-05 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.094639+0000", 2023-09-23T20:13:14.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.050057175000000002 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.094650+0000", 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.078e-05 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.644628+0000", 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.54997875299999999 2023-09-23T20:13:14.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:02.644638+0000", 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0051000000000001e-05 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:125547 8.f 8:f06e005a:::benchmark_data_smithi135_460343_object41314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:02.542323+0000", 2023-09-23T20:13:14.690 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 131.97487606199999, 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.61188724500000002, 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 125547 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.691 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542323+0000", 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542323+0000", 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542322+0000", 2023-09-23T20:13:14.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542328+0000", 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6019999999999996e-06 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542338+0000", 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.008e-05 2023-09-23T20:13:14.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542344+0000", 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1589999999999998e-06 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.829850+0000", 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28750529699999999 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.840856+0000", 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011006592000000001 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.927237+0000", 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.086380278000000005 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:13:14.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.930065+0000", 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028284019999999998 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.934871+0000", 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004806388 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.980254+0000", 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.045382458000000001 2023-09-23T20:13:14.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:03.075878+0000", 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.095624656000000002 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:03.148645+0000", 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.072766843999999997 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:03.149741+0000", 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001095256 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:03.154210+0000", 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0044692960000000002 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.698 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:13:14.699 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:13:14.699 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:13:14.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:14.700 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:14.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:14.800 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:14.801 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:14.856 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:14.937 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:14.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:14.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:14.965 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:13:15.000 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:13:15.034 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:13:15.044 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:15.047 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 1 16 3309 3293 205.678 205.812 0.00589383 0.00484535 2023-09-23T20:13:15.073 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:13:15.104 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:15.113 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:13:15.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2188783549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:15.155 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:13:15.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2188783549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:15.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2188783549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:15.500 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:13:15.501 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:13:15.501 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:13:15.501 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:13:15.501 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:13:15.501 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:13:15.501 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:13:15.501 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:13:15.501 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:13:15.502 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:13:15.896 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:15.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:16.004 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:16.048 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 2 16 5690 5674 177.206 148.812 0.00389076 0.00563089 2023-09-23T20:13:16.058 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:16.149 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:16.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:16.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:15 smithi135 ceph-mon[221113]: pgmap v1153: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:16.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:15 smithi135 ceph-mon[213871]: pgmap v1153: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:16.257 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:16.302 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:15 smithi144 ceph-mon[194833]: pgmap v1153: 109 pgs: 109 active+clean; 4.7 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:13:16.308 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:16.392 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:16.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:16.497 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:16.523 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:16.569 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:16.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:16.678 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:16.738 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:17.048 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 3 16 8307 8291 172.655 163.562 0.00511543 0.00578271 2023-09-23T20:13:17.057 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:13:17.061 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:17.143 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:17.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:17.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:17.253 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:17.317 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:18.048 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 4 16 11187 11171 174.487 180 0.00386485 0.00572539 2023-09-23T20:13:18.364 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:18.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:18 smithi135 ceph-mon[221113]: pgmap v1154: 109 pgs: 109 active+clean; 56 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 6.2 MiB/s wr, 99 op/s 2023-09-23T20:13:18.366 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:18 smithi135 ceph-mon[213871]: pgmap v1154: 109 pgs: 109 active+clean; 56 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 6.2 MiB/s wr, 99 op/s 2023-09-23T20:13:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:18.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:18.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:18 smithi144 ceph-mon[194833]: pgmap v1154: 109 pgs: 109 active+clean; 56 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 6.2 MiB/s wr, 99 op/s 2023-09-23T20:13:18.469 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:18.506 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:19.048 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 5 16 13378 13362 166.976 136.938 0.00299979 0.00598317 2023-09-23T20:13:19.101 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:13:19.465 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:19.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:19.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:19.751 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:19.817 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:19.965 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:19.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:20.048 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 6 16 15697 15681 163.301 144.938 0.00954337 0.00611982 2023-09-23T20:13:20.063 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:20 smithi135 ceph-mon[213871]: pgmap v1155: 109 pgs: 109 active+clean; 720 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:13:20.063 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:20 smithi135 ceph-mon[221113]: pgmap v1155: 109 pgs: 109 active+clean; 720 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:13:20.064 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:20.078 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:20.122 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:20.198 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:20.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:20.208 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:20.303 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:20.348 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:20 smithi144 ceph-mon[194833]: pgmap v1155: 109 pgs: 109 active+clean; 720 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:13:20.352 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:20.425 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:20.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:20.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:20.533 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:20.606 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:20.690 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:13:20.690 INFO:teuthology.orchestra.run.smithi135.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:13:20.690 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:13:21.050 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 7 16 18006 17990 160.546 144.312 0.00393636 0.0062218 2023-09-23T20:13:21.158 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:21.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:21.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:21.265 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:21.337 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:22.050 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 8 16 20649 20633 161.125 165.188 0.00637848 0.00619952 2023-09-23T20:13:22.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:22 smithi144 ceph-mon[194833]: pgmap v1156: 109 pgs: 109 active+clean; 720 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:13:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:22 smithi135 ceph-mon[213871]: pgmap v1156: 109 pgs: 109 active+clean; 720 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:13:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:22 smithi135 ceph-mon[221113]: pgmap v1156: 109 pgs: 109 active+clean; 720 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:13:23.050 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 9 16 22713 22697 157.555 129 0.00415728 0.00633936 2023-09-23T20:13:23.088 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:23.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:23.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:23.196 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:23.253 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:23.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:13:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:13:23] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-09-23T20:13:24.050 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 10 16 24342 24326 151.982 101.812 0.00184687 0.00653109 2023-09-23T20:13:24.097 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:13:24.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:24 smithi135 ceph-mon[221113]: pgmap v1157: 109 pgs: 109 active+clean; 813 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:13:24.371 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/279749133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:13:24.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:24 smithi135 ceph-mon[213871]: pgmap v1157: 109 pgs: 109 active+clean; 813 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:13:24.372 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/279749133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:13:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:24 smithi144 ceph-mon[194833]: pgmap v1157: 109 pgs: 109 active+clean; 813 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:13:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/279749133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:13:24.598 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:24.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:24.705 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:24.730 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:24.771 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:24.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:24.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:24.876 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:24.931 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:25.050 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 11 16 26199 26183 148.716 116.062 0.0065301 0.00671875 2023-09-23T20:13:25.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:25.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/279749133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:13:25.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T20:13:25.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[213871]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:25.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/279749133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:13:25.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T20:13:25.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:25 smithi135 ceph-mon[221113]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.246 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:25.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:25.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:25.352 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:25.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:25.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/279749133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:13:25.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T20:13:25.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.386 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:25 smithi144 ceph-mon[194833]: failed to encode map e412 with expected crc 2023-09-23T20:13:25.405 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:25.473 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:25.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:25.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:25.580 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:25.661 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:25.829 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:13:25.830 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:13:25.834 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:25.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:25.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:25.946 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:25.958 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:13:25.990 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:13:25.990 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:26.022 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:13:26.050 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 12 16 28182 28166 146.651 123.938 0.0053112 0.00681369 2023-09-23T20:13:26.058 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:13:26.073 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:26.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:26.094 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[213871]: pgmap v1159: 109 pgs: 109 active+clean; 1.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 144 MiB/s wr, 2.30k op/s 2023-09-23T20:13:26.094 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[213871]: 1.14 scrub starts 2023-09-23T20:13:26.095 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[213871]: 1.14 scrub ok 2023-09-23T20:13:26.095 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[213871]: 1.13 scrub starts 2023-09-23T20:13:26.095 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[213871]: 1.13 scrub ok 2023-09-23T20:13:26.095 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2135814506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:26.095 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[221113]: pgmap v1159: 109 pgs: 109 active+clean; 1.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 144 MiB/s wr, 2.30k op/s 2023-09-23T20:13:26.095 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[221113]: 1.14 scrub starts 2023-09-23T20:13:26.095 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[221113]: 1.14 scrub ok 2023-09-23T20:13:26.096 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[221113]: 1.13 scrub starts 2023-09-23T20:13:26.096 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[221113]: 1.13 scrub ok 2023-09-23T20:13:26.096 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2135814506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:26.100 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:13:26.139 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:13:26.175 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:13:26.180 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:26.212 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:13:26.226 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:13:26.226 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:13:26.226 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:13:26.226 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:13:26.226 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:13:26.226 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:13:26.226 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:13:26.226 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:13:26.237 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:26.322 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:26.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:26.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:26 smithi144 ceph-mon[194833]: pgmap v1159: 109 pgs: 109 active+clean; 1.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 144 MiB/s wr, 2.30k op/s 2023-09-23T20:13:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:26 smithi144 ceph-mon[194833]: 1.14 scrub starts 2023-09-23T20:13:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:26 smithi144 ceph-mon[194833]: 1.14 scrub ok 2023-09-23T20:13:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:26 smithi144 ceph-mon[194833]: 1.13 scrub starts 2023-09-23T20:13:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:26 smithi144 ceph-mon[194833]: 1.13 scrub ok 2023-09-23T20:13:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2135814506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:26.426 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:26.478 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:26.548 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:26.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:26.655 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:26.700 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:26.789 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:26.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:26.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:26.895 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:26.955 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:27.050 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 13 16 30094 30078 144.562 119.5 0.00950973 0.00691177 2023-09-23T20:13:27.108 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:27 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:13:27.109 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:27 smithi144 ceph-mon[194833]: 1.d scrub ok 2023-09-23T20:13:27.109 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:27 smithi144 ceph-mon[194833]: 1.4 scrub starts 2023-09-23T20:13:27.109 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:27 smithi144 ceph-mon[194833]: 1.4 scrub ok 2023-09-23T20:13:27.109 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:27 smithi144 ceph-mon[194833]: pgmap v1160: 109 pgs: 109 active+clean; 1.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-09-23T20:13:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:13:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[213871]: 1.d scrub ok 2023-09-23T20:13:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[213871]: 1.4 scrub starts 2023-09-23T20:13:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[213871]: 1.4 scrub ok 2023-09-23T20:13:27.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[213871]: pgmap v1160: 109 pgs: 109 active+clean; 1.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-09-23T20:13:27.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:13:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[221113]: 1.d scrub ok 2023-09-23T20:13:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[221113]: 1.4 scrub starts 2023-09-23T20:13:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[221113]: 1.4 scrub ok 2023-09-23T20:13:27.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:27 smithi135 ceph-mon[221113]: pgmap v1160: 109 pgs: 109 active+clean; 1.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-09-23T20:13:27.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:13:27.449 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:27.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:27.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:27.554 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:27.623 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:27.636 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:13:27.915 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:27.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:27.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:28.035 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:28.050 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 14 16 31606 31590 140.986 94.5 0.011256 0.00708767 2023-09-23T20:13:28.057 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:13:28.057 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:13:28.058 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-09-23T20:13:28.058 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:13:28.074 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:28.126 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:28.141 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:28.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:28.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:28.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:28 smithi135 ceph-mon[213871]: 1.11 scrub starts 2023-09-23T20:13:28.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:28 smithi135 ceph-mon[213871]: 1.11 scrub ok 2023-09-23T20:13:28.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:28 smithi135 ceph-mon[213871]: 1.b scrub starts 2023-09-23T20:13:28.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:28 smithi135 ceph-mon[213871]: 1.b scrub ok 2023-09-23T20:13:28.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:28 smithi135 ceph-mon[221113]: 1.11 scrub starts 2023-09-23T20:13:28.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:28 smithi135 ceph-mon[221113]: 1.11 scrub ok 2023-09-23T20:13:28.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:28 smithi135 ceph-mon[221113]: 1.b scrub starts 2023-09-23T20:13:28.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:28 smithi135 ceph-mon[221113]: 1.b scrub ok 2023-09-23T20:13:28.250 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:28.355 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:28 smithi144 ceph-mon[194833]: 1.11 scrub starts 2023-09-23T20:13:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:28 smithi144 ceph-mon[194833]: 1.11 scrub ok 2023-09-23T20:13:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:28 smithi144 ceph-mon[194833]: 1.b scrub starts 2023-09-23T20:13:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:28 smithi144 ceph-mon[194833]: 1.b scrub ok 2023-09-23T20:13:28.474 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:28.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:28.582 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:28.630 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:28.641 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:28.712 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:28.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:28.819 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:28.869 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:28.946 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:28.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:29.051 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 15 16 33290 33274 138.603 105.25 0.00424346 0.0072088 2023-09-23T20:13:29.054 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:29.102 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:29.188 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:29.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:29.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:29.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:29 smithi135 ceph-mon[213871]: pgmap v1161: 109 pgs: 109 active+clean; 1.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-09-23T20:13:29.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:29 smithi135 ceph-mon[221113]: pgmap v1161: 109 pgs: 109 active+clean; 1.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-09-23T20:13:29.296 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:29.323 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:29.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:29 smithi144 ceph-mon[194833]: pgmap v1161: 109 pgs: 109 active+clean; 1.9 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-09-23T20:13:29.416 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:29.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:29.519 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:29.574 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:29.823 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:29.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:29.929 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:13:29.930 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:29.995 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:30.024 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:30.051 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 16 16 34702 34686 135.452 88.25 0.0138962 0.00737622 2023-09-23T20:13:30.380 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:30.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:30.489 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:30.539 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:30.583 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:30.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:30.689 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:30.717 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:31.052 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 17 16 36065 36049 132.487 85.1875 0.00778146 0.00754261 2023-09-23T20:13:31.083 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:31.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:31.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:31.192 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:31.236 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:31.329 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:31.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:31.437 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:31.540 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:31.645 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:31.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:31.752 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:31.782 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:31.796 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:31.863 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:31.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:31.973 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:32.025 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:32.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 18 16 37609 37593 130.488 96.5 0.00745487 0.00765746 2023-09-23T20:13:32.075 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:32.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:32.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:32.180 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:32.221 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:32.222 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:13:32.222 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:13:32.222 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:13:32.222 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:13:32.222 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:13:32.293 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:32.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:32.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:32.356 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:32 smithi135 ceph-mon[221113]: pgmap v1162: 109 pgs: 109 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T20:13:32.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:32 smithi135 ceph-mon[213871]: pgmap v1162: 109 pgs: 109 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T20:13:32.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:32 smithi144 ceph-mon[194833]: pgmap v1162: 109 pgs: 109 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T20:13:32.399 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:32.460 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:33.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 19 16 38900 38884 127.867 80.6875 0.00585978 0.00781638 2023-09-23T20:13:33.103 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:13:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:33 smithi144 ceph-mon[194833]: 1.12 scrub starts 2023-09-23T20:13:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:33 smithi144 ceph-mon[194833]: 1.12 scrub ok 2023-09-23T20:13:33.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:33 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2174443487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:33.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:33 smithi135 ceph-mon[213871]: 1.12 scrub starts 2023-09-23T20:13:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:33 smithi135 ceph-mon[213871]: 1.12 scrub ok 2023-09-23T20:13:33.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:33 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2174443487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:33 smithi135 ceph-mon[221113]: 1.12 scrub starts 2023-09-23T20:13:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:33 smithi135 ceph-mon[221113]: 1.12 scrub ok 2023-09-23T20:13:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:33 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2174443487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:33.852 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:33.857 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:13:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:13:33] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-09-23T20:13:33.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:33.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:33.960 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:34.016 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:34.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:13:34.053645+0000 min lat: 0.00116484 max lat: 0.304574 avg lat: 0.00790004 2023-09-23T20:13:34.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:13:34.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 20 16 40505 40489 126.489 100.312 0.00585381 0.00790004 2023-09-23T20:13:34.112 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:34.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: pgmap v1163: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T20:13:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:13:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:13:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2174443487' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T20:13:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: 1.f scrub starts 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[213871]: 1.f scrub ok 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: pgmap v1163: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2174443487' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T20:13:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: 1.f scrub starts 2023-09-23T20:13:34.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:34 smithi135 ceph-mon[221113]: 1.f scrub ok 2023-09-23T20:13:34.223 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:34.260 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:34.302 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:34.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: pgmap v1163: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T20:13:34.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:13:34.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:13:34.308 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2174443487' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:34.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T20:13:34.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: failed to encode map e413 with expected crc 2023-09-23T20:13:34.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: 1.f scrub starts 2023-09-23T20:13:34.309 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:34 smithi144 ceph-mon[194833]: 1.f scrub ok 2023-09-23T20:13:34.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:34.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:34.411 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:34.481 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:34.584 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:13:35.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 21 16 41786 41770 124.278 80.0625 0.00244509 0.00801196 2023-09-23T20:13:35.277 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:35.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:35.386 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:35.438 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:35.520 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:35.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:35.629 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:35.678 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:35.744 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:35.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:35.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:35.853 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:35.932 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:36.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 22 16 43145 43129 122.49 84.9375 0.0152451 0.00815952 2023-09-23T20:13:36.065 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:36.086 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:36 smithi144 ceph-mon[194833]: pgmap v1165: 109 pgs: 109 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:13:36.231 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:36.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:36.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:36.337 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:36.404 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:36 smithi135 ceph-mon[213871]: pgmap v1165: 109 pgs: 109 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:13:36.404 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:36 smithi135 ceph-mon[221113]: pgmap v1165: 109 pgs: 109 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:13:36.412 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:36.507 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:36.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:36.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:36.564 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:13:36.594 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:13:36.597 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:13:36.607 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:13:36.617 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:36.649 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:13:36.664 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:36.683 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:13:36.724 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:13:36.737 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:36.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:36.765 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:13:36.810 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:13:36.844 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:36.845 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:13:36.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:13:36.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:13:36.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:13:36.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:13:36.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:13:36.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:13:36.852 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:13:36.853 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:13:36.900 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:36.980 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:36.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:36.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:37.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 23 16 44495 44479 120.833 84.375 0.00803437 0.00827172 2023-09-23T20:13:37.091 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:37.104 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:37 smithi135 ceph-mon[213871]: 1.10 scrub starts 2023-09-23T20:13:37.105 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:37 smithi135 ceph-mon[213871]: 1.10 scrub ok 2023-09-23T20:13:37.105 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:37 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2395830685' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:37.130 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:37.216 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:37.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:37.328 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:37.333 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:13:37.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:37 smithi135 ceph-mon[221113]: 1.10 scrub starts 2023-09-23T20:13:37.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:37 smithi135 ceph-mon[221113]: 1.10 scrub ok 2023-09-23T20:13:37.366 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:37 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2395830685' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:37.369 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:37 smithi144 ceph-mon[194833]: 1.10 scrub starts 2023-09-23T20:13:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:37 smithi144 ceph-mon[194833]: 1.10 scrub ok 2023-09-23T20:13:37.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:37 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2395830685' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:38.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 24 16 45755 45739 119.079 78.75 0.00933139 0.00839269 2023-09-23T20:13:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:38 smithi144 ceph-mon[194833]: from='client.46082 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:38 smithi144 ceph-mon[194833]: pgmap v1166: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:13:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:38 smithi135 ceph-mon[213871]: from='client.46082 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:38 smithi135 ceph-mon[213871]: pgmap v1166: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:13:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:38 smithi135 ceph-mon[221113]: from='client.46082 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:38 smithi135 ceph-mon[221113]: pgmap v1166: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:13:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:38.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:38.473 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1165,"stamp":"2023-09-23T20:13:35.009190+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2573675313,"num_objects":39229,"num_object_clones":0,"num_object_copies":117687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39229,"num_whiteouts":0,"num_read":6002,"num_read_kb":13248,"num_write":40119,"num_write_kb":2521699,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":209,"num_bytes_recovered":9030920,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":43616,"ondisk_log_size":43616,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":333,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11938412,"kb_used_data":8015924,"kb_used_omap":32,"kb_used_meta":3922399,"kb_avail":644277652,"statfs":{"total":671965249536,"available":659740315648,"internally_reserved":0,"allocated":8208306176,"data_stored":8196518740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":4016537094},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,3,6,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1059016001,"num_objects":16163,"num_object_clones":0,"num_object_copies":48489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16163,"num_whiteouts":0,"num_read":-75,"num_read_kb":-140,"num_write":16127,"num_write_kb":1034129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3,"num_bytes_recovered":-705256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003223"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":411,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.095765+0000","last_change":"2023-09-23T20:07:18.956531+0000","last_active":"2023-09-23T20:13:24.095765+0000","last_peered":"2023-09-23T20:13:24.095765+0000","last_clean":"2023-09-23T20:13:24.095765+0000","last_became_active":"2023-09-23T20:07:18.956189+0000","last_became_peered":"2023-09-23T20:07:18.956189+0000","last_unstale":"2023-09-23T20:13:24.095765+0000","last_undegraded":"2023-09-23T20:13:24.095765+0000","last_fullsized":"2023-09-23T20:13:24.095765+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:40.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":242,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.093902+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:13:24.093902+0000","last_peered":"2023-09-23T20:13:24.093902+0000","last_clean":"2023-09-23T20:13:24.093902+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:13:24.093902+0000","last_undegraded":"2023-09-23T20:13:24.093902+0000","last_fullsized":"2023-09-23T20:13:24.093902+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":109,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.092758+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:13:24.092758+0000","last_peered":"2023-09-23T20:13:24.092758+0000","last_clean":"2023-09-23T20:13:24.092758+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:13:24.092758+0000","last_undegraded":"2023-09-23T20:13:24.092758+0000","last_fullsized":"2023-09-23T20:13:24.092758+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":121,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.097535+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:13:24.097535+0000","last_peered":"2023-09-23T20:13:24.097535+0000","last_clean":"2023-09-23T20:13:24.097535+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:13:24.097535+0000","last_undegraded":"2023-09-23T20:13:24.097535+0000","last_fullsized":"2023-09-23T20:13:24.097535+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":242,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.092955+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:13:24.092955+0000","last_peered":"2023-09-23T20:13:24.092955+0000","last_clean":"2023-09-23T20:13:24.092955+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:13:24.092955+0000","last_undegraded":"2023-09-23T20:13:24.092955+0000","last_fullsized":"2023-09-23T20:13:24.092955+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":358,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.102400+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:13:24.102400+0000","last_peered":"2023-09-23T20:13:24.102400+0000","last_clean":"2023-09-23T20:13:24.102400+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:13:24.102400+0000","last_undegraded":"2023-09-23T20:13:24.102400+0000","last_fullsized":"2023-09-23T20:13:24.102400+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":339,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.102294+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:13:24.102294+0000","last_peered":"2023-09-23T20:13:24.102294+0000","last_clean":"2023-09-23T20:13:24.102294+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:13:24.102294+0000","last_undegraded":"2023-09-23T20:13:24.102294+0000","last_fullsized":"2023-09-23T20:13:24.102294+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":256,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:11.734062+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:13:11.734062+0000","last_peered":"2023-09-23T20:13:11.734062+0000","last_clean":"2023-09-23T20:13:11.734062+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:13:11.734062+0000","last_undegraded":"2023-09-23T20:13:11.734062+0000","last_fullsized":"2023-09-23T20:13:11.734062+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":744,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:25.109189+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:13:25.109189+0000","last_peered":"2023-09-23T20:13:25.109189+0000","last_clean":"2023-09-23T20:13:25.109189+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:13:25.109189+0000","last_undegraded":"2023-09-23T20:13:25.109189+0000","last_fullsized":"2023-09-23T20:13:25.109189+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":412,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.100171+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:13:24.100171+0000","last_peered":"2023-09-23T20:13:24.100171+0000","last_clean":"2023-09-23T20:13:24.100171+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:13:24.100171+0000","last_undegraded":"2023-09-23T20:13:24.100171+0000","last_fullsized":"2023-09-23T20:13:24.100171+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":275,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:11.734106+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:13:11.734106+0000","last_peered":"2023-09-23T20:13:11.734106+0000","last_clean":"2023-09-23T20:13:11.734106+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:13:11.734106+0000","last_undegraded":"2023-09-23T20:13:11.734106+0000","last_fullsized":"2023-09-23T20:13:11.734106+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":106,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.097496+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:13:24.097496+0000","last_peered":"2023-09-23T20:13:24.097496+0000","last_clean":"2023-09-23T20:13:24.097496+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:13:24.097496+0000","last_undegraded":"2023-09-23T20:13:24.097496+0000","last_fullsized":"2023-09-23T20:13:24.097496+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":109,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.094558+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:13:24.094558+0000","last_peered":"2023-09-23T20:13:24.094558+0000","last_clean":"2023-09-23T20:13:24.094558+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:13:24.094558+0000","last_undegraded":"2023-09-23T20:13:24.094558+0000","last_fullsized":"2023-09-23T20:13:24.094558+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":467,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096841+0000","last_change":"2023-09-23T20:07:37.062795+0000","last_active":"2023-09-23T20:13:24.096841+0000","last_peered":"2023-09-23T20:13:24.096841+0000","last_clean":"2023-09-23T20:13:24.096841+0000","last_became_active":"2023-09-23T20:07:36.981273+0000","last_became_peered":"2023-09-23T20:07:36.981273+0000","last_unstale":"2023-09-23T20:13:24.096841+0000","last_undegraded":"2023-09-23T20:13:24.096841+0000","last_fullsized":"2023-09-23T20:13:24.096841+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:16.902341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":409,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096076+0000","last_change":"2023-09-23T20:07:18.954831+0000","last_active":"2023-09-23T20:13:24.096076+0000","last_peered":"2023-09-23T20:13:24.096076+0000","last_clean":"2023-09-23T20:13:24.096076+0000","last_became_active":"2023-09-23T20:07:18.954387+0000","last_became_peered":"2023-09-23T20:07:18.954387+0000","last_unstale":"2023-09-23T20:13:24.096076+0000","last_undegraded":"2023-09-23T20:13:24.096076+0000","last_fullsized":"2023-09-23T20:13:24.096076+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:42.295551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":772,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:27.219489+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:13:27.219489+0000","last_peered":"2023-09-23T20:13:27.219489+0000","last_clean":"2023-09-23T20:13:27.219489+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:13:27.219489+0000","last_undegraded":"2023-09-23T20:13:27.219489+0000","last_fullsized":"2023-09-23T20:13:27.219489+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":106,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.097131+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:13:24.097131+0000","last_peered":"2023-09-23T20:13:24.097131+0000","last_clean":"2023-09-23T20:13:24.097131+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:13:24.097131+0000","last_undegraded":"2023-09-23T20:13:24.097131+0000","last_fullsized":"2023-09-23T20:13:24.097131+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":266,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.093681+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:13:24.093681+0000","last_peered":"2023-09-23T20:13:24.093681+0000","last_clean":"2023-09-23T20:13:24.093681+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:13:24.093681+0000","last_undegraded":"2023-09-23T20:13:24.093681+0000","last_fullsized":"2023-09-23T20:13:24.093681+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":451,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098563+0000","last_change":"2023-09-23T20:12:57.746858+0000","last_active":"2023-09-23T20:13:24.098563+0000","last_peered":"2023-09-23T20:13:24.098563+0000","last_clean":"2023-09-23T20:13:24.098563+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:13:24.098563+0000","last_undegraded":"2023-09-23T20:13:24.098563+0000","last_fullsized":"2023-09-23T20:13:24.098563+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.079134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.079134+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":121,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096859+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:13:24.096859+0000","last_peered":"2023-09-23T20:13:24.096859+0000","last_clean":"2023-09-23T20:13:24.096859+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:13:24.096859+0000","last_undegraded":"2023-09-23T20:13:24.096859+0000","last_fullsized":"2023-09-23T20:13:24.096859+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":411,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.101005+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:13:24.101005+0000","last_peered":"2023-09-23T20:13:24.101005+0000","last_clean":"2023-09-23T20:13:24.101005+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:13:24.101005+0000","last_undegraded":"2023-09-23T20:13:24.101005+0000","last_fullsized":"2023-09-23T20:13:24.101005+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":568,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:25.715415+0000","last_change":"2023-09-23T20:13:25.715415+0000","last_active":"2023-09-23T20:13:25.715415+0000","last_peered":"2023-09-23T20:13:25.715415+0000","last_clean":"2023-09-23T20:13:25.715415+0000","last_became_active":"2023-09-23T20:12:57.738891+0000","last_became_peered":"2023-09-23T20:12:57.738891+0000","last_unstale":"2023-09-23T20:13:25.715415+0000","last_undegraded":"2023-09-23T20:13:25.715415+0000","last_fullsized":"2023-09-23T20:13:25.715415+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:03.625145+0000","scrub_duration":0.0084372379999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":100,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.099934+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:13:24.099934+0000","last_peered":"2023-09-23T20:13:24.099934+0000","last_clean":"2023-09-23T20:13:24.099934+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:13:24.099934+0000","last_undegraded":"2023-09-23T20:13:24.099934+0000","last_fullsized":"2023-09-23T20:13:24.099934+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":399,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:11.784178+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:13:11.784178+0000","last_peered":"2023-09-23T20:13:11.784178+0000","last_clean":"2023-09-23T20:13:11.784178+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:13:11.784178+0000","last_undegraded":"2023-09-23T20:13:11.784178+0000","last_fullsized":"2023-09-23T20:13:11.784178+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":530,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:31.285090+0000","last_change":"2023-09-23T20:13:31.285021+0000","last_active":"2023-09-23T20:13:31.285090+0000","last_peered":"2023-09-23T20:13:31.285090+0000","last_clean":"2023-09-23T20:13:31.285090+0000","last_became_active":"2023-09-23T20:12:57.744840+0000","last_became_peered":"2023-09-23T20:12:57.744840+0000","last_unstale":"2023-09-23T20:13:31.285090+0000","last_undegraded":"2023-09-23T20:13:31.285090+0000","last_fullsized":"2023-09-23T20:13:31.285090+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:32.440549+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":56,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098350+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:13:24.098350+0000","last_peered":"2023-09-23T20:13:24.098350+0000","last_clean":"2023-09-23T20:13:24.098350+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:13:24.098350+0000","last_undegraded":"2023-09-23T20:13:24.098350+0000","last_fullsized":"2023-09-23T20:13:24.098350+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":42,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:11.784260+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:13:11.784260+0000","last_peered":"2023-09-23T20:13:11.784260+0000","last_clean":"2023-09-23T20:13:11.784260+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:13:11.784260+0000","last_undegraded":"2023-09-23T20:13:11.784260+0000","last_fullsized":"2023-09-23T20:13:11.784260+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"413'2462","reported_seq":2476,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.285847+0000","last_change":"2023-09-23T20:13:02.794951+0000","last_active":"2023-09-23T20:13:33.285847+0000","last_peered":"2023-09-23T20:13:33.285847+0000","last_clean":"2023-09-23T20:13:33.285847+0000","last_became_active":"2023-09-23T20:13:02.794601+0000","last_became_peered":"2023-09-23T20:13:02.794601+0000","last_unstale":"2023-09-23T20:13:33.285847+0000","last_undegraded":"2023-09-23T20:13:33.285847+0000","last_fullsized":"2023-09-23T20:13:33.285847+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2462,"log_dups_size":0,"ondisk_log_size":2462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:25.583752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161349632,"num_objects":2462,"num_object_clones":0,"num_object_copies":7386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2462,"num_write_kb":157568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":653,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:25.898233+0000","last_change":"2023-09-23T20:13:25.898164+0000","last_active":"2023-09-23T20:13:25.898233+0000","last_peered":"2023-09-23T20:13:25.898233+0000","last_clean":"2023-09-23T20:13:25.898233+0000","last_became_active":"2023-09-23T20:12:57.739435+0000","last_became_peered":"2023-09-23T20:12:57.739435+0000","last_unstale":"2023-09-23T20:13:25.898233+0000","last_undegraded":"2023-09-23T20:13:25.898233+0000","last_fullsized":"2023-09-23T20:13:25.898233+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.172294+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":104,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098164+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:13:24.098164+0000","last_peered":"2023-09-23T20:13:24.098164+0000","last_clean":"2023-09-23T20:13:24.098164+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:13:24.098164+0000","last_undegraded":"2023-09-23T20:13:24.098164+0000","last_fullsized":"2023-09-23T20:13:24.098164+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":421,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098670+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:13:24.098670+0000","last_peered":"2023-09-23T20:13:24.098670+0000","last_clean":"2023-09-23T20:13:24.098670+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:13:24.098670+0000","last_undegraded":"2023-09-23T20:13:24.098670+0000","last_fullsized":"2023-09-23T20:13:24.098670+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"413'2524","reported_seq":2539,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:34.055401+0000","last_change":"2023-09-23T20:13:02.737552+0000","last_active":"2023-09-23T20:13:34.055401+0000","last_peered":"2023-09-23T20:13:34.055401+0000","last_clean":"2023-09-23T20:13:34.055401+0000","last_became_active":"2023-09-23T20:13:02.736984+0000","last_became_peered":"2023-09-23T20:13:02.736984+0000","last_unstale":"2023-09-23T20:13:34.055401+0000","last_undegraded":"2023-09-23T20:13:34.055401+0000","last_fullsized":"2023-09-23T20:13:34.055401+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2524,"log_dups_size":0,"ondisk_log_size":2524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:21.010081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165412864,"num_objects":2524,"num_object_clones":0,"num_object_copies":7572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2524,"num_write_kb":161536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":746,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:11.734301+0000","last_change":"2023-09-23T20:12:57.740105+0000","last_active":"2023-09-23T20:13:11.734301+0000","last_peered":"2023-09-23T20:13:11.734301+0000","last_clean":"2023-09-23T20:13:11.734301+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:13:11.734301+0000","last_undegraded":"2023-09-23T20:13:11.734301+0000","last_fullsized":"2023-09-23T20:13:11.734301+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:01:45.743986+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:01:45.743986+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":546134,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":105,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.097613+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:13:24.097613+0000","last_peered":"2023-09-23T20:13:24.097613+0000","last_clean":"2023-09-23T20:13:24.097613+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:13:24.097613+0000","last_undegraded":"2023-09-23T20:13:24.097613+0000","last_fullsized":"2023-09-23T20:13:24.097613+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":469,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.095815+0000","last_change":"2023-09-23T20:11:17.697083+0000","last_active":"2023-09-23T20:13:24.095815+0000","last_peered":"2023-09-23T20:13:24.095815+0000","last_clean":"2023-09-23T20:13:24.095815+0000","last_became_active":"2023-09-23T20:11:17.696720+0000","last_became_peered":"2023-09-23T20:11:17.696720+0000","last_unstale":"2023-09-23T20:13:24.095815+0000","last_undegraded":"2023-09-23T20:13:24.095815+0000","last_fullsized":"2023-09-23T20:13:24.095815+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.738910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"413'2463","reported_seq":2477,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.296649+0000","last_change":"2023-09-23T20:13:02.212695+0000","last_active":"2023-09-23T20:13:33.296649+0000","last_peered":"2023-09-23T20:13:33.296649+0000","last_clean":"2023-09-23T20:13:33.296649+0000","last_became_active":"2023-09-23T20:13:02.212400+0000","last_became_peered":"2023-09-23T20:13:02.212400+0000","last_unstale":"2023-09-23T20:13:33.296649+0000","last_undegraded":"2023-09-23T20:13:33.296649+0000","last_fullsized":"2023-09-23T20:13:33.296649+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2463,"log_dups_size":0,"ondisk_log_size":2463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:13.554058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161415168,"num_objects":2463,"num_object_clones":0,"num_object_copies":7389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2463,"num_write_kb":157632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":485,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.256489+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:13:33.256489+0000","last_peered":"2023-09-23T20:13:33.256489+0000","last_clean":"2023-09-23T20:13:33.256489+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:13:33.256489+0000","last_undegraded":"2023-09-23T20:13:33.256489+0000","last_fullsized":"2023-09-23T20:13:33.256489+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":104,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.097346+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:13:24.097346+0000","last_peered":"2023-09-23T20:13:24.097346+0000","last_clean":"2023-09-23T20:13:24.097346+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:13:24.097346+0000","last_undegraded":"2023-09-23T20:13:24.097346+0000","last_fullsized":"2023-09-23T20:13:24.097346+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":118,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.094635+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:13:24.094635+0000","last_peered":"2023-09-23T20:13:24.094635+0000","last_clean":"2023-09-23T20:13:24.094635+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:13:24.094635+0000","last_undegraded":"2023-09-23T20:13:24.094635+0000","last_fullsized":"2023-09-23T20:13:24.094635+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"413'2521","reported_seq":2535,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.299783+0000","last_change":"2023-09-23T20:13:02.211796+0000","last_active":"2023-09-23T20:13:33.299783+0000","last_peered":"2023-09-23T20:13:33.299783+0000","last_clean":"2023-09-23T20:13:33.299783+0000","last_became_active":"2023-09-23T20:13:02.211407+0000","last_became_peered":"2023-09-23T20:13:02.211407+0000","last_unstale":"2023-09-23T20:13:33.299783+0000","last_undegraded":"2023-09-23T20:13:33.299783+0000","last_fullsized":"2023-09-23T20:13:33.299783+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2521,"log_dups_size":0,"ondisk_log_size":2521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:11.120676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165216256,"num_objects":2521,"num_object_clones":0,"num_object_copies":7563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2521,"num_write_kb":161344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":641,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096567+0000","last_change":"2023-09-23T20:12:57.740764+0000","last_active":"2023-09-23T20:13:24.096567+0000","last_peered":"2023-09-23T20:13:24.096567+0000","last_clean":"2023-09-23T20:13:24.096567+0000","last_became_active":"2023-09-23T20:12:57.740404+0000","last_became_peered":"2023-09-23T20:12:57.740404+0000","last_unstale":"2023-09-23T20:13:24.096567+0000","last_undegraded":"2023-09-23T20:13:24.096567+0000","last_fullsized":"2023-09-23T20:13:24.096567+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:05.339351+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":451,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098997+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:13:24.098997+0000","last_peered":"2023-09-23T20:13:24.098997+0000","last_clean":"2023-09-23T20:13:24.098997+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:13:24.098997+0000","last_undegraded":"2023-09-23T20:13:24.098997+0000","last_fullsized":"2023-09-23T20:13:24.098997+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":109,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.093604+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:13:24.093604+0000","last_peered":"2023-09-23T20:13:24.093604+0000","last_clean":"2023-09-23T20:13:24.093604+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:13:24.093604+0000","last_undegraded":"2023-09-23T20:13:24.093604+0000","last_fullsized":"2023-09-23T20:13:24.093604+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"413'2488","reported_seq":2502,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.594506+0000","last_change":"2023-09-23T20:13:02.737123+0000","last_active":"2023-09-23T20:13:33.594506+0000","last_peered":"2023-09-23T20:13:33.594506+0000","last_clean":"2023-09-23T20:13:33.594506+0000","last_became_active":"2023-09-23T20:13:02.736852+0000","last_became_peered":"2023-09-23T20:13:02.736852+0000","last_unstale":"2023-09-23T20:13:33.594506+0000","last_undegraded":"2023-09-23T20:13:33.594506+0000","last_fullsized":"2023-09-23T20:13:33.594506+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2488,"log_dups_size":0,"ondisk_log_size":2488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:02.273532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163053568,"num_objects":2488,"num_object_clones":0,"num_object_copies":7464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2488,"num_write_kb":159232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":795,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.099839+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:13:24.099839+0000","last_peered":"2023-09-23T20:13:24.099839+0000","last_clean":"2023-09-23T20:13:24.099839+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:13:24.099839+0000","last_undegraded":"2023-09-23T20:13:24.099839+0000","last_fullsized":"2023-09-23T20:13:24.099839+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":103,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096041+0000","last_change":"2023-09-23T20:08:04.756228+0000","last_active":"2023-09-23T20:13:24.096041+0000","last_peered":"2023-09-23T20:13:24.096041+0000","last_clean":"2023-09-23T20:13:24.096041+0000","last_became_active":"2023-09-23T20:08:04.755730+0000","last_became_peered":"2023-09-23T20:08:04.755730+0000","last_unstale":"2023-09-23T20:13:24.096041+0000","last_undegraded":"2023-09-23T20:13:24.096041+0000","last_fullsized":"2023-09-23T20:13:24.096041+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:05.117038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":439,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096917+0000","last_change":"2023-09-23T20:07:18.955270+0000","last_active":"2023-09-23T20:13:24.096917+0000","last_peered":"2023-09-23T20:13:24.096917+0000","last_clean":"2023-09-23T20:13:24.096917+0000","last_became_active":"2023-09-23T20:07:18.954654+0000","last_became_peered":"2023-09-23T20:07:18.954654+0000","last_unstale":"2023-09-23T20:13:24.096917+0000","last_undegraded":"2023-09-23T20:13:24.096917+0000","last_fullsized":"2023-09-23T20:13:24.096917+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:07.483590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"413'2493","reported_seq":2508,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:34.027837+0000","last_change":"2023-09-23T20:13:02.794915+0000","last_active":"2023-09-23T20:13:34.027837+0000","last_peered":"2023-09-23T20:13:34.027837+0000","last_clean":"2023-09-23T20:13:34.027837+0000","last_became_active":"2023-09-23T20:13:02.794591+0000","last_became_peered":"2023-09-23T20:13:02.794591+0000","last_unstale":"2023-09-23T20:13:34.027837+0000","last_undegraded":"2023-09-23T20:13:34.027837+0000","last_fullsized":"2023-09-23T20:13:34.027837+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2493,"log_dups_size":0,"ondisk_log_size":2493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:28.806237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163381248,"num_objects":2493,"num_object_clones":0,"num_object_copies":7479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2493,"num_write_kb":159552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":760,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098480+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:13:24.098480+0000","last_peered":"2023-09-23T20:13:24.098480+0000","last_clean":"2023-09-23T20:13:24.098480+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:13:24.098480+0000","last_undegraded":"2023-09-23T20:13:24.098480+0000","last_fullsized":"2023-09-23T20:13:24.098480+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":379,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.095729+0000","last_change":"2023-09-23T20:07:18.956381+0000","last_active":"2023-09-23T20:13:24.095729+0000","last_peered":"2023-09-23T20:13:24.095729+0000","last_clean":"2023-09-23T20:13:24.095729+0000","last_became_active":"2023-09-23T20:07:18.955186+0000","last_became_peered":"2023-09-23T20:07:18.955186+0000","last_unstale":"2023-09-23T20:13:24.095729+0000","last_undegraded":"2023-09-23T20:13:24.095729+0000","last_fullsized":"2023-09-23T20:13:24.095729+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:57.763718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":409,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096397+0000","last_change":"2023-09-23T20:07:18.963064+0000","last_active":"2023-09-23T20:13:24.096397+0000","last_peered":"2023-09-23T20:13:24.096397+0000","last_clean":"2023-09-23T20:13:24.096397+0000","last_became_active":"2023-09-23T20:07:18.962633+0000","last_became_peered":"2023-09-23T20:07:18.962633+0000","last_unstale":"2023-09-23T20:13:24.096397+0000","last_undegraded":"2023-09-23T20:13:24.096397+0000","last_fullsized":"2023-09-23T20:13:24.096397+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:27.848618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"413'2480","reported_seq":2494,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.611136+0000","last_change":"2023-09-23T20:13:02.212538+0000","last_active":"2023-09-23T20:13:33.611136+0000","last_peered":"2023-09-23T20:13:33.611136+0000","last_clean":"2023-09-23T20:13:33.611136+0000","last_became_active":"2023-09-23T20:13:02.212305+0000","last_became_peered":"2023-09-23T20:13:02.212305+0000","last_unstale":"2023-09-23T20:13:33.611136+0000","last_undegraded":"2023-09-23T20:13:33.611136+0000","last_fullsized":"2023-09-23T20:13:33.611136+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2480,"log_dups_size":0,"ondisk_log_size":2480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:10.419377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162529280,"num_objects":2480,"num_object_clones":0,"num_object_copies":7440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2480,"num_write_kb":158720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":568,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:27.750109+0000","last_change":"2023-09-23T20:13:27.750109+0000","last_active":"2023-09-23T20:13:27.750109+0000","last_peered":"2023-09-23T20:13:27.750109+0000","last_clean":"2023-09-23T20:13:27.750109+0000","last_became_active":"2023-09-23T20:12:57.745568+0000","last_became_peered":"2023-09-23T20:12:57.745568+0000","last_unstale":"2023-09-23T20:13:27.750109+0000","last_undegraded":"2023-09-23T20:13:27.750109+0000","last_fullsized":"2023-09-23T20:13:27.750109+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:09.736440+0000","scrub_duration":0.0052404440000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":403,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.095630+0000","last_change":"2023-09-23T20:07:36.981225+0000","last_active":"2023-09-23T20:13:24.095630+0000","last_peered":"2023-09-23T20:13:24.095630+0000","last_clean":"2023-09-23T20:13:24.095630+0000","last_became_active":"2023-09-23T20:07:36.980915+0000","last_became_peered":"2023-09-23T20:07:36.980915+0000","last_unstale":"2023-09-23T20:13:24.095630+0000","last_undegraded":"2023-09-23T20:13:24.095630+0000","last_fullsized":"2023-09-23T20:13:24.095630+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:35.246266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":439,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096689+0000","last_change":"2023-09-23T20:07:18.955972+0000","last_active":"2023-09-23T20:13:24.096689+0000","last_peered":"2023-09-23T20:13:24.096689+0000","last_clean":"2023-09-23T20:13:24.096689+0000","last_became_active":"2023-09-23T20:07:18.955387+0000","last_became_peered":"2023-09-23T20:07:18.955387+0000","last_unstale":"2023-09-23T20:13:24.096689+0000","last_undegraded":"2023-09-23T20:13:24.096689+0000","last_fullsized":"2023-09-23T20:13:24.096689+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:19.353585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"412'2290","reported_seq":2304,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:31.151512+0000","last_change":"2023-09-23T20:13:02.209574+0000","last_active":"2023-09-23T20:13:31.151512+0000","last_peered":"2023-09-23T20:13:31.151512+0000","last_clean":"2023-09-23T20:13:31.151512+0000","last_became_active":"2023-09-23T20:13:02.209307+0000","last_became_peered":"2023-09-23T20:13:02.209307+0000","last_unstale":"2023-09-23T20:13:31.151512+0000","last_undegraded":"2023-09-23T20:13:31.151512+0000","last_fullsized":"2023-09-23T20:13:31.151512+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2290,"log_dups_size":0,"ondisk_log_size":2290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:18.271180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150077440,"num_objects":2290,"num_object_clones":0,"num_object_copies":6870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2290,"num_write_kb":146560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":751,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098830+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:13:24.098830+0000","last_peered":"2023-09-23T20:13:24.098830+0000","last_clean":"2023-09-23T20:13:24.098830+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:13:24.098830+0000","last_undegraded":"2023-09-23T20:13:24.098830+0000","last_fullsized":"2023-09-23T20:13:24.098830+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":43,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.085617+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:13:24.085617+0000","last_peered":"2023-09-23T20:13:24.085617+0000","last_clean":"2023-09-23T20:13:24.085617+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:13:24.085617+0000","last_undegraded":"2023-09-23T20:13:24.085617+0000","last_fullsized":"2023-09-23T20:13:24.085617+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":104,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.093868+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:13:24.093868+0000","last_peered":"2023-09-23T20:13:24.093868+0000","last_clean":"2023-09-23T20:13:24.093868+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:13:24.093868+0000","last_undegraded":"2023-09-23T20:13:24.093868+0000","last_fullsized":"2023-09-23T20:13:24.093868+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"413'2436","reported_seq":2450,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.275114+0000","last_change":"2023-09-23T20:13:02.211701+0000","last_active":"2023-09-23T20:13:33.275114+0000","last_peered":"2023-09-23T20:13:33.275114+0000","last_clean":"2023-09-23T20:13:33.275114+0000","last_became_active":"2023-09-23T20:13:02.211349+0000","last_became_peered":"2023-09-23T20:13:02.211349+0000","last_unstale":"2023-09-23T20:13:33.275114+0000","last_undegraded":"2023-09-23T20:13:33.275114+0000","last_fullsized":"2023-09-23T20:13:33.275114+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2436,"log_dups_size":0,"ondisk_log_size":2436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:35.001450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159645696,"num_objects":2436,"num_object_clones":0,"num_object_copies":7308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2436,"num_write_kb":155904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":681,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.086346+0000","last_change":"2023-09-23T20:11:20.857010+0000","last_active":"2023-09-23T20:13:24.086346+0000","last_peered":"2023-09-23T20:13:24.086346+0000","last_clean":"2023-09-23T20:13:24.086346+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:13:24.086346+0000","last_undegraded":"2023-09-23T20:13:24.086346+0000","last_fullsized":"2023-09-23T20:13:24.086346+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:01:57.172074+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:01:57.172074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015118009999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":546,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.094150+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:13:24.094150+0000","last_peered":"2023-09-23T20:13:24.094150+0000","last_clean":"2023-09-23T20:13:24.094150+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:13:24.094150+0000","last_undegraded":"2023-09-23T20:13:24.094150+0000","last_fullsized":"2023-09-23T20:13:24.094150+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":413,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.100935+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:13:24.100935+0000","last_peered":"2023-09-23T20:13:24.100935+0000","last_clean":"2023-09-23T20:13:24.100935+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:13:24.100935+0000","last_undegraded":"2023-09-23T20:13:24.100935+0000","last_fullsized":"2023-09-23T20:13:24.100935+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.091704+0000","last_change":"2023-09-23T20:07:18.953029+0000","last_active":"2023-09-23T20:13:33.091704+0000","last_peered":"2023-09-23T20:13:33.091704+0000","last_clean":"2023-09-23T20:13:33.091704+0000","last_became_active":"2023-09-23T20:07:18.951420+0000","last_became_peered":"2023-09-23T20:07:18.951420+0000","last_unstale":"2023-09-23T20:13:33.091704+0000","last_undegraded":"2023-09-23T20:13:33.091704+0000","last_fullsized":"2023-09-23T20:13:33.091704+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:53.324489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"413'2586","reported_seq":2600,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.589230+0000","last_change":"2023-09-23T20:13:02.796250+0000","last_active":"2023-09-23T20:13:33.589230+0000","last_peered":"2023-09-23T20:13:33.589230+0000","last_clean":"2023-09-23T20:13:33.589230+0000","last_became_active":"2023-09-23T20:13:02.795989+0000","last_became_peered":"2023-09-23T20:13:02.795989+0000","last_unstale":"2023-09-23T20:13:33.589230+0000","last_undegraded":"2023-09-23T20:13:33.589230+0000","last_fullsized":"2023-09-23T20:13:33.589230+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2586,"log_dups_size":0,"ondisk_log_size":2586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:49.452140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169476096,"num_objects":2586,"num_object_clones":0,"num_object_copies":7758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2586,"num_write_kb":165504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":756,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098539+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:13:24.098539+0000","last_peered":"2023-09-23T20:13:24.098539+0000","last_clean":"2023-09-23T20:13:24.098539+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:13:24.098539+0000","last_undegraded":"2023-09-23T20:13:24.098539+0000","last_fullsized":"2023-09-23T20:13:24.098539+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1033,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098486+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:13:24.098486+0000","last_peered":"2023-09-23T20:13:24.098486+0000","last_clean":"2023-09-23T20:13:24.098486+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:13:24.098486+0000","last_undegraded":"2023-09-23T20:13:24.098486+0000","last_fullsized":"2023-09-23T20:13:24.098486+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":432,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096154+0000","last_change":"2023-09-23T20:07:18.940612+0000","last_active":"2023-09-23T20:13:24.096154+0000","last_peered":"2023-09-23T20:13:24.096154+0000","last_clean":"2023-09-23T20:13:24.096154+0000","last_became_active":"2023-09-23T20:07:18.939557+0000","last_became_peered":"2023-09-23T20:07:18.939557+0000","last_unstale":"2023-09-23T20:13:24.096154+0000","last_undegraded":"2023-09-23T20:13:24.096154+0000","last_fullsized":"2023-09-23T20:13:24.096154+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:51.752371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.094071+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:13:24.094071+0000","last_peered":"2023-09-23T20:13:24.094071+0000","last_clean":"2023-09-23T20:13:24.094071+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:13:24.094071+0000","last_undegraded":"2023-09-23T20:13:24.094071+0000","last_fullsized":"2023-09-23T20:13:24.094071+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"413'2419","reported_seq":2433,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.299920+0000","last_change":"2023-09-23T20:13:02.212686+0000","last_active":"2023-09-23T20:13:33.299920+0000","last_peered":"2023-09-23T20:13:33.299920+0000","last_clean":"2023-09-23T20:13:33.299920+0000","last_became_active":"2023-09-23T20:13:02.212438+0000","last_became_peered":"2023-09-23T20:13:02.212438+0000","last_unstale":"2023-09-23T20:13:33.299920+0000","last_undegraded":"2023-09-23T20:13:33.299920+0000","last_fullsized":"2023-09-23T20:13:33.299920+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2419,"log_dups_size":0,"ondisk_log_size":2419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:09.919399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158531584,"num_objects":2419,"num_object_clones":0,"num_object_copies":7257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2419,"num_write_kb":154816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":651,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096973+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:13:24.096973+0000","last_peered":"2023-09-23T20:13:24.096973+0000","last_clean":"2023-09-23T20:13:24.096973+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:13:24.096973+0000","last_undegraded":"2023-09-23T20:13:24.096973+0000","last_fullsized":"2023-09-23T20:13:24.096973+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1049,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:15.914957+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:13:15.914957+0000","last_peered":"2023-09-23T20:13:15.914957+0000","last_clean":"2023-09-23T20:13:15.914957+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:13:15.914957+0000","last_undegraded":"2023-09-23T20:13:15.914957+0000","last_fullsized":"2023-09-23T20:13:15.914957+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":110,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.093091+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:13:24.093091+0000","last_peered":"2023-09-23T20:13:24.093091+0000","last_clean":"2023-09-23T20:13:24.093091+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:13:24.093091+0000","last_undegraded":"2023-09-23T20:13:24.093091+0000","last_fullsized":"2023-09-23T20:13:24.093091+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":104,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.093834+0000","last_change":"2023-09-23T20:07:18.951247+0000","last_active":"2023-09-23T20:13:24.093834+0000","last_peered":"2023-09-23T20:13:24.093834+0000","last_clean":"2023-09-23T20:13:24.093834+0000","last_became_active":"2023-09-23T20:07:18.950496+0000","last_became_peered":"2023-09-23T20:07:18.950496+0000","last_unstale":"2023-09-23T20:13:24.093834+0000","last_undegraded":"2023-09-23T20:13:24.093834+0000","last_fullsized":"2023-09-23T20:13:24.093834+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:23.598137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"413'2451","reported_seq":2465,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.296492+0000","last_change":"2023-09-23T20:13:02.794985+0000","last_active":"2023-09-23T20:13:33.296492+0000","last_peered":"2023-09-23T20:13:33.296492+0000","last_clean":"2023-09-23T20:13:33.296492+0000","last_became_active":"2023-09-23T20:13:02.794652+0000","last_became_peered":"2023-09-23T20:13:02.794652+0000","last_unstale":"2023-09-23T20:13:33.296492+0000","last_undegraded":"2023-09-23T20:13:33.296492+0000","last_fullsized":"2023-09-23T20:13:33.296492+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2451,"log_dups_size":0,"ondisk_log_size":2451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:32.459920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160628736,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2451,"num_write_kb":156864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1321,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.099071+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:13:24.099071+0000","last_peered":"2023-09-23T20:13:24.099071+0000","last_clean":"2023-09-23T20:13:24.099071+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:13:24.099071+0000","last_undegraded":"2023-09-23T20:13:24.099071+0000","last_fullsized":"2023-09-23T20:13:24.099071+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":588,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096542+0000","last_change":"2023-09-23T20:12:54.043697+0000","last_active":"2023-09-23T20:13:24.096542+0000","last_peered":"2023-09-23T20:13:24.096542+0000","last_clean":"2023-09-23T20:13:24.096542+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:13:24.096542+0000","last_undegraded":"2023-09-23T20:13:24.096542+0000","last_fullsized":"2023-09-23T20:13:24.096542+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:01.598813+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":278,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098574+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:13:24.098574+0000","last_peered":"2023-09-23T20:13:24.098574+0000","last_clean":"2023-09-23T20:13:24.098574+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:13:24.098574+0000","last_undegraded":"2023-09-23T20:13:24.098574+0000","last_fullsized":"2023-09-23T20:13:24.098574+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":308,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096573+0000","last_change":"2023-09-23T20:07:36.974978+0000","last_active":"2023-09-23T20:13:24.096573+0000","last_peered":"2023-09-23T20:13:24.096573+0000","last_clean":"2023-09-23T20:13:24.096573+0000","last_became_active":"2023-09-23T20:07:36.974655+0000","last_became_peered":"2023-09-23T20:07:36.974655+0000","last_unstale":"2023-09-23T20:13:24.096573+0000","last_undegraded":"2023-09-23T20:13:24.096573+0000","last_fullsized":"2023-09-23T20:13:24.096573+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:04.389246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"413'2434","reported_seq":2448,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.288126+0000","last_change":"2023-09-23T20:13:02.795014+0000","last_active":"2023-09-23T20:13:33.288126+0000","last_peered":"2023-09-23T20:13:33.288126+0000","last_clean":"2023-09-23T20:13:33.288126+0000","last_became_active":"2023-09-23T20:13:02.794587+0000","last_became_peered":"2023-09-23T20:13:02.794587+0000","last_unstale":"2023-09-23T20:13:33.288126+0000","last_undegraded":"2023-09-23T20:13:33.288126+0000","last_fullsized":"2023-09-23T20:13:33.288126+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2434,"log_dups_size":0,"ondisk_log_size":2434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:08.535861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2434,"num_write_kb":155776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":551,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.093654+0000","last_change":"2023-09-23T20:12:38.758800+0000","last_active":"2023-09-23T20:13:24.093654+0000","last_peered":"2023-09-23T20:13:24.093654+0000","last_clean":"2023-09-23T20:13:24.093654+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:13:24.093654+0000","last_undegraded":"2023-09-23T20:13:24.093654+0000","last_fullsized":"2023-09-23T20:13:24.093654+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:56.065108+0000","scrub_duration":0.001864159,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":849,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096434+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:13:24.096434+0000","last_peered":"2023-09-23T20:13:24.096434+0000","last_clean":"2023-09-23T20:13:24.096434+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:13:24.096434+0000","last_undegraded":"2023-09-23T20:13:24.096434+0000","last_fullsized":"2023-09-23T20:13:24.096434+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":399,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.095841+0000","last_change":"2023-09-23T20:11:17.705411+0000","last_active":"2023-09-23T20:13:24.095841+0000","last_peered":"2023-09-23T20:13:24.095841+0000","last_clean":"2023-09-23T20:13:24.095841+0000","last_became_active":"2023-09-23T20:11:17.705124+0000","last_became_peered":"2023-09-23T20:11:17.705124+0000","last_unstale":"2023-09-23T20:13:24.095841+0000","last_undegraded":"2023-09-23T20:13:24.095841+0000","last_fullsized":"2023-09-23T20:13:24.095841+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:03.483026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":109,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.094620+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:13:24.094620+0000","last_peered":"2023-09-23T20:13:24.094620+0000","last_clean":"2023-09-23T20:13:24.094620+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:13:24.094620+0000","last_undegraded":"2023-09-23T20:13:24.094620+0000","last_fullsized":"2023-09-23T20:13:24.094620+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"413'2467","reported_seq":2481,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.602941+0000","last_change":"2023-09-23T20:13:02.736405+0000","last_active":"2023-09-23T20:13:33.602941+0000","last_peered":"2023-09-23T20:13:33.602941+0000","last_clean":"2023-09-23T20:13:33.602941+0000","last_became_active":"2023-09-23T20:13:02.736146+0000","last_became_peered":"2023-09-23T20:13:02.736146+0000","last_unstale":"2023-09-23T20:13:33.602941+0000","last_undegraded":"2023-09-23T20:13:33.602941+0000","last_fullsized":"2023-09-23T20:13:33.602941+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2467,"log_dups_size":0,"ondisk_log_size":2467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:41.788802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161677312,"num_objects":2467,"num_object_clones":0,"num_object_copies":7401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2467,"num_write_kb":157888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":631,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096956+0000","last_change":"2023-09-23T20:12:49.763399+0000","last_active":"2023-09-23T20:13:24.096956+0000","last_peered":"2023-09-23T20:13:24.096956+0000","last_clean":"2023-09-23T20:13:24.096956+0000","last_became_active":"2023-09-23T20:09:56.042525+0000","last_became_peered":"2023-09-23T20:09:56.042525+0000","last_unstale":"2023-09-23T20:13:24.096956+0000","last_undegraded":"2023-09-23T20:13:24.096956+0000","last_fullsized":"2023-09-23T20:13:24.096956+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:23.024903+0000","scrub_duration":0.001589575,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":113,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.097422+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:13:24.097422+0000","last_peered":"2023-09-23T20:13:24.097422+0000","last_clean":"2023-09-23T20:13:24.097422+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:13:24.097422+0000","last_undegraded":"2023-09-23T20:13:24.097422+0000","last_fullsized":"2023-09-23T20:13:24.097422+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":109,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.094000+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:13:24.094000+0000","last_peered":"2023-09-23T20:13:24.094000+0000","last_clean":"2023-09-23T20:13:24.094000+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:13:24.094000+0000","last_undegraded":"2023-09-23T20:13:24.094000+0000","last_fullsized":"2023-09-23T20:13:24.094000+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":239,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098990+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:13:24.098990+0000","last_peered":"2023-09-23T20:13:24.098990+0000","last_clean":"2023-09-23T20:13:24.098990+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:13:24.098990+0000","last_undegraded":"2023-09-23T20:13:24.098990+0000","last_fullsized":"2023-09-23T20:13:24.098990+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"413'2484","reported_seq":2498,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T20:13:33.581610+0000","last_change":"2023-09-23T20:13:02.213807+0000","last_active":"2023-09-23T20:13:33.581610+0000","last_peered":"2023-09-23T20:13:33.581610+0000","last_clean":"2023-09-23T20:13:33.581610+0000","last_became_active":"2023-09-23T20:13:02.213521+0000","last_became_peered":"2023-09-23T20:13:02.213521+0000","last_unstale":"2023-09-23T20:13:33.581610+0000","last_undegraded":"2023-09-23T20:13:33.581610+0000","last_fullsized":"2023-09-23T20:13:33.581610+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2484,"log_dups_size":0,"ondisk_log_size":2484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:09.558945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162791424,"num_objects":2484,"num_object_clones":0,"num_object_copies":7452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2484,"num_write_kb":158976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":859,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096262+0000","last_change":"2023-09-23T20:12:53.737343+0000","last_active":"2023-09-23T20:13:24.096262+0000","last_peered":"2023-09-23T20:13:24.096262+0000","last_clean":"2023-09-23T20:13:24.096262+0000","last_became_active":"2023-09-23T20:10:52.686550+0000","last_became_peered":"2023-09-23T20:10:52.686550+0000","last_unstale":"2023-09-23T20:13:24.096262+0000","last_undegraded":"2023-09-23T20:13:24.096262+0000","last_fullsized":"2023-09-23T20:13:24.096262+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.611543+0000","scrub_duration":0.0015738379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":255,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098020+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:13:24.098020+0000","last_peered":"2023-09-23T20:13:24.098020+0000","last_clean":"2023-09-23T20:13:24.098020+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:13:24.098020+0000","last_undegraded":"2023-09-23T20:13:24.098020+0000","last_fullsized":"2023-09-23T20:13:24.098020+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":451,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.094532+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:13:24.094532+0000","last_peered":"2023-09-23T20:13:24.094532+0000","last_clean":"2023-09-23T20:13:24.094532+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:13:24.094532+0000","last_undegraded":"2023-09-23T20:13:24.094532+0000","last_fullsized":"2023-09-23T20:13:24.094532+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":43,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:21.180800+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:13:21.180800+0000","last_peered":"2023-09-23T20:13:21.180800+0000","last_clean":"2023-09-23T20:13:21.180800+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:13:21.180800+0000","last_undegraded":"2023-09-23T20:13:21.180800+0000","last_fullsized":"2023-09-23T20:13:21.180800+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"412'2203","reported_seq":2217,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:31.170617+0000","last_change":"2023-09-23T20:13:02.737701+0000","last_active":"2023-09-23T20:13:31.170617+0000","last_peered":"2023-09-23T20:13:31.170617+0000","last_clean":"2023-09-23T20:13:31.170617+0000","last_became_active":"2023-09-23T20:13:02.737397+0000","last_became_peered":"2023-09-23T20:13:02.737397+0000","last_unstale":"2023-09-23T20:13:31.170617+0000","last_undegraded":"2023-09-23T20:13:31.170617+0000","last_fullsized":"2023-09-23T20:13:31.170617+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":2203,"log_dups_size":0,"ondisk_log_size":2203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.355044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144375808,"num_objects":2203,"num_object_clones":0,"num_object_copies":6609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2203,"num_write_kb":140992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":736,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:26.021476+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:13:26.021476+0000","last_peered":"2023-09-23T20:13:26.021476+0000","last_clean":"2023-09-23T20:13:26.021476+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:13:26.021476+0000","last_undegraded":"2023-09-23T20:13:26.021476+0000","last_fullsized":"2023-09-23T20:13:26.021476+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":114,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098112+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:13:24.098112+0000","last_peered":"2023-09-23T20:13:24.098112+0000","last_clean":"2023-09-23T20:13:24.098112+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:13:24.098112+0000","last_undegraded":"2023-09-23T20:13:24.098112+0000","last_fullsized":"2023-09-23T20:13:24.098112+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":113,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.094406+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:13:24.094406+0000","last_peered":"2023-09-23T20:13:24.094406+0000","last_clean":"2023-09-23T20:13:24.094406+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:13:24.094406+0000","last_undegraded":"2023-09-23T20:13:24.094406+0000","last_fullsized":"2023-09-23T20:13:24.094406+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":372,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.095718+0000","last_change":"2023-09-23T20:01:52.442987+0000","last_active":"2023-09-23T20:13:24.095718+0000","last_peered":"2023-09-23T20:13:24.095718+0000","last_clean":"2023-09-23T20:13:24.095718+0000","last_became_active":"2023-09-23T20:01:52.442663+0000","last_became_peered":"2023-09-23T20:01:52.442663+0000","last_unstale":"2023-09-23T20:13:24.095718+0000","last_undegraded":"2023-09-23T20:13:24.095718+0000","last_fullsized":"2023-09-23T20:13:24.095718+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:52.923053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":187,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:12.474976+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:13:12.474976+0000","last_peered":"2023-09-23T20:13:12.474976+0000","last_clean":"2023-09-23T20:13:12.474976+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:13:12.474976+0000","last_undegraded":"2023-09-23T20:13:12.474976+0000","last_fullsized":"2023-09-23T20:13:12.474976+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":179,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.093492+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:13:24.093492+0000","last_peered":"2023-09-23T20:13:24.093492+0000","last_clean":"2023-09-23T20:13:24.093492+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:13:24.093492+0000","last_undegraded":"2023-09-23T20:13:24.093492+0000","last_fullsized":"2023-09-23T20:13:24.093492+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":416,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.098852+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:13:24.098852+0000","last_peered":"2023-09-23T20:13:24.098852+0000","last_clean":"2023-09-23T20:13:24.098852+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:13:24.098852+0000","last_undegraded":"2023-09-23T20:13:24.098852+0000","last_fullsized":"2023-09-23T20:13:24.098852+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":429,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.096216+0000","last_change":"2023-09-23T20:07:18.955925+0000","last_active":"2023-09-23T20:13:24.096216+0000","last_peered":"2023-09-23T20:13:24.096216+0000","last_clean":"2023-09-23T20:13:24.096216+0000","last_became_active":"2023-09-23T20:07:18.955500+0000","last_became_peered":"2023-09-23T20:07:18.955500+0000","last_unstale":"2023-09-23T20:13:24.096216+0000","last_undegraded":"2023-09-23T20:13:24.096216+0000","last_fullsized":"2023-09-23T20:13:24.096216+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:23.332334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":406,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.100842+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:13:24.100842+0000","last_peered":"2023-09-23T20:13:24.100842+0000","last_clean":"2023-09-23T20:13:24.100842+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:13:24.100842+0000","last_undegraded":"2023-09-23T20:13:24.100842+0000","last_fullsized":"2023-09-23T20:13:24.100842+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":409,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.095679+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:13:24.095679+0000","last_peered":"2023-09-23T20:13:24.095679+0000","last_clean":"2023-09-23T20:13:24.095679+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:13:24.095679+0000","last_undegraded":"2023-09-23T20:13:24.095679+0000","last_fullsized":"2023-09-23T20:13:24.095679+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":113,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:12.474440+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:13:12.474440+0000","last_peered":"2023-09-23T20:13:12.474440+0000","last_clean":"2023-09-23T20:13:12.474440+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:13:12.474440+0000","last_undegraded":"2023-09-23T20:13:12.474440+0000","last_fullsized":"2023-09-23T20:13:12.474440+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":270,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T20:13:12.475152+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:13:12.475152+0000","last_peered":"2023-09-23T20:13:12.475152+0000","last_clean":"2023-09-23T20:13:12.475152+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:13:12.475152+0000","last_undegraded":"2023-09-23T20:13:12.475152+0000","last_fullsized":"2023-09-23T20:13:12.475152+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":258,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.102222+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:13:24.102222+0000","last_peered":"2023-09-23T20:13:24.102222+0000","last_clean":"2023-09-23T20:13:24.102222+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:13:24.102222+0000","last_undegraded":"2023-09-23T20:13:24.102222+0000","last_fullsized":"2023-09-23T20:13:24.102222+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":459,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T20:13:24.095627+0000","last_change":"2023-09-23T20:07:18.954964+0000","last_active":"2023-09-23T20:13:24.095627+0000","last_peered":"2023-09-23T20:13:24.095627+0000","last_clean":"2023-09-23T20:13:24.095627+0000","last_became_active":"2023-09-23T20:07:18.954449+0000","last_became_peered":"2023-09-23T20:07:18.954449+0000","last_unstale":"2023-09-23T20:13:24.095627+0000","last_undegraded":"2023-09-23T20:13:24.095627+0000","last_fullsized":"2023-09-23T20:13:24.095627+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:42.006470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":15616,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":398750,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":626639,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2569076736,"num_objects":39201,"num_object_clones":0,"num_object_copies":117603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39201,"num_write_kb":2508864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7631536128,"data_stored":7631536128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39201,"ondisk_log_size":39201,"up":48,"acting":48,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234389,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1767120,"kb_used_data":1395272,"kb_used_omap":0,"kb_used_meta":371840,"kb_avail":91978032,"statfs":{"total":95995035648,"available":94185504768,"internally_reserved":0,"allocated":1428758528,"data_stored":1427481316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":380764160},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.0019999999999998,"15min":2.2599999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":6.3630000000000004,"5min":55.390000000000001,"15min":62.061999999999998},"last":4.0970000000000004},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.0960000000000001,"15min":2.3090000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.19900000000000001},"max":{"1min":6.0049999999999999,"5min":56.664000000000001,"15min":62.389000000000003},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":8.8119999999999994,"5min":3.4590000000000001,"15min":3.8889999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":91.335999999999999,"5min":91.335999999999999,"15min":104.16800000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":11.324,"5min":4.0149999999999997,"15min":4.1639999999999997},"min":{"1min":0.29399999999999998,"5min":0.249,"15min":0.223},"max":{"1min":91.516999999999996,"5min":91.516999999999996,"15min":104.003},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":9.4710000000000001,"5min":5.3049999999999997,"15min":4.2329999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":67.760000000000005,"5min":92.319000000000003,"15min":105.842},"last":1.2410000000000001},{"interface":"front","average":{"1min":10.063000000000001,"5min":5.7400000000000002,"15min":4.7290000000000001},"min":{"1min":0.27700000000000002,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":67.488,"5min":92.049000000000007,"15min":105.66},"last":1}]},{"osd":3,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.5950000000000002,"15min":4.4379999999999997},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":8.2720000000000002,"5min":94.953000000000003,"15min":146.38800000000001},"last":1.786},{"interface":"front","average":{"1min":1.4370000000000001,"5min":2.444,"15min":4.375},"min":{"1min":0.29699999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.5369999999999999,"5min":93.423000000000002,"15min":146.851},"last":1.5740000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:13:11 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.48899999999999999,"15min":0.72999999999999998},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.22500000000000001},"max":{"1min":1.2869999999999999,"5min":3.2069999999999999,"15min":19.215},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.54900000000000004,"15min":0.85099999999999998},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":2.75,"5min":2.75,"15min":41.822000000000003},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:13:11 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.65900000000000003,"15min":1.006},"min":{"1min":0.29899999999999999,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":5.1580000000000004,"5min":5.1580000000000004,"15min":35.488},"last":1.2909999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.61399999999999999,"15min":0.86799999999999999},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":1.7290000000000001,"5min":4.2030000000000003,"15min":34.896000000000001},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.70299999999999996,"15min":0.83399999999999996},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":3.4849999999999999,"5min":3.4849999999999999,"15min":41.902999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.55800000000000005,"15min":0.64000000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30399999999999999},"max":{"1min":2.3500000000000001,"5min":2.4590000000000001,"15min":6.3620000000000001},"last":1.123}]}]},{"osd":6,"up_from":0,"seq":1078036791430,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762588,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2096820,"kb_used_data":1448684,"kb_used_omap":7,"kb_used_meta":648120,"kb_avail":91648332,"statfs":{"total":95995035648,"available":93847891968,"internally_reserved":0,"allocated":1483452416,"data_stored":1481506730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":663675706},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.8089999999999999,"15min":2.9550000000000001},"min":{"1min":0.20899999999999999,"5min":0.20200000000000001,"15min":0.16900000000000001},"max":{"1min":6.5590000000000002,"5min":28.510000000000002,"15min":144.273},"last":0.248},{"interface":"front","average":{"1min":2.573,"5min":2.0779999999999998,"15min":3.3149999999999999},"min":{"1min":0.217,"5min":0.19800000000000001,"15min":0.19400000000000001},"max":{"1min":20.997,"5min":20.997,"15min":144.321},"last":3.0699999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":3.766,"15min":4.4660000000000002},"min":{"1min":0.21099999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":24.207000000000001,"5min":58.499000000000002,"15min":134.72499999999999},"last":0.312},{"interface":"front","average":{"1min":3.8029999999999999,"5min":3.524,"15min":4.5970000000000004},"min":{"1min":0.223,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":13.105,"5min":58.741,"15min":135.74600000000001},"last":0.26800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.1360000000000001,"15min":3.1040000000000001},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.22},"max":{"1min":12.384,"5min":25.963999999999999,"15min":56.564},"last":1.329},{"interface":"front","average":{"1min":1.335,"5min":1.6519999999999999,"15min":3.0270000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.247},"max":{"1min":9.1760000000000002,"5min":12.723000000000001,"15min":56.939999999999998},"last":2.9020000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.54,"15min":1.798},"min":{"1min":0.379,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":6.3979999999999997,"5min":18.087,"15min":35.497999999999998},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.373,"5min":1.341,"15min":1.9359999999999999},"min":{"1min":0.35899999999999999,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":9.0280000000000005,"5min":11.08,"15min":35.521000000000001},"last":2.9670000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":2.157,"15min":2.351},"min":{"1min":0.40300000000000002,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":9.0540000000000003,"5min":83.510999999999996,"15min":83.510999999999996},"last":2.9399999999999999},{"interface":"front","average":{"1min":1.24,"5min":2.3570000000000002,"15min":2.4449999999999998},"min":{"1min":0.38400000000000001,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":9.4290000000000003,"5min":83.353999999999999,"15min":83.353999999999999},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":1.8959999999999999,"15min":2.0009999999999999},"min":{"1min":0.42299999999999999,"5min":0.38500000000000001,"15min":0.35899999999999999},"max":{"1min":11.868,"5min":18.117999999999999,"15min":18.117999999999999},"last":1.0089999999999999},{"interface":"front","average":{"1min":1.8260000000000001,"5min":1.6799999999999999,"15min":2.5499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":11.109,"5min":11.109,"15min":20.734999999999999},"last":3.0449999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.6910000000000001,"15min":2.1030000000000002},"min":{"1min":0.39500000000000002,"5min":0.33300000000000002,"15min":0.30399999999999999},"max":{"1min":9.4730000000000008,"5min":25.937000000000001,"15min":25.937000000000001},"last":0.375},{"interface":"front","average":{"1min":1.46,"5min":1.6599999999999999,"15min":1.891},"min":{"1min":0.45800000000000002,"5min":0.373,"15min":0.30199999999999999},"max":{"1min":6.4210000000000003,"5min":18.053999999999998,"15min":18.053999999999998},"last":0.437}]}]},{"osd":0,"up_from":94,"seq":403726926125,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1467804,"kb_used_data":831572,"kb_used_omap":8,"kb_used_meta":636215,"kb_avail":92277348,"statfs":{"total":95995035648,"available":94492004352,"internally_reserved":0,"allocated":851529728,"data_stored":849599304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":651484769},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":3.0899999999999999,"15min":3.8639999999999999},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":6.5199999999999996,"5min":72.534000000000006,"15min":94.055999999999997},"last":2.2850000000000001},{"interface":"front","average":{"1min":2.4540000000000002,"5min":3.6890000000000001,"15min":4.1440000000000001},"min":{"1min":0.218,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":14.513999999999999,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.86799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:13:10 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":3.4140000000000001,"15min":4.8840000000000003},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":6.6550000000000002,"5min":102.65300000000001,"15min":270.51100000000002},"last":2.2229999999999999},{"interface":"front","average":{"1min":1.8120000000000001,"5min":3.6869999999999998,"15min":4.734},"min":{"1min":0.28699999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.298999999999999,"5min":96.578999999999994,"15min":269.69299999999998},"last":1.258}]},{"osd":3,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":2.8220000000000001,"15min":5.6210000000000004},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":10.082000000000001,"5min":32.204999999999998,"15min":145.72800000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.8680000000000001,"5min":3.1120000000000001,"15min":5.6890000000000001},"min":{"1min":0.29999999999999999,"5min":0.27800000000000002,"15min":0.23999999999999999},"max":{"1min":9.3970000000000002,"5min":33.457000000000001,"15min":145.50700000000001},"last":0.70099999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.294,"15min":2.206},"min":{"1min":0.41399999999999998,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":6.0149999999999997,"5min":15.688000000000001,"15min":148.20599999999999},"last":1.198},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.3160000000000001,"15min":2.0169999999999999},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.27600000000000002},"max":{"1min":4.843,"5min":8.8490000000000002,"15min":148.11000000000001},"last":0.65700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.99099999999999999,"15min":1.913},"min":{"1min":0.307,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":6.5599999999999996,"5min":6.5599999999999996,"15min":64.725999999999999},"last":2.4550000000000001},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.4379999999999999,"15min":1.9510000000000001},"min":{"1min":0.40899999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.0620000000000003,"5min":15.717000000000001,"15min":27.609999999999999},"last":1.3109999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.5589999999999999,"15min":1.6319999999999999},"min":{"1min":0.51400000000000001,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":6.0839999999999996,"5min":15.744999999999999,"15min":15.744999999999999},"last":1.284},{"interface":"front","average":{"1min":1.675,"5min":1.633,"15min":1.8260000000000001},"min":{"1min":0.437,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":8.3919999999999995,"5min":13.470000000000001,"15min":13.766999999999999},"last":0.68000000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.087,"15min":1.865},"min":{"1min":0.45700000000000002,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":6.5899999999999999,"5min":6.5899999999999999,"15min":65.081999999999994},"last":2.484},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.4890000000000001,"15min":1.9590000000000001},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.1020000000000003,"5min":15.771000000000001,"15min":15.771000000000001},"last":1.3380000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599063,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1314040,"kb_used_data":569200,"kb_used_omap":12,"kb_used_meta":744819,"kb_avail":92431112,"statfs":{"total":95995035648,"available":94649458688,"internally_reserved":0,"allocated":582860800,"data_stored":580907973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":762695678},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.379,"15min":3.004},"min":{"1min":0.252,"5min":0.221,"15min":0.17299999999999999},"max":{"1min":5.6890000000000001,"5min":14.794,"15min":76.251999999999995},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.742,"5min":1.861,"15min":3.081},"min":{"1min":0.22,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":11.109999999999999,"5min":18.561,"15min":76.590000000000003},"last":0.435}]},{"osd":1,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":4.0880000000000001,"5min":2.702,"15min":3.8079999999999998},"min":{"1min":0.27400000000000002,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":26.808,"5min":32.677999999999997,"15min":94.768000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.4870000000000001,"5min":2.1840000000000002,"15min":3.625},"min":{"1min":0.315,"5min":0.252,"15min":0.21099999999999999},"max":{"1min":26.943999999999999,"5min":32.582000000000001,"15min":96.808000000000007},"last":0.312}]},{"osd":3,"last update":"Sat Sep 23 20:12:36 2023","interfaces":[{"interface":"back","average":{"1min":3.395,"5min":2.5600000000000001,"15min":4.2320000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":14.131,"5min":14.131,"15min":107.381},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.5259999999999998,"5min":2.25,"15min":3.508},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23699999999999999},"max":{"1min":14.778,"5min":20.308,"15min":106.301},"last":0.28599999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.278,"15min":2.0419999999999998},"min":{"1min":0.317,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":2.0339999999999998,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.5579999999999998,"5min":1.7070000000000001,"15min":2.2349999999999999},"min":{"1min":0.35699999999999998,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":12.728999999999999,"5min":13.387,"15min":76.161000000000001},"last":0.71799999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.6799999999999999,"15min":2.665},"min":{"1min":0.36499999999999999,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":2.0680000000000001,"5min":29.433,"15min":172.505},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.8640000000000001,"5min":2.149,"15min":2.883},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29099999999999998},"max":{"1min":11.037000000000001,"5min":29.196999999999999,"15min":172.375},"last":1.8540000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:12:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.6799999999999999,"15min":1.641},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":14.664,"5min":14.664,"15min":14.664},"last":2.6819999999999999},{"interface":"front","average":{"1min":2.214,"5min":1.8640000000000001,"15min":1.9259999999999999},"min":{"1min":0.42099999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":11.081,"5min":14.279999999999999,"15min":14.279999999999999},"last":1.5229999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.8320000000000001,"15min":1.944},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":12.645,"5min":13.481999999999999,"15min":23.079999999999998},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.4650000000000001,"15min":1.887},"min":{"1min":0.41499999999999998,"5min":0.39900000000000002,"15min":0.36299999999999999},"max":{"1min":2.1280000000000001,"5min":14.698,"15min":20.614999999999998},"last":0.69899999999999995}]}]},{"osd":3,"up_from":109,"seq":468151435537,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2103932,"kb_used_data":1381300,"kb_used_omap":4,"kb_used_meta":722619,"kb_avail":91641220,"statfs":{"total":95995035648,"available":93840609280,"internally_reserved":0,"allocated":1414451200,"data_stored":1412311847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":739961965},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":4.1029999999999998,"15min":3.4489999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":4.9340000000000002,"5min":74.323999999999998,"15min":74.323999999999998},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.9690000000000001,"5min":4.0149999999999997,"15min":3.7639999999999998},"min":{"1min":0.26500000000000001,"5min":0.20100000000000001,"15min":0.19600000000000001},"max":{"1min":10.484,"5min":74.447000000000003,"15min":74.447000000000003},"last":0.96899999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":4.7510000000000003,"15min":4.1319999999999997},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":5.5949999999999998,"5min":163.49700000000001,"15min":163.49700000000001},"last":9.6790000000000003},{"interface":"front","average":{"1min":2.6779999999999999,"5min":5.258,"15min":4.4859999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":9.9760000000000009,"5min":163.512,"15min":163.512},"last":3.2200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1629999999999998,"5min":6.6520000000000001,"15min":4.351},"min":{"1min":0.27400000000000002,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":14.9,"5min":163.83500000000001,"15min":163.83500000000001},"last":2.919},{"interface":"front","average":{"1min":1.252,"5min":6.0380000000000003,"15min":4.0140000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":4.2439999999999998,"5min":163.92099999999999,"15min":163.92099999999999},"last":1.7350000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.202,"15min":3.1429999999999998},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":3.6259999999999999,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.98999999999999999},{"interface":"front","average":{"1min":1.714,"5min":2.8420000000000001,"15min":3.5510000000000002},"min":{"1min":0.308,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":7.5759999999999996,"5min":58.408000000000001,"15min":110.295},"last":1.0029999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.6160000000000001,"15min":3.395},"min":{"1min":0.40300000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":7.6200000000000001,"5min":58.506,"15min":113.881},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.321,"5min":2.8919999999999999,"15min":3.2959999999999998},"min":{"1min":0.36299999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":4.274,"5min":58.643000000000001,"15min":114.598},"last":3.0070000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":2.625,"15min":3.6059999999999999},"min":{"1min":0.41399999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":4.3120000000000003,"5min":58.588000000000001,"15min":82.281999999999996},"last":3.04},{"interface":"front","average":{"1min":0.88700000000000001,"5min":2.2389999999999999,"15min":3.7599999999999998},"min":{"1min":0.376,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":3.6720000000000002,"5min":58.701000000000001,"15min":82.233000000000004},"last":1.044}]},{"osd":7,"last update":"Sat Sep 23 20:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":2.9359999999999999,"15min":3.516},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.29599999999999999},"max":{"1min":4.335,"5min":58.686,"15min":138.035},"last":2.9529999999999998},{"interface":"front","average":{"1min":1.52,"5min":2.6989999999999998,"15min":3.5070000000000001},"min":{"1min":0.41899999999999998,"5min":0.38400000000000001,"15min":0.32600000000000001},"max":{"1min":4.125,"5min":58.667999999999999,"15min":138.054},"last":1.0229999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019299,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1176588,"kb_used_data":803012,"kb_used_omap":0,"kb_used_meta":373568,"kb_avail":92568564,"statfs":{"total":95995035648,"available":94790209536,"internally_reserved":0,"allocated":822284288,"data_stored":821011160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":382533632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:23 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.0960000000000001,"15min":2.198},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.189},"max":{"1min":14.063000000000001,"5min":14.063000000000001,"15min":89.004999999999995},"last":1.651},{"interface":"front","average":{"1min":1.345,"5min":1.1479999999999999,"15min":2.161},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":14.048999999999999,"5min":14.048999999999999,"15min":89.024000000000001},"last":0.52000000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:13:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7450000000000001,"5min":2.613,"15min":5.7300000000000004},"min":{"1min":0.313,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":29.245999999999999,"5min":138.47999999999999,"15min":161.613},"last":1.7350000000000001},{"interface":"front","average":{"1min":2.5179999999999998,"5min":2.7360000000000002,"15min":5.8319999999999999},"min":{"1min":0.312,"5min":0.217,"15min":0.217},"max":{"1min":29.309000000000001,"5min":138.429,"15min":162.017},"last":1.581}]},{"osd":2,"last update":"Sat Sep 23 20:13:09 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":5.7850000000000001,"15min":6.6779999999999999},"min":{"1min":0.34399999999999997,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":8.75,"5min":154.94999999999999,"15min":182.96700000000001},"last":3.3170000000000002},{"interface":"front","average":{"1min":1.1319999999999999,"5min":5.3360000000000003,"15min":6.4400000000000004},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":5.0650000000000004,"5min":155.51300000000001,"15min":183.244},"last":1.718}]},{"osd":3,"last update":"Sat Sep 23 20:13:27 2023","interfaces":[{"interface":"back","average":{"1min":7.2599999999999998,"5min":2.4399999999999999,"15min":6.6050000000000004},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.255},"max":{"1min":100.361,"5min":100.361,"15min":266.54700000000003},"last":3.1920000000000002},{"interface":"front","average":{"1min":7.0869999999999997,"5min":2.625,"15min":5.5250000000000004},"min":{"1min":0.35999999999999999,"5min":0.33600000000000002,"15min":0.28599999999999998},"max":{"1min":100.20699999999999,"5min":100.20699999999999,"15min":184.22300000000001},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:13:23 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47399999999999998,"15min":2.1240000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":0.752,"5min":3.3290000000000002,"15min":266.59899999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.45900000000000002,"15min":2.1459999999999999},"min":{"1min":0.31900000000000001,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":0.73999999999999999,"5min":1.722,"15min":266.72899999999998},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.50900000000000001,"15min":0.51400000000000001},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.23499999999999999},"max":{"1min":2.9350000000000001,"5min":2.9350000000000001,"15min":4.0410000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.51100000000000001,"15min":0.58899999999999997},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.247},"max":{"1min":0.81100000000000005,"5min":1.6240000000000001,"15min":4.2889999999999997},"last":3.234}]},{"osd":7,"last update":"Sat Sep 23 20:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.496,"15min":1.4119999999999999},"min":{"1min":0.37,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":0.82799999999999996,"5min":1.5960000000000001,"15min":76.507000000000005},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.55400000000000005,"15min":1.5389999999999999},"min":{"1min":0.373,"5min":0.27200000000000002,"15min":0.222},"max":{"1min":1.5429999999999999,"5min":3.2890000000000001,"15min":76.462000000000003},"last":1.6930000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757663,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2012108,"kb_used_data":1586884,"kb_used_omap":0,"kb_used_meta":425216,"kb_avail":91733044,"statfs":{"total":95995035648,"available":93934637056,"internally_reserved":0,"allocated":1624969216,"data_stored":1623700410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":435421184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.79900000000000004,"15min":1.722},"min":{"1min":0.22600000000000001,"5min":0.224,"15min":0.216},"max":{"1min":6.1029999999999998,"5min":8.6739999999999995,"15min":121.94799999999999},"last":3.3500000000000001},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.111,"15min":2.004},"min":{"1min":0.28799999999999998,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":4.2750000000000004,"5min":10.327,"15min":121.964},"last":2.9710000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.2480000000000002,"15min":5.9630000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":20.337,"5min":75.204999999999998,"15min":266.197},"last":3.1899999999999999},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.2330000000000001,"15min":6.1399999999999997},"min":{"1min":0.28199999999999997,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":23.497,"5min":75.231999999999999,"15min":266.14999999999998},"last":3.0619999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:13:01 2023","interfaces":[{"interface":"back","average":{"1min":11.291,"5min":6.4000000000000004,"15min":5.8250000000000002},"min":{"1min":0.36799999999999999,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":120.297,"5min":240.57300000000001,"15min":240.57300000000001},"last":3.5219999999999998},{"interface":"front","average":{"1min":11.327,"5min":6.4740000000000002,"15min":5.8840000000000003},"min":{"1min":0.41499999999999998,"5min":0.374,"15min":0.313},"max":{"1min":120.325,"5min":240.464,"15min":240.464},"last":3.5489999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.631,"15min":4.8849999999999998},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.608000000000001,"5min":12.608000000000001,"15min":201.50899999999999},"last":0.505},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.0820000000000001,"15min":4.6719999999999997},"min":{"1min":0.313,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":1.9299999999999999,"5min":7.3739999999999997,"15min":201.58099999999999},"last":0.70499999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:13:18 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.49299999999999999,"15min":1.4099999999999999},"min":{"1min":0.245,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":1.206,"5min":4.0510000000000002,"15min":191.78899999999999},"last":3.004},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.45100000000000001,"15min":1.319},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":0.78900000000000003,"5min":1.4359999999999999,"15min":191.821},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.52200000000000002,"15min":0.63100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":1.7410000000000001,"5min":3.278,"15min":9.9009999999999998},"last":3.2200000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.47299999999999998,"15min":0.51900000000000002},"min":{"1min":0.315,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.841,"5min":1.841,"15min":5.0119999999999996},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:13:01 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.51000000000000001,"15min":1.486},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26200000000000001},"max":{"1min":0.70999999999999996,"5min":2.3069999999999999,"15min":191.84299999999999},"last":3.4700000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.54400000000000004,"15min":1.645},"min":{"1min":0.32800000000000001,"5min":0.318,"15min":0.26300000000000001},"max":{"1min":0.77100000000000002,"5min":3.1440000000000001,"15min":191.69},"last":3.0369999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":796065792,"data_stored":796065792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1493893120,"data_stored":1493893120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":592314368,"data_stored":592314368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1186004992,"data_stored":1186004992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":644677632,"data_stored":644677632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1455226880,"data_stored":1455226880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1463353344,"data_stored":1463353344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:13:38.481 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:13:38.481 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-09-23T20:13:38.755 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:38.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:38.862 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:38.904 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:38.982 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:38.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:38.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:39.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 25 16 46931 46915 117.256 73.5 0.00210299 0.00849749 2023-09-23T20:13:39.088 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:39.112 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.113 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.115 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:39 smithi144 ceph-mon[194833]: pgmap v1167: 109 pgs: 109 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T20:13:39.396 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:39.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:39.402 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.402 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:39 smithi135 ceph-mon[213871]: pgmap v1167: 109 pgs: 109 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T20:13:39.403 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.403 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.403 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T20:13:39.403 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:39 smithi135 ceph-mon[221113]: pgmap v1167: 109 pgs: 109 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T20:13:39.503 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:39.569 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:39.655 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:39.673 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:39.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:39.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:39.706 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:39.784 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:39.856 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:40.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 26 16 48217 48201 115.837 80.375 0.00562664 0.00862808 2023-09-23T20:13:40.169 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:40.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[221113]: failed to encode map e414 with expected crc 2023-09-23T20:13:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[221113]: failed to encode map e414 with expected crc 2023-09-23T20:13:40.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T20:13:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-09-23T20:13:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T20:13:40.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[221113]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T20:13:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[213871]: failed to encode map e414 with expected crc 2023-09-23T20:13:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[213871]: failed to encode map e414 with expected crc 2023-09-23T20:13:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T20:13:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-09-23T20:13:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T20:13:40.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:40 smithi135 ceph-mon[213871]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T20:13:40.233 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:13:40.234 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:13:40.287 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:40.353 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:40 smithi144 ceph-mon[194833]: failed to encode map e414 with expected crc 2023-09-23T20:13:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:40 smithi144 ceph-mon[194833]: failed to encode map e414 with expected crc 2023-09-23T20:13:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T20:13:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-09-23T20:13:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T20:13:40.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:40 smithi144 ceph-mon[194833]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T20:13:40.427 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:40.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:40.534 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:40.583 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:40.658 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:40.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:40.766 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:40.821 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:40.862 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:13:40.910 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:40.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:41.018 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:41.054 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 27 16 49589 49573 114.723 85.75 0.00836423 0.00871106 2023-09-23T20:13:41.112 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:41.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[213871]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[213871]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T20:13:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[213871]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[213871]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[213871]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[213871]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[213871]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[213871]: pgmap v1170: 109 pgs: 109 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[221113]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[221113]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[221113]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[221113]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[221113]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[221113]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[221113]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:41 smithi135 ceph-mon[221113]: pgmap v1170: 109 pgs: 109 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T20:13:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:41 smithi144 ceph-mon[194833]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:41 smithi144 ceph-mon[194833]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T20:13:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:41 smithi144 ceph-mon[194833]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:41 smithi144 ceph-mon[194833]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:41 smithi144 ceph-mon[194833]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:41 smithi144 ceph-mon[194833]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:41 smithi144 ceph-mon[194833]: failed to encode map e415 with expected crc 2023-09-23T20:13:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:41 smithi144 ceph-mon[194833]: pgmap v1170: 109 pgs: 109 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T20:13:42.053 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 28 16 50681 50665 113.063 68.25 0.0184388 0.00883887 2023-09-23T20:13:42.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:42 smithi135 ceph-mon[221113]: 1.7 scrub starts 2023-09-23T20:13:42.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:42 smithi135 ceph-mon[221113]: 1.7 scrub ok 2023-09-23T20:13:42.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:42 smithi135 ceph-mon[213871]: 1.7 scrub starts 2023-09-23T20:13:42.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:42 smithi135 ceph-mon[213871]: 1.7 scrub ok 2023-09-23T20:13:42.555 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:42.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:42.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:42 smithi144 ceph-mon[194833]: 1.7 scrub starts 2023-09-23T20:13:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:42 smithi144 ceph-mon[194833]: 1.7 scrub ok 2023-09-23T20:13:42.664 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:42.716 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:42.799 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:42.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:42.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:42.839 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:13:42.903 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:43.004 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:43.057 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 29 16 51815 51799 111.596 70.875 0.00863965 0.00895523 2023-09-23T20:13:43.141 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:43.171 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:43.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:43.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:43.277 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:43.319 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:43.420 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:13:43.461 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:43.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:43.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:43.567 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:43.664 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:43.779 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 9 pg_num to 26 2023-09-23T20:13:43.782 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:43.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:43.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:43.903 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:13:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:13:43] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2023-09-23T20:13:43.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:43 smithi135 ceph-mon[213871]: pgmap v1171: 109 pgs: 109 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T20:13:43.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2536987786' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:13:43.904 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:43 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:13:43.904 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:43 smithi135 ceph-mon[221113]: pgmap v1171: 109 pgs: 109 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T20:13:43.904 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2536987786' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:13:43.904 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:43 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:13:43.932 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:43 smithi144 ceph-mon[194833]: pgmap v1171: 109 pgs: 109 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T20:13:43.932 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2536987786' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:13:43.932 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:43 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:13:43.952 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:44.057 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 30 16 52963 52947 110.268 71.75 0.0128468 0.00906381 2023-09-23T20:13:44.779 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:13:45.006 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[221113]: failed to encode map e416 with expected crc 2023-09-23T20:13:45.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[221113]: failed to encode map e416 with expected crc 2023-09-23T20:13:45.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:13:45.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[221113]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T20:13:45.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/219752234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:13:45.008 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[213871]: failed to encode map e416 with expected crc 2023-09-23T20:13:45.008 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[213871]: failed to encode map e416 with expected crc 2023-09-23T20:13:45.008 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:13:45.008 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[213871]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T20:13:45.008 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:44 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/219752234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:13:45.057 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 31 16 54359 54343 109.525 87.25 0.0133048 0.00912506 2023-09-23T20:13:45.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:44 smithi144 ceph-mon[194833]: failed to encode map e416 with expected crc 2023-09-23T20:13:45.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:44 smithi144 ceph-mon[194833]: failed to encode map e416 with expected crc 2023-09-23T20:13:45.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:44 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:13:45.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:44 smithi144 ceph-mon[194833]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T20:13:45.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:44 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/219752234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:13:45.757 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: 1.c scrub starts 2023-09-23T20:13:45.757 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: 1.c scrub ok 2023-09-23T20:13:45.972 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:46.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:46.057 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 32 16 55134 55118 107.616 48.4375 0.00306874 0.00921648 2023-09-23T20:13:46.105 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:46.129 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:46.129 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.129 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/219752234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:13:46.129 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T20:13:46.129 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: pgmap v1174: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: 1.c scrub starts 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: 1.c scrub ok 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/219752234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:13:46.130 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T20:13:46.131 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.131 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.131 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: pgmap v1174: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:13:46.131 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:45 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:13:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: 1.c scrub starts 2023-09-23T20:13:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: 1.c scrub ok 2023-09-23T20:13:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:46.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/219752234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:13:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T20:13:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: failed to encode map e417 with expected crc 2023-09-23T20:13:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: pgmap v1174: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:13:46.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:45 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:13:46.132 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:46.182 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:46.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:46.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:46.285 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:46.582 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:13:46.600 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:13:46.618 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:13:46.637 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:13:46.657 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:13:46.675 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:13:46.693 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:13:46.711 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:13:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:13:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:13:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:13:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:13:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:13:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:13:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:13:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:13:47.057 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 33 16 55279 55263 104.631 9.0625 0.00137517 0.0095037 2023-09-23T20:13:47.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:46 smithi144 ceph-mon[194833]: failed to encode map e418 with expected crc 2023-09-23T20:13:47.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:46 smithi144 ceph-mon[194833]: failed to encode map e418 with expected crc 2023-09-23T20:13:47.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:46 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:13:47.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:46 smithi144 ceph-mon[194833]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T20:13:47.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3070820144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:47.138 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:47.185 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:47.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:47.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:47.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[221113]: failed to encode map e418 with expected crc 2023-09-23T20:13:47.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[221113]: failed to encode map e418 with expected crc 2023-09-23T20:13:47.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:13:47.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[221113]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T20:13:47.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3070820144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:47.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[213871]: failed to encode map e418 with expected crc 2023-09-23T20:13:47.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[213871]: failed to encode map e418 with expected crc 2023-09-23T20:13:47.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:13:47.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[213871]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T20:13:47.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3070820144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:47.290 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:47.348 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:47.440 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:47.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:47.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:47.549 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:47.593 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:47.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T20:13:47.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.777 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.778 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:47 smithi144 ceph-mon[194833]: pgmap v1177: 119 pgs: 1 peering, 10 unknown, 108 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:13:47.807 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:47.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:47.920 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:47.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[221113]: pgmap v1177: 119 pgs: 1 peering, 10 unknown, 108 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: failed to encode map e419 with expected crc 2023-09-23T20:13:47.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:47 smithi135 ceph-mon[213871]: pgmap v1177: 119 pgs: 1 peering, 10 unknown, 108 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:13:47.988 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:48.057 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 34 16 56911 56895 104.553 102 0.00771115 0.00956012 2023-09-23T20:13:48.084 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:48.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:48.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:48.193 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:48.196 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:13:48.246 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:48.349 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:48.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:48.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:48.461 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:48.517 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:48.533 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:13:48.533 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:13:48.533 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.533 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.36186.0:57679 9.14 e419/418 9:2bf17d87:::benchmark_data_smithi135_549536_object57678:head v 419'3529, mlcod=419'3527)", 2023-09-23T20:13:48.533 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:48.521969+0000", 2023-09-23T20:13:48.533 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0088289260000000008, 2023-09-23T20:13:48.533 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0088672360000000006, 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.521969+0000", 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:13:48.534 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.521969+0000", 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.521968+0000", 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:13:48.535 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.522013+0000", 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5052999999999997e-05 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.522036+0000", 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2121e-05 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.522043+0000", 2023-09-23T20:13:48.536 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5290000000000003e-06 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.522106+0000", 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3205000000000002e-05 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.522128+0000", 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2099999999999998e-05 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.537 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:13:48.538 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.538 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.538 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.538 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.36186.0:57681 9.c e419/407 9:3bd5430b:::benchmark_data_smithi135_549536_object57680:head v 419'3470, mlcod=419'3468)", 2023-09-23T20:13:48.538 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:48.525498+0000", 2023-09-23T20:13:48.538 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0053001039999999999, 2023-09-23T20:13:48.538 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005422474, 2023-09-23T20:13:48.538 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:13:48.539 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:13:48.539 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:13:48.539 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.539 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:13:48.539 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.525498+0000", 2023-09-23T20:13:48.539 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:13:48.539 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.525498+0000", 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.525497+0000", 2023-09-23T20:13:48.540 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.525532+0000", 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4989e-05 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:13:48.541 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.525544+0000", 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1863e-05 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.525549+0000", 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.87e-06 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.542 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.526942+0000", 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0013927150000000001 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.526965+0000", 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.319e-05 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.543 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.36186.0:57687 9.c e419/407 9:33c5b4c2:::benchmark_data_smithi135_549536_object57686:head v 419'3471, mlcod=419'3468)", 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:48.526927+0000", 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.0038708530000000001, 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040712589999999998, 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:13:48.544 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.526927+0000", 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.526927+0000", 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:13:48.545 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.526927+0000", 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.526960+0000", 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2767e-05 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.526973+0000", 2023-09-23T20:13:48.546 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2931999999999999e-05 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.526979+0000", 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3409999999999999e-06 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.529271+0000", 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002291642 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.547 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.529293+0000", 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2932e-05 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.36186.0:57691 9.e e419/407 9:7d0b6a0d:::benchmark_data_smithi135_549536_object57690:head v 419'3555, mlcod=419'3551)", 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:48.530062+0000", 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.00073653, 2023-09-23T20:13:48.548 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0010028179999999999, 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.530062+0000", 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:13:48.549 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.530062+0000", 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.530061+0000", 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:13:48.550 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.530089+0000", 2023-09-23T20:13:48.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7917999999999999e-05 2023-09-23T20:13:48.551 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.551 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.551 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:13:48.551 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.530103+0000", 2023-09-23T20:13:48.551 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4487e-05 2023-09-23T20:13:48.551 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.551 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.530109+0000", 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8479999999999997e-06 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.530147+0000", 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8234e-05 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:13:48.552 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:13:48.553 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:48.530167+0000", 2023-09-23T20:13:48.553 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9745000000000002e-05 2023-09-23T20:13:48.553 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.553 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:13:48.553 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.553 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:13:48.553 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2023-09-23T20:13:48.554 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 4 2023-09-23T20:13:48.554 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:13:48.782 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:48.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:48.824 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:48.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:48.948 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:49.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 35 16 58580 58564 104.546 104.312 0.00743504 0.0095619 2023-09-23T20:13:49.087 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:49.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:49.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:49.191 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:49.218 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:49.260 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:49.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:49.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:49.364 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:49.391 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:49.496 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:49.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:49.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:49.570 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:13:49.604 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:49.654 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:49.870 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:49.871 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:13:49.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:49.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:49.975 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:50.001 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:50.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 36 16 59230 59214 102.771 40.625 0.00493113 0.00972602 2023-09-23T20:13:50.082 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:50.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:50.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:50.194 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:50.256 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:50 smithi144 ceph-mon[194833]: pgmap v1178: 119 pgs: 1 peering, 3 unknown, 115 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:13:50.264 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:50.407 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:50.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:50.461 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:13:50.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:50 smithi135 ceph-mon[213871]: pgmap v1178: 119 pgs: 1 peering, 3 unknown, 115 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:13:50.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:50 smithi135 ceph-mon[221113]: pgmap v1178: 119 pgs: 1 peering, 3 unknown, 115 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:13:50.518 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:50.586 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:50.641 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:50.708 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:50.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:50.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:50.814 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:50.867 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:51.004 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:51.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:51.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 37 16 60812 60796 102.665 98.875 0.0123268 0.00973585 2023-09-23T20:13:51.115 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:51.163 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:51.242 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:51.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:51.308 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:13:51.308 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:13:51.308 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:13:51.308 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-09-23T20:13:51.309 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-09-23T20:13:51.347 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:51.436 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:51.540 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:51.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:51.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:51.653 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:51.709 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:52.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 38 16 61798 61782 101.585 61.625 0.0117561 0.00983988 2023-09-23T20:13:52.092 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:13:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:52 smithi144 ceph-mon[194833]: pgmap v1179: 119 pgs: 1 peering, 1 unknown, 117 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2686994589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:52.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:52 smithi135 ceph-mon[221113]: pgmap v1179: 119 pgs: 1 peering, 1 unknown, 117 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:52.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2686994589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:52.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:52.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:52 smithi135 ceph-mon[213871]: pgmap v1179: 119 pgs: 1 peering, 1 unknown, 117 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:52.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2686994589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:52.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:13:52.536 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:52.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:52.645 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:13:52.700 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:52.821 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:13:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:52.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:13:52.897 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:13:52.897 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:13:52.897 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:13:52.897 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:13:52.898 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:13:52.928 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:52.974 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:53.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 39 16 63374 63358 101.506 98.5 0.0100837 0.00984752 2023-09-23T20:13:53.175 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:53.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:53.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:53.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:53.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T20:13:53.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:13:53.225 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.227 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:53.227 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:53.227 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:13:53.287 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:53.360 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:13:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T20:13:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: failed to encode map e420 with expected crc 2023-09-23T20:13:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:53.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:13:53.460 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:53.479 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:53.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:53.558 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:13:53.577 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:13:53.583 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:53.595 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:13:53.610 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:53.615 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:13:53.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:13:53.654 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:13:53.655 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:53.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:53.674 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:13:53.757 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:13:53.759 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:53.782 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:13:53.782 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:13:53.782 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:13:53.782 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:13:53.782 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:13:53.782 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:13:53.782 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:13:53.783 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:13:53.811 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:13:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:13:53] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-09-23T20:13:53.820 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:54.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:13:54.058915+0000 min lat: 0.00116484 max lat: 1.11419 avg lat: 0.00991596 2023-09-23T20:13:54.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:13:54.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 40 16 64404 64388 100.578 64.375 0.00344597 0.00991596 2023-09-23T20:13:54.059 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:54.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:54.137 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:13:54.166 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:54.215 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:54.274 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:54.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:54.379 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:54 smithi144 ceph-mon[194833]: pgmap v1181: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3060430742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:54 smithi144 ceph-mon[194833]: failed to encode map e421 with expected crc 2023-09-23T20:13:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:54 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:13:54.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:54 smithi144 ceph-mon[194833]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T20:13:54.411 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:54.465 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:54.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[221113]: pgmap v1181: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3060430742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[221113]: failed to encode map e421 with expected crc 2023-09-23T20:13:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:13:54.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[221113]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T20:13:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[213871]: pgmap v1181: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:13:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3060430742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:13:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[213871]: failed to encode map e421 with expected crc 2023-09-23T20:13:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:13:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:54 smithi135 ceph-mon[213871]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T20:13:54.547 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:13:54.570 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:54.597 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:55.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 41 16 64654 64638 98.5058 15.625 0.00243243 0.00989032 2023-09-23T20:13:55.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[213871]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[213871]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[213871]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[213871]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[213871]: failed to encode map e422 with expected crc 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[213871]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[221113]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[221113]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[221113]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[221113]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[221113]: failed to encode map e422 with expected crc 2023-09-23T20:13:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[221113]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T20:13:55.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:55 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:13:55.222 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:13:55.346 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:55.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:55.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:55 smithi144 ceph-mon[194833]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:55 smithi144 ceph-mon[194833]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:55 smithi144 ceph-mon[194833]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:55 smithi144 ceph-mon[194833]: failed to encode map e421 with expected crc 2023-09-23T20:13:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:55 smithi144 ceph-mon[194833]: failed to encode map e422 with expected crc 2023-09-23T20:13:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:55 smithi144 ceph-mon[194833]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T20:13:55.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:55 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:13:55.453 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:55.483 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:55.621 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:55.628 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:55.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:55.733 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:55.767 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:55.951 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:55.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:56.052 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:56.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 42 16 64896 64880 96.5208 15.125 0.00596696 0.0103439 2023-09-23T20:13:56.090 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:56.102 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:56 smithi144 ceph-mon[194833]: failed to encode map e422 with expected crc 2023-09-23T20:13:56.102 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:56 smithi144 ceph-mon[194833]: failed to encode map e422 with expected crc 2023-09-23T20:13:56.102 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:56 smithi144 ceph-mon[194833]: pgmap v1184: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 43 KiB/s, 0 objects/s recovering 2023-09-23T20:13:56.102 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:56 smithi144 ceph-mon[194833]: failed to encode map e423 with expected crc 2023-09-23T20:13:56.218 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[213871]: failed to encode map e422 with expected crc 2023-09-23T20:13:56.218 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[213871]: failed to encode map e422 with expected crc 2023-09-23T20:13:56.218 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[213871]: pgmap v1184: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 43 KiB/s, 0 objects/s recovering 2023-09-23T20:13:56.218 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[213871]: failed to encode map e423 with expected crc 2023-09-23T20:13:56.219 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:13:56.219 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[213871]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T20:13:56.219 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[221113]: failed to encode map e422 with expected crc 2023-09-23T20:13:56.219 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[221113]: failed to encode map e422 with expected crc 2023-09-23T20:13:56.219 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[221113]: pgmap v1184: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 43 KiB/s, 0 objects/s recovering 2023-09-23T20:13:56.219 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[221113]: failed to encode map e423 with expected crc 2023-09-23T20:13:56.219 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:13:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:56 smithi135 ceph-mon[221113]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T20:13:56.258 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:56.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:56.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:56.366 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:56 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:13:56.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:56 smithi144 ceph-mon[194833]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T20:13:56.396 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:56.450 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:56.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:56.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:56.531 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:13:56.531 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:13:56.554 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:13:56.584 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:56.631 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:13:56.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:56.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:13:56.735 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:56.754 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.4 primary-affinity to 0 (802) 2023-09-23T20:13:56.771 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:56.814 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:56.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:56.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:56.917 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:56.945 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:57.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 43 16 65201 65185 94.7198 19.0625 0.00238839 0.010323 2023-09-23T20:13:57.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.211 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/126020369' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: failed to encode map e424 with expected crc 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.212 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/126020369' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:13:57.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:13:57.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: failed to encode map e424 with expected crc 2023-09-23T20:13:57.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:13:57.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T20:13:57.213 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:13:57.241 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:57.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:57.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:57.344 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:13:57.365 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:13:57.371 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:57.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: failed to encode map e423 with expected crc 2023-09-23T20:13:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/126020369' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:13:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:13:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: failed to encode map e424 with expected crc 2023-09-23T20:13:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:13:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T20:13:57.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:13:57.407 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:13:57.769 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:13:57.886 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:13:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:57.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:13:57.993 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:58.028 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:58.040 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:58.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 44 16 65432 65416 92.8954 14.4375 0.00339015 0.0107322 2023-09-23T20:13:58.103 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[221113]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.103 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[221113]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.103 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[221113]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.104 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[221113]: pgmap v1187: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:13:58.104 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:58.104 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:58.104 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:13:58.104 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[221113]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T20:13:58.290 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:13:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:58.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:13:58.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[213871]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[213871]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[213871]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[213871]: pgmap v1187: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:13:58.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:58.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:58.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:13:58.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:58 smithi135 ceph-mon[213871]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T20:13:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:58 smithi144 ceph-mon[194833]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:58 smithi144 ceph-mon[194833]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:58 smithi144 ceph-mon[194833]: failed to encode map e424 with expected crc 2023-09-23T20:13:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:58 smithi144 ceph-mon[194833]: pgmap v1187: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:13:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:13:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:58 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:13:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:58 smithi144 ceph-mon[194833]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T20:13:58.395 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:13:58.437 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:58.534 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:13:58.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:58.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:13:58.637 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:13:58.667 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:58.748 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:13:58.764 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:13:58.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:58.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:13:58.868 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:13:58.910 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:58.986 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:13:58.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:58.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:13:59.058 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 45 16 65602 65586 91.0675 10.625 0.00226394 0.0108481 2023-09-23T20:13:59.091 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:13:59.127 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:59.361 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.362 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3810067302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:13:59.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:59.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: failed to encode map e426 with expected crc 2023-09-23T20:13:59.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3810067302' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:13:59.363 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:13:59 smithi144 ceph-mon[194833]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T20:13:59.377 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:13:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:13:59.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:13:59.394 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.395 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3810067302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: failed to encode map e426 with expected crc 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3810067302' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[221113]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.396 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e425 with expected crc 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3810067302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: failed to encode map e426 with expected crc 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3810067302' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:13:59.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:13:59 smithi135 ceph-mon[213871]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T20:13:59.484 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:13:59.513 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:13:59.915 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:13:59.916 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:14:00.059 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 46 16 65925 65909 89.5265 20.1875 0.0279986 0.0111647 2023-09-23T20:14:00.270 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:00.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:00.376 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:00 smithi144 ceph-mon[194833]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:00 smithi144 ceph-mon[194833]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:00 smithi144 ceph-mon[194833]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:00 smithi144 ceph-mon[194833]: pgmap v1190: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 29 MiB/s wr, 471 op/s; 1914/195927 objects misplaced (0.977%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:14:00.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:00 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:00.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:00 smithi144 ceph-mon[194833]: failed to encode map e427 with expected crc 2023-09-23T20:14:00.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:00 smithi144 ceph-mon[194833]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T20:14:00.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[213871]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[213871]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[213871]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[213871]: pgmap v1190: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 29 MiB/s wr, 471 op/s; 1914/195927 objects misplaced (0.977%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:14:00.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:00.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[213871]: failed to encode map e427 with expected crc 2023-09-23T20:14:00.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[213871]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T20:14:00.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[221113]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[221113]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[221113]: failed to encode map e426 with expected crc 2023-09-23T20:14:00.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[221113]: pgmap v1190: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 29 MiB/s wr, 471 op/s; 1914/195927 objects misplaced (0.977%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:14:00.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:00.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[221113]: failed to encode map e427 with expected crc 2023-09-23T20:14:00.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:00 smithi135 ceph-mon[221113]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T20:14:00.452 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:00.568 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:00.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:00.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:00.674 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:00.705 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:00.744 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:14:00.751 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:00.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:00.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:00.765 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:14:00.785 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:14:00.809 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:14:00.833 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:14:00.855 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:00.884 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:00.934 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:14:00.968 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:14:00.985 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:00.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:00.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:01.027 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:14:01.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:14:01.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:14:01.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:14:01.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:14:01.045 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:14:01.046 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:14:01.046 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:14:01.046 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:14:01.059 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 47 16 66753 66737 88.7228 51.75 0.0154724 0.0112666 2023-09-23T20:14:01.095 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:01.144 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:01.244 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:01 smithi135 ceph-mon[221113]: failed to encode map e427 with expected crc 2023-09-23T20:14:01.244 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:01 smithi135 ceph-mon[221113]: failed to encode map e427 with expected crc 2023-09-23T20:14:01.244 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:01 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/589280127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:01.247 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:01 smithi135 ceph-mon[213871]: failed to encode map e427 with expected crc 2023-09-23T20:14:01.247 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:01 smithi135 ceph-mon[213871]: failed to encode map e427 with expected crc 2023-09-23T20:14:01.247 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:01 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/589280127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:01.452 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:01 smithi144 ceph-mon[194833]: failed to encode map e427 with expected crc 2023-09-23T20:14:01.452 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:01 smithi144 ceph-mon[194833]: failed to encode map e427 with expected crc 2023-09-23T20:14:01.452 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:01 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/589280127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:01.474 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:01.484 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:14:01.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:01.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:01.588 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:01.638 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:01.757 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:01.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:01.875 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:01.952 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:02.027 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:02.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:02.059 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 48 16 67945 67929 88.4262 74.5 0.0120811 0.0113042 2023-09-23T20:14:02.134 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:02.137 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:14:02.186 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:02.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:02 smithi135 ceph-mon[213871]: pgmap v1192: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 116 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 22 MiB/s wr, 354 op/s; 1914/196272 objects misplaced (0.975%); 3.3 MiB/s, 51 objects/s recovering 2023-09-23T20:14:02.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:02 smithi135 ceph-mon[221113]: pgmap v1192: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 116 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 22 MiB/s wr, 354 op/s; 1914/196272 objects misplaced (0.975%); 3.3 MiB/s, 51 objects/s recovering 2023-09-23T20:14:02.288 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:02.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:02.399 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:02.419 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:14:02.444 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:02 smithi144 ceph-mon[194833]: pgmap v1192: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 116 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 22 MiB/s wr, 354 op/s; 1914/196272 objects misplaced (0.975%); 3.3 MiB/s, 51 objects/s recovering 2023-09-23T20:14:02.445 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:02.512 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:02.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:02.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:02.615 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:02.678 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:02.818 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:02.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:02.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:02.917 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:14:02.917 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:14:02.918 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-09-23T20:14:02.918 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:14:02.926 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:03.022 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:03.059 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 49 16 69027 69011 88.0016 67.625 0.0132307 0.011357 2023-09-23T20:14:03.217 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:03.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:03.239 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:03 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:03.239 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:03 smithi135 ceph-mon[213871]: pgmap v1193: 119 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 116 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 19 MiB/s wr, 299 op/s; 5073/197127 objects misplaced (2.573%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:14:03.239 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:03 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:03.240 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:03 smithi135 ceph-mon[221113]: pgmap v1193: 119 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 116 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 19 MiB/s wr, 299 op/s; 5073/197127 objects misplaced (2.573%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:14:03.337 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:03.389 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:03.402 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:03.465 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:03.467 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:03 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:03.467 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:03 smithi144 ceph-mon[194833]: pgmap v1193: 119 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 116 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 19 MiB/s wr, 299 op/s; 5073/197127 objects misplaced (2.573%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:14:03.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:03.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:03.569 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:03.608 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:03.636 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:03.657 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:03.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:03.761 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:03.795 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:03.961 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:03.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:03.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:03.969 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:14:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:14:03] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-09-23T20:14:04.065 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 50 16 69838 69822 87.2454 50.6875 0.0113043 0.0114572 2023-09-23T20:14:04.070 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:04.106 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:04.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:04 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:04.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:04 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:04.565 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:04 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:05.065 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 51 16 71067 71051 87.0407 76.8125 0.00612719 0.0114853 2023-09-23T20:14:05.230 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:14:05.276 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:14:05.276 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:14:05.276 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:14:05.276 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:14:05.276 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:14:05.305 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:05 smithi144 ceph-mon[194833]: pgmap v1194: 119 pgs: 3 active+recovering+undersized+remapped, 116 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 8453/207756 objects misplaced (4.069%); 10 MiB/s, 164 objects/s recovering 2023-09-23T20:14:05.305 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:05 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:14:05.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:05 smithi135 ceph-mon[213871]: pgmap v1194: 119 pgs: 3 active+recovering+undersized+remapped, 116 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 8453/207756 objects misplaced (4.069%); 10 MiB/s, 164 objects/s recovering 2023-09-23T20:14:05.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:05 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:14:05.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:05 smithi135 ceph-mon[221113]: pgmap v1194: 119 pgs: 3 active+recovering+undersized+remapped, 116 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 8453/207756 objects misplaced (4.069%); 10 MiB/s, 164 objects/s recovering 2023-09-23T20:14:05.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:05 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:14:06.026 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:06.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:06.065 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 52 16 71761 71745 86.2012 43.375 0.00370999 0.0114552 2023-09-23T20:14:06.132 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:06.174 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:06.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[213871]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[213871]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:14:06.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[213871]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T20:14:06.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[213871]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[221113]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[221113]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:14:06.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[221113]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T20:14:06.459 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:06 smithi135 ceph-mon[221113]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.557 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:06.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:06.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:06 smithi144 ceph-mon[194833]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:06 smithi144 ceph-mon[194833]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:06 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:14:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:06 smithi144 ceph-mon[194833]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T20:14:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:06 smithi144 ceph-mon[194833]: failed to encode map e428 with expected crc 2023-09-23T20:14:06.660 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:06.693 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:06.791 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:14:07.065 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 53 16 71761 71745 84.5752 0 - 0.0114552 2023-09-23T20:14:07.218 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:14:07.437 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:07.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:07.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[213871]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[213871]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T20:14:07.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[213871]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[213871]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[213871]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.446 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[213871]: pgmap v1197: 119 pgs: 3 active+recovering+undersized+remapped, 116 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 37 MiB/s wr, 589 op/s; 8453/209148 objects misplaced (4.042%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T20:14:07.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:14:07.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:07.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[221113]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[221113]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T20:14:07.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[221113]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[221113]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[221113]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[221113]: pgmap v1197: 119 pgs: 3 active+recovering+undersized+remapped, 116 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 37 MiB/s wr, 589 op/s; 8453/209148 objects misplaced (4.042%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T20:14:07.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:14:07.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:07.483 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:07 smithi144 ceph-mon[194833]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.483 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:07 smithi144 ceph-mon[194833]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T20:14:07.483 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:07 smithi144 ceph-mon[194833]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.483 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:07 smithi144 ceph-mon[194833]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.483 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:07 smithi144 ceph-mon[194833]: failed to encode map e429 with expected crc 2023-09-23T20:14:07.483 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:07 smithi144 ceph-mon[194833]: pgmap v1197: 119 pgs: 3 active+recovering+undersized+remapped, 116 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 37 MiB/s wr, 589 op/s; 8453/209148 objects misplaced (4.042%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T20:14:07.483 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:14:07.483 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:07.547 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:07.601 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:07.719 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:07.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:07.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:07.828 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:07.906 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:08.065 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 54 16 72578 72562 83.9546 25.5312 0.00444347 0.011897 2023-09-23T20:14:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[213871]: failed to encode map e430 with expected crc 2023-09-23T20:14:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[213871]: failed to encode map e430 with expected crc 2023-09-23T20:14:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:14:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[213871]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T20:14:08.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[213871]: failed to encode map e430 with expected crc 2023-09-23T20:14:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[221113]: failed to encode map e430 with expected crc 2023-09-23T20:14:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[221113]: failed to encode map e430 with expected crc 2023-09-23T20:14:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:14:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[221113]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T20:14:08.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:08 smithi135 ceph-mon[221113]: failed to encode map e430 with expected crc 2023-09-23T20:14:08.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:08 smithi144 ceph-mon[194833]: failed to encode map e430 with expected crc 2023-09-23T20:14:08.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:08 smithi144 ceph-mon[194833]: failed to encode map e430 with expected crc 2023-09-23T20:14:08.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:14:08.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:08 smithi144 ceph-mon[194833]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T20:14:08.530 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:08 smithi144 ceph-mon[194833]: failed to encode map e430 with expected crc 2023-09-23T20:14:09.066 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 55 16 73372 73356 83.3305 49.625 0.00229112 0.011967 2023-09-23T20:14:09.098 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:09.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:09.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:09.201 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:14:09.433 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:09.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:09 smithi135 ceph-mon[213871]: pgmap v1199: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 4442/217365 objects degraded (2.044%); 6764/217365 objects misplaced (3.112%); 20 MiB/s, 319 objects/s recovering 2023-09-23T20:14:09.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:09 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:09.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:09 smithi135 ceph-mon[221113]: pgmap v1199: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 4442/217365 objects degraded (2.044%); 6764/217365 objects misplaced (3.112%); 20 MiB/s, 319 objects/s recovering 2023-09-23T20:14:09.465 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:09 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:09.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:09 smithi144 ceph-mon[194833]: pgmap v1199: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 4442/217365 objects degraded (2.044%); 6764/217365 objects misplaced (3.112%); 20 MiB/s, 319 objects/s recovering 2023-09-23T20:14:09.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:09 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:09.605 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:14:09.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:09.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:14:09.711 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:09.791 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:10.066 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 56 16 74497 74481 83.098 70.3125 0.0102258 0.0120292 2023-09-23T20:14:10.208 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:14:10.208 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:14:10.208 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:14:10.208 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:14:10.208 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.208 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52213 8.d 8:bac999e7:::benchmark_data_smithi135_460343_object52212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T20:14:10.208 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.894700+0000", 2023-09-23T20:14:10.208 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.30946517000001, 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1738284119999998, 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52213 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.209 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894700+0000", 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894700+0000", 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.210 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894699+0000", 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894728+0000", 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9312e-05 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894737+0000", 2023-09-23T20:14:10.211 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3720000000000005e-06 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894742+0000", 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6650000000000002e-06 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894789+0000", 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7170999999999999e-05 2023-09-23T20:14:10.212 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894928+0000", 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000138978 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.895072+0000", 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014409199999999999 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.213 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.897222+0000", 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021498049999999999 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.897297+0000", 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5024000000000005e-05 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.214 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.904254+0000", 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.006957759 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.904265+0000", 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0658e-05 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.925985+0000", 2023-09-23T20:14:10.215 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.021720427 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.925997+0000", 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1661999999999999e-05 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068429+0000", 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.142431854 2023-09-23T20:14:10.216 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068528+0000", 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9085000000000004e-05 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52220 8.8 8:1576796a:::benchmark_data_smithi135_460343_object52219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:14:10.217 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.910095+0000", 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.29406941600001, 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.155063207, 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52220 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.218 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910095+0000", 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910095+0000", 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.219 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910094+0000", 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910138+0000", 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4589999999999998e-05 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.220 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910150+0000", 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1629e-05 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910157+0000", 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0049999999999998e-06 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.911108+0000", 2023-09-23T20:14:10.221 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00095094199999999998 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.911114+0000", 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.533e-06 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.930596+0000", 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0194816419999999 2023-09-23T20:14:10.222 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.930607+0000", 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1243999999999999e-05 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.050602+0000", 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1199942940000001 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.223 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.050759+0000", 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015762999999999999 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.050943+0000", 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018359100000000001 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.224 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.058845+0000", 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0079017199999999992 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.062852+0000", 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004007399 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.062931+0000", 2023-09-23T20:14:10.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8728999999999998e-05 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.062985+0000", 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4354000000000002e-05 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065159+0000", 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021735790000000001 2023-09-23T20:14:10.226 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52237 8.8 8:1c4a366e:::benchmark_data_smithi135_460343_object52236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.938063+0000", 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.26610130900002, 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1294171959999999, 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.227 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52237 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938063+0000", 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938063+0000", 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938063+0000", 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.229 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938133+0000", 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9815999999999997e-05 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938144+0000", 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1532000000000001e-05 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.230 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938150+0000", 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.28e-06 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938195+0000", 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4767e-05 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.231 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938201+0000", 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1149999999999999e-06 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931044+0000", 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99284272399999995 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931053+0000", 2023-09-23T20:14:10.232 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6689999999999995e-06 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.054684+0000", 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1236309849999999 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.054796+0000", 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0001121 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.233 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.054912+0000", 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115862 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.062894+0000", 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0079826789999999995 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.234 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065219+0000", 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023241949999999998 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065297+0000", 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8922000000000001e-05 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.235 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065347+0000", 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9633999999999999e-05 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.067481+0000", 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002133556 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.236 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52241 8.d 8:b3c3dddb:::benchmark_data_smithi135_460343_object52240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.941349+0000", 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.26281538699999, 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1495913739999999, 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:14:10.237 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52241 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941349+0000", 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941349+0000", 2023-09-23T20:14:10.238 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941348+0000", 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941394+0000", 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6112000000000002e-05 2023-09-23T20:14:10.239 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941409+0000", 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4915e-05 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941415+0000", 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1809999999999998e-06 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.240 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941468+0000", 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2883000000000002e-05 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941476+0000", 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0490000000000004e-06 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.241 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.926017+0000", 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98454064299999999 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.926023+0000", 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2179999999999996e-06 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.242 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068547+0000", 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1425238929999999 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068641+0000", 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3647000000000006e-05 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068731+0000", 2023-09-23T20:14:10.243 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0111999999999994e-05 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.075257+0000", 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0065266220000000002 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.075267+0000", 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8490000000000001e-06 2023-09-23T20:14:10.244 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.088158+0000", 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012890798 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.088216+0000", 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7862999999999998e-05 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.245 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.090941+0000", 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027247870000000002 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52250 8.8 8:17b1c085:::benchmark_data_smithi135_460343_object52249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.949594+0000", 2023-09-23T20:14:10.246 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.25457042099998, 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.129605008, 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52250 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.247 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949594+0000", 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949594+0000", 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.248 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949594+0000", 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949668+0000", 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3799000000000003e-05 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.249 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949677+0000", 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3519999999999996e-06 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949682+0000", 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7160000000000002e-06 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.250 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949736+0000", 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4299999999999998e-05 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949746+0000", 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0351e-05 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.251 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931077+0000", 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98133084199999998 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931085+0000", 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5859999999999999e-06 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.058877+0000", 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1277923919999999 2023-09-23T20:14:10.252 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.059001+0000", 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012386999999999999 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.059159+0000", 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000157589 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.253 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065259+0000", 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0061000919999999997 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.067619+0000", 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023606730000000002 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.254 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.076559+0000", 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0089399140000000002 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.076612+0000", 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.223e-05 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.079199+0000", 2023-09-23T20:14:10.255 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025878229999999999 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52251 8.8 8:198971ba:::benchmark_data_smithi135_460343_object52250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.952872+0000", 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 430.25129324300002, 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.129071428, 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.256 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52251 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952872+0000", 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.257 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952872+0000", 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952871+0000", 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.258 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952933+0000", 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2553999999999998e-05 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952943+0000", 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.054e-06 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.259 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952948+0000", 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1710000000000001e-06 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952978+0000", 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0403e-05 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952985+0000", 2023-09-23T20:14:10.260 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3980000000000002e-06 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931106+0000", 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97812022600000004 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931113+0000", 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5549999999999997e-06 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.261 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.061064+0000", 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.129951087 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.061176+0000", 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111876 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.262 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.061331+0000", 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000154353 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.067493+0000", 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0061627449999999999 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.067671+0000", 2023-09-23T20:14:10.263 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017801899999999999 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.079252+0000", 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011580228999999999 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.079288+0000", 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6318e-05 2023-09-23T20:14:10.264 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.081943+0000", 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002655059 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:65653 8.d 8:bc798c3d:::benchmark_data_smithi135_460343_object65652:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:17.919923+0000", 2023-09-23T20:14:10.265 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 412.28424142, 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.187381153, 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 65653 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.266 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919923+0000", 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919923+0000", 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.267 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919923+0000", 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919972+0000", 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9447000000000003e-05 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919980+0000", 2023-09-23T20:14:10.268 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8860000000000004e-06 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919985+0000", 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1150000000000002e-06 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920041+0000", 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5970000000000001e-05 2023-09-23T20:14:10.269 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920193+0000", 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000151831 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920376+0000", 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018274 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.270 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.922919+0000", 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025428830000000001 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.925658+0000", 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027392150000000001 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.271 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.939445+0000", 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013786678 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.939456+0000", 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.166e-05 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107196+0000", 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1677396499999999 2023-09-23T20:14:10.272 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107305+0000", 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010861699999999999 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:65657 8.d 8:bc23b147:::benchmark_data_smithi135_460343_object65656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:17.956099+0000", 2023-09-23T20:14:10.273 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 412.24806615099999, 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1600642969999999, 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 65657 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.274 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956099+0000", 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956099+0000", 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.275 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956098+0000", 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956131+0000", 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3785000000000003e-05 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956146+0000", 2023-09-23T20:14:10.276 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.473e-05 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956152+0000", 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5740000000000003e-06 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956190+0000", 2023-09-23T20:14:10.277 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8167999999999998e-05 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956198+0000", 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5180000000000004e-06 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107324+0000", 2023-09-23T20:14:10.278 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1511252599999999 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107430+0000", 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000106363 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107554+0000", 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012434300000000001 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.279 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.111957+0000", 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0044023949999999999 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.111987+0000", 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0077999999999999e-05 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.280 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.113834+0000", 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018473949999999999 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.113906+0000", 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1398999999999999e-05 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.116163+0000", 2023-09-23T20:14:10.281 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022574090000000002 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:58570 9.15 9:aa4646a7:::benchmark_data_smithi135_549536_object58569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:49.058520+0000", 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 21.145645013999999, 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.37502988300000001, 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.282 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 58570 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058520+0000", 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.283 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058520+0000", 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058519+0000", 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.284 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058618+0000", 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8367000000000002e-05 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058631+0000", 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3662000000000001e-05 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.285 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058637+0000", 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4439999999999999e-06 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058684+0000", 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7615999999999999e-05 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.286 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058789+0000", 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010464899999999999 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058923+0000", 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013415500000000001 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.338396+0000", 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27947285300000002 2023-09-23T20:14:10.287 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.338402+0000", 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3470000000000004e-06 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433456+0000", 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.095053173000000005 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.288 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433508+0000", 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2732000000000003e-05 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433550+0000", 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1349e-05 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.289 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:58573 9.15 9:ad189949:::benchmark_data_smithi135_549536_object58572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:49.058664+0000", 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 21.145500670000001, 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.377179341, 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.290 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 58573 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058664+0000", 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058664+0000", 2023-09-23T20:14:10.291 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058658+0000", 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999938 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058771+0000", 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113026 2023-09-23T20:14:10.292 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058779+0000", 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3149999999999992e-06 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058785+0000", 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7189999999999998e-06 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.293 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.335939+0000", 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.277154452 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336099+0000", 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000159736 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.294 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.337277+0000", 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0011776759999999999 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433560+0000", 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.096283481000000004 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.295 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435710+0000", 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021502259999999999 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435738+0000", 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7231999999999999e-05 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.296 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435783+0000", 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5704000000000002e-05 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435843+0000", 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0053000000000003e-05 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.297 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:58575 9.17 9:e80be3e6:::benchmark_data_smithi135_549536_object58574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:49.058916+0000", 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 21.145248756000001, 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.37711894899999998, 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.298 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 58575 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058916+0000", 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058916+0000", 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.299 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058915+0000", 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058963+0000", 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7434e-05 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.300 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058971+0000", 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9629999999999998e-06 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058988+0000", 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7312e-05 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.301 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336234+0000", 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27724562600000002 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336345+0000", 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111279 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336489+0000", 2023-09-23T20:14:10.302 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000143852 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.339342+0000", 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028530299999999999 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.339346+0000", 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8680000000000001e-06 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.303 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435909+0000", 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.096563021999999998 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435995+0000", 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6460999999999998e-05 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.304 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.436035+0000", 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9669999999999998e-05 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:58577 9.18 9:194cab73:::benchmark_data_smithi135_549536_object58576:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:49.058999+0000", 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 21.145165317, 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.37468497299999998, 2023-09-23T20:14:10.305 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 58577 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058999+0000", 2023-09-23T20:14:10.306 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058999+0000", 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.058999+0000", 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.307 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059039+0000", 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0429000000000002e-05 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059047+0000", 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.605e-06 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.308 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.309 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059052+0000", 2023-09-23T20:14:10.309 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5920000000000002e-06 2023-09-23T20:14:10.309 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.309 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.309 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.309 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.060041+0000", 2023-09-23T20:14:10.309 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00098975600000000006 2023-09-23T20:14:10.310 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.310 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.310 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.310 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.060154+0000", 2023-09-23T20:14:10.310 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011264699999999999 2023-09-23T20:14:10.310 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336001+0000", 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27584725199999999 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.338007+0000", 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020059399999999999 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.311 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.338383+0000", 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00037596100000000001 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433595+0000", 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.095211603000000006 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.312 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433646+0000", 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1091999999999998e-05 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433684+0000", 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8671000000000003e-05 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.313 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:58580 9.5 9:a5d2123c:::benchmark_data_smithi135_549536_object58579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:49.059075+0000", 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 21.145089513999999, 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.37674133100000001, 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 58580 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.314 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059075+0000", 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059075+0000", 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.315 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059075+0000", 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059101+0000", 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6509e-05 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.316 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059108+0000", 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4829999999999996e-06 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.059114+0000", 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.84e-06 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.060025+0000", 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00091092199999999999 2023-09-23T20:14:10.317 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.060128+0000", 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103934 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336001+0000", 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27587277599999999 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.318 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.338408+0000", 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0024063790000000002 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435677+0000", 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.097269662000000007 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.319 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435728+0000", 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0489999999999999e-05 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435773+0000", 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5297999999999999e-05 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435817+0000", 2023-09-23T20:14:10.320 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3529999999999998e-05 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.36186.0:58579 9.d e419/407 9:b9366b54:::benchmark_data_smithi135_549536_object58578:head v 419'3711, mlcod=419'3709)", 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:49.060569+0000", 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 21.143595448999999, 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.375100086, 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.321 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.060569+0000", 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.060569+0000", 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.322 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.060569+0000", 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.060591+0000", 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2524e-05 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.323 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336020+0000", 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27542912200000003 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336028+0000", 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4610000000000002e-06 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.324 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.337352+0000", 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001324083 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.337370+0000", 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8558000000000001e-05 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435633+0000", 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.098262365000000004 2023-09-23T20:14:10.325 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.435669+0000", 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6650999999999999e-05 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:58584 9.6 9:626cc9f3:::benchmark_data_smithi135_549536_object58583:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.326 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:49.065488+0000", 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 21.138676898, 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.36806173199999997, 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 58584 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.327 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.065488+0000", 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.065488+0000", 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.328 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.065474+0000", 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999862 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.065535+0000", 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0522000000000001e-05 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.335933+0000", 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27039813000000001 2023-09-23T20:14:10.329 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.335947+0000", 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3713000000000001e-05 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336018+0000", 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1614999999999998e-05 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.330 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336147+0000", 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000128603 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.336537+0000", 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00039042800000000001 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.331 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.338335+0000", 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001797866 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.339783+0000", 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014483650000000001 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433474+0000", 2023-09-23T20:14:10.332 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.093690651 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433511+0000", 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7045000000000002e-05 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:49.433550+0000", 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8452999999999999e-05 2023-09-23T20:14:10.333 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:61640 9.17 9:ee0b1a98:::benchmark_data_smithi135_549536_object61639:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:51.678219+0000", 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 18.525945369999999, 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.31436195, 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.334 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 61640 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678219+0000", 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.335 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678219+0000", 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678219+0000", 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.336 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678343+0000", 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012460300000000001 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678352+0000", 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5040000000000002e-06 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.337 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678355+0000", 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7349999999999998e-06 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678400+0000", 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4761000000000001e-05 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.338 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678485+0000", 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4625999999999999e-05 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678627+0000", 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014223600000000001 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.683300+0000", 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046726459999999999 2023-09-23T20:14:10.339 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.685076+0000", 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001776775 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992414+0000", 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.30733809200000001 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.340 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992505+0000", 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0832e-05 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992581+0000", 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6023e-05 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.341 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:61643 9.7 9:e1c5fd7c:::benchmark_data_smithi135_549536_object61642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:51.678541+0000", 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 18.525623932999999, 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.314161574, 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.342 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 61643 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678541+0000", 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.343 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678541+0000", 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678540+0000", 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678588+0000", 2023-09-23T20:14:10.344 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7713999999999998e-05 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678595+0000", 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.486e-06 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678600+0000", 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9230000000000001e-06 2023-09-23T20:14:10.345 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678807+0000", 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020683100000000001 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678928+0000", 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000120714 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.346 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679066+0000", 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000138014 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.683314+0000", 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0042478209999999997 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.347 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.685111+0000", 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0017973189999999999 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992622+0000", 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.30751128500000002 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992667+0000", 2023-09-23T20:14:10.348 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4799000000000003e-05 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992702+0000", 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5261e-05 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.349 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:61646 9.17 9:eea3a3ea:::benchmark_data_smithi135_549536_object61645:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:51.678611+0000", 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 18.52555358, 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.314110852, 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 61646 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.350 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678611+0000", 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678611+0000", 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.351 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678611+0000", 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678660+0000", 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9100000000000001e-05 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678688+0000", 2023-09-23T20:14:10.352 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7630000000000001e-05 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678692+0000", 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.938e-06 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679667+0000", 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00097414799999999998 2023-09-23T20:14:10.353 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679768+0000", 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010170600000000001 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679913+0000", 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014441000000000001 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.354 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.683319+0000", 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0034061080000000001 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.685128+0000", 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001808785 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.355 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992641+0000", 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.30751297900000002 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992686+0000", 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5005e-05 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.992722+0000", 2023-09-23T20:14:10.356 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6449000000000003e-05 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:61647 9.f 9:f1ce2a85:::benchmark_data_smithi135_549536_object61646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:51.678699+0000", 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 18.525466048999998, 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.318630199, 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.357 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 61647 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678699+0000", 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.358 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678699+0000", 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678698+0000", 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.359 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678758+0000", 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0087999999999998e-05 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678778+0000", 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9952000000000001e-05 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.678784+0000", 2023-09-23T20:14:10.360 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0680000000000002e-06 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.680196+0000", 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001411549 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.680295+0000", 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9505000000000004e-05 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.361 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.680413+0000", 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000117796 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.683330+0000", 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00291712 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.362 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.687563+0000", 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0042325499999999999 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.997230+0000", 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.30966691400000002 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.363 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.997284+0000", 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3980000000000002e-05 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.997329+0000", 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5207999999999998e-05 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.364 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:61650 9.17 9:eed5fbf2:::benchmark_data_smithi135_549536_object61649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:13:51.679022+0000", 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 18.525142481, 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.31841828100000003, 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 61650 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.365 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679022+0000", 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679022+0000", 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.366 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679022+0000", 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679051+0000", 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9281e-05 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.367 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679060+0000", 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6770000000000001e-06 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.679065+0000", 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9459999999999997e-06 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:10.368 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.682049+0000", 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00298388 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.682162+0000", 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011335 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:14:10.369 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.682281+0000", 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011914899999999999 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.683487+0000", 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001205827 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.685154+0000", 2023-09-23T20:14:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0016670109999999999 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.997366+0000", 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.31221250900000003 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.997409+0000", 2023-09-23T20:14:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2333999999999998e-05 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:13:51.997441+0000", 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1846e-05 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:10.372 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:14:10.373 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:10 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 4442/217365 objects degraded (2.044%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:10.373 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:10 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:10.420 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:10.468 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:10 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 4442/217365 objects degraded (2.044%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:10 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:10.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:10 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 4442/217365 objects degraded (2.044%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:10.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:10 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:10.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:10.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:10.575 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:10.652 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:10.760 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:10.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:10.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:10.868 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:10.920 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:10.995 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:14:11.001 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:11.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:11.006 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:14:11.065 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 57 16 75431 75415 82.6643 58.375 0.00871326 0.0120801 2023-09-23T20:14:11.106 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:11.142 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:11.229 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:11 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:11.230 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:11 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3831157358' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:11.230 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:11 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:11.230 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:11 smithi144 ceph-mon[194833]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:14:11.230 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:11 smithi144 ceph-mon[194833]: pgmap v1200: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 605 op/s; 4442/218655 objects degraded (2.032%); 6066/218655 objects misplaced (2.774%); 16 MiB/s, 262 objects/s recovering 2023-09-23T20:14:11.275 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:14:11.312 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:11.312 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3831157358' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:11.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:11.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[213871]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:14:11.313 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[213871]: pgmap v1200: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 605 op/s; 4442/218655 objects degraded (2.032%); 6066/218655 objects misplaced (2.774%); 16 MiB/s, 262 objects/s recovering 2023-09-23T20:14:11.313 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:11.314 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3831157358' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:11.314 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:11.314 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[221113]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:14:11.314 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:11 smithi135 ceph-mon[221113]: pgmap v1200: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 605 op/s; 4442/218655 objects degraded (2.032%); 6066/218655 objects misplaced (2.774%); 16 MiB/s, 262 objects/s recovering 2023-09-23T20:14:11.785 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:14:12.066 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 58 16 76506 76490 82.3974 67.1875 0.0153995 0.0121326 2023-09-23T20:14:12.625 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:14:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:12 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:14:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:12 smithi144 ceph-mon[194833]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:12 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:12 smithi144 ceph-mon[194833]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T20:14:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:12 smithi144 ceph-mon[194833]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:12 smithi144 ceph-mon[194833]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:12 smithi144 ceph-mon[194833]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:14:12.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[221113]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:12.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[221113]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T20:14:12.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[221113]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[221113]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[221113]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:14:12.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[213871]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:12.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[213871]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T20:14:12.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[213871]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[213871]: failed to encode map e431 with expected crc 2023-09-23T20:14:12.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:12 smithi135 ceph-mon[213871]: failed to encode map e431 with expected crc 2023-09-23T20:14:13.068 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 59 16 77499 77483 82.0497 62.0625 0.0169467 0.0121824 2023-09-23T20:14:13.830 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:13.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:13.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:13.939 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:14:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:14:13] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-09-23T20:14:13.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:13 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:13.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:13 smithi135 ceph-mon[213871]: pgmap v1202: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 114 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 4442/220230 objects degraded (2.017%); 5322/220230 objects misplaced (2.417%); 18 MiB/s, 285 objects/s recovering 2023-09-23T20:14:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:13 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:13 smithi135 ceph-mon[221113]: pgmap v1202: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 114 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 4442/220230 objects degraded (2.017%); 5322/220230 objects misplaced (2.417%); 18 MiB/s, 285 objects/s recovering 2023-09-23T20:14:14.007 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:14.068 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:14:14.069007+0000 min lat: 0.00116484 max lat: 2.14573 avg lat: 0.012196 2023-09-23T20:14:14.068 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:14:14.068 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 60 16 78732 78716 81.9665 77.0625 0.012132 0.012196 2023-09-23T20:14:14.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:13 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:14.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:13 smithi144 ceph-mon[194833]: pgmap v1202: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 114 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 4442/220230 objects degraded (2.017%); 5322/220230 objects misplaced (2.417%); 18 MiB/s, 285 objects/s recovering 2023-09-23T20:14:14.654 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:14.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:14.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:14.760 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:14.789 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:15.069 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 61 16 79458 79442 81.3657 45.375 0.00660251 0.0122741 2023-09-23T20:14:15.152 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:15.156 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1199,"stamp":"2023-09-23T20:14:09.019356+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4751366067,"num_objects":72455,"num_object_clones":0,"num_object_copies":217365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4442,"num_objects_misplaced":6764,"num_objects_unfound":0,"num_objects_dirty":72455,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":73334,"num_write_kb":4648053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2638,"num_bytes_recovered":167848635,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111572,"ondisk_log_size":111572,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":367,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18466252,"kb_used_data":14330324,"kb_used_omap":32,"kb_used_meta":4135839,"kb_avail":637749812,"statfs":{"total":671965249536,"available":653055807488,"internally_reserved":0,"allocated":14674251776,"data_stored":14662157180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33274,"internal_metadata":4235099654},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,9,0,4,0,0,12],"upper_bound":256},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":442105856,"num_objects":6746,"num_object_clones":0,"num_object_copies":20238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4442,"num_objects_misplaced":1691,"num_objects_unfound":0,"num_objects_dirty":6746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6746,"num_write_kb":431744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1920,"num_bytes_recovered":125829120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001851"},"pg_stats":[{"pgid":"9.10","version":"430'3951","reported_seq":6492,"reported_epoch":430,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:14:07.938433+0000","last_change":"2023-09-23T20:13:58.895343+0000","last_active":"2023-09-23T20:14:07.938433+0000","last_peered":"2023-09-23T20:14:07.938433+0000","last_clean":"2023-09-23T20:13:53.416242+0000","last_became_active":"2023-09-23T20:13:58.867163+0000","last_became_peered":"2023-09-23T20:13:58.867163+0000","last_unstale":"2023-09-23T20:14:07.938433+0000","last_undegraded":"2023-09-23T20:14:07.938433+0000","last_fullsized":"2023-09-23T20:13:58.742873+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3951,"log_dups_size":0,"ondisk_log_size":3951,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146210816,"num_objects":2231,"num_object_clones":0,"num_object_copies":6693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2233,"num_objects_unfound":0,"num_objects_dirty":2231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2231,"num_write_kb":142784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":828,"num_bytes_recovered":54263808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,7],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1116},{"shards":"0,1,2,7","objects":1},{"shards":"0,7","objects":1114}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":320,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.078473+0000","last_change":"2023-09-23T20:13:57.751561+0000","last_active":"2023-09-23T20:13:59.078473+0000","last_peered":"2023-09-23T20:13:59.078473+0000","last_clean":"2023-09-23T20:13:59.078473+0000","last_became_active":"2023-09-23T20:13:57.751227+0000","last_became_peered":"2023-09-23T20:13:57.751227+0000","last_unstale":"2023-09-23T20:13:59.078473+0000","last_undegraded":"2023-09-23T20:13:59.078473+0000","last_fullsized":"2023-09-23T20:13:59.078473+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:00.196397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":256,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745956+0000","last_change":"2023-09-23T20:08:04.757024+0000","last_active":"2023-09-23T20:13:58.745956+0000","last_peered":"2023-09-23T20:13:58.745956+0000","last_clean":"2023-09-23T20:13:58.745956+0000","last_became_active":"2023-09-23T20:08:04.756678+0000","last_became_peered":"2023-09-23T20:08:04.756678+0000","last_unstale":"2023-09-23T20:13:58.745956+0000","last_undegraded":"2023-09-23T20:13:58.745956+0000","last_fullsized":"2023-09-23T20:13:58.745956+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:24.850014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.13","version":"430'3993","reported_seq":4047,"reported_epoch":430,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:14:08.540856+0000","last_change":"2023-09-23T20:14:07.309099+0000","last_active":"2023-09-23T20:14:08.540856+0000","last_peered":"2023-09-23T20:14:08.540856+0000","last_clean":"2023-09-23T20:14:05.142062+0000","last_became_active":"2023-09-23T20:14:07.304693+0000","last_became_peered":"2023-09-23T20:14:07.304693+0000","last_unstale":"2023-09-23T20:14:08.540856+0000","last_undegraded":"2023-09-23T20:14:07.259027+0000","last_fullsized":"2023-09-23T20:14:07.197546+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3993,"log_dups_size":0,"ondisk_log_size":3993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148144128,"num_objects":2261,"num_object_clones":0,"num_object_copies":6783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4442,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2261,"num_write_kb":144672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,0],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"0,7","objects":40},{"shards":"1,3,7","objects":2221}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":123,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745714+0000","last_change":"2023-09-23T20:07:18.942370+0000","last_active":"2023-09-23T20:13:58.745714+0000","last_peered":"2023-09-23T20:13:58.745714+0000","last_clean":"2023-09-23T20:13:58.745714+0000","last_became_active":"2023-09-23T20:07:18.942150+0000","last_became_peered":"2023-09-23T20:07:18.942150+0000","last_unstale":"2023-09-23T20:13:58.745714+0000","last_undegraded":"2023-09-23T20:13:58.745714+0000","last_fullsized":"2023-09-23T20:13:58.745714+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:30.790602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":138,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206784+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:14:06.206784+0000","last_peered":"2023-09-23T20:14:06.206784+0000","last_clean":"2023-09-23T20:14:06.206784+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:14:06.206784+0000","last_undegraded":"2023-09-23T20:14:06.206784+0000","last_fullsized":"2023-09-23T20:14:06.206784+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"427'3969","reported_seq":4521,"reported_epoch":427,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:14:04.074481+0000","last_change":"2023-09-23T20:13:59.885259+0000","last_active":"2023-09-23T20:14:04.074481+0000","last_peered":"2023-09-23T20:14:04.074481+0000","last_clean":"2023-09-23T20:13:56.394433+0000","last_became_active":"2023-09-23T20:13:59.851808+0000","last_became_peered":"2023-09-23T20:13:59.851808+0000","last_unstale":"2023-09-23T20:14:04.074481+0000","last_undegraded":"2023-09-23T20:14:04.074481+0000","last_fullsized":"2023-09-23T20:13:59.746009+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3969,"log_dups_size":0,"ondisk_log_size":3969,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145424384,"num_objects":2219,"num_object_clones":0,"num_object_copies":6657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3902,"num_objects_unfound":0,"num_objects_dirty":2219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2219,"num_write_kb":142016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":170,"num_bytes_recovered":11141120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[1,0],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"0,1","objects":268},{"shards":"1,2,3","objects":1951}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":256,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.744683+0000","last_change":"2023-09-23T20:08:04.755725+0000","last_active":"2023-09-23T20:13:58.744683+0000","last_peered":"2023-09-23T20:13:58.744683+0000","last_clean":"2023-09-23T20:13:58.744683+0000","last_became_active":"2023-09-23T20:08:04.755432+0000","last_became_peered":"2023-09-23T20:08:04.755432+0000","last_unstale":"2023-09-23T20:13:58.744683+0000","last_undegraded":"2023-09-23T20:13:58.744683+0000","last_fullsized":"2023-09-23T20:13:58.744683+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:53.604550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":375,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237385+0000","last_change":"2023-09-23T20:07:18.967775+0000","last_active":"2023-09-23T20:14:06.237385+0000","last_peered":"2023-09-23T20:14:06.237385+0000","last_clean":"2023-09-23T20:14:06.237385+0000","last_became_active":"2023-09-23T20:07:18.967460+0000","last_became_peered":"2023-09-23T20:07:18.967460+0000","last_unstale":"2023-09-23T20:14:06.237385+0000","last_undegraded":"2023-09-23T20:14:06.237385+0000","last_fullsized":"2023-09-23T20:14:06.237385+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:01.694302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":356,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237306+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:14:06.237306+0000","last_peered":"2023-09-23T20:14:06.237306+0000","last_clean":"2023-09-23T20:14:06.237306+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:14:06.237306+0000","last_undegraded":"2023-09-23T20:14:06.237306+0000","last_fullsized":"2023-09-23T20:14:06.237306+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":273,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.177372+0000","last_change":"2023-09-23T20:07:18.956022+0000","last_active":"2023-09-23T20:14:05.177372+0000","last_peered":"2023-09-23T20:14:05.177372+0000","last_clean":"2023-09-23T20:14:05.177372+0000","last_became_active":"2023-09-23T20:07:18.955368+0000","last_became_peered":"2023-09-23T20:07:18.955368+0000","last_unstale":"2023-09-23T20:14:05.177372+0000","last_undegraded":"2023-09-23T20:14:05.177372+0000","last_fullsized":"2023-09-23T20:14:05.177372+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:03.678210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":760,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.815702+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:13:59.815702+0000","last_peered":"2023-09-23T20:13:59.815702+0000","last_clean":"2023-09-23T20:13:59.815702+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:13:59.815702+0000","last_undegraded":"2023-09-23T20:13:59.815702+0000","last_fullsized":"2023-09-23T20:13:59.815702+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":429,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237069+0000","last_change":"2023-09-23T20:07:36.979502+0000","last_active":"2023-09-23T20:14:06.237069+0000","last_peered":"2023-09-23T20:14:06.237069+0000","last_clean":"2023-09-23T20:14:06.237069+0000","last_became_active":"2023-09-23T20:07:36.979167+0000","last_became_peered":"2023-09-23T20:07:36.979167+0000","last_unstale":"2023-09-23T20:14:06.237069+0000","last_undegraded":"2023-09-23T20:14:06.237069+0000","last_fullsized":"2023-09-23T20:14:06.237069+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:22.362729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":292,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.176730+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:14:05.176730+0000","last_peered":"2023-09-23T20:14:05.176730+0000","last_clean":"2023-09-23T20:14:05.176730+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:14:05.176730+0000","last_undegraded":"2023-09-23T20:14:05.176730+0000","last_fullsized":"2023-09-23T20:14:05.176730+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":123,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206792+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:14:06.206792+0000","last_peered":"2023-09-23T20:14:06.206792+0000","last_clean":"2023-09-23T20:14:06.206792+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:14:06.206792+0000","last_undegraded":"2023-09-23T20:14:06.206792+0000","last_fullsized":"2023-09-23T20:14:06.206792+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745539+0000","last_change":"2023-09-23T20:07:18.948334+0000","last_active":"2023-09-23T20:13:58.745539+0000","last_peered":"2023-09-23T20:13:58.745539+0000","last_clean":"2023-09-23T20:13:58.745539+0000","last_became_active":"2023-09-23T20:07:18.946355+0000","last_became_peered":"2023-09-23T20:07:18.946355+0000","last_unstale":"2023-09-23T20:13:58.745539+0000","last_undegraded":"2023-09-23T20:13:58.745539+0000","last_fullsized":"2023-09-23T20:13:58.745539+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:44.219149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":376,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206243+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:14:06.206243+0000","last_peered":"2023-09-23T20:14:06.206243+0000","last_clean":"2023-09-23T20:14:06.206243+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:14:06.206243+0000","last_undegraded":"2023-09-23T20:14:06.206243+0000","last_fullsized":"2023-09-23T20:14:06.206243+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":318,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.078551+0000","last_change":"2023-09-23T20:13:57.751349+0000","last_active":"2023-09-23T20:13:59.078551+0000","last_peered":"2023-09-23T20:13:59.078551+0000","last_clean":"2023-09-23T20:13:59.078551+0000","last_became_active":"2023-09-23T20:13:57.751061+0000","last_became_peered":"2023-09-23T20:13:57.751061+0000","last_unstale":"2023-09-23T20:13:59.078551+0000","last_undegraded":"2023-09-23T20:13:59.078551+0000","last_fullsized":"2023-09-23T20:13:59.078551+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:51.912413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.19","version":"430'4100","reported_seq":4141,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.698907+0000","last_change":"2023-09-23T20:13:46.811423+0000","last_active":"2023-09-23T20:14:08.698907+0000","last_peered":"2023-09-23T20:14:08.698907+0000","last_clean":"2023-09-23T20:14:08.698907+0000","last_became_active":"2023-09-23T20:13:46.808266+0000","last_became_peered":"2023-09-23T20:13:46.808266+0000","last_unstale":"2023-09-23T20:14:08.698907+0000","last_undegraded":"2023-09-23T20:14:08.698907+0000","last_fullsized":"2023-09-23T20:14:08.698907+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4100,"log_dups_size":0,"ondisk_log_size":4100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153649152,"num_objects":2345,"num_object_clones":0,"num_object_copies":7035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2345,"num_write_kb":150048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":789,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236350+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:14:06.236350+0000","last_peered":"2023-09-23T20:14:06.236350+0000","last_clean":"2023-09-23T20:14:06.236350+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:14:06.236350+0000","last_undegraded":"2023-09-23T20:14:06.236350+0000","last_fullsized":"2023-09-23T20:14:06.236350+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":123,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.238491+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:14:06.238491+0000","last_peered":"2023-09-23T20:14:06.238491+0000","last_clean":"2023-09-23T20:14:06.238491+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:14:06.238491+0000","last_undegraded":"2023-09-23T20:14:06.238491+0000","last_fullsized":"2023-09-23T20:14:06.238491+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":280,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745757+0000","last_change":"2023-09-23T20:08:04.752962+0000","last_active":"2023-09-23T20:13:58.745757+0000","last_peered":"2023-09-23T20:13:58.745757+0000","last_clean":"2023-09-23T20:13:58.745757+0000","last_became_active":"2023-09-23T20:08:04.752622+0000","last_became_peered":"2023-09-23T20:08:04.752622+0000","last_unstale":"2023-09-23T20:13:58.745757+0000","last_undegraded":"2023-09-23T20:13:58.745757+0000","last_fullsized":"2023-09-23T20:13:58.745757+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:27.717674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.18","version":"430'4006","reported_seq":4047,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.268022+0000","last_change":"2023-09-23T20:13:46.824838+0000","last_active":"2023-09-23T20:14:08.268022+0000","last_peered":"2023-09-23T20:14:08.268022+0000","last_clean":"2023-09-23T20:14:08.268022+0000","last_became_active":"2023-09-23T20:13:46.820709+0000","last_became_peered":"2023-09-23T20:13:46.820709+0000","last_unstale":"2023-09-23T20:14:08.268022+0000","last_undegraded":"2023-09-23T20:14:08.268022+0000","last_fullsized":"2023-09-23T20:14:08.268022+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4006,"log_dups_size":0,"ondisk_log_size":4006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150110208,"num_objects":2291,"num_object_clones":0,"num_object_copies":6873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2291,"num_write_kb":146592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":478,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236457+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:14:06.236457+0000","last_peered":"2023-09-23T20:14:06.236457+0000","last_clean":"2023-09-23T20:14:06.236457+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:14:06.236457+0000","last_undegraded":"2023-09-23T20:14:06.236457+0000","last_fullsized":"2023-09-23T20:14:06.236457+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":138,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206221+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:14:06.206221+0000","last_peered":"2023-09-23T20:14:06.206221+0000","last_clean":"2023-09-23T20:14:06.206221+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:14:06.206221+0000","last_undegraded":"2023-09-23T20:14:06.206221+0000","last_fullsized":"2023-09-23T20:14:06.206221+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":428,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237114+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:14:06.237114+0000","last_peered":"2023-09-23T20:14:06.237114+0000","last_clean":"2023-09-23T20:14:06.237114+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:14:06.237114+0000","last_undegraded":"2023-09-23T20:14:06.237114+0000","last_fullsized":"2023-09-23T20:14:06.237114+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":582,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745205+0000","last_change":"2023-09-23T20:13:25.715415+0000","last_active":"2023-09-23T20:13:58.745205+0000","last_peered":"2023-09-23T20:13:58.745205+0000","last_clean":"2023-09-23T20:13:58.745205+0000","last_became_active":"2023-09-23T20:12:57.738891+0000","last_became_peered":"2023-09-23T20:12:57.738891+0000","last_unstale":"2023-09-23T20:13:58.745205+0000","last_undegraded":"2023-09-23T20:13:58.745205+0000","last_fullsized":"2023-09-23T20:13:58.745205+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:03.625145+0000","scrub_duration":0.0084372379999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":117,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206451+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:14:06.206451+0000","last_peered":"2023-09-23T20:14:06.206451+0000","last_clean":"2023-09-23T20:14:06.206451+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:14:06.206451+0000","last_undegraded":"2023-09-23T20:14:06.206451+0000","last_fullsized":"2023-09-23T20:14:06.206451+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":416,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.176747+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:14:05.176747+0000","last_peered":"2023-09-23T20:14:05.176747+0000","last_clean":"2023-09-23T20:14:05.176747+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:14:05.176747+0000","last_undegraded":"2023-09-23T20:14:05.176747+0000","last_fullsized":"2023-09-23T20:14:05.176747+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":547,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236567+0000","last_change":"2023-09-23T20:13:31.285021+0000","last_active":"2023-09-23T20:14:06.236567+0000","last_peered":"2023-09-23T20:14:06.236567+0000","last_clean":"2023-09-23T20:14:06.236567+0000","last_became_active":"2023-09-23T20:12:57.744840+0000","last_became_peered":"2023-09-23T20:12:57.744840+0000","last_unstale":"2023-09-23T20:14:06.236567+0000","last_undegraded":"2023-09-23T20:14:06.236567+0000","last_fullsized":"2023-09-23T20:14:06.236567+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:32.440549+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":73,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236252+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:14:06.236252+0000","last_peered":"2023-09-23T20:14:06.236252+0000","last_clean":"2023-09-23T20:14:06.236252+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:14:06.236252+0000","last_undegraded":"2023-09-23T20:14:06.236252+0000","last_fullsized":"2023-09-23T20:14:06.236252+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":60,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.179277+0000","last_change":"2023-09-23T20:11:17.697042+0000","last_active":"2023-09-23T20:14:05.179277+0000","last_peered":"2023-09-23T20:14:05.179277+0000","last_clean":"2023-09-23T20:14:05.179277+0000","last_became_active":"2023-09-23T20:11:17.696717+0000","last_became_peered":"2023-09-23T20:11:17.696717+0000","last_unstale":"2023-09-23T20:14:05.179277+0000","last_undegraded":"2023-09-23T20:14:05.179277+0000","last_fullsized":"2023-09-23T20:14:05.179277+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:13.262913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"430'4047","reported_seq":4089,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.271688+0000","last_change":"2023-09-23T20:13:46.790587+0000","last_active":"2023-09-23T20:14:08.271688+0000","last_peered":"2023-09-23T20:14:08.271688+0000","last_clean":"2023-09-23T20:14:08.271688+0000","last_became_active":"2023-09-23T20:13:46.788387+0000","last_became_peered":"2023-09-23T20:13:46.788387+0000","last_unstale":"2023-09-23T20:14:08.271688+0000","last_undegraded":"2023-09-23T20:14:08.271688+0000","last_fullsized":"2023-09-23T20:14:08.271688+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4047,"log_dups_size":0,"ondisk_log_size":4047,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2317,"num_write_kb":148288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":670,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206596+0000","last_change":"2023-09-23T20:13:25.898164+0000","last_active":"2023-09-23T20:14:06.206596+0000","last_peered":"2023-09-23T20:14:06.206596+0000","last_clean":"2023-09-23T20:14:06.206596+0000","last_became_active":"2023-09-23T20:12:57.739435+0000","last_became_peered":"2023-09-23T20:12:57.739435+0000","last_unstale":"2023-09-23T20:14:06.206596+0000","last_undegraded":"2023-09-23T20:14:06.206596+0000","last_fullsized":"2023-09-23T20:14:06.206596+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.172294+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":121,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237768+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:14:06.237768+0000","last_peered":"2023-09-23T20:14:06.237768+0000","last_clean":"2023-09-23T20:14:06.237768+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:14:06.237768+0000","last_undegraded":"2023-09-23T20:14:06.237768+0000","last_fullsized":"2023-09-23T20:14:06.237768+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":438,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236307+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:14:06.236307+0000","last_peered":"2023-09-23T20:14:06.236307+0000","last_clean":"2023-09-23T20:14:06.236307+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:14:06.236307+0000","last_undegraded":"2023-09-23T20:14:06.236307+0000","last_fullsized":"2023-09-23T20:14:06.236307+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"427'3903","reported_seq":3941,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:14:04.065263+0000","last_change":"2023-09-23T20:13:46.814963+0000","last_active":"2023-09-23T20:14:04.065263+0000","last_peered":"2023-09-23T20:14:04.065263+0000","last_clean":"2023-09-23T20:14:04.065263+0000","last_became_active":"2023-09-23T20:13:46.813313+0000","last_became_peered":"2023-09-23T20:13:46.813313+0000","last_unstale":"2023-09-23T20:14:04.065263+0000","last_undegraded":"2023-09-23T20:14:04.065263+0000","last_fullsized":"2023-09-23T20:14:04.065263+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3903,"log_dups_size":0,"ondisk_log_size":3903,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142770176,"num_objects":2178,"num_object_clones":0,"num_object_copies":6534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2178,"num_write_kb":139424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":775,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.176629+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:14:05.176629+0000","last_peered":"2023-09-23T20:14:05.176629+0000","last_clean":"2023-09-23T20:14:05.176629+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:14:05.176629+0000","last_undegraded":"2023-09-23T20:14:05.176629+0000","last_fullsized":"2023-09-23T20:14:05.176629+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":122,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237511+0000","last_change":"2023-09-23T20:07:37.063406+0000","last_active":"2023-09-23T20:14:06.237511+0000","last_peered":"2023-09-23T20:14:06.237511+0000","last_clean":"2023-09-23T20:14:06.237511+0000","last_became_active":"2023-09-23T20:07:36.970726+0000","last_became_peered":"2023-09-23T20:07:36.970726+0000","last_unstale":"2023-09-23T20:14:06.237511+0000","last_undegraded":"2023-09-23T20:14:06.237511+0000","last_fullsized":"2023-09-23T20:14:06.237511+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:10.104486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":442,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.078420+0000","last_change":"2023-09-23T20:13:57.750981+0000","last_active":"2023-09-23T20:13:59.078420+0000","last_peered":"2023-09-23T20:13:59.078420+0000","last_clean":"2023-09-23T20:13:59.078420+0000","last_became_active":"2023-09-23T20:13:57.750543+0000","last_became_peered":"2023-09-23T20:13:57.750543+0000","last_unstale":"2023-09-23T20:13:59.078420+0000","last_undegraded":"2023-09-23T20:13:59.078420+0000","last_fullsized":"2023-09-23T20:13:59.078420+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:02.488251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"430'4017","reported_seq":4059,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.307273+0000","last_change":"2023-09-23T20:13:46.802046+0000","last_active":"2023-09-23T20:14:08.307273+0000","last_peered":"2023-09-23T20:14:08.307273+0000","last_clean":"2023-09-23T20:14:08.307273+0000","last_became_active":"2023-09-23T20:13:46.800187+0000","last_became_peered":"2023-09-23T20:13:46.800187+0000","last_unstale":"2023-09-23T20:14:08.307273+0000","last_undegraded":"2023-09-23T20:14:08.307273+0000","last_fullsized":"2023-09-23T20:14:08.307273+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4017,"log_dups_size":0,"ondisk_log_size":4017,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151191552,"num_objects":2307,"num_object_clones":0,"num_object_copies":6921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2307,"num_write_kb":147648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":502,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237129+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:14:06.237129+0000","last_peered":"2023-09-23T20:14:06.237129+0000","last_clean":"2023-09-23T20:14:06.237129+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:14:06.237129+0000","last_undegraded":"2023-09-23T20:14:06.237129+0000","last_fullsized":"2023-09-23T20:14:06.237129+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":121,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206693+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:14:06.206693+0000","last_peered":"2023-09-23T20:14:06.206693+0000","last_clean":"2023-09-23T20:14:06.206693+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:14:06.206693+0000","last_undegraded":"2023-09-23T20:14:06.206693+0000","last_fullsized":"2023-09-23T20:14:06.206693+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":133,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745597+0000","last_change":"2023-09-23T20:07:36.977257+0000","last_active":"2023-09-23T20:13:58.745597+0000","last_peered":"2023-09-23T20:13:58.745597+0000","last_clean":"2023-09-23T20:13:58.745597+0000","last_became_active":"2023-09-23T20:07:36.976864+0000","last_became_peered":"2023-09-23T20:07:36.976864+0000","last_unstale":"2023-09-23T20:13:58.745597+0000","last_undegraded":"2023-09-23T20:13:58.745597+0000","last_fullsized":"2023-09-23T20:13:58.745597+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:45.500176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"430'4131","reported_seq":4171,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.308019+0000","last_change":"2023-09-23T20:13:46.799208+0000","last_active":"2023-09-23T20:14:08.308019+0000","last_peered":"2023-09-23T20:14:08.308019+0000","last_clean":"2023-09-23T20:14:08.308019+0000","last_became_active":"2023-09-23T20:13:46.795630+0000","last_became_peered":"2023-09-23T20:13:46.795630+0000","last_unstale":"2023-09-23T20:14:08.308019+0000","last_undegraded":"2023-09-23T20:14:08.308019+0000","last_fullsized":"2023-09-23T20:14:08.308019+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4131,"log_dups_size":0,"ondisk_log_size":4131,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153944064,"num_objects":2349,"num_object_clones":0,"num_object_copies":7047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2349,"num_write_kb":150336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":633,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206685+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:14:06.206685+0000","last_peered":"2023-09-23T20:14:06.206685+0000","last_clean":"2023-09-23T20:14:06.206685+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:14:06.206685+0000","last_undegraded":"2023-09-23T20:14:06.206685+0000","last_fullsized":"2023-09-23T20:14:06.206685+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":468,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236753+0000","last_change":"2023-09-23T20:07:18.969373+0000","last_active":"2023-09-23T20:14:06.236753+0000","last_peered":"2023-09-23T20:14:06.236753+0000","last_clean":"2023-09-23T20:14:06.236753+0000","last_became_active":"2023-09-23T20:07:18.969048+0000","last_became_peered":"2023-09-23T20:07:18.969048+0000","last_unstale":"2023-09-23T20:14:06.236753+0000","last_undegraded":"2023-09-23T20:14:06.236753+0000","last_fullsized":"2023-09-23T20:14:06.236753+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:03.732633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":124,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.746615+0000","last_change":"2023-09-23T20:07:18.940275+0000","last_active":"2023-09-23T20:13:59.746615+0000","last_peered":"2023-09-23T20:13:59.746615+0000","last_clean":"2023-09-23T20:13:59.746615+0000","last_became_active":"2023-09-23T20:07:18.939349+0000","last_became_peered":"2023-09-23T20:07:18.939349+0000","last_unstale":"2023-09-23T20:13:59.746615+0000","last_undegraded":"2023-09-23T20:13:59.746615+0000","last_fullsized":"2023-09-23T20:13:59.746615+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:02.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"430'4033","reported_seq":4074,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.569198+0000","last_change":"2023-09-23T20:13:46.797007+0000","last_active":"2023-09-23T20:14:08.569198+0000","last_peered":"2023-09-23T20:14:08.569198+0000","last_clean":"2023-09-23T20:14:08.569198+0000","last_became_active":"2023-09-23T20:13:46.793442+0000","last_became_peered":"2023-09-23T20:13:46.793442+0000","last_unstale":"2023-09-23T20:14:08.569198+0000","last_undegraded":"2023-09-23T20:14:08.569198+0000","last_fullsized":"2023-09-23T20:14:08.569198+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4033,"log_dups_size":0,"ondisk_log_size":4033,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149946368,"num_objects":2288,"num_object_clones":0,"num_object_copies":6864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2288,"num_write_kb":146432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":812,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206371+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:14:06.206371+0000","last_peered":"2023-09-23T20:14:06.206371+0000","last_clean":"2023-09-23T20:14:06.206371+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:14:06.206371+0000","last_undegraded":"2023-09-23T20:14:06.206371+0000","last_fullsized":"2023-09-23T20:14:06.206371+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":174,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.176072+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:14:05.176072+0000","last_peered":"2023-09-23T20:14:05.176072+0000","last_clean":"2023-09-23T20:14:05.176072+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:14:05.176072+0000","last_undegraded":"2023-09-23T20:14:05.176072+0000","last_fullsized":"2023-09-23T20:14:05.176072+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":348,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.078642+0000","last_change":"2023-09-23T20:13:57.751859+0000","last_active":"2023-09-23T20:13:59.078642+0000","last_peered":"2023-09-23T20:13:59.078642+0000","last_clean":"2023-09-23T20:13:59.078642+0000","last_became_active":"2023-09-23T20:13:57.751605+0000","last_became_peered":"2023-09-23T20:13:57.751605+0000","last_unstale":"2023-09-23T20:13:59.078642+0000","last_undegraded":"2023-09-23T20:13:59.078642+0000","last_fullsized":"2023-09-23T20:13:59.078642+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:00.697072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.0","version":"427'3895","reported_seq":3937,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:14:04.082863+0000","last_change":"2023-09-23T20:13:46.825574+0000","last_active":"2023-09-23T20:14:04.082863+0000","last_peered":"2023-09-23T20:14:04.082863+0000","last_clean":"2023-09-23T20:14:04.082863+0000","last_became_active":"2023-09-23T20:13:46.781741+0000","last_became_peered":"2023-09-23T20:13:46.781741+0000","last_unstale":"2023-09-23T20:14:04.082863+0000","last_undegraded":"2023-09-23T20:14:04.082863+0000","last_fullsized":"2023-09-23T20:14:04.082863+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3895,"log_dups_size":0,"ondisk_log_size":3895,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142540800,"num_objects":2175,"num_object_clones":0,"num_object_copies":6525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2175,"num_write_kb":139200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":777,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236353+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:14:06.236353+0000","last_peered":"2023-09-23T20:14:06.236353+0000","last_clean":"2023-09-23T20:14:06.236353+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:14:06.236353+0000","last_undegraded":"2023-09-23T20:14:06.236353+0000","last_fullsized":"2023-09-23T20:14:06.236353+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":288,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.078492+0000","last_change":"2023-09-23T20:13:57.751429+0000","last_active":"2023-09-23T20:13:59.078492+0000","last_peered":"2023-09-23T20:13:59.078492+0000","last_clean":"2023-09-23T20:13:59.078492+0000","last_became_active":"2023-09-23T20:13:57.751167+0000","last_became_peered":"2023-09-23T20:13:57.751167+0000","last_unstale":"2023-09-23T20:13:59.078492+0000","last_undegraded":"2023-09-23T20:13:59.078492+0000","last_fullsized":"2023-09-23T20:13:59.078492+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:29.323402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":319,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:14:01.025144+0000","last_change":"2023-09-23T20:13:57.748537+0000","last_active":"2023-09-23T20:14:01.025144+0000","last_peered":"2023-09-23T20:14:01.025144+0000","last_clean":"2023-09-23T20:14:01.025144+0000","last_became_active":"2023-09-23T20:13:57.748177+0000","last_became_peered":"2023-09-23T20:13:57.748177+0000","last_unstale":"2023-09-23T20:14:01.025144+0000","last_undegraded":"2023-09-23T20:14:01.025144+0000","last_fullsized":"2023-09-23T20:14:01.025144+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:19.737724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"430'4036","reported_seq":4077,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.436513+0000","last_change":"2023-09-23T20:13:46.794622+0000","last_active":"2023-09-23T20:14:08.436513+0000","last_peered":"2023-09-23T20:14:08.436513+0000","last_clean":"2023-09-23T20:14:08.436513+0000","last_became_active":"2023-09-23T20:13:46.792301+0000","last_became_peered":"2023-09-23T20:13:46.792301+0000","last_unstale":"2023-09-23T20:14:08.436513+0000","last_undegraded":"2023-09-23T20:14:08.436513+0000","last_fullsized":"2023-09-23T20:14:08.436513+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4036,"log_dups_size":0,"ondisk_log_size":4036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150962176,"num_objects":2303,"num_object_clones":0,"num_object_copies":6909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2303,"num_write_kb":147424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":582,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745333+0000","last_change":"2023-09-23T20:13:27.750109+0000","last_active":"2023-09-23T20:13:58.745333+0000","last_peered":"2023-09-23T20:13:58.745333+0000","last_clean":"2023-09-23T20:13:58.745333+0000","last_became_active":"2023-09-23T20:12:57.745568+0000","last_became_peered":"2023-09-23T20:12:57.745568+0000","last_unstale":"2023-09-23T20:13:58.745333+0000","last_undegraded":"2023-09-23T20:13:58.745333+0000","last_fullsized":"2023-09-23T20:13:58.745333+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:09.736440+0000","scrub_duration":0.0052404440000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":319,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:14:01.025047+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:14:01.025047+0000","last_peered":"2023-09-23T20:14:01.025047+0000","last_clean":"2023-09-23T20:14:01.025047+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:14:01.025047+0000","last_undegraded":"2023-09-23T20:14:01.025047+0000","last_fullsized":"2023-09-23T20:14:01.025047+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":348,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.078700+0000","last_change":"2023-09-23T20:13:57.750876+0000","last_active":"2023-09-23T20:13:59.078700+0000","last_peered":"2023-09-23T20:13:59.078700+0000","last_clean":"2023-09-23T20:13:59.078700+0000","last_became_active":"2023-09-23T20:13:57.750586+0000","last_became_peered":"2023-09-23T20:13:57.750586+0000","last_unstale":"2023-09-23T20:13:59.078700+0000","last_undegraded":"2023-09-23T20:13:59.078700+0000","last_fullsized":"2023-09-23T20:13:59.078700+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:14.881201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"428'4035","reported_seq":4074,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.925206+0000","last_change":"2023-09-23T20:13:46.788577+0000","last_active":"2023-09-23T20:14:05.925206+0000","last_peered":"2023-09-23T20:14:05.925206+0000","last_clean":"2023-09-23T20:14:05.925206+0000","last_became_active":"2023-09-23T20:13:46.786812+0000","last_became_peered":"2023-09-23T20:13:46.786812+0000","last_unstale":"2023-09-23T20:14:05.925206+0000","last_undegraded":"2023-09-23T20:14:05.925206+0000","last_fullsized":"2023-09-23T20:14:05.925206+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4035,"log_dups_size":0,"ondisk_log_size":4035,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149749760,"num_objects":2285,"num_object_clones":0,"num_object_copies":6855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2285,"num_write_kb":146240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":768,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236759+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:14:06.236759+0000","last_peered":"2023-09-23T20:14:06.236759+0000","last_clean":"2023-09-23T20:14:06.236759+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:14:06.236759+0000","last_undegraded":"2023-09-23T20:14:06.236759+0000","last_fullsized":"2023-09-23T20:14:06.236759+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":60,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.176968+0000","last_change":"2023-09-23T20:11:17.697403+0000","last_active":"2023-09-23T20:14:05.176968+0000","last_peered":"2023-09-23T20:14:05.176968+0000","last_clean":"2023-09-23T20:14:05.176968+0000","last_became_active":"2023-09-23T20:11:17.697159+0000","last_became_peered":"2023-09-23T20:11:17.697159+0000","last_unstale":"2023-09-23T20:14:05.176968+0000","last_undegraded":"2023-09-23T20:14:05.176968+0000","last_fullsized":"2023-09-23T20:14:05.176968+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:42.854848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":118,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.746866+0000","last_change":"2023-09-23T20:07:18.961372+0000","last_active":"2023-09-23T20:13:58.746866+0000","last_peered":"2023-09-23T20:13:58.746866+0000","last_clean":"2023-09-23T20:13:58.746866+0000","last_became_active":"2023-09-23T20:07:18.961068+0000","last_became_peered":"2023-09-23T20:07:18.961068+0000","last_unstale":"2023-09-23T20:13:58.746866+0000","last_undegraded":"2023-09-23T20:13:58.746866+0000","last_fullsized":"2023-09-23T20:13:58.746866+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:29.040886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"430'4500","reported_seq":4531,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.287391+0000","last_change":"2023-09-23T20:13:02.211701+0000","last_active":"2023-09-23T20:14:08.287391+0000","last_peered":"2023-09-23T20:14:08.287391+0000","last_clean":"2023-09-23T20:14:08.287391+0000","last_became_active":"2023-09-23T20:13:02.211349+0000","last_became_peered":"2023-09-23T20:13:02.211349+0000","last_unstale":"2023-09-23T20:14:08.287391+0000","last_undegraded":"2023-09-23T20:14:08.287391+0000","last_fullsized":"2023-09-23T20:14:08.287391+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4500,"log_dups_size":0,"ondisk_log_size":4500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:35.001450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912000,"num_objects":4500,"num_object_clones":0,"num_object_copies":13500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4500,"num_write_kb":288000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":710,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.179372+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:14:05.179372+0000","last_peered":"2023-09-23T20:14:05.179372+0000","last_clean":"2023-09-23T20:14:05.179372+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:14:05.179372+0000","last_undegraded":"2023-09-23T20:14:05.179372+0000","last_fullsized":"2023-09-23T20:14:05.179372+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":561,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.746069+0000","last_change":"2023-09-23T20:07:52.781869+0000","last_active":"2023-09-23T20:13:58.746069+0000","last_peered":"2023-09-23T20:13:58.746069+0000","last_clean":"2023-09-23T20:13:58.746069+0000","last_became_active":"2023-09-23T20:07:52.781156+0000","last_became_peered":"2023-09-23T20:07:52.781156+0000","last_unstale":"2023-09-23T20:13:58.746069+0000","last_undegraded":"2023-09-23T20:13:58.746069+0000","last_fullsized":"2023-09-23T20:13:58.746069+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:32.777601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":430,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237132+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:14:06.237132+0000","last_peered":"2023-09-23T20:14:06.237132+0000","last_clean":"2023-09-23T20:14:06.237132+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:14:06.237132+0000","last_undegraded":"2023-09-23T20:14:06.237132+0000","last_fullsized":"2023-09-23T20:14:06.237132+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.818675+0000","last_change":"2023-09-23T20:13:40.151946+0000","last_active":"2023-09-23T20:13:59.818675+0000","last_peered":"2023-09-23T20:13:59.818675+0000","last_clean":"2023-09-23T20:13:59.818675+0000","last_became_active":"2023-09-23T20:13:40.151634+0000","last_became_peered":"2023-09-23T20:13:40.151634+0000","last_unstale":"2023-09-23T20:13:59.818675+0000","last_undegraded":"2023-09-23T20:13:59.818675+0000","last_fullsized":"2023-09-23T20:13:59.818675+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:42.394323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"430'4109","reported_seq":4151,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.543639+0000","last_change":"2023-09-23T20:13:46.785589+0000","last_active":"2023-09-23T20:14:08.543639+0000","last_peered":"2023-09-23T20:14:08.543639+0000","last_clean":"2023-09-23T20:14:08.543639+0000","last_became_active":"2023-09-23T20:13:46.781975+0000","last_became_peered":"2023-09-23T20:13:46.781975+0000","last_unstale":"2023-09-23T20:14:08.543639+0000","last_undegraded":"2023-09-23T20:14:08.543639+0000","last_fullsized":"2023-09-23T20:14:08.543639+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4109,"log_dups_size":0,"ondisk_log_size":4109,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154238976,"num_objects":2353,"num_object_clones":0,"num_object_copies":7059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2353,"num_write_kb":150624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":773,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236465+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:14:06.236465+0000","last_peered":"2023-09-23T20:14:06.236465+0000","last_clean":"2023-09-23T20:14:06.236465+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:14:06.236465+0000","last_undegraded":"2023-09-23T20:14:06.236465+0000","last_fullsized":"2023-09-23T20:14:06.236465+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1050,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236590+0000","last_change":"2023-09-23T20:07:37.226535+0000","last_active":"2023-09-23T20:14:06.236590+0000","last_peered":"2023-09-23T20:14:06.236590+0000","last_clean":"2023-09-23T20:14:06.236590+0000","last_became_active":"2023-09-23T20:07:18.950844+0000","last_became_peered":"2023-09-23T20:07:18.950844+0000","last_unstale":"2023-09-23T20:14:06.236590+0000","last_undegraded":"2023-09-23T20:14:06.236590+0000","last_fullsized":"2023-09-23T20:14:06.236590+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:01.872761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":341,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.078574+0000","last_change":"2023-09-23T20:13:57.751304+0000","last_active":"2023-09-23T20:13:59.078574+0000","last_peered":"2023-09-23T20:13:59.078574+0000","last_clean":"2023-09-23T20:13:59.078574+0000","last_became_active":"2023-09-23T20:13:57.750999+0000","last_became_peered":"2023-09-23T20:13:57.750999+0000","last_unstale":"2023-09-23T20:13:59.078574+0000","last_undegraded":"2023-09-23T20:13:59.078574+0000","last_fullsized":"2023-09-23T20:13:59.078574+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:43.815519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":124,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745985+0000","last_change":"2023-09-23T20:07:18.962677+0000","last_active":"2023-09-23T20:13:58.745985+0000","last_peered":"2023-09-23T20:13:58.745985+0000","last_clean":"2023-09-23T20:13:58.745985+0000","last_became_active":"2023-09-23T20:07:18.962456+0000","last_became_peered":"2023-09-23T20:07:18.962456+0000","last_unstale":"2023-09-23T20:13:58.745985+0000","last_undegraded":"2023-09-23T20:13:58.745985+0000","last_fullsized":"2023-09-23T20:13:58.745985+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:15.419586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"430'4492","reported_seq":4523,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.312893+0000","last_change":"2023-09-23T20:13:02.212686+0000","last_active":"2023-09-23T20:14:08.312893+0000","last_peered":"2023-09-23T20:14:08.312893+0000","last_clean":"2023-09-23T20:14:08.312893+0000","last_became_active":"2023-09-23T20:13:02.212438+0000","last_became_peered":"2023-09-23T20:13:02.212438+0000","last_unstale":"2023-09-23T20:14:08.312893+0000","last_undegraded":"2023-09-23T20:14:08.312893+0000","last_fullsized":"2023-09-23T20:14:08.312893+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4492,"log_dups_size":0,"ondisk_log_size":4492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:09.919399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294387712,"num_objects":4492,"num_object_clones":0,"num_object_copies":13476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4492,"num_write_kb":287488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":668,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206281+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:14:06.206281+0000","last_peered":"2023-09-23T20:14:06.206281+0000","last_clean":"2023-09-23T20:14:06.206281+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:14:06.206281+0000","last_undegraded":"2023-09-23T20:14:06.206281+0000","last_fullsized":"2023-09-23T20:14:06.206281+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1065,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.815064+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:13:59.815064+0000","last_peered":"2023-09-23T20:13:59.815064+0000","last_clean":"2023-09-23T20:13:59.815064+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:13:59.815064+0000","last_undegraded":"2023-09-23T20:13:59.815064+0000","last_fullsized":"2023-09-23T20:13:59.815064+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":124,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.744922+0000","last_change":"2023-09-23T20:07:18.950732+0000","last_active":"2023-09-23T20:13:58.744922+0000","last_peered":"2023-09-23T20:13:58.744922+0000","last_clean":"2023-09-23T20:13:58.744922+0000","last_became_active":"2023-09-23T20:07:18.950344+0000","last_became_peered":"2023-09-23T20:07:18.950344+0000","last_unstale":"2023-09-23T20:13:58.744922+0000","last_undegraded":"2023-09-23T20:13:58.744922+0000","last_fullsized":"2023-09-23T20:13:58.744922+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:41.466963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":24,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.815510+0000","last_change":"2023-09-23T20:13:40.153551+0000","last_active":"2023-09-23T20:13:59.815510+0000","last_peered":"2023-09-23T20:13:59.815510+0000","last_clean":"2023-09-23T20:13:59.815510+0000","last_became_active":"2023-09-23T20:13:40.153265+0000","last_became_peered":"2023-09-23T20:13:40.153265+0000","last_unstale":"2023-09-23T20:13:59.815510+0000","last_undegraded":"2023-09-23T20:13:59.815510+0000","last_fullsized":"2023-09-23T20:13:59.815510+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:21.894619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"430'3983","reported_seq":4025,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.283883+0000","last_change":"2023-09-23T20:13:46.814034+0000","last_active":"2023-09-23T20:14:08.283883+0000","last_peered":"2023-09-23T20:14:08.283883+0000","last_clean":"2023-09-23T20:14:08.283883+0000","last_became_active":"2023-09-23T20:13:46.812300+0000","last_became_peered":"2023-09-23T20:13:46.812300+0000","last_unstale":"2023-09-23T20:14:08.283883+0000","last_undegraded":"2023-09-23T20:14:08.283883+0000","last_fullsized":"2023-09-23T20:14:08.283883+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3983,"log_dups_size":0,"ondisk_log_size":3983,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148602880,"num_objects":2267,"num_object_clones":0,"num_object_copies":6801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2267,"num_write_kb":145120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1338,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236569+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:14:06.236569+0000","last_peered":"2023-09-23T20:14:06.236569+0000","last_clean":"2023-09-23T20:14:06.236569+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:14:06.236569+0000","last_undegraded":"2023-09-23T20:14:06.236569+0000","last_fullsized":"2023-09-23T20:14:06.236569+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":605,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.239047+0000","last_change":"2023-09-23T20:12:54.043697+0000","last_active":"2023-09-23T20:14:06.239047+0000","last_peered":"2023-09-23T20:14:06.239047+0000","last_clean":"2023-09-23T20:14:06.239047+0000","last_became_active":"2023-09-23T20:09:56.050384+0000","last_became_peered":"2023-09-23T20:09:56.050384+0000","last_unstale":"2023-09-23T20:14:06.239047+0000","last_undegraded":"2023-09-23T20:14:06.239047+0000","last_fullsized":"2023-09-23T20:14:06.239047+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:01.598813+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":295,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236685+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:14:06.236685+0000","last_peered":"2023-09-23T20:14:06.236685+0000","last_clean":"2023-09-23T20:14:06.236685+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:14:06.236685+0000","last_undegraded":"2023-09-23T20:14:06.236685+0000","last_fullsized":"2023-09-23T20:14:06.236685+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":227,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206580+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:14:06.206580+0000","last_peered":"2023-09-23T20:14:06.206580+0000","last_clean":"2023-09-23T20:14:06.206580+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:14:06.206580+0000","last_undegraded":"2023-09-23T20:14:06.206580+0000","last_fullsized":"2023-09-23T20:14:06.206580+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"430'4411","reported_seq":4442,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.302168+0000","last_change":"2023-09-23T20:13:02.795014+0000","last_active":"2023-09-23T20:14:08.302168+0000","last_peered":"2023-09-23T20:14:08.302168+0000","last_clean":"2023-09-23T20:14:08.302168+0000","last_became_active":"2023-09-23T20:13:02.794587+0000","last_became_peered":"2023-09-23T20:13:02.794587+0000","last_unstale":"2023-09-23T20:14:08.302168+0000","last_undegraded":"2023-09-23T20:14:08.302168+0000","last_fullsized":"2023-09-23T20:14:08.302168+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4411,"log_dups_size":0,"ondisk_log_size":4411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:08.535861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289079296,"num_objects":4411,"num_object_clones":0,"num_object_copies":13233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4411,"num_write_kb":282304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":566,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.745637+0000","last_change":"2023-09-23T20:12:38.758800+0000","last_active":"2023-09-23T20:13:58.745637+0000","last_peered":"2023-09-23T20:13:58.745637+0000","last_clean":"2023-09-23T20:13:58.745637+0000","last_became_active":"2023-09-23T20:09:56.051160+0000","last_became_peered":"2023-09-23T20:09:56.051160+0000","last_unstale":"2023-09-23T20:13:58.745637+0000","last_undegraded":"2023-09-23T20:13:58.745637+0000","last_fullsized":"2023-09-23T20:13:58.745637+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:56.065108+0000","scrub_duration":0.001864159,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":866,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.238872+0000","last_change":"2023-09-23T20:10:10.708427+0000","last_active":"2023-09-23T20:14:06.238872+0000","last_peered":"2023-09-23T20:14:06.238872+0000","last_clean":"2023-09-23T20:14:06.238872+0000","last_became_active":"2023-09-23T20:10:10.707975+0000","last_became_peered":"2023-09-23T20:10:10.707975+0000","last_unstale":"2023-09-23T20:14:06.238872+0000","last_undegraded":"2023-09-23T20:14:06.238872+0000","last_fullsized":"2023-09-23T20:14:06.238872+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:52.749146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":374,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.176457+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:14:05.176457+0000","last_peered":"2023-09-23T20:14:05.176457+0000","last_clean":"2023-09-23T20:14:05.176457+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:14:05.176457+0000","last_undegraded":"2023-09-23T20:14:05.176457+0000","last_fullsized":"2023-09-23T20:14:05.176457+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.746644+0000","last_change":"2023-09-23T20:07:18.951455+0000","last_active":"2023-09-23T20:13:58.746644+0000","last_peered":"2023-09-23T20:13:58.746644+0000","last_clean":"2023-09-23T20:13:58.746644+0000","last_became_active":"2023-09-23T20:07:18.950606+0000","last_became_peered":"2023-09-23T20:07:18.950606+0000","last_unstale":"2023-09-23T20:13:58.746644+0000","last_undegraded":"2023-09-23T20:13:58.746644+0000","last_fullsized":"2023-09-23T20:13:58.746644+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:28.987902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"430'4526","reported_seq":4557,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.709331+0000","last_change":"2023-09-23T20:13:02.736405+0000","last_active":"2023-09-23T20:14:08.709331+0000","last_peered":"2023-09-23T20:14:08.709331+0000","last_clean":"2023-09-23T20:14:08.709331+0000","last_became_active":"2023-09-23T20:13:02.736146+0000","last_became_peered":"2023-09-23T20:13:02.736146+0000","last_unstale":"2023-09-23T20:14:08.709331+0000","last_undegraded":"2023-09-23T20:14:08.709331+0000","last_fullsized":"2023-09-23T20:14:08.709331+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4526,"log_dups_size":0,"ondisk_log_size":4526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:41.788802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296615936,"num_objects":4526,"num_object_clones":0,"num_object_copies":13578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4526,"num_write_kb":289664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":602,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.193021+0000","last_change":"2023-09-23T20:13:57.748021+0000","last_active":"2023-09-23T20:14:06.193021+0000","last_peered":"2023-09-23T20:14:06.193021+0000","last_clean":"2023-09-23T20:14:06.193021+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:14:06.193021+0000","last_undegraded":"2023-09-23T20:14:06.193021+0000","last_fullsized":"2023-09-23T20:14:06.193021+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":130,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.206753+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:14:06.206753+0000","last_peered":"2023-09-23T20:14:06.206753+0000","last_clean":"2023-09-23T20:14:06.206753+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:14:06.206753+0000","last_undegraded":"2023-09-23T20:14:06.206753+0000","last_fullsized":"2023-09-23T20:14:06.206753+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":124,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.746001+0000","last_change":"2023-09-23T20:07:18.951382+0000","last_active":"2023-09-23T20:13:58.746001+0000","last_peered":"2023-09-23T20:13:58.746001+0000","last_clean":"2023-09-23T20:13:58.746001+0000","last_became_active":"2023-09-23T20:07:18.950566+0000","last_became_peered":"2023-09-23T20:07:18.950566+0000","last_unstale":"2023-09-23T20:13:58.746001+0000","last_undegraded":"2023-09-23T20:13:58.746001+0000","last_fullsized":"2023-09-23T20:13:58.746001+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:24.943189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":256,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236523+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:14:06.236523+0000","last_peered":"2023-09-23T20:14:06.236523+0000","last_clean":"2023-09-23T20:14:06.236523+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:14:06.236523+0000","last_undegraded":"2023-09-23T20:14:06.236523+0000","last_fullsized":"2023-09-23T20:14:06.236523+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"430'4653","reported_seq":4684,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.704078+0000","last_change":"2023-09-23T20:13:02.213807+0000","last_active":"2023-09-23T20:14:08.704078+0000","last_peered":"2023-09-23T20:14:08.704078+0000","last_clean":"2023-09-23T20:14:08.704078+0000","last_became_active":"2023-09-23T20:13:02.213521+0000","last_became_peered":"2023-09-23T20:13:02.213521+0000","last_unstale":"2023-09-23T20:14:08.704078+0000","last_undegraded":"2023-09-23T20:14:08.704078+0000","last_fullsized":"2023-09-23T20:14:08.704078+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4653,"log_dups_size":0,"ondisk_log_size":4653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:09.558945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304939008,"num_objects":4653,"num_object_clones":0,"num_object_copies":13959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4653,"num_write_kb":297792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":638,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:14:03.071177+0000","last_change":"2023-09-23T20:14:05.194683+0000","last_active":"2023-09-23T20:14:03.071177+0000","last_peered":"2023-09-23T20:14:03.071177+0000","last_clean":"2023-09-23T20:14:03.071177+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:14:03.071177+0000","last_undegraded":"2023-09-23T20:14:03.071177+0000","last_fullsized":"2023-09-23T20:14:03.071177+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":272,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237524+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:14:06.237524+0000","last_peered":"2023-09-23T20:14:06.237524+0000","last_clean":"2023-09-23T20:14:06.237524+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:14:06.237524+0000","last_undegraded":"2023-09-23T20:14:06.237524+0000","last_fullsized":"2023-09-23T20:14:06.237524+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":465,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.746576+0000","last_change":"2023-09-23T20:07:37.993549+0000","last_active":"2023-09-23T20:13:58.746576+0000","last_peered":"2023-09-23T20:13:58.746576+0000","last_clean":"2023-09-23T20:13:58.746576+0000","last_became_active":"2023-09-23T20:07:37.992375+0000","last_became_peered":"2023-09-23T20:07:37.992375+0000","last_unstale":"2023-09-23T20:13:58.746576+0000","last_undegraded":"2023-09-23T20:13:58.746576+0000","last_fullsized":"2023-09-23T20:13:58.746576+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:33.200401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":60,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.176876+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:14:05.176876+0000","last_peered":"2023-09-23T20:14:05.176876+0000","last_clean":"2023-09-23T20:14:05.176876+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:14:05.176876+0000","last_undegraded":"2023-09-23T20:14:05.176876+0000","last_fullsized":"2023-09-23T20:14:05.176876+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"428'4324","reported_seq":4355,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.862363+0000","last_change":"2023-09-23T20:13:02.737701+0000","last_active":"2023-09-23T20:14:05.862363+0000","last_peered":"2023-09-23T20:14:05.862363+0000","last_clean":"2023-09-23T20:14:05.862363+0000","last_became_active":"2023-09-23T20:13:02.737397+0000","last_became_peered":"2023-09-23T20:13:02.737397+0000","last_unstale":"2023-09-23T20:14:05.862363+0000","last_undegraded":"2023-09-23T20:14:05.862363+0000","last_fullsized":"2023-09-23T20:14:05.862363+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4324,"log_dups_size":0,"ondisk_log_size":4324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.355044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283377664,"num_objects":4324,"num_object_clones":0,"num_object_copies":12972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4324,"num_write_kb":276736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":753,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.177050+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:14:05.177050+0000","last_peered":"2023-09-23T20:14:05.177050+0000","last_clean":"2023-09-23T20:14:05.177050+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:14:05.177050+0000","last_undegraded":"2023-09-23T20:14:05.177050+0000","last_fullsized":"2023-09-23T20:14:05.177050+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":131,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237926+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:14:06.237926+0000","last_peered":"2023-09-23T20:14:06.237926+0000","last_clean":"2023-09-23T20:14:06.237926+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:14:06.237926+0000","last_undegraded":"2023-09-23T20:14:06.237926+0000","last_fullsized":"2023-09-23T20:14:06.237926+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":128,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:58.746451+0000","last_change":"2023-09-23T20:07:18.972115+0000","last_active":"2023-09-23T20:13:58.746451+0000","last_peered":"2023-09-23T20:13:58.746451+0000","last_clean":"2023-09-23T20:13:58.746451+0000","last_became_active":"2023-09-23T20:07:18.950164+0000","last_became_peered":"2023-09-23T20:07:18.950164+0000","last_unstale":"2023-09-23T20:13:58.746451+0000","last_undegraded":"2023-09-23T20:13:58.746451+0000","last_fullsized":"2023-09-23T20:13:58.746451+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:01.244873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":213,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236670+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:14:06.236670+0000","last_peered":"2023-09-23T20:14:06.236670+0000","last_clean":"2023-09-23T20:14:06.236670+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:14:06.236670+0000","last_undegraded":"2023-09-23T20:14:06.236670+0000","last_fullsized":"2023-09-23T20:14:06.236670+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"430'4045","reported_seq":6956,"reported_epoch":430,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:14:08.722260+0000","last_change":"2023-09-23T20:13:57.866715+0000","last_active":"2023-09-23T20:14:08.722260+0000","last_peered":"2023-09-23T20:14:08.722260+0000","last_clean":"2023-09-23T20:13:54.750093+0000","last_became_active":"2023-09-23T20:13:57.803898+0000","last_became_peered":"2023-09-23T20:13:57.803898+0000","last_unstale":"2023-09-23T20:14:08.722260+0000","last_undegraded":"2023-09-23T20:14:08.722260+0000","last_fullsized":"2023-09-23T20:13:57.744246+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4045,"log_dups_size":0,"ondisk_log_size":4045,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150732800,"num_objects":2300,"num_object_clones":0,"num_object_copies":6900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":629,"num_objects_unfound":0,"num_objects_dirty":2300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2300,"num_write_kb":147200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1430,"num_bytes_recovered":93716480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3],"avail_no_missing":["7","3","4"],"object_location_counts":[{"shards":"3,4,7","objects":629},{"shards":"3,7","objects":1671}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":204,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.816058+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:13:59.816058+0000","last_peered":"2023-09-23T20:13:59.816058+0000","last_clean":"2023-09-23T20:13:59.816058+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:13:59.816058+0000","last_undegraded":"2023-09-23T20:13:59.816058+0000","last_fullsized":"2023-09-23T20:13:59.816058+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":195,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.815862+0000","last_change":"2023-09-23T20:10:52.685406+0000","last_active":"2023-09-23T20:13:59.815862+0000","last_peered":"2023-09-23T20:13:59.815862+0000","last_clean":"2023-09-23T20:13:59.815862+0000","last_became_active":"2023-09-23T20:10:52.685079+0000","last_became_peered":"2023-09-23T20:10:52.685079+0000","last_unstale":"2023-09-23T20:13:59.815862+0000","last_undegraded":"2023-09-23T20:13:59.815862+0000","last_fullsized":"2023-09-23T20:13:59.815862+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:35.837303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"430'4092","reported_seq":4131,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.302722+0000","last_change":"2023-09-23T20:13:46.809608+0000","last_active":"2023-09-23T20:14:08.302722+0000","last_peered":"2023-09-23T20:14:08.302722+0000","last_clean":"2023-09-23T20:14:08.302722+0000","last_became_active":"2023-09-23T20:13:46.807679+0000","last_became_peered":"2023-09-23T20:13:46.807679+0000","last_unstale":"2023-09-23T20:14:08.302722+0000","last_undegraded":"2023-09-23T20:14:08.302722+0000","last_fullsized":"2023-09-23T20:14:08.302722+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4092,"log_dups_size":0,"ondisk_log_size":4092,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151388160,"num_objects":2310,"num_object_clones":0,"num_object_copies":6930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2310,"num_write_kb":147840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":433,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.236592+0000","last_change":"2023-09-23T20:07:19.098637+0000","last_active":"2023-09-23T20:14:06.236592+0000","last_peered":"2023-09-23T20:14:06.236592+0000","last_clean":"2023-09-23T20:14:06.236592+0000","last_became_active":"2023-09-23T20:07:18.968553+0000","last_became_peered":"2023-09-23T20:07:18.968553+0000","last_unstale":"2023-09-23T20:14:06.236592+0000","last_undegraded":"2023-09-23T20:14:06.236592+0000","last_fullsized":"2023-09-23T20:14:06.236592+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:48.131624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":448,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-09-23T20:14:05.176149+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:14:05.176149+0000","last_peered":"2023-09-23T20:14:05.176149+0000","last_clean":"2023-09-23T20:14:05.176149+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:14:05.176149+0000","last_undegraded":"2023-09-23T20:14:05.176149+0000","last_fullsized":"2023-09-23T20:14:05.176149+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.17","version":"430'3995","reported_seq":4036,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.311604+0000","last_change":"2023-09-23T20:13:46.814598+0000","last_active":"2023-09-23T20:14:08.311604+0000","last_peered":"2023-09-23T20:14:08.311604+0000","last_clean":"2023-09-23T20:14:08.311604+0000","last_became_active":"2023-09-23T20:13:46.812685+0000","last_became_peered":"2023-09-23T20:13:46.812685+0000","last_unstale":"2023-09-23T20:14:08.311604+0000","last_undegraded":"2023-09-23T20:14:08.311604+0000","last_fullsized":"2023-09-23T20:14:08.311604+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3995,"log_dups_size":0,"ondisk_log_size":3995,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149749760,"num_objects":2285,"num_object_clones":0,"num_object_copies":6855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2285,"num_write_kb":146240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":423,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237001+0000","last_change":"2023-09-23T20:07:19.095526+0000","last_active":"2023-09-23T20:14:06.237001+0000","last_peered":"2023-09-23T20:14:06.237001+0000","last_clean":"2023-09-23T20:14:06.237001+0000","last_became_active":"2023-09-23T20:07:18.968812+0000","last_became_peered":"2023-09-23T20:07:18.968812+0000","last_unstale":"2023-09-23T20:14:06.237001+0000","last_undegraded":"2023-09-23T20:14:06.237001+0000","last_fullsized":"2023-09-23T20:14:06.237001+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:53.958859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":426,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.208177+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:14:06.208177+0000","last_peered":"2023-09-23T20:14:06.208177+0000","last_clean":"2023-09-23T20:14:06.208177+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:14:06.208177+0000","last_undegraded":"2023-09-23T20:14:06.208177+0000","last_fullsized":"2023-09-23T20:14:06.208177+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"427'3941","reported_seq":3983,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:14:04.034735+0000","last_change":"2023-09-23T20:13:46.840429+0000","last_active":"2023-09-23T20:14:04.034735+0000","last_peered":"2023-09-23T20:14:04.034735+0000","last_clean":"2023-09-23T20:14:04.034735+0000","last_became_active":"2023-09-23T20:13:46.829550+0000","last_became_peered":"2023-09-23T20:13:46.829550+0000","last_unstale":"2023-09-23T20:14:04.034735+0000","last_undegraded":"2023-09-23T20:14:04.034735+0000","last_fullsized":"2023-09-23T20:14:04.034735+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3941,"log_dups_size":0,"ondisk_log_size":3941,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145260544,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2217,"num_write_kb":141856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":129,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.815201+0000","last_change":"2023-09-23T20:08:04.754524+0000","last_active":"2023-09-23T20:13:59.815201+0000","last_peered":"2023-09-23T20:13:59.815201+0000","last_clean":"2023-09-23T20:13:59.815201+0000","last_became_active":"2023-09-23T20:08:04.754192+0000","last_became_peered":"2023-09-23T20:08:04.754192+0000","last_unstale":"2023-09-23T20:13:59.815201+0000","last_undegraded":"2023-09-23T20:13:59.815201+0000","last_fullsized":"2023-09-23T20:13:59.815201+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.705925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":286,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.815143+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:13:59.815143+0000","last_peered":"2023-09-23T20:13:59.815143+0000","last_clean":"2023-09-23T20:13:59.815143+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:13:59.815143+0000","last_undegraded":"2023-09-23T20:13:59.815143+0000","last_fullsized":"2023-09-23T20:13:59.815143+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"430'3970","reported_seq":4011,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T20:14:08.311689+0000","last_change":"2023-09-23T20:13:46.818935+0000","last_active":"2023-09-23T20:14:08.311689+0000","last_peered":"2023-09-23T20:14:08.311689+0000","last_clean":"2023-09-23T20:14:08.311689+0000","last_became_active":"2023-09-23T20:13:46.815125+0000","last_became_peered":"2023-09-23T20:13:46.815125+0000","last_unstale":"2023-09-23T20:14:08.311689+0000","last_undegraded":"2023-09-23T20:14:08.311689+0000","last_fullsized":"2023-09-23T20:14:08.311689+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":3970,"log_dups_size":0,"ondisk_log_size":3970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146800640,"num_objects":2240,"num_object_clones":0,"num_object_copies":6720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2240,"num_write_kb":143360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":275,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T20:14:06.237291+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:14:06.237291+0000","last_peered":"2023-09-23T20:14:06.237291+0000","last_clean":"2023-09-23T20:14:06.237291+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:14:06.237291+0000","last_undegraded":"2023-09-23T20:14:06.237291+0000","last_fullsized":"2023-09-23T20:14:06.237291+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":368,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T20:13:59.078381+0000","last_change":"2023-09-23T20:13:57.756348+0000","last_active":"2023-09-23T20:13:59.078381+0000","last_peered":"2023-09-23T20:13:59.078381+0000","last_clean":"2023-09-23T20:13:59.078381+0000","last_became_active":"2023-09-23T20:13:57.756049+0000","last_became_peered":"2023-09-23T20:13:57.756049+0000","last_unstale":"2023-09-23T20:13:59.078381+0000","last_undegraded":"2023-09-23T20:13:59.078381+0000","last_fullsized":"2023-09-23T20:13:59.078381+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:02.849428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":4746575872,"num_objects":72427,"num_object_clones":0,"num_object_copies":217281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4442,"num_objects_misplaced":6764,"num_objects_unfound":0,"num_objects_dirty":72427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":72427,"num_write_kb":4635328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2433,"num_bytes_recovered":159449088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14400880640,"data_stored":14400880640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107157,"ondisk_log_size":107157,"up":78,"acting":74,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234396,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3116156,"kb_used_data":2716276,"kb_used_omap":0,"kb_used_meta":399872,"kb_avail":90628996,"statfs":{"total":95995035648,"available":92804091904,"internally_reserved":0,"allocated":2781466624,"data_stored":2780160752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":409468928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":1.9930000000000001,"15min":2.0670000000000002},"min":{"1min":0.27100000000000002,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":4.0970000000000004,"5min":55.390000000000001,"15min":60.140000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.008,"5min":2.028,"15min":2.1179999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":3.097,"5min":56.664000000000001,"15min":62.389000000000003},"last":0.77700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:13:45 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":2.9550000000000001,"15min":3.8439999999999999},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.468,"5min":91.335999999999999,"15min":104.16800000000001},"last":1.2509999999999999},{"interface":"front","average":{"1min":0.999,"5min":3.5870000000000002,"15min":4.0839999999999996},"min":{"1min":0.32100000000000001,"5min":0.249,"15min":0.223},"max":{"1min":2.8559999999999999,"5min":91.516999999999996,"15min":104.003},"last":1.0489999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":3.9740000000000002,"15min":3.8450000000000002},"min":{"1min":0.38800000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":3.8239999999999998,"5min":67.760000000000005,"15min":105.842},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.355,"5min":4.3019999999999996,"15min":4.3049999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":7.4199999999999999,"5min":67.488,"15min":105.66},"last":0.84499999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":1.3049999999999999,"15min":3.831},"min":{"1min":0.40699999999999997,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":9.1259999999999994,"5min":9.1259999999999994,"15min":146.38800000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.667,"5min":1.3240000000000001,"15min":3.8199999999999998},"min":{"1min":0.371,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":13.942,"5min":13.942,"15min":146.851},"last":0.871}]},{"osd":4,"last update":"Sat Sep 23 20:13:11 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.48899999999999999,"15min":0.72999999999999998},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.22500000000000001},"max":{"1min":1.2869999999999999,"5min":3.2069999999999999,"15min":19.215},"last":0.313},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.54900000000000004,"15min":0.85099999999999998},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":2.75,"5min":2.75,"15min":41.822000000000003},"last":0.432}]},{"osd":5,"last update":"Sat Sep 23 20:13:11 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.65900000000000003,"15min":1.006},"min":{"1min":0.29899999999999999,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":5.1580000000000004,"5min":5.1580000000000004,"15min":35.488},"last":1.155},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.61399999999999999,"15min":0.86799999999999999},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":1.7290000000000001,"5min":4.2030000000000003,"15min":34.896000000000001},"last":0.60599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":0.78200000000000003,"15min":0.85599999999999998},"min":{"1min":0.38700000000000001,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":3.407,"5min":3.4849999999999999,"15min":41.902999999999999},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.60499999999999998,"15min":0.65200000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":2.0459999999999998,"5min":2.4590000000000001,"15min":6.3620000000000001},"last":0.45000000000000001}]}]},{"osd":6,"up_from":0,"seq":1078036791436,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762594,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3244948,"kb_used_data":2560716,"kb_used_omap":7,"kb_used_meta":684216,"kb_avail":90500204,"statfs":{"total":95995035648,"available":92672208896,"internally_reserved":0,"allocated":2622173184,"data_stored":2620182564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":700638010},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.8089999999999999,"15min":2.9550000000000001},"min":{"1min":0.20899999999999999,"5min":0.20200000000000001,"15min":0.16900000000000001},"max":{"1min":6.5590000000000002,"5min":28.510000000000002,"15min":144.273},"last":2.0489999999999999},{"interface":"front","average":{"1min":2.573,"5min":2.0779999999999998,"15min":3.3149999999999999},"min":{"1min":0.217,"5min":0.19800000000000001,"15min":0.19400000000000001},"max":{"1min":20.997,"5min":20.997,"15min":144.321},"last":2.4390000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":3.766,"15min":4.4660000000000002},"min":{"1min":0.21099999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":24.207000000000001,"5min":58.499000000000002,"15min":134.72499999999999},"last":8.9909999999999997},{"interface":"front","average":{"1min":3.8029999999999999,"5min":3.524,"15min":4.5970000000000004},"min":{"1min":0.223,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":13.105,"5min":58.741,"15min":135.74600000000001},"last":1.4830000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.1360000000000001,"15min":3.1040000000000001},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.22},"max":{"1min":12.384,"5min":25.963999999999999,"15min":56.564},"last":10.321},{"interface":"front","average":{"1min":1.335,"5min":1.6519999999999999,"15min":3.0270000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.247},"max":{"1min":9.1760000000000002,"5min":12.723000000000001,"15min":56.939999999999998},"last":4.0309999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.54,"15min":1.798},"min":{"1min":0.379,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":6.3979999999999997,"5min":18.087,"15min":35.497999999999998},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.373,"5min":1.341,"15min":1.9359999999999999},"min":{"1min":0.35899999999999999,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":9.0280000000000005,"5min":11.08,"15min":35.521000000000001},"last":2.52}]},{"osd":5,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":2.157,"15min":2.351},"min":{"1min":0.40300000000000002,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":9.0540000000000003,"5min":83.510999999999996,"15min":83.510999999999996},"last":2.4940000000000002},{"interface":"front","average":{"1min":1.24,"5min":2.3570000000000002,"15min":2.4449999999999998},"min":{"1min":0.38400000000000001,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":9.4290000000000003,"5min":83.353999999999999,"15min":83.353999999999999},"last":9.1080000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:13:48 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":1.452,"15min":1.909},"min":{"1min":0.41899999999999998,"5min":0.38500000000000001,"15min":0.35899999999999999},"max":{"1min":3.5510000000000002,"5min":18.117999999999999,"15min":18.117999999999999},"last":1.153},{"interface":"front","average":{"1min":1.335,"5min":1.3959999999999999,"15min":2.4089999999999998},"min":{"1min":0.44600000000000001,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":3.1720000000000002,"5min":11.109,"15min":20.734999999999999},"last":2.5550000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.6910000000000001,"15min":2.1030000000000002},"min":{"1min":0.39500000000000002,"5min":0.33300000000000002,"15min":0.30399999999999999},"max":{"1min":9.4730000000000008,"5min":25.937000000000001,"15min":25.937000000000001},"last":9.077},{"interface":"front","average":{"1min":1.46,"5min":1.6599999999999999,"15min":1.891},"min":{"1min":0.45800000000000002,"5min":0.373,"15min":0.30199999999999999},"max":{"1min":6.4210000000000003,"5min":18.053999999999998,"15min":18.053999999999998},"last":1.2070000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926132,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2164456,"kb_used_data":1490720,"kb_used_omap":8,"kb_used_meta":673719,"kb_avail":91580696,"statfs":{"total":95995035648,"available":93778632704,"internally_reserved":0,"allocated":1526497280,"data_stored":1524509348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":689888865},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":2.4849999999999999,"15min":3.698},"min":{"1min":0.22,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.0759999999999996,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.23000000000000001},{"interface":"front","average":{"1min":1.105,"5min":3.044,"15min":3.9119999999999999},"min":{"1min":0.245,"5min":0.20200000000000001,"15min":0.183},"max":{"1min":4.9480000000000004,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.432}]},{"osd":2,"last update":"Sat Sep 23 20:13:10 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":3.4140000000000001,"15min":4.8840000000000003},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":6.6550000000000002,"5min":102.65300000000001,"15min":270.51100000000002},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.8120000000000001,"5min":3.6869999999999998,"15min":4.734},"min":{"1min":0.28699999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.298999999999999,"5min":96.578999999999994,"15min":269.69299999999998},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:13:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.3220000000000001,"15min":4.5679999999999996},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21299999999999999},"max":{"1min":4.9059999999999997,"5min":10.082000000000001,"15min":133.422},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.7410000000000001,"5min":1.7250000000000001,"15min":4.6970000000000001},"min":{"1min":0.29199999999999998,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":9.5440000000000005,"5min":17.384,"15min":134.249},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.294,"15min":2.206},"min":{"1min":0.41399999999999998,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":6.0149999999999997,"5min":15.688000000000001,"15min":148.20599999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.3160000000000001,"15min":2.0169999999999999},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.27600000000000002},"max":{"1min":4.843,"5min":8.8490000000000002,"15min":148.11000000000001},"last":0.497}]},{"osd":5,"last update":"Sat Sep 23 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.99099999999999999,"15min":1.913},"min":{"1min":0.307,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":6.5599999999999996,"5min":6.5599999999999996,"15min":64.725999999999999},"last":52.061},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.4379999999999999,"15min":1.9510000000000001},"min":{"1min":0.40899999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.0620000000000003,"5min":15.717000000000001,"15min":27.609999999999999},"last":51.741}]},{"osd":6,"last update":"Sat Sep 23 20:13:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.2070000000000001,"15min":1.581},"min":{"1min":0.41999999999999998,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":2.117,"5min":15.744999999999999,"15min":15.744999999999999},"last":0.372},{"interface":"front","average":{"1min":1.032,"5min":1.204,"15min":1.768},"min":{"1min":0.41999999999999998,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":4.8810000000000002,"5min":8.3919999999999995,"15min":13.766999999999999},"last":0.72699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.087,"15min":1.865},"min":{"1min":0.45700000000000002,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":6.5899999999999999,"5min":6.5899999999999999,"15min":65.081999999999994},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.4890000000000001,"15min":1.9590000000000001},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":6.1020000000000003,"5min":15.771000000000001,"15min":15.771000000000001},"last":0.53300000000000003}]}]},{"osd":2,"up_from":104,"seq":446676599070,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1892072,"kb_used_data":1116896,"kb_used_omap":12,"kb_used_meta":775155,"kb_avail":91853080,"statfs":{"total":95995035648,"available":94057553920,"internally_reserved":0,"allocated":1143701504,"data_stored":1141696471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":793759742},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.379,"15min":3.004},"min":{"1min":0.252,"5min":0.221,"15min":0.17299999999999999},"max":{"1min":5.6890000000000001,"5min":14.794,"15min":76.251999999999995},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.742,"5min":1.861,"15min":3.081},"min":{"1min":0.22,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":11.109999999999999,"5min":18.561,"15min":76.590000000000003},"last":0.26600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":4.0880000000000001,"5min":2.702,"15min":3.8079999999999998},"min":{"1min":0.27400000000000002,"5min":0.19600000000000001,"15min":0.184},"max":{"1min":26.808,"5min":32.677999999999997,"15min":94.768000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.4870000000000001,"5min":2.1840000000000002,"15min":3.625},"min":{"1min":0.315,"5min":0.252,"15min":0.21099999999999999},"max":{"1min":26.943999999999999,"5min":32.582000000000001,"15min":96.808000000000007},"last":0.54800000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.667,"15min":3.9220000000000002},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":5.298,"5min":14.131,"15min":107.381},"last":1.2869999999999999},{"interface":"front","average":{"1min":0.79200000000000004,"5min":1.5549999999999999,"15min":3.2130000000000001},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.23699999999999999},"max":{"1min":5.4610000000000003,"5min":20.308,"15min":106.301},"last":0.73399999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.278,"15min":2.0419999999999998},"min":{"1min":0.317,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":2.0339999999999998,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.5579999999999998,"5min":1.7070000000000001,"15min":2.2349999999999999},"min":{"1min":0.35699999999999998,"5min":0.33500000000000002,"15min":0.26700000000000002},"max":{"1min":12.728999999999999,"5min":13.387,"15min":76.161000000000001},"last":0.56399999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.6799999999999999,"15min":2.665},"min":{"1min":0.36499999999999999,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":2.0680000000000001,"5min":29.433,"15min":172.505},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.8640000000000001,"5min":2.149,"15min":2.883},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29099999999999998},"max":{"1min":11.037000000000001,"5min":29.196999999999999,"15min":172.375},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:13:51 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":1.375,"15min":1.611},"min":{"1min":0.45500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.9660000000000002,"5min":14.664,"15min":14.664},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.4119999999999999,"15min":1.859},"min":{"1min":0.40799999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.75,"5min":11.081,"15min":14.279999999999999},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 20:13:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.8320000000000001,"15min":1.944},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":12.645,"5min":13.481999999999999,"15min":23.079999999999998},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.4650000000000001,"15min":1.887},"min":{"1min":0.41499999999999998,"5min":0.39900000000000002,"15min":0.36299999999999999},"max":{"1min":2.1280000000000001,"5min":14.698,"15min":20.614999999999998},"last":0.64700000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435544,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3154820,"kb_used_data":2408380,"kb_used_omap":4,"kb_used_meta":746427,"kb_avail":90590332,"statfs":{"total":95995035648,"available":92764499968,"internally_reserved":0,"allocated":2466181120,"data_stored":2463975116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":764341357},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.649,"15min":3.3660000000000001},"min":{"1min":0.25,"5min":0.186,"15min":0.182},"max":{"1min":3.9529999999999998,"5min":53.593000000000004,"15min":74.323999999999998},"last":0.379},{"interface":"front","average":{"1min":0.86899999999999999,"5min":1.708,"15min":3.605},"min":{"1min":0.26000000000000001,"5min":0.20100000000000001,"15min":0.19600000000000001},"max":{"1min":4.2389999999999999,"5min":53.478000000000002,"15min":74.447000000000003},"last":0.23200000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:13:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":4.226,"15min":4.109},"min":{"1min":0.23100000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":9.6790000000000003,"5min":163.49700000000001,"15min":163.49700000000001},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.0880000000000001,"5min":4.7240000000000002,"15min":4.383},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":3.9249999999999998,"5min":163.512,"15min":163.512},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:13:41 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":6.1710000000000003,"15min":4.2889999999999997},"min":{"1min":0.29199999999999998,"5min":0.24199999999999999,"15min":0.219},"max":{"1min":2.919,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.365,"5min":5.6900000000000004,"15min":3.9319999999999999},"min":{"1min":0.33300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":6.8380000000000001,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.54000000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":2.0459999999999998,"15min":3.085},"min":{"1min":0.35999999999999999,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":2.9039999999999999,"5min":58.472999999999999,"15min":109.66800000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.3100000000000001,"15min":3.464},"min":{"1min":0.42699999999999999,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":4.133,"5min":58.408000000000001,"15min":110.295},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":2.3580000000000001,"15min":3.306},"min":{"1min":0.47799999999999998,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":4.0919999999999996,"5min":58.506,"15min":113.881},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.208,"5min":2.258,"15min":3.2879999999999998},"min":{"1min":0.45800000000000002,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":3.1219999999999999,"5min":58.643000000000001,"15min":114.598},"last":0.80800000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:13:54 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.1519999999999999,"15min":3.323},"min":{"1min":0.40300000000000002,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":5.1790000000000003,"5min":5.9080000000000004,"15min":82.281999999999996},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.208,"5min":1.014,"15min":3.4569999999999999},"min":{"1min":0.44500000000000001,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":5.1950000000000003,"5min":6.0730000000000004,"15min":82.233000000000004},"last":0.434}]},{"osd":7,"last update":"Sat Sep 23 20:13:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":2.2709999999999999,"15min":3.4980000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":2.9529999999999998,"5min":58.686,"15min":138.035},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.077,"5min":2.2469999999999999,"15min":3.4470000000000001},"min":{"1min":0.48599999999999999,"5min":0.38400000000000001,"15min":0.32600000000000001},"max":{"1min":3.8199999999999998,"5min":58.667999999999999,"15min":138.054},"last":0.57099999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019306,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1653304,"kb_used_data":1262064,"kb_used_omap":0,"kb_used_meta":391232,"kb_avail":92091848,"statfs":{"total":95995035648,"available":94302052352,"internally_reserved":0,"allocated":1292353536,"data_stored":1291051748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":400621568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:23 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.0960000000000001,"15min":2.198},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.189},"max":{"1min":14.063000000000001,"5min":14.063000000000001,"15min":89.004999999999995},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.345,"5min":1.1479999999999999,"15min":2.161},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":14.048999999999999,"5min":14.048999999999999,"15min":89.024000000000001},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:13:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7450000000000001,"5min":2.613,"15min":5.7300000000000004},"min":{"1min":0.313,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":29.245999999999999,"5min":138.47999999999999,"15min":161.613},"last":0.33100000000000002},{"interface":"front","average":{"1min":2.5179999999999998,"5min":2.7360000000000002,"15min":5.8319999999999999},"min":{"1min":0.312,"5min":0.217,"15min":0.217},"max":{"1min":29.309000000000001,"5min":138.429,"15min":162.017},"last":0.77200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:13:09 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":5.7850000000000001,"15min":6.6779999999999999},"min":{"1min":0.34399999999999997,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":8.75,"5min":154.94999999999999,"15min":182.96700000000001},"last":1.7549999999999999},{"interface":"front","average":{"1min":1.1319999999999999,"5min":5.3360000000000003,"15min":6.4400000000000004},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":5.0650000000000004,"5min":155.51300000000001,"15min":183.244},"last":0.51500000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:13:27 2023","interfaces":[{"interface":"back","average":{"1min":7.2599999999999998,"5min":2.4399999999999999,"15min":6.6050000000000004},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.255},"max":{"1min":100.361,"5min":100.361,"15min":266.54700000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":7.0869999999999997,"5min":2.625,"15min":5.5250000000000004},"min":{"1min":0.35999999999999999,"5min":0.33600000000000002,"15min":0.28599999999999998},"max":{"1min":100.20699999999999,"5min":100.20699999999999,"15min":184.22300000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:13:23 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47399999999999998,"15min":2.1240000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":0.752,"5min":3.3290000000000002,"15min":266.59899999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.45900000000000002,"15min":2.1459999999999999},"min":{"1min":0.31900000000000001,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":0.73999999999999999,"5min":1.722,"15min":266.72899999999998},"last":0.496}]},{"osd":6,"last update":"Sat Sep 23 20:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.51100000000000001,"15min":0.51700000000000002},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":0.78800000000000003,"5min":2.9350000000000001,"15min":4.0410000000000004},"last":0.438},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.55600000000000005,"15min":0.60499999999999998},"min":{"1min":0.36599999999999999,"5min":0.33100000000000002,"15min":0.247},"max":{"1min":3.234,"5min":3.234,"15min":4.2889999999999997},"last":0.55800000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.496,"15min":1.4119999999999999},"min":{"1min":0.37,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":0.82799999999999996,"5min":1.5960000000000001,"15min":76.507000000000005},"last":1.6819999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.55400000000000005,"15min":1.5389999999999999},"min":{"1min":0.373,"5min":0.27200000000000002,"15min":0.222},"max":{"1min":1.5429999999999999,"5min":3.2890000000000001,"15min":76.462000000000003},"last":1.716}]}]},{"osd":5,"up_from":38,"seq":163208757670,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3240496,"kb_used_data":2775272,"kb_used_omap":0,"kb_used_meta":465216,"kb_avail":90504656,"statfs":{"total":95995035648,"available":92676767744,"internally_reserved":0,"allocated":2841878528,"data_stored":2840581181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":476381184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,12],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.79900000000000004,"15min":1.722},"min":{"1min":0.22600000000000001,"5min":0.224,"15min":0.216},"max":{"1min":6.1029999999999998,"5min":8.6739999999999995,"15min":121.94799999999999},"last":3.0760000000000001},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.111,"15min":2.004},"min":{"1min":0.28799999999999998,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":4.2750000000000004,"5min":10.327,"15min":121.964},"last":3.169}]},{"osd":1,"last update":"Sat Sep 23 20:13:44 2023","interfaces":[{"interface":"back","average":{"1min":7.7249999999999996,"5min":3.234,"15min":5.8879999999999999},"min":{"1min":0.28599999999999998,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":51.792999999999999,"5min":75.204999999999998,"15min":266.197},"last":1.607},{"interface":"front","average":{"1min":7.3410000000000002,"5min":3.2090000000000001,"15min":6.0730000000000004},"min":{"1min":0.434,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":51.427999999999997,"5min":75.231999999999999,"15min":266.14999999999998},"last":3.1890000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:14:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":3.4980000000000002,"15min":5.0810000000000004},"min":{"1min":0.36599999999999999,"5min":0.313,"15min":0.26400000000000001},"max":{"1min":3.5219999999999998,"5min":120.297,"15min":240.57300000000001},"last":1.639},{"interface":"front","average":{"1min":1.339,"5min":3.5990000000000002,"15min":5.149},"min":{"1min":0.39900000000000002,"5min":0.374,"15min":0.313},"max":{"1min":3.609,"5min":120.325,"15min":240.464},"last":1.6919999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:13:44 2023","interfaces":[{"interface":"back","average":{"1min":8.0939999999999994,"5min":2.6110000000000002,"15min":4.782},"min":{"1min":0.33700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":63.508000000000003,"5min":63.508000000000003,"15min":201.50899999999999},"last":2.4449999999999998},{"interface":"front","average":{"1min":8.2490000000000006,"5min":2.3660000000000001,"15min":4.5860000000000003},"min":{"1min":0.41199999999999998,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":65.251000000000005,"5min":65.251000000000005,"15min":201.58099999999999},"last":2.423}]},{"osd":4,"last update":"Sat Sep 23 20:13:18 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.49299999999999999,"15min":1.4099999999999999},"min":{"1min":0.245,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":1.206,"5min":4.0510000000000002,"15min":191.78899999999999},"last":3.343},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.45100000000000001,"15min":1.319},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":0.78900000000000003,"5min":1.4359999999999999,"15min":191.821},"last":2.367}]},{"osd":6,"last update":"Sat Sep 23 20:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.58999999999999997,"15min":0.65300000000000002},"min":{"1min":0.375,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":3.2200000000000002,"5min":3.278,"15min":9.9009999999999998},"last":3.2090000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.50800000000000001,"15min":0.53300000000000003},"min":{"1min":0.307,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.2930000000000001,"5min":2.2930000000000001,"15min":5.0119999999999996},"last":2.3969999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:14:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.627,"15min":1.5109999999999999},"min":{"1min":0.44,"5min":0.28799999999999998,"15min":0.26200000000000001},"max":{"1min":3.4700000000000002,"5min":3.4700000000000002,"15min":191.84299999999999},"last":3.0579999999999998},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.622,"15min":1.5760000000000001},"min":{"1min":0.45900000000000002,"5min":0.32500000000000001,"15min":0.26300000000000001},"max":{"1min":3.0369999999999999,"5min":3.1440000000000001,"15min":191.69},"last":3.375}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1531314176,"data_stored":1531314176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2625241088,"data_stored":2625241088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1147797504,"data_stored":1147797504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2333540352,"data_stored":2333540352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1245970432,"data_stored":1245970432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2742091776,"data_stored":2742091776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2774925312,"data_stored":2774925312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:14:15.164 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:14:15.164 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 36 2023-09-23T20:14:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:15.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:15.272 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:14:15.324 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:15.446 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:14:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:15.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:14:15.552 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:15.602 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:15.691 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:14:15.691 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:14:15.691 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.691 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79767 9.11 9.43a167b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.691 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.674853+0000", 2023-09-23T20:14:15.691 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.014673822, 2023-09-23T20:14:15.691 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.014701274, 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79767 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.674853+0000", 2023-09-23T20:14:15.692 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.674853+0000", 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.674852+0000", 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.674908+0000", 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5701000000000001e-05 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.674920+0000", 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2294e-05 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.694 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.674926+0000", 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5200000000000003e-06 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.674985+0000", 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8755000000000002e-05 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.674995+0000", 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0256e-05 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79417 9.11 9.6b109fd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.035673+0000", 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.65385435400000003, 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.65394882499999996, 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.696 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79417 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.035673+0000", 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.035673+0000", 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.035671+0000", 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.035731+0000", 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9292000000000001e-05 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.035747+0000", 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6245000000000001e-05 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.035754+0000", 2023-09-23T20:14:15.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4420000000000001e-06 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.035809+0000", 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4539000000000001e-05 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.035816+0000", 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0199999999999997e-06 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.700 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79496 9.11 9.94a55711 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.117245+0000", 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.57228195299999995, 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.57241792800000002, 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.701 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79496 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.117245+0000", 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.117245+0000", 2023-09-23T20:14:15.702 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.117244+0000", 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.117274+0000", 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9458000000000002e-05 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.703 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.117282+0000", 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2779999999999993e-06 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.117287+0000", 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9690000000000002e-06 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.704 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.117338+0000", 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1097999999999998e-05 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.117347+0000", 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.303e-06 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.705 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79523 9.11 9.e57b06b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.149946+0000", 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.53958129499999996, 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.53975461000000002, 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79523 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.706 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.149946+0000", 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.149946+0000", 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.707 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.149945+0000", 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.149976+0000", 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0932999999999997e-05 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.149987+0000", 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1586999999999999e-05 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.149993+0000", 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9909999999999999e-06 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.150076+0000", 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2455999999999998e-05 2023-09-23T20:14:15.709 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.150087+0000", 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0964999999999999e-05 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79644 9.11 9.53f9e831 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.710 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.345835+0000", 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.343691619, 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.343903923, 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79644 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.711 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.345835+0000", 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.345835+0000", 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.712 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.345834+0000", 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.345901+0000", 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6526000000000006e-05 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.345918+0000", 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7864e-05 2023-09-23T20:14:15.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.345926+0000", 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2740000000000002e-06 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.345993+0000", 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7323000000000006e-05 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.714 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.346002+0000", 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5550000000000003e-06 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79771 9.19 9:9f0f2d97:::benchmark_data_smithi135_549536_object79770:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.689314+0000", 2023-09-23T20:14:15.715 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.00021250199999999999, 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00049284899999999996, 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79771 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.716 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689314+0000", 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689314+0000", 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689305+0000", 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999909 2023-09-23T20:14:15.717 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689345+0000", 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9428999999999998e-05 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689361+0000", 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.677e-05 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.718 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689367+0000", 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.84e-06 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689423+0000", 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5801000000000003e-05 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689550+0000", 2023-09-23T20:14:15.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012740399999999999 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.689669+0000", 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000118762 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79311 9.11 9.bfa6bbf1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.720 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:14.939852+0000", 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.74967449200000003, 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.74999830300000003, 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79311 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.721 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.939852+0000", 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.939852+0000", 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.939852+0000", 2023-09-23T20:14:15.722 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.939888+0000", 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6053999999999997e-05 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.939896+0000", 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4829999999999999e-06 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.723 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.939902+0000", 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5500000000000002e-06 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.941689+0000", 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001787542 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.941698+0000", 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3329999999999997e-06 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79612 9.11 9.5e824731 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.298639+0000", 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.390888134, 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.39128044499999998, 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79612 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.298639+0000", 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.298639+0000", 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.298637+0000", 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.727 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.298682+0000", 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4842999999999997e-05 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.298700+0000", 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7795000000000002e-05 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.298708+0000", 2023-09-23T20:14:15.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1280000000000008e-06 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.298775+0000", 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6298999999999997e-05 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.298786+0000", 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1238e-05 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.729 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79222 9.11 9.4029fff1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:14.866022+0000", 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.82350531800000004, 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.82396124699999995, 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.730 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79222 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.866022+0000", 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.866022+0000", 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.731 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.866020+0000", 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.866172+0000", 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000151798 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.732 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.866188+0000", 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6303e-05 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.866195+0000", 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8040000000000004e-06 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.866246+0000", 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0967000000000002e-05 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.866257+0000", 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0577000000000001e-05 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.734 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79197 9.11 9:8fe57809:::benchmark_data_smithi135_549536_object79196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:14.733786+0000", 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.95574072600000004, 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.95629707500000005, 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79197 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.735 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.733786+0000", 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.733786+0000", 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.736 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.733785+0000", 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.733851+0000", 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6240000000000003e-05 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.733876+0000", 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4643000000000001e-05 2023-09-23T20:14:15.737 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.733883+0000", 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0929999999999997e-06 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.733960+0000", 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7554999999999999e-05 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.738 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.734124+0000", 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000164007 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.734294+0000", 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016978499999999999 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.736241+0000", 2023-09-23T20:14:15.739 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0019467589999999999 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.759657+0000", 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.023416381 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.759668+0000", 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1147e-05 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.740 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79547 9.11 9.346c0871 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.189102+0000", 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.50042497699999999, 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.50107111000000004, 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.741 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79547 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.189102+0000", 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.742 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.189102+0000", 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.189091+0000", 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999886 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.189138+0000", 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7688000000000001e-05 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.743 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.189156+0000", 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7742999999999999e-05 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.189165+0000", 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3790000000000006e-06 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.744 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.189226+0000", 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1957000000000003e-05 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.189234+0000", 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.401e-06 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.745 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79670 9.11 9.a2dd77b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.396717+0000", 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.292809818, 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.293524112, 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79670 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.746 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.396717+0000", 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.396717+0000", 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.396716+0000", 2023-09-23T20:14:15.747 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.396763+0000", 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6554999999999997e-05 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.396774+0000", 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0957e-05 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.748 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.396778+0000", 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8160000000000001e-06 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.396842+0000", 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3304000000000002e-05 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.749 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.396849+0000", 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2660000000000004e-06 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79472 9.11 9.d919ccf1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.087101+0000", 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.60242643500000004, 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.60320887999999995, 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.750 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79472 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.087101+0000", 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.751 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.087101+0000", 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.087099+0000", 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.752 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.087147+0000", 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8093999999999997e-05 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.087158+0000", 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0482000000000001e-05 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.087163+0000", 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7790000000000003e-06 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.753 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.087212+0000", 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9212999999999998e-05 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.087224+0000", 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2004000000000001e-05 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.754 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79200 9.11 9.c8397a71 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:14.848168+0000", 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.84135913500000004, 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.84220742500000001, 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79200 2023-09-23T20:14:15.755 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.848168+0000", 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.848168+0000", 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.756 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.848167+0000", 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.848218+0000", 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1240999999999999e-05 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.757 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.848230+0000", 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2534999999999999e-05 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.848237+0000", 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5289999999999997e-06 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.848272+0000", 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5636000000000002e-05 2023-09-23T20:14:15.758 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:14.848281+0000", 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2279999999999998e-06 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79601 9.11 9.be0e1ff1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.279375+0000", 2023-09-23T20:14:15.759 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.41015179699999998, 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.41105170800000002, 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79601 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.760 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.279375+0000", 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.279375+0000", 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.279374+0000", 2023-09-23T20:14:15.761 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.279424+0000", 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9761e-05 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.279442+0000", 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8482e-05 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.762 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.279449+0000", 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3350000000000001e-06 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.279940+0000", 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00049092500000000002 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.763 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.279950+0000", 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9760000000000007e-06 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:79507 9.11 9.a945a931 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:15.134125+0000", 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.55540147500000003, 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.55633805300000005, 2023-09-23T20:14:15.764 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "delayed", 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 79507 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.134125+0000", 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.765 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.134125+0000", 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.134123+0000", 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.766 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.134170+0000", 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6789999999999998e-05 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.134195+0000", 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4980000000000001e-05 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.767 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.134205+0000", 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4749999999999999e-06 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.134274+0000", 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9119e-05 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:15.134286+0000", 2023-09-23T20:14:15.768 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1783000000000001e-05 2023-09-23T20:14:15.769 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.769 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:14:15.769 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.769 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:14:15.769 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2023-09-23T20:14:15.769 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 16 2023-09-23T20:14:15.769 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:14:15.857 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:15.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:15.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[213871]: failed to encode map e432 with expected crc 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[213871]: failed to encode map e432 with expected crc 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[213871]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[213871]: failed to encode map e432 with expected crc 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[213871]: pgmap v1204: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 114 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 4442/233664 objects degraded (1.901%); 4136/233664 objects misplaced (1.770%); 23 MiB/s, 363 objects/s recovering 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[221113]: failed to encode map e432 with expected crc 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[221113]: failed to encode map e432 with expected crc 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[221113]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[221113]: failed to encode map e432 with expected crc 2023-09-23T20:14:15.862 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:15 smithi135 ceph-mon[221113]: pgmap v1204: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 114 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 4442/233664 objects degraded (1.901%); 4136/233664 objects misplaced (1.770%); 23 MiB/s, 363 objects/s recovering 2023-09-23T20:14:15.962 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:16.038 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:16.070 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 62 16 80090 80074 80.6895 39.5 0.0101332 0.0123881 2023-09-23T20:14:16.105 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:15 smithi144 ceph-mon[194833]: failed to encode map e432 with expected crc 2023-09-23T20:14:16.106 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:15 smithi144 ceph-mon[194833]: failed to encode map e432 with expected crc 2023-09-23T20:14:16.106 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:15 smithi144 ceph-mon[194833]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T20:14:16.106 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:15 smithi144 ceph-mon[194833]: failed to encode map e432 with expected crc 2023-09-23T20:14:16.106 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:15 smithi144 ceph-mon[194833]: pgmap v1204: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 114 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 4442/233664 objects degraded (1.901%); 4136/233664 objects misplaced (1.770%); 23 MiB/s, 363 objects/s recovering 2023-09-23T20:14:16.127 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:16.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:16.163 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:14:16.201 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:14:16.239 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:16.246 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:14:16.282 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:14:16.290 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:16.323 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:14:16.509 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:16.660 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:14:16.694 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:14:16.731 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:14:16.741 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:14:16.741 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:14:16.741 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:14:16.741 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:14:16.741 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:14:16.742 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:14:16.742 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:14:16.742 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:14:16.955 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:16.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:17.043 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:16 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:17.043 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:16 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4442/233664 objects degraded (1.901%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:17.043 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:16 smithi144 ceph-mon[194833]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.043 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:16 smithi144 ceph-mon[194833]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.043 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:16 smithi144 ceph-mon[194833]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.044 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:16 smithi144 ceph-mon[194833]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T20:14:17.044 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:16 smithi144 ceph-mon[194833]: 9.2 deep-scrub starts 2023-09-23T20:14:17.044 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2430011324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:17.061 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:17.067 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:14:17.070 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 63 16 80898 80882 80.2103 50.5 0.0152138 0.0124626 2023-09-23T20:14:17.125 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:14:17.127 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:17.173 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:17.173 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4442/233664 objects degraded (1.901%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:17.173 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[221113]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.173 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[221113]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.174 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[221113]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.174 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[221113]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T20:14:17.174 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[221113]: 9.2 deep-scrub starts 2023-09-23T20:14:17.174 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2430011324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:17.174 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:17.174 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4442/233664 objects degraded (1.901%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:17.174 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[213871]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.174 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[213871]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.175 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[213871]: failed to encode map e433 with expected crc 2023-09-23T20:14:17.175 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[213871]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T20:14:17.175 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[213871]: 9.2 deep-scrub starts 2023-09-23T20:14:17.175 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2430011324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:17.357 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:17.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:17.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:17.462 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:17.527 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:17.780 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:17 smithi135 ceph-mon[213871]: pgmap v1206: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s; 4442/234951 objects degraded (1.891%); 4136/234951 objects misplaced (1.760%); 13 MiB/s, 210 objects/s recovering 2023-09-23T20:14:17.814 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:14:18.070 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 64 16 81962 81946 79.9961 66.5 0.00383455 0.0124922 2023-09-23T20:14:18.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:17 smithi144 ceph-mon[194833]: pgmap v1206: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s; 4442/234951 objects degraded (1.891%); 4136/234951 objects misplaced (1.760%); 13 MiB/s, 210 objects/s recovering 2023-09-23T20:14:18.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:17 smithi135 ceph-mon[221113]: pgmap v1206: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s; 4442/234951 objects degraded (1.891%); 4136/234951 objects misplaced (1.760%); 13 MiB/s, 210 objects/s recovering 2023-09-23T20:14:18.826 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:18.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:18.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:18.938 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:18.994 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:19.071 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 65 16 82984 82968 79.748 63.875 0.0130304 0.0125351 2023-09-23T20:14:19.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:18 smithi144 ceph-mon[194833]: 1.5 deep-scrub starts 2023-09-23T20:14:19.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:18 smithi135 ceph-mon[221113]: 1.5 deep-scrub starts 2023-09-23T20:14:19.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:18 smithi135 ceph-mon[213871]: 1.5 deep-scrub starts 2023-09-23T20:14:19.596 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:19.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:19.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:19.700 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:19.746 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:19 smithi144 ceph-mon[194833]: 9.2 scrub starts 2023-09-23T20:14:19.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:19 smithi144 ceph-mon[194833]: pgmap v1207: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 4442/245160 objects degraded (1.812%); 3239/245160 objects misplaced (1.321%); 14 MiB/s, 221 objects/s recovering 2023-09-23T20:14:19.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:19 smithi135 ceph-mon[221113]: 9.2 scrub starts 2023-09-23T20:14:19.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:19 smithi135 ceph-mon[221113]: pgmap v1207: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 4442/245160 objects degraded (1.812%); 3239/245160 objects misplaced (1.321%); 14 MiB/s, 221 objects/s recovering 2023-09-23T20:14:19.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:19 smithi135 ceph-mon[213871]: 9.2 scrub starts 2023-09-23T20:14:19.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:19 smithi135 ceph-mon[213871]: pgmap v1207: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 4442/245160 objects degraded (1.812%); 3239/245160 objects misplaced (1.321%); 14 MiB/s, 221 objects/s recovering 2023-09-23T20:14:20.074 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 66 16 83815 83799 79.3222 51.9375 0.0167739 0.0126028 2023-09-23T20:14:20.074 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:20.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:20.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:20.186 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:20.214 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:20.293 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:20.532 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:20.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:20.609 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:14:20.610 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:14:20.642 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:20.709 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:20.844 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:20.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:20.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:20.948 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:21.048 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:21.074 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 67 16 84849 84833 79.1028 64.625 0.0107186 0.0126365 2023-09-23T20:14:21.314 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:21.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:21.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:21.351 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 9 pg_num to 36 2023-09-23T20:14:21.451 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:21.511 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:21.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:21.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:21.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:21 smithi144 ceph-mon[194833]: 9.2 deep-scrub starts 2023-09-23T20:14:21.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:21 smithi144 ceph-mon[194833]: pgmap v1208: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4442/246723 objects degraded (1.800%); 2581/246723 objects misplaced (1.046%); 13 MiB/s, 210 objects/s recovering 2023-09-23T20:14:21.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2947438921' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T20:14:21.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:21.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[213871]: 9.2 deep-scrub starts 2023-09-23T20:14:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[213871]: pgmap v1208: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4442/246723 objects degraded (1.800%); 2581/246723 objects misplaced (1.046%); 13 MiB/s, 210 objects/s recovering 2023-09-23T20:14:21.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2947438921' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T20:14:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:21.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[221113]: 9.2 deep-scrub starts 2023-09-23T20:14:21.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[221113]: pgmap v1208: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4442/246723 objects degraded (1.800%); 2581/246723 objects misplaced (1.046%); 13 MiB/s, 210 objects/s recovering 2023-09-23T20:14:21.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2947438921' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T20:14:21.824 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:14:22.038 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:14:22.074 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 68 16 85370 85354 78.4185 32.5625 0.00389292 0.012676 2023-09-23T20:14:22.241 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:14:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4442/246723 objects degraded (1.800%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2947438921' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4442/246723 objects degraded (1.800%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2947438921' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T20:14:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:22.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:22.556 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:22.556 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4442/246723 objects degraded (1.800%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:22.556 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.556 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2947438921' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-09-23T20:14:22.556 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: failed to encode map e434 with expected crc 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:22.557 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:23.074 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 69 16 86137 86121 77.9769 47.9375 0.00717317 0.0128005 2023-09-23T20:14:23.169 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:23.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:23.272 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:23.300 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:23.382 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:23.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:23.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[213871]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[213871]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[213871]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T20:14:23.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[213871]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[213871]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[213871]: pgmap v1211: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 4442/248022 objects degraded (1.791%); 2581/248022 objects misplaced (1.041%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[221113]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[221113]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[221113]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[221113]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[221113]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:23.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[221113]: pgmap v1211: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 4442/248022 objects degraded (1.791%); 2581/248022 objects misplaced (1.041%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:14:23.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T20:14:23.487 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:23.567 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:23 smithi144 ceph-mon[194833]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:23 smithi144 ceph-mon[194833]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:23 smithi144 ceph-mon[194833]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T20:14:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:23 smithi144 ceph-mon[194833]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:23 smithi144 ceph-mon[194833]: failed to encode map e435 with expected crc 2023-09-23T20:14:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:23 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:23 smithi144 ceph-mon[194833]: pgmap v1211: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 4442/248022 objects degraded (1.791%); 2581/248022 objects misplaced (1.041%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:14:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T20:14:23.735 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:23.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:23.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:23.844 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:23.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:14:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:14:23] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-09-23T20:14:24.036 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:24.074 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 70 16 86690 86674 77.3568 34.5625 0.00365162 0.0128777 2023-09-23T20:14:24.112 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:24.367 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:14:24.367 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:14:24.634 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:24.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:24.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:24.738 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:14:24.773 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:24.858 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:14:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:24.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:14:24.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[213871]: 9.2 scrub starts 2023-09-23T20:14:24.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[213871]: failed to encode map e436 with expected crc 2023-09-23T20:14:24.914 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T20:14:24.915 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[213871]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T20:14:24.915 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[213871]: failed to encode map e436 with expected crc 2023-09-23T20:14:24.915 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[213871]: failed to encode map e436 with expected crc 2023-09-23T20:14:24.915 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[213871]: failed to encode map e436 with expected crc 2023-09-23T20:14:24.915 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[221113]: 9.2 scrub starts 2023-09-23T20:14:24.915 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[221113]: failed to encode map e436 with expected crc 2023-09-23T20:14:24.915 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T20:14:24.915 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[221113]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T20:14:24.916 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[221113]: failed to encode map e436 with expected crc 2023-09-23T20:14:24.916 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[221113]: failed to encode map e436 with expected crc 2023-09-23T20:14:24.916 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:24 smithi135 ceph-mon[221113]: failed to encode map e436 with expected crc 2023-09-23T20:14:24.962 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:25.002 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:25.074 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 71 16 86972 86956 76.5157 17.625 0.00861442 0.013065 2023-09-23T20:14:25.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:24 smithi144 ceph-mon[194833]: 9.2 scrub starts 2023-09-23T20:14:25.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:24 smithi144 ceph-mon[194833]: failed to encode map e436 with expected crc 2023-09-23T20:14:25.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T20:14:25.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:24 smithi144 ceph-mon[194833]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T20:14:25.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:24 smithi144 ceph-mon[194833]: failed to encode map e436 with expected crc 2023-09-23T20:14:25.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:24 smithi144 ceph-mon[194833]: failed to encode map e436 with expected crc 2023-09-23T20:14:25.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:24 smithi144 ceph-mon[194833]: failed to encode map e436 with expected crc 2023-09-23T20:14:25.453 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:25.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:25.461 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:14:25.559 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:25.622 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:25.778 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:14:26.045 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:14:26.075 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 72 16 88297 88281 76.6023 82.8125 0.124889 0.0130323 2023-09-23T20:14:26.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:25 smithi144 ceph-mon[194833]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:25 smithi144 ceph-mon[194833]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:25 smithi144 ceph-mon[194833]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T20:14:26.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:25 smithi144 ceph-mon[194833]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:25 smithi144 ceph-mon[194833]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4253383819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:14:26.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:25 smithi144 ceph-mon[194833]: pgmap v1214: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 115 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 4442/258915 objects degraded (1.716%); 1274/258915 objects misplaced (0.492%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:14:26.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[213871]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[213871]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[213871]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T20:14:26.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[213871]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[213871]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4253383819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:14:26.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[213871]: pgmap v1214: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 115 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 4442/258915 objects degraded (1.716%); 1274/258915 objects misplaced (0.492%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:14:26.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[221113]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[221113]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[221113]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T20:14:26.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[221113]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[221113]: failed to encode map e437 with expected crc 2023-09-23T20:14:26.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4253383819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:14:26.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:25 smithi135 ceph-mon[221113]: pgmap v1214: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 115 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 4442/258915 objects degraded (1.716%); 1274/258915 objects misplaced (0.492%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:14:26.524 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:26.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:26.628 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:26.687 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:26.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: 9.c scrub starts 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4253383819' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[213871]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: 9.c scrub starts 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4253383819' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: failed to encode map e438 with expected crc 2023-09-23T20:14:26.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:26 smithi135 ceph-mon[221113]: failed to encode map e438 with expected crc 2023-09-23T20:14:27.075 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 73 16 89709 89693 76.7617 88.25 0.00253964 0.0130102 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: failed to encode map e438 with expected crc 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: 9.c scrub starts 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: failed to encode map e438 with expected crc 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4253383819' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: failed to encode map e438 with expected crc 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: failed to encode map e438 with expected crc 2023-09-23T20:14:27.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:26 smithi144 ceph-mon[194833]: failed to encode map e438 with expected crc 2023-09-23T20:14:27.493 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:27.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:27.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:27.599 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:27.673 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:27.858 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:27 smithi135 ceph-mon[213871]: pgmap v1216: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 8 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s; 4442/259965 objects degraded (1.709%); 1274/259965 objects misplaced (0.490%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T20:14:27.859 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:27 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4442/258915 objects degraded (1.716%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:27.859 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:27 smithi135 ceph-mon[221113]: pgmap v1216: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 8 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s; 4442/259965 objects degraded (1.709%); 1274/259965 objects misplaced (0.490%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T20:14:27.860 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:27 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4442/258915 objects degraded (1.716%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:27.862 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:27.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:27.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:27.969 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:28.041 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:27 smithi144 ceph-mon[194833]: pgmap v1216: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 8 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s; 4442/259965 objects degraded (1.709%); 1274/259965 objects misplaced (0.490%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T20:14:28.041 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:27 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4442/258915 objects degraded (1.716%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:28.055 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:28.076 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 74 16 91066 91050 76.8699 84.8125 0.00947399 0.0130046 2023-09-23T20:14:28.205 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:14:28.225 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:28.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:28.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:28.249 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:14:28.284 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:14:28.331 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:14:28.340 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:28.357 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:14:28.358 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:14:28.358 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 2, 4, 5, 3, 1, 7] 2023-09-23T20:14:28.358 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd out 1 2023-09-23T20:14:28.369 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:14:28.412 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:14:28.419 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:28.452 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:14:28.503 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:14:28.523 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:14:28.523 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:14:28.523 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:14:28.523 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:14:28.523 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:14:28.524 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:14:28.524 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:14:28.524 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:14:28.560 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:28.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:28.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:28.643 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:14:28.643 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:14:28.643 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:14:28.644 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:14:28.680 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:28.737 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:28.919 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:28.925 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3461400625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:28.925 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:28 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:28.925 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:28 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:28.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:28.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3461400625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:28.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:28 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:28.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:28 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:28.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3461400625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:28.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:28 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:28.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:28 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:29.026 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:14:29.076 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 75 16 92426 92410 76.9782 85 0.0018587 0.0129866 2023-09-23T20:14:29.081 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:29.173 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:14:29.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:29.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:14:29.278 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:29.303 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:29.345 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:29.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:29.448 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:29.484 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:29.618 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:14:29.797 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:29 smithi135 ceph-mon[213871]: pgmap v1217: 129 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 124 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 4442/272274 objects degraded (1.631%); 1274/272274 objects misplaced (0.468%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T20:14:29.853 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:29.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:29.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:29.959 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:14:29.963 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:30.031 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:30.076 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 76 16 93452 93436 76.8091 64.125 0.0109187 0.0130153 2023-09-23T20:14:30.127 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:30.130 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:29 smithi135 ceph-mon[221113]: pgmap v1217: 129 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 124 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 4442/272274 objects degraded (1.631%); 1274/272274 objects misplaced (0.468%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T20:14:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:30.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:30.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:29 smithi144 ceph-mon[194833]: pgmap v1217: 129 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 124 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 4442/272274 objects degraded (1.631%); 1274/272274 objects misplaced (0.468%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T20:14:30.169 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:30.232 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:30.284 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:30.365 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:30.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:30.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:30.469 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:30.525 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:30.584 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:14:30.707 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:30.711 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:30.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:30.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:30.820 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:30.864 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:30.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:30.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:30.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:30.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:30.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:30.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:30.999 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:31.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:31.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:31.053 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:31.053 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:31.053 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:31.076 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 77 16 94780 94764 76.8894 83 0.00432559 0.0129797 2023-09-23T20:14:31.104 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:31.132 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:31.432 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:31.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:31.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:31.536 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:31.569 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:31.620 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:31.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:31.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:31.724 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:31.754 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:31.921 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:31.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:31.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:32.000 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:14:32.030 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:32.047 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[221113]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.047 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[221113]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.047 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[221113]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T20:14:32.047 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[221113]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.047 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[221113]: pgmap v1219: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 4442/274362 objects degraded (1.619%); 148/274362 objects misplaced (0.054%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:14:32.047 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[213871]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.048 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[213871]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.048 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[213871]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T20:14:32.048 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[213871]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.048 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:31 smithi135 ceph-mon[213871]: pgmap v1219: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 4442/274362 objects degraded (1.619%); 148/274362 objects misplaced (0.054%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:14:32.075 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:31 smithi144 ceph-mon[194833]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.075 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:31 smithi144 ceph-mon[194833]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.075 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:31 smithi144 ceph-mon[194833]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T20:14:32.075 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:31 smithi144 ceph-mon[194833]: failed to encode map e439 with expected crc 2023-09-23T20:14:32.075 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:31 smithi144 ceph-mon[194833]: pgmap v1219: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 4442/274362 objects degraded (1.619%); 148/274362 objects misplaced (0.054%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:14:32.076 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 78 16 95339 95323 76.3516 34.9375 0.0106651 0.0130928 2023-09-23T20:14:32.084 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:32.175 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:32.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:32.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:32.283 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:32.345 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:32.416 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:32.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:32.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:32.520 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:32.580 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[221113]: failed to encode map e440 with expected crc 2023-09-23T20:14:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[221113]: failed to encode map e440 with expected crc 2023-09-23T20:14:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[221113]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T20:14:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[221113]: failed to encode map e440 with expected crc 2023-09-23T20:14:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[221113]: failed to encode map e440 with expected crc 2023-09-23T20:14:32.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4442/274362 objects degraded (1.619%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:32.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[213871]: failed to encode map e440 with expected crc 2023-09-23T20:14:32.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[213871]: failed to encode map e440 with expected crc 2023-09-23T20:14:32.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[213871]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T20:14:32.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[213871]: failed to encode map e440 with expected crc 2023-09-23T20:14:32.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[213871]: failed to encode map e440 with expected crc 2023-09-23T20:14:32.976 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:32 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4442/274362 objects degraded (1.619%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:33.077 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 79 16 96507 96491 76.3086 73 0.00720484 0.0131009 2023-09-23T20:14:33.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:32 smithi144 ceph-mon[194833]: failed to encode map e440 with expected crc 2023-09-23T20:14:33.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:32 smithi144 ceph-mon[194833]: failed to encode map e440 with expected crc 2023-09-23T20:14:33.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:32 smithi144 ceph-mon[194833]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T20:14:33.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:32 smithi144 ceph-mon[194833]: failed to encode map e440 with expected crc 2023-09-23T20:14:33.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:32 smithi144 ceph-mon[194833]: failed to encode map e440 with expected crc 2023-09-23T20:14:33.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:32 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4442/274362 objects degraded (1.619%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:14:33.238 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:33.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:33.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:33.324 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:33.351 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:33.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:33 smithi135 conmon[213845]: 2023-09-23T20:14:33.100+0000 7f9708921700 -1 mon.a@0(leader).osd e440 definitely_dead 0 2023-09-23T20:14:33.427 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:33.578 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:33.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:33.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:33.682 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:33.686 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:14:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:14:33] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-09-23T20:14:33.762 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:33.863 INFO:teuthology.orchestra.run.smithi135.stderr:marked out osd.1. 2023-09-23T20:14:33.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:33 smithi144 ceph-mon[194833]: pgmap v1221: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s; 4442/276159 objects degraded (1.608%); 148/276159 objects misplaced (0.054%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:14:33.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:33 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/159017738' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:14:33.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:33 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:14:33.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:33 smithi144 ceph-mon[194833]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:14:33.882 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:33.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:33.916 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:14:33.916 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:14:33.916 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:14:33.916 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:14:33.916 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:14:33.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:33 smithi135 ceph-mon[213871]: pgmap v1221: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s; 4442/276159 objects degraded (1.608%); 148/276159 objects misplaced (0.054%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:14:33.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:33 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/159017738' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:14:33.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:33 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:14:33.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:33 smithi135 ceph-mon[213871]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:14:33.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:33 smithi135 ceph-mon[221113]: pgmap v1221: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s; 4442/276159 objects degraded (1.608%); 148/276159 objects misplaced (0.054%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:14:33.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:33 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/159017738' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:14:33.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:33 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T20:14:33.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:33 smithi135 ceph-mon[221113]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T20:14:33.988 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:34.018 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:34.077 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:14:34.077822+0000 min lat: 0.00116484 max lat: 2.14573 avg lat: 0.0130886 2023-09-23T20:14:34.077 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:14:34.077 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 80 16 97547 97531 76.1672 65 0.00793779 0.0130886 2023-09-23T20:14:34.168 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:34.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:34.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:34.273 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:34.300 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:34.344 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:34.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:34.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:34.448 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:34.476 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:34.623 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:34.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:34.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:34.732 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:34.759 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:34.857 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:14:35.077 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 81 16 97547 97531 75.2271 0 - 0.0130886 2023-09-23T20:14:35.090 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.091 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:14:35.091 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: osdmap e441: 8 total, 8 up, 6 in 2023-09-23T20:14:35.091 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.091 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.091 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.091 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.091 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.091 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1032521497' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: osdmap e441: 8 total, 8 up, 6 in 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.092 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1032521497' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:35.093 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:34 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:35.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T20:14:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: osdmap e441: 8 total, 8 up, 6 in 2023-09-23T20:14:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: failed to encode map e441 with expected crc 2023-09-23T20:14:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1032521497' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:35.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:34 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:35.509 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:35.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:35.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:35.613 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:35.641 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:35.790 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:35.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:35.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:35.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:35.911 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:14:35.979 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:36.077 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 82 16 97547 97531 74.31 0 - 0.0130886 2023-09-23T20:14:36.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: osdmap e442: 8 total, 8 up, 6 in 2023-09-23T20:14:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: pgmap v1224: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 127 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 4010/289293 objects misplaced (1.386%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:14:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:14:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:14:36.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:35 smithi144 ceph-mon[194833]: osdmap e443: 8 total, 8 up, 6 in 2023-09-23T20:14:36.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: osdmap e442: 8 total, 8 up, 6 in 2023-09-23T20:14:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: pgmap v1224: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 127 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 4010/289293 objects misplaced (1.386%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[221113]: osdmap e443: 8 total, 8 up, 6 in 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:36.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: osdmap e442: 8 total, 8 up, 6 in 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: failed to encode map e442 with expected crc 2023-09-23T20:14:36.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: pgmap v1224: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 127 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 4010/289293 objects misplaced (1.386%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:14:36.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:14:36.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:14:36.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:35 smithi135 ceph-mon[213871]: osdmap e443: 8 total, 8 up, 6 in 2023-09-23T20:14:36.384 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:36.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:36.489 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:14:36.543 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:14:36.543 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:36.649 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:14:36.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:36.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:14:36.765 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:36.820 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:36.993 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:37.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:37.077 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 83 16 98250 98234 73.9441 14.6458 0.00580328 0.013483 2023-09-23T20:14:37.101 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:37.106 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:14:37.131 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4442/276159 objects degraded (1.608%), 1 pg degraded) 2023-09-23T20:14:37.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: 9.11 starting backfill to osd.4 from (0'0,421'3780] MIN to 440'4814 2023-09-23T20:14:37.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:36 smithi144 ceph-mon[194833]: 9.12 starting backfill to osd.2 from (0'0,423'3823] MIN to 440'4784 2023-09-23T20:14:37.151 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:37.156 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:14:37.180 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:14:37.203 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:14:37.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4442/276159 objects degraded (1.608%), 1 pg degraded) 2023-09-23T20:14:37.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: 9.11 starting backfill to osd.4 from (0'0,421'3780] MIN to 440'4814 2023-09-23T20:14:37.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[221113]: 9.12 starting backfill to osd.2 from (0'0,423'3823] MIN to 440'4784 2023-09-23T20:14:37.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4442/276159 objects degraded (1.608%), 1 pg degraded) 2023-09-23T20:14:37.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: 9.11 starting backfill to osd.4 from (0'0,421'3780] MIN to 440'4814 2023-09-23T20:14:37.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: failed to encode map e443 with expected crc 2023-09-23T20:14:37.224 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:36 smithi135 ceph-mon[213871]: 9.12 starting backfill to osd.2 from (0'0,423'3823] MIN to 440'4784 2023-09-23T20:14:37.224 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:14:37.245 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:14:37.267 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:14:37.268 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:14:37.268 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:14:37.268 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:14:37.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:14:37.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:14:37.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:14:37.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:14:37.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:14:37.362 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:37.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:37.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:37.466 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:37.494 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:37.541 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:37.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:37.645 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:37.674 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:37.717 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:37.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:37.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:37.727 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:14:37.819 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:37.847 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:37.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: osdmap e444: 8 total, 8 up, 6 in 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: pgmap v1227: 129 pgs: 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 121 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 7.0 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 6191/290619 objects degraded (2.130%); 4010/290619 objects misplaced (1.380%); 4.5 MiB/s, 75 objects/s recovering 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1055188400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:37.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: osdmap e444: 8 total, 8 up, 6 in 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: failed to encode map e444 with expected crc 2023-09-23T20:14:37.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: pgmap v1227: 129 pgs: 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 121 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 7.0 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 6191/290619 objects degraded (2.130%); 4010/290619 objects misplaced (1.380%); 4.5 MiB/s, 75 objects/s recovering 2023-09-23T20:14:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1055188400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:37.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:38.077 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 84 16 98426 98410 73.1949 11 0.00768939 0.0136321 2023-09-23T20:14:38.128 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: failed to encode map e444 with expected crc 2023-09-23T20:14:38.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: osdmap e444: 8 total, 8 up, 6 in 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: failed to encode map e444 with expected crc 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: failed to encode map e444 with expected crc 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: failed to encode map e444 with expected crc 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: failed to encode map e444 with expected crc 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: failed to encode map e444 with expected crc 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: failed to encode map e444 with expected crc 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: pgmap v1227: 129 pgs: 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 121 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 7.0 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 6191/290619 objects degraded (2.130%); 4010/290619 objects misplaced (1.380%); 4.5 MiB/s, 75 objects/s recovering 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1055188400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:38.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:38.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:38.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:38.253 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:38.305 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:38.319 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:38.402 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:38.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:38.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:38.508 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:38.552 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:38.638 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:38.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:38.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:38.744 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:38.790 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:38.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 6191/290619 objects degraded (2.130%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:38.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[221113]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[221113]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[221113]: osdmap e445: 8 total, 8 up, 6 in 2023-09-23T20:14:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[221113]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[221113]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[221113]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[221113]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 6191/290619 objects degraded (2.130%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:38.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[213871]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[213871]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[213871]: osdmap e445: 8 total, 8 up, 6 in 2023-09-23T20:14:38.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[213871]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[213871]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[213871]: failed to encode map e445 with expected crc 2023-09-23T20:14:38.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:38 smithi135 ceph-mon[213871]: failed to encode map e445 with expected crc 2023-09-23T20:14:39.073 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:39.079 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 85 16 99509 99493 73.1287 67.6875 0.010778 0.0136692 2023-09-23T20:14:39.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:39.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:39.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:38 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 6191/290619 objects degraded (2.130%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:39.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:38 smithi144 ceph-mon[194833]: failed to encode map e445 with expected crc 2023-09-23T20:14:39.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:38 smithi144 ceph-mon[194833]: failed to encode map e445 with expected crc 2023-09-23T20:14:39.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:38 smithi144 ceph-mon[194833]: osdmap e445: 8 total, 8 up, 6 in 2023-09-23T20:14:39.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:38 smithi144 ceph-mon[194833]: failed to encode map e445 with expected crc 2023-09-23T20:14:39.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:38 smithi144 ceph-mon[194833]: failed to encode map e445 with expected crc 2023-09-23T20:14:39.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:38 smithi144 ceph-mon[194833]: failed to encode map e445 with expected crc 2023-09-23T20:14:39.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:38 smithi144 ceph-mon[194833]: failed to encode map e445 with expected crc 2023-09-23T20:14:39.182 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:39.209 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:39.252 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:39.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:39.355 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:39.419 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:39.775 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:39.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:39.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:39.885 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:39.966 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:39.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:39 smithi135 ceph-mon[221113]: pgmap v1229: 129 pgs: 1 active+recovery_wait+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 104 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 6.7 KiB/s rd, 32 MiB/s wr, 513 op/s; 49529/295707 objects degraded (16.749%); 9004/295707 objects misplaced (3.045%); 5.8 MiB/s, 111 objects/s recovering 2023-09-23T20:14:39.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:39 smithi135 ceph-mon[213871]: pgmap v1229: 129 pgs: 1 active+recovery_wait+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 104 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 6.7 KiB/s rd, 32 MiB/s wr, 513 op/s; 49529/295707 objects degraded (16.749%); 9004/295707 objects misplaced (3.045%); 5.8 MiB/s, 111 objects/s recovering 2023-09-23T20:14:40.079 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 86 16 100335 100319 72.8787 51.625 0.0080824 0.0137171 2023-09-23T20:14:40.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:39 smithi144 ceph-mon[194833]: pgmap v1229: 129 pgs: 1 active+recovery_wait+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 104 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 6.7 KiB/s rd, 32 MiB/s wr, 513 op/s; 49529/295707 objects degraded (16.749%); 9004/295707 objects misplaced (3.045%); 5.8 MiB/s, 111 objects/s recovering 2023-09-23T20:14:40.153 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:14:40.215 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:40.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:40.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:40.321 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:40.401 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:40.497 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:40.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:40.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:40.527 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:14:40.527 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:14:40.603 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:14:40.687 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:40.703 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7] out_osds: [6, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:14:40.703 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:14:40.747 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:40.807 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:14:40.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:40.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:14:40.918 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:40.943 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:14:40.984 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:41.055 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:40 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4202959014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:14:41.056 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:40 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:14:41.056 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:40 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4202959014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:14:41.056 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:40 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:14:41.077 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:41.079 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 87 16 101267 101251 72.7105 58.25 0.0118766 0.0137494 2023-09-23T20:14:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:41.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:41.188 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:41.241 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:41.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:40 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4202959014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:14:41.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:40 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:14:41.752 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:41.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:41.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:41.856 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:41.899 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:42.047 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:42.049 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:42.049 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:14:42.049 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: osdmap e446: 8 total, 8 up, 6 in 2023-09-23T20:14:42.049 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.049 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.050 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.050 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.050 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.050 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.050 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[213871]: pgmap v1231: 129 pgs: 2 active+recovery_wait+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 103 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 5.4 KiB/s rd, 28 MiB/s wr, 458 op/s; 49529/296385 objects degraded (16.711%); 12050/296385 objects misplaced (4.066%); 4.7 MiB/s, 90 objects/s recovering 2023-09-23T20:14:42.050 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: osdmap e446: 8 total, 8 up, 6 in 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.051 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:41 smithi135 ceph-mon[221113]: pgmap v1231: 129 pgs: 2 active+recovery_wait+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 103 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 5.4 KiB/s rd, 28 MiB/s wr, 458 op/s; 49529/296385 objects degraded (16.711%); 12050/296385 objects misplaced (4.066%); 4.7 MiB/s, 90 objects/s recovering 2023-09-23T20:14:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:42.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:42.079 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 88 16 101971 101955 72.3843 44 0.00407993 0.0138001 2023-09-23T20:14:42.153 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:42.208 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: osdmap e446: 8 total, 8 up, 6 in 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: failed to encode map e446 with expected crc 2023-09-23T20:14:42.209 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:41 smithi144 ceph-mon[194833]: pgmap v1231: 129 pgs: 2 active+recovery_wait+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 103 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 5.4 KiB/s rd, 28 MiB/s wr, 458 op/s; 49529/296385 objects degraded (16.711%); 12050/296385 objects misplaced (4.066%); 4.7 MiB/s, 90 objects/s recovering 2023-09-23T20:14:42.212 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:42.620 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:42.638 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:14:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:42.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:42.740 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:42.818 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:42.981 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:42.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:43.008 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:43.081 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 89 16 102876 102860 72.2055 56.5625 0.00787101 0.0138442 2023-09-23T20:14:43.097 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:43.160 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:43.266 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:43.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:43.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:43.371 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:43.446 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:43.548 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:43.563 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:14:43.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:43.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:43.598 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:14:43.631 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:14:43.663 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:14:43.664 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:43.693 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:14:43.714 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:43.733 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:14:43.737 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:43.775 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:14:43.812 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:14:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:14:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:14:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:14:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:14:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:14:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:14:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:14:43.814 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:14:43.825 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:43.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:43.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:43.841 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 20:14:43 smithi144 conmon[173765]: logger=cleanup t=2023-09-23T20:14:43.552304525Z level=info msg="Completed cleanup jobs" duration=124.420744ms 2023-09-23T20:14:43.842 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:43 smithi144 ceph-mon[194833]: pgmap v1232: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 24 MiB/s wr, 389 op/s; 49529/297822 objects degraded (16.630%); 12050/297822 objects misplaced (4.046%); 4.0 MiB/s, 74 objects/s recovering 2023-09-23T20:14:43.842 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/816384072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:43.844 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:14:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:14:43] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-09-23T20:14:43.845 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:43 smithi135 ceph-mon[221113]: pgmap v1232: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 24 MiB/s wr, 389 op/s; 49529/297822 objects degraded (16.630%); 12050/297822 objects misplaced (4.046%); 4.0 MiB/s, 74 objects/s recovering 2023-09-23T20:14:43.846 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/816384072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:43.847 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:43 smithi135 ceph-mon[213871]: pgmap v1232: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 24 MiB/s wr, 389 op/s; 49529/297822 objects degraded (16.630%); 12050/297822 objects misplaced (4.046%); 4.0 MiB/s, 74 objects/s recovering 2023-09-23T20:14:43.847 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/816384072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:43.941 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:44.002 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:44.081 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total time run: 90.0126 2023-09-23T20:14:44.081 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total writes made: 103852 2023-09-23T20:14:44.081 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Write size: 65536 2023-09-23T20:14:44.081 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object size: 65536 2023-09-23T20:14:44.081 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Bandwidth (MB/sec): 72.1093 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Bandwidth: 41.4696 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max bandwidth (MB/sec): 205.812 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average IOPS: 1153 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev IOPS: 663.525 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max IOPS: 3293 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min IOPS: 0 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average Latency(s): 0.0138636 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Latency(s): 0.0621581 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max latency(s): 2.42448 2023-09-23T20:14:44.082 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min latency(s): 0.00116484 2023-09-23T20:14:44.089 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:14:44.369 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:14:44.676 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:44.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:44.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:44.784 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:44.837 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:44.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:44 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 49529/297822 objects degraded (16.630%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:44.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:44 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 49529/297822 objects degraded (16.630%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:45.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:44 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 49529/297822 objects degraded (16.630%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:45.399 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:45.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:45.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:45.504 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:45.579 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:45.709 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:45.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:45.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:45.819 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:45.871 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:45 smithi135 ceph-mon[221113]: pgmap v1233: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 49 MiB/s wr, 783 op/s; 49529/309429 objects degraded (16.007%); 12024/309429 objects misplaced (3.886%); 5.4 MiB/s, 94 objects/s recovering 2023-09-23T20:14:45.871 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:45 smithi135 ceph-mon[213871]: pgmap v1233: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 49 MiB/s wr, 783 op/s; 49529/309429 objects degraded (16.007%); 12024/309429 objects misplaced (3.886%); 5.4 MiB/s, 94 objects/s recovering 2023-09-23T20:14:45.878 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:45 smithi144 ceph-mon[194833]: pgmap v1233: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 49 MiB/s wr, 783 op/s; 49529/309429 objects degraded (16.007%); 12024/309429 objects misplaced (3.886%); 5.4 MiB/s, 94 objects/s recovering 2023-09-23T20:14:45.881 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:45.992 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:45.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:46.098 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:46.157 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:46.200 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:46.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:46.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:46.306 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:46.352 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:46.699 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:46.723 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:46.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:46.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:46.819 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:14:46.838 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:46.896 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:47.030 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1875625649' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:14:47.030 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:14:47.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1875625649' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:14:47.031 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:14:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1875625649' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:14:47.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:14:47.362 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:14:47.384 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:47.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:47.497 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:47.565 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:47.921 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:14:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:47 smithi144 ceph-mon[194833]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:47 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:14:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:47 smithi144 ceph-mon[194833]: osdmap e447: 8 total, 8 up, 6 in 2023-09-23T20:14:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:47 smithi144 ceph-mon[194833]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:47 smithi144 ceph-mon[194833]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:47 smithi144 ceph-mon[194833]: pgmap v1235: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 38 MiB/s wr, 644 op/s; 49529/309561 objects degraded (16.000%); 12024/309561 objects misplaced (3.884%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:14:48.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[213871]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:14:48.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[213871]: osdmap e447: 8 total, 8 up, 6 in 2023-09-23T20:14:48.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[213871]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[213871]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[213871]: pgmap v1235: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 38 MiB/s wr, 644 op/s; 49529/309561 objects degraded (16.000%); 12024/309561 objects misplaced (3.884%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:14:48.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[221113]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:14:48.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[221113]: osdmap e447: 8 total, 8 up, 6 in 2023-09-23T20:14:48.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[221113]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.228 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[221113]: failed to encode map e447 with expected crc 2023-09-23T20:14:48.229 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:47 smithi135 ceph-mon[221113]: pgmap v1235: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 38 MiB/s wr, 644 op/s; 49529/309561 objects degraded (16.000%); 12024/309561 objects misplaced (3.884%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.36186.0:107708 9.13 e447/429 9:ca24183e:::benchmark_data_smithi135_549536_object3853:head v 447'5073, mlcod=411'882)", 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:14:48.230973+0000", 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0041269599999999998, 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041507230000000003, 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:14:48.238 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.230973+0000", 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.230973+0000", 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:14:48.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.230973+0000", 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.230978+0000", 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9370000000000003e-06 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.230984+0000", 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8009999999999998e-06 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.230989+0000", 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6990000000000002e-06 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.232364+0000", 2023-09-23T20:14:48.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001375135 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.232403+0000", 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9065e-05 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:14:48.242 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 1 2023-09-23T20:14:48.243 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:14:48.243 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:48.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:48.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:48.349 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:48.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:48.932 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:48.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:48.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:49.036 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:49.085 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:49.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:49 smithi135 ceph-mon[221113]: pgmap v1236: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 125 B/s rd, 39 MiB/s wr, 1.08k op/s; 49275/300513 objects degraded (16.397%); 11502/300513 objects misplaced (3.827%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:14:49.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:49 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 49275/300513 objects degraded (16.397%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:49.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:49 smithi135 ceph-mon[213871]: pgmap v1236: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 125 B/s rd, 39 MiB/s wr, 1.08k op/s; 49275/300513 objects degraded (16.397%); 11502/300513 objects misplaced (3.827%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:14:49.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:49 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 49275/300513 objects degraded (16.397%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:49.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:49 smithi144 ceph-mon[194833]: pgmap v1236: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 125 B/s rd, 39 MiB/s wr, 1.08k op/s; 49275/300513 objects degraded (16.397%); 11502/300513 objects misplaced (3.827%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:14:49.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:49 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 49275/300513 objects degraded (16.397%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:49.701 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:49.722 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:49.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:49.739 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:14:49.837 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:49.904 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:50.070 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:50.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:50.176 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:50.177 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:50.277 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:50.577 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:50.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:50.691 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:50.749 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:14:50.782 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:50.855 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3833372331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:50.856 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:50 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:50.856 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3833372331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:50.856 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:50 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:50.887 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:50.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:50.997 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:51.054 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:51.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3833372331' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:51.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:50 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:14:51.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:14:51.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:51.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: osdmap e448: 8 total, 8 up, 6 in 2023-09-23T20:14:51.907 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.908 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[213871]: pgmap v1238: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 36 MiB/s wr, 1.05k op/s; 49275/300180 objects degraded (16.415%); 11502/300180 objects misplaced (3.832%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:14:51.908 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:14:51.908 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:51.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: osdmap e448: 8 total, 8 up, 6 in 2023-09-23T20:14:51.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: failed to encode map e448 with expected crc 2023-09-23T20:14:51.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:51 smithi135 ceph-mon[221113]: pgmap v1238: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 36 MiB/s wr, 1.05k op/s; 49275/300180 objects degraded (16.415%); 11502/300180 objects misplaced (3.832%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:14:52.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:14:52.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:14:52.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: osdmap e448: 8 total, 8 up, 6 in 2023-09-23T20:14:52.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: failed to encode map e448 with expected crc 2023-09-23T20:14:52.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: failed to encode map e448 with expected crc 2023-09-23T20:14:52.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: failed to encode map e448 with expected crc 2023-09-23T20:14:52.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: failed to encode map e448 with expected crc 2023-09-23T20:14:52.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: failed to encode map e448 with expected crc 2023-09-23T20:14:52.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: failed to encode map e448 with expected crc 2023-09-23T20:14:52.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:51 smithi144 ceph-mon[194833]: pgmap v1238: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 36 MiB/s wr, 1.05k op/s; 49275/300180 objects degraded (16.415%); 11502/300180 objects misplaced (3.832%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T20:14:52.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:52.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:52.532 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:52.592 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:52.676 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:52.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:14:52.782 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:14:52.838 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:52.967 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:14:52.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:52.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:14:53.075 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:53.132 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:53.135 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:53.237 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:53.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:53.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:53.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:53.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:53.245 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:53 smithi144 ceph-mon[194833]: pgmap v1239: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 643 op/s; 49275/298416 objects degraded (16.512%); 11502/298416 objects misplaced (3.854%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:14:53.345 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:53.405 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:53.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:53.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:53 smithi135 ceph-mon[221113]: pgmap v1239: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 643 op/s; 49275/298416 objects degraded (16.512%); 11502/298416 objects misplaced (3.854%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:14:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:14:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:53 smithi135 ceph-mon[213871]: pgmap v1239: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 643 op/s; 49275/298416 objects degraded (16.512%); 11502/298416 objects misplaced (3.854%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:14:53.533 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:53.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:53.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:53.647 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:14:53.712 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:53.831 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:14:53.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:53.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:14:53.929 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:14:53.934 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:14:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:14:53] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2023-09-23T20:14:53.936 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:53.971 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:14:54.012 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:14:54.025 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:54.044 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:14:54.086 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:14:54.125 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:14:54.156 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:54.158 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:14:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:54.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:54.196 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:14:54.197 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:14:54.198 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:14:54.198 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:14:54.198 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:14:54.198 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:14:54.198 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:14:54.198 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:14:54.198 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:14:54.260 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:54.284 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:14:54.328 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:54.414 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:54.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:54.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:54.524 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:54.589 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:54.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3039749900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:54.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3039749900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:55.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3039749900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:14:55.619 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:55.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:55.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:55.729 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:14:55.786 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:55 smithi135 ceph-mon[213871]: pgmap v1240: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 124 B/s rd, 3.2 MiB/s wr, 1.04k op/s; 48975/286356 objects degraded (17.103%); 10917/286356 objects misplaced (3.812%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:14:55.786 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:55 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3214560430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:14:55.786 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:55 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:14:55.786 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:55 smithi135 ceph-mon[221113]: pgmap v1240: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 124 B/s rd, 3.2 MiB/s wr, 1.04k op/s; 48975/286356 objects degraded (17.103%); 10917/286356 objects misplaced (3.812%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:14:55.786 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:55 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3214560430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:14:55.786 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:55 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:14:55.787 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:55.799 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:14:56.090 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:14:56.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:56.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:14:56.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:55 smithi144 ceph-mon[194833]: pgmap v1240: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 124 B/s rd, 3.2 MiB/s wr, 1.04k op/s; 48975/286356 objects degraded (17.103%); 10917/286356 objects misplaced (3.812%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:14:56.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:55 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3214560430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:14:56.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:55 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:14:56.199 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:56.276 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:56.356 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:56.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:56.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:56.464 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:56.490 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:56.581 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:56.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:56.694 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:56.746 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:56.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:56 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 48975/286356 objects degraded (17.103%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:56.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:56 smithi144 ceph-mon[194833]: failed to encode map e449 with expected crc 2023-09-23T20:14:56.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:56 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:14:56.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:56 smithi144 ceph-mon[194833]: osdmap e449: 8 total, 8 up, 6 in 2023-09-23T20:14:56.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:56 smithi144 ceph-mon[194833]: failed to encode map e449 with expected crc 2023-09-23T20:14:56.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:56 smithi144 ceph-mon[194833]: failed to encode map e449 with expected crc 2023-09-23T20:14:56.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:56 smithi144 ceph-mon[194833]: failed to encode map e449 with expected crc 2023-09-23T20:14:56.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:56 smithi144 ceph-mon[194833]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.003 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 48975/286356 objects degraded (17.103%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:57.004 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[213871]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.004 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:14:57.004 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[213871]: osdmap e449: 8 total, 8 up, 6 in 2023-09-23T20:14:57.004 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[213871]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.004 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[213871]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.004 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[213871]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.004 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[213871]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.005 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 48975/286356 objects degraded (17.103%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:14:57.005 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[221113]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.005 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:14:57.005 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[221113]: osdmap e449: 8 total, 8 up, 6 in 2023-09-23T20:14:57.005 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[221113]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.005 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[221113]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.006 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[221113]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.006 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:56 smithi135 ceph-mon[221113]: failed to encode map e449 with expected crc 2023-09-23T20:14:57.016 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:57.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:57.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:57.032 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:14:57.048 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:14:57.049 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:14:57.124 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:14:57.179 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:57.295 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:14:57.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:57.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:14:57.403 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:57.429 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:57.734 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:14:57.776 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:57.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:57.874 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:57.888 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:57.917 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:14:57.941 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:58.047 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:58.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:58.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:58.053 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:14:57 smithi144 ceph-mon[194833]: pgmap v1242: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 48975/284550 objects degraded (17.211%); 10917/284550 objects misplaced (3.837%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:14:58.136 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:58.153 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:14:58.232 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:14:57 smithi135 ceph-mon[213871]: pgmap v1242: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 48975/284550 objects degraded (17.211%); 10917/284550 objects misplaced (3.837%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:14:58.232 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:14:57 smithi135 ceph-mon[221113]: pgmap v1242: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 48975/284550 objects degraded (17.211%); 10917/284550 objects misplaced (3.837%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:14:58.239 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:58.457 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:14:58.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:58.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:14:58.567 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:14:58.625 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:58.718 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:14:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:58.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:14:58.835 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:14:58.886 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:58.967 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:14:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:58.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:14:59.071 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:14:59.098 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:59.653 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:14:59.676 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:14:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:59.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:14:59.789 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:14:59.863 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:14:59.923 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:14:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:14:59.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:00.029 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:00.092 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:00.111 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:00 smithi144 ceph-mon[194833]: pgmap v1243: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 48693/272691 objects degraded (17.856%); 10295/272691 objects misplaced (3.775%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:15:00.336 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:00.426 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:00.430 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:00 smithi135 ceph-mon[213871]: pgmap v1243: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 48693/272691 objects degraded (17.856%); 10295/272691 objects misplaced (3.775%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:15:00.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:00.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:00.441 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:00 smithi135 ceph-mon[221113]: pgmap v1243: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 48693/272691 objects degraded (17.856%); 10295/272691 objects misplaced (3.775%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:15:00.533 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:00.598 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:00.937 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:00.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:00.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:01.046 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:01.111 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:01.181 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:01.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:01.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:01.291 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:01.341 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:01 smithi144 ceph-mon[194833]: pgmap v1244: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 946 op/s; 48693/271770 objects degraded (17.917%); 10295/271770 objects misplaced (3.788%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:15:01.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:01 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 48693/271770 objects degraded (17.917%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:15:01.456 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:01.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:01.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:01 smithi135 ceph-mon[213871]: pgmap v1244: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 946 op/s; 48693/271770 objects degraded (17.917%); 10295/271770 objects misplaced (3.788%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:15:01.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:01 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 48693/271770 objects degraded (17.917%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:15:01.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:01 smithi135 ceph-mon[221113]: pgmap v1244: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 946 op/s; 48693/271770 objects degraded (17.917%); 10295/271770 objects misplaced (3.788%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:15:01.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:01 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 48693/271770 objects degraded (17.917%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:15:01.573 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:01.604 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:02.182 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/676537619' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:02.182 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2168048132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:15:02.186 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:15:02.195 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:15:02.445 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/676537619' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:02.445 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2168048132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:15:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/676537619' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2168048132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:15:02.687 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:02.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:02.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:02.794 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:02.857 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:02.963 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:02.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:03.076 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:03.151 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:03.188 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:03.188 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/676537619' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:15:03.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2168048132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:15:03.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: osdmap e450: 8 total, 8 up, 6 in 2023-09-23T20:15:03.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.189 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[213871]: pgmap v1246: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 657 op/s; 48693/270570 objects degraded (17.996%); 10295/270570 objects misplaced (3.805%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:15:03.278 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:03.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:03.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:03.401 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:03.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/676537619' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:15:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2168048132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:15:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: osdmap e450: 8 total, 8 up, 6 in 2023-09-23T20:15:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:03 smithi135 ceph-mon[221113]: pgmap v1246: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 657 op/s; 48693/270570 objects degraded (17.996%); 10295/270570 objects misplaced (3.805%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:15:03.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:03.487 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.487 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.487 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/676537619' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:15:03.487 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2168048132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:15:03.487 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: osdmap e450: 8 total, 8 up, 6 in 2023-09-23T20:15:03.487 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.487 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: failed to encode map e450 with expected crc 2023-09-23T20:15:03.487 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:03 smithi144 ceph-mon[194833]: pgmap v1246: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 657 op/s; 48693/270570 objects degraded (17.996%); 10295/270570 objects misplaced (3.805%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:15:03.489 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:03.535 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:03.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:03.648 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:03.713 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:03.826 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:03.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:03.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:03.937 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:03.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:15:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:15:03] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2023-09-23T20:15:04.005 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:04.080 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:04.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:04.186 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:04.216 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:04.535 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:04.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:04.646 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:04.713 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:04.917 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:04.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:04.961 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:15:04.987 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:15:04.994 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:15:05.025 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:05.030 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:15:05.060 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:15:05.085 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:05.095 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:15:05.148 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:15:05.178 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:05.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:05.207 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:15:05.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2938631269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:05.226 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2938631269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:05.243 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:15:05.243 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:175181 7.7 7:eb856231:::benchmark_data_smithi135_404353_object78119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.719599+0000", 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 571.52073239499998, 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.34012610199999999, 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.244 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 175181 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719599+0000", 2023-09-23T20:15:05.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719599+0000", 2023-09-23T20:15:05.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719597+0000", 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719604+0000", 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9439999999999999e-06 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.247 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719617+0000", 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3339999999999999e-05 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719625+0000", 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5700000000000004e-06 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.248 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728365+0000", 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0087402499999999998 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728520+0000", 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000155014 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728656+0000", 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013616600000000001 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.731769+0000", 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003113265 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038834+0000", 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30706430699999998 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038876+0000", 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2216999999999999e-05 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059552+0000", 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.020676305999999998 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059607+0000", 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4753000000000002e-05 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059725+0000", 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117835 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:175178 7.4 e290/240 7:2901d72c:::benchmark_data_smithi135_404353_object78116:head v 290'10878, mlcod=290'10875)", 2023-09-23T20:15:05.253 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.719930+0000", 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 571.52040127700002, 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31795096099999998, 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719930+0000", 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.254 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719930+0000", 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719929+0000", 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.255 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.256 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719935+0000", 2023-09-23T20:15:05.256 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7559999999999996e-06 2023-09-23T20:15:05.256 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.256 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719941+0000", 2023-09-23T20:15:05.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9610000000000002e-06 2023-09-23T20:15:05.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.719945+0000", 2023-09-23T20:15:05.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7280000000000001e-06 2023-09-23T20:15:05.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.723900+0000", 2023-09-23T20:15:05.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0039551480000000003 2023-09-23T20:15:05.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.723923+0000", 2023-09-23T20:15:05.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3629e-05 2023-09-23T20:15:05.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037838+0000", 2023-09-23T20:15:05.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31391448599999999 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037881+0000", 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2777000000000001e-05 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:175183 7.4 e290/240 7:299b4e88:::benchmark_data_smithi135_404353_object78121:head v 290'10879, mlcod=290'10875)", 2023-09-23T20:15:05.261 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.724977+0000", 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 571.515353979, 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.31294654799999999, 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724977+0000", 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724977+0000", 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724976+0000", 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724983+0000", 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7970000000000003e-06 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.724994+0000", 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1429e-05 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.725001+0000", 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7009999999999999e-06 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.730362+0000", 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053613630000000001 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.730376+0000", 2023-09-23T20:15:05.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3526000000000001e-05 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037888+0000", 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30751193100000002 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037924+0000", 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5802e-05 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.266 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.35358.0:175182 7.5 e290/240 7:a10814f3:::benchmark_data_smithi135_404353_object78120:head v 290'11086, mlcod=290'11083)", 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.728954+0000", 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 571.51137701000005, 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30892793499999999, 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728954+0000", 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728954+0000", 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728953+0000", 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728959+0000", 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5990000000000001e-06 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728974+0000", 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4636e-05 2023-09-23T20:15:05.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.728980+0000", 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7680000000000003e-06 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.729049+0000", 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8345999999999999e-05 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.729068+0000", 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9057000000000001e-05 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037848+0000", 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30878053300000002 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037882+0000", 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3633000000000001e-05 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35358.0:175190 7.7 7:eddb5a12:::benchmark_data_smithi135_404353_object78128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.732026+0000", 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 571.508305178, 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.32787697900000001, 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.272 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35358", 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/2667957301", 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 175190 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732026+0000", 2023-09-23T20:15:05.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732026+0000", 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732024+0000", 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732031+0000", 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3180000000000001e-06 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037829+0000", 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30579754199999998 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037843+0000", 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4378e-05 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038899+0000", 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001056083 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.039026+0000", 2023-09-23T20:15:05.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000126594 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.039123+0000", 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6921000000000003e-05 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.042110+0000", 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029867309999999998 2023-09-23T20:15:05.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.042985+0000", 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000875116 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.047677+0000", 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046925919999999998 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.278 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059761+0000", 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01208389 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059805+0000", 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3761999999999999e-05 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.059903+0000", 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7800000000000006e-05 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.35358.0:175181 7.7 e290/274 ondisk, result = 0)", 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:05:33.732356+0000", 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 571.50797462399998, 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30652289999999999, 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.280 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732356+0000", 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732356+0000", 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732355+0000", 2023-09-23T20:15:05.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:15:05.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:33.732361+0000", 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8690000000000001e-06 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037823+0000", 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.30546166600000002 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.037835+0000", 2023-09-23T20:15:05.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.168e-05 2023-09-23T20:15:05.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038861+0000", 2023-09-23T20:15:05.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010267570000000001 2023-09-23T20:15:05.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038869+0000", 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6520000000000006e-06 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:05:34.038879+0000", 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0169e-05 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.286 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65675 8.1 8:8413818a:::benchmark_data_smithi135_460343_object65674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:15:05.286 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:19.118032+0000", 2023-09-23T20:15:05.286 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 466.12229863499999, 2023-09-23T20:15:05.286 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5792248369999999, 2023-09-23T20:15:05.286 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.286 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.286 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65675 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118032+0000", 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118032+0000", 2023-09-23T20:15:05.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118031+0000", 2023-09-23T20:15:05.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118053+0000", 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2157000000000001e-05 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118063+0000", 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0210000000000001e-05 2023-09-23T20:15:05.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118071+0000", 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2199999999999992e-06 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118136+0000", 2023-09-23T20:15:05.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4697999999999996e-05 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.118148+0000", 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1849e-05 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.676405+0000", 2023-09-23T20:15:05.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5582569479999999 2023-09-23T20:15:05.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.678641+0000", 2023-09-23T20:15:05.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022364920000000001 2023-09-23T20:15:05.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.678889+0000", 2023-09-23T20:15:05.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000247607 2023-09-23T20:15:05.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.686930+0000", 2023-09-23T20:15:05.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080408010000000002 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.688012+0000", 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001082118 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.693954+0000", 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0059426150000000001 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.694023+0000", 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9093999999999999e-05 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.697257+0000", 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0032336639999999998 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.295 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:91666 8.1 8:8cda5659:::benchmark_data_smithi135_460343_object7433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:14.988938+0000", 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 410.25139326200002, 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.349132473, 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.296 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 91666 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988938+0000", 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988938+0000", 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988936+0000", 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988944+0000", 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6179999999999995e-06 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988958+0000", 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4174e-05 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.988967+0000", 2023-09-23T20:15:05.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4209999999999995e-06 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.993525+0000", 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045585570000000004 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.993654+0000", 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012858600000000001 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:14.993783+0000", 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000129159 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.002953+0000", 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0091700099999999993 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.002991+0000", 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8149999999999999e-05 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.335673+0000", 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.33268188599999998 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.335729+0000", 2023-09-23T20:15:05.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5519999999999997e-05 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:15.338070+0000", 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023415509999999999 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:113327 8.1 8:8edd1613:::benchmark_data_smithi135_460343_object29094:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:58.101036+0000", 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 367.13929497399999, 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.23996662499999999, 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 113327 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101036+0000", 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101036+0000", 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101034+0000", 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101042+0000", 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3629999999999994e-06 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.306 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101056+0000", 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3973e-05 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.101065+0000", 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4689999999999997e-06 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.102649+0000", 2023-09-23T20:15:05.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015837010000000001 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.102742+0000", 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3016999999999994e-05 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.102857+0000", 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000115259 2023-09-23T20:15:05.308 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.109123+0000", 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0062665530000000002 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.114004+0000", 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048803420000000002 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.339586+0000", 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.225582702 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.339641+0000", 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5003999999999997e-05 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:58.341003+0000", 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001361129 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:114810 8.1 8:890a0c99:::benchmark_data_smithi135_460343_object30577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:01.208771+0000", 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 364.03155986500002, 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24141304099999999, 2023-09-23T20:15:05.311 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 114810 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208771+0000", 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208771+0000", 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208769+0000", 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208777+0000", 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7319999999999998e-06 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208790+0000", 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3232e-05 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.314 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.315 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.208799+0000", 2023-09-23T20:15:05.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4120000000000007e-06 2023-09-23T20:15:05.315 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.315 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.315 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.217359+0000", 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085597390000000002 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.217501+0000", 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014166100000000001 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.217635+0000", 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013428600000000001 2023-09-23T20:15:05.316 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.317 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.317 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.317 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.224173+0000", 2023-09-23T20:15:05.317 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065379990000000001 2023-09-23T20:15:05.317 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.317 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.317 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.317 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.227602+0000", 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034295369999999999 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.448962+0000", 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.22135990799999999 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.449025+0000", 2023-09-23T20:15:05.318 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2816000000000003e-05 2023-09-23T20:15:05.319 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.319 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.319 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.319 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.450184+0000", 2023-09-23T20:15:05.319 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0011589580000000001 2023-09-23T20:15:05.319 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.319 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.319 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:114816 8.1 8:86e3ef4c:::benchmark_data_smithi135_460343_object30583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:01.212087+0000", 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 364.02824441899998, 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.248140421, 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.320 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.321 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 114816 2023-09-23T20:15:05.321 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.321 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.321 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.321 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.321 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212087+0000", 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212087+0000", 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.322 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212085+0000", 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212093+0000", 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5459999999999992e-06 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.323 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212110+0000", 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7144000000000001e-05 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.212120+0000", 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2590000000000006e-06 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.221012+0000", 2023-09-23T20:15:05.324 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088927590000000001 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.221160+0000", 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014748699999999999 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.221300+0000", 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000140536 2023-09-23T20:15:05.325 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.224275+0000", 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029750200000000001 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.227642+0000", 2023-09-23T20:15:05.326 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033664469999999998 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.458959+0000", 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.231317518 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.327 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.459016+0000", 2023-09-23T20:15:05.328 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6428999999999998e-05 2023-09-23T20:15:05.328 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.328 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.328 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.328 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:01.460227+0000", 2023-09-23T20:15:05.328 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001211061 2023-09-23T20:15:05.328 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.328 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:137726 8.1 8:8067c0eb:::benchmark_data_smithi135_460343_object53493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:41.383562+0000", 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 203.85676863800001, 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17975639500000001, 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.329 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 137726 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383562+0000", 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.330 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383562+0000", 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383561+0000", 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.331 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383567+0000", 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4130000000000003e-06 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383582+0000", 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.438e-05 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383589+0000", 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3649999999999998e-06 2023-09-23T20:15:05.332 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383630+0000", 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1159999999999999e-05 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383742+0000", 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000111358 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.333 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.383836+0000", 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4891999999999999e-05 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.386813+0000", 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029765730000000002 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.334 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.387070+0000", 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000256514 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.562338+0000", 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17526850099999999 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.335 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.562399+0000", 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0739000000000002e-05 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.563319+0000", 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00091990500000000005 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.336 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:137728 8.1 8:801befe5:::benchmark_data_smithi135_460343_object53495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:41.388353+0000", 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 203.851978344, 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17817260300000001, 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 137728 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.337 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388353+0000", 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388353+0000", 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.338 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388351+0000", 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388360+0000", 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6460000000000005e-06 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388380+0000", 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9233999999999999e-05 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388388+0000", 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.789e-06 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388439+0000", 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0358999999999997e-05 2023-09-23T20:15:05.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.341 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.341 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.341 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388610+0000", 2023-09-23T20:15:05.341 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017112299999999999 2023-09-23T20:15:05.341 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.341 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.341 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.341 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.388733+0000", 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012366100000000001 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.391543+0000", 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028094040000000002 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.342 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.391953+0000", 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00041040100000000002 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.565545+0000", 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17359206199999999 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.565610+0000", 2023-09-23T20:15:05.343 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4317000000000001e-05 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:41.566525+0000", 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00091557699999999999 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.344 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:144171 8.1 8:88e198c0:::benchmark_data_smithi135_460343_object59938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:55.556999+0000", 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 189.683331735, 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.24187935099999999, 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 144171 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.345 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.556999+0000", 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.556999+0000", 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.346 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.556998+0000", 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557005+0000", 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5349999999999999e-06 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.347 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557020+0000", 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4883e-05 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557028+0000", 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3869999999999997e-06 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557086+0000", 2023-09-23T20:15:05.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8470000000000001e-05 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557266+0000", 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000180199 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.557436+0000", 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000169603 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.560273+0000", 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00283665 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.789439+0000", 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.229166656 2023-09-23T20:15:05.350 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.798080+0000", 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0086407440000000005 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.798142+0000", 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2162999999999994e-05 2023-09-23T20:15:05.351 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:55.798879+0000", 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00073647100000000004 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:157147 8.1 8:8e5789f8:::benchmark_data_smithi135_460343_object72914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T20:15:05.352 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:18.285993+0000", 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 166.95433759700001, 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.21334767800000001, 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 157147 2023-09-23T20:15:05.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.285993+0000", 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.285993+0000", 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.355 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.355 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.285992+0000", 2023-09-23T20:15:05.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:05.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.355 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.355 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.355 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.285998+0000", 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1319999999999999e-06 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286020+0000", 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2356000000000001e-05 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286030+0000", 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3470000000000005e-06 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286072+0000", 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2633999999999999e-05 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.357 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286235+0000", 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000162917 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.286388+0000", 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015320399999999999 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.289566+0000", 2023-09-23T20:15:05.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031780200000000002 2023-09-23T20:15:05.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.299076+0000", 2023-09-23T20:15:05.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0095100389999999996 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.498675+0000", 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.199598674 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.498733+0000", 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.8239999999999998e-05 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.499341+0000", 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060763400000000002 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.361 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:157158 8.1 8:8cb317c3:::benchmark_data_smithi135_460343_object72925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:18.301097+0000", 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 166.93923412999999, 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.19862486200000001, 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.362 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 157158 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301097+0000", 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301097+0000", 2023-09-23T20:15:05.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.365 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301095+0000", 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301103+0000", 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8720000000000002e-06 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.366 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301116+0000", 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2967e-05 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301124+0000", 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3059999999999998e-06 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.301981+0000", 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00085613599999999996 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.302131+0000", 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015003799999999999 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.302275+0000", 2023-09-23T20:15:05.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000143905 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.304842+0000", 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025672690000000001 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.311933+0000", 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007091445 2023-09-23T20:15:05.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.499390+0000", 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.18745663900000001 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.499438+0000", 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7888e-05 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.499722+0000", 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028388600000000001 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:157160 8.1 8:8e424eab:::benchmark_data_smithi135_460343_object72927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:18.305597+0000", 2023-09-23T20:15:05.371 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 166.93473359500001, 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.203750138, 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 157160 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305597+0000", 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305597+0000", 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305596+0000", 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305604+0000", 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3690000000000007e-06 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305620+0000", 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6052000000000001e-05 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305627+0000", 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0509999999999999e-06 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305673+0000", 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6063000000000003e-05 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.375 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305830+0000", 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000156362 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.305947+0000", 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117244 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.311879+0000", 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005932629 2023-09-23T20:15:05.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.313100+0000", 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001220715 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.508732+0000", 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.195631429 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.508787+0000", 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5819000000000002e-05 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:18.509347+0000", 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00056003599999999998 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.378 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:161045 8.1 8:8cd8880e:::benchmark_data_smithi135_460343_object76812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:12:23.836273+0000", 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 161.40405774199999, 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.18956387599999999, 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:05.379 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 161045 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836273+0000", 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836273+0000", 2023-09-23T20:15:05.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836272+0000", 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836280+0000", 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3860000000000007e-06 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836297+0000", 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7323e-05 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836306+0000", 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9109999999999999e-06 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836357+0000", 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1103999999999997e-05 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836501+0000", 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014376100000000001 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:15:05.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.836599+0000", 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8263000000000004e-05 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.840941+0000", 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0043419169999999998 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:05.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:23.843289+0000", 2023-09-23T20:15:05.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023477950000000002 2023-09-23T20:15:05.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.385 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:05.385 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:24.025186+0000", 2023-09-23T20:15:05.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.181896575 2023-09-23T20:15:05.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:24.025236+0000", 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.075e-05 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:12:24.025837+0000", 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060075700000000005 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.386 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.387 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.387 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.36186.0:107896 9.16 e447/442 9:694b3ee0:::benchmark_data_smithi135_549536_object4041:head v 447'5238, mlcod=0'0)", 2023-09-23T20:15:05.387 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:14:48.410096+0000", 2023-09-23T20:15:05.387 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 16.830234769, 2023-09-23T20:15:05.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.170605861, 2023-09-23T20:15:05.387 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.410096+0000", 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.410096+0000", 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.410096+0000", 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967296 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.410100+0000", 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1420000000000003e-06 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.410105+0000", 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.293e-06 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.410110+0000", 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5190000000000003e-06 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.410175+0000", 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4906999999999994e-05 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.410194+0000", 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8799000000000001e-05 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.580666+0000", 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17047285300000001 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.580702+0000", 2023-09-23T20:15:05.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5578999999999999e-05 2023-09-23T20:15:05.393 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.393 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.393 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.394 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:05.395 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2938631269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:05.395 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.36186.0:107894 9.19 e447/418 9:9e56605a:::benchmark_data_smithi135_549536_object4039:head v 447'5163, mlcod=447'5161)", 2023-09-23T20:15:05.395 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:14:48.411813+0000", 2023-09-23T20:15:05.395 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 16.828517451, 2023-09-23T20:15:05.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17021281599999999, 2023-09-23T20:15:05.395 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.411813+0000", 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.411813+0000", 2023-09-23T20:15:05.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.411813+0000", 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.411817+0000", 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1699999999999999e-06 2023-09-23T20:15:05.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.411824+0000", 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7129999999999999e-06 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.411828+0000", 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3290000000000004e-06 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.574855+0000", 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.163026951 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.574891+0000", 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5138000000000001e-05 2023-09-23T20:15:05.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.581994+0000", 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0071035320000000001 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:48.582026+0000", 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2234999999999999e-05 2023-09-23T20:15:05.400 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.401 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.401 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.401 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:05.401 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:05.401 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:15:05.401 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:15:05.402 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:15:05.402 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:15:05.402 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:15:05.402 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:15:05.402 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:15:05.402 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:15:05.402 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:15:05.402 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:15:05.453 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:06.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:05 smithi144 ceph-mon[194833]: pgmap v1247: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 981 op/s; 48236/259074 objects degraded (18.619%); 9661/259074 objects misplaced (3.729%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:15:06.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:05 smithi135 ceph-mon[213871]: pgmap v1247: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 981 op/s; 48236/259074 objects degraded (18.619%); 9661/259074 objects misplaced (3.729%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:15:06.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:05 smithi135 ceph-mon[221113]: pgmap v1247: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 981 op/s; 48236/259074 objects degraded (18.619%); 9661/259074 objects misplaced (3.729%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:15:06.551 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:15:06.708 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:15:06.733 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:06.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:06.851 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:06.906 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:06.966 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:07.012 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:07.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:07.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:07.119 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:07.169 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:07 smithi144 ceph-mon[194833]: pgmap v1248: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 48236/257532 objects degraded (18.730%); 9661/257532 objects misplaced (3.751%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:15:07.169 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:07 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 48236/257532 objects degraded (18.730%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:15:07.177 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:07.380 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:07.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:07.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:07 smithi135 ceph-mon[221113]: pgmap v1248: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 48236/257532 objects degraded (18.730%); 9661/257532 objects misplaced (3.751%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:15:07.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:07 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 48236/257532 objects degraded (18.730%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:15:07.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:07 smithi135 ceph-mon[213871]: pgmap v1248: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 48236/257532 objects degraded (18.730%); 9661/257532 objects misplaced (3.751%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:15:07.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:07 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 48236/257532 objects degraded (18.730%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T20:15:07.486 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:07.527 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:07.583 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:07.622 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:07.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:07.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:07.730 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:07.780 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:07.880 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:07.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:07.988 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:08.014 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:08.056 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:08.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:08.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:08.165 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:08.177 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:08.177 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:08.213 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:08.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:08.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:08.447 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:08.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:08.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:08.455 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:08.455 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:08.554 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:08.619 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:08.627 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:08.789 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:15:08.924 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:08.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:08.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:09.033 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:09.094 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:09.345 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:09 smithi135 ceph-mon[213871]: pgmap v1249: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 47695/246570 objects degraded (19.343%); 9044/246570 objects misplaced (3.668%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:15:09.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:09 smithi135 ceph-mon[221113]: pgmap v1249: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 47695/246570 objects degraded (19.343%); 9044/246570 objects misplaced (3.668%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:15:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:09 smithi144 ceph-mon[194833]: pgmap v1249: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 47695/246570 objects degraded (19.343%); 9044/246570 objects misplaced (3.668%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:15:10.057 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:10.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:10.170 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:10.230 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:10.335 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:10.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:10.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:10.447 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:10.495 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:10.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3296409754' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:15:10.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:15:10.568 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:15:10.577 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:10.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:10.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:10.684 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:10.764 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:10.862 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:10.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:10.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3296409754' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:15:10.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:15:10.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3296409754' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:15:10.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:15:10.976 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:11.034 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:11.226 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:11.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:11.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:11.338 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:11.367 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:11.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[213871]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:15:11.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[213871]: osdmap e451: 8 total, 8 up, 6 in 2023-09-23T20:15:11.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[213871]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[213871]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[213871]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[213871]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[213871]: pgmap v1251: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 930 op/s; 47695/245781 objects degraded (19.405%); 9044/245781 objects misplaced (3.680%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T20:15:11.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[221113]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:15:11.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[221113]: osdmap e451: 8 total, 8 up, 6 in 2023-09-23T20:15:11.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[221113]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[221113]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[221113]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[221113]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:11 smithi135 ceph-mon[221113]: pgmap v1251: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 930 op/s; 47695/245781 objects degraded (19.405%); 9044/245781 objects misplaced (3.680%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T20:15:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:11 smithi144 ceph-mon[194833]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:11 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:15:11.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:11 smithi144 ceph-mon[194833]: osdmap e451: 8 total, 8 up, 6 in 2023-09-23T20:15:11.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:11 smithi144 ceph-mon[194833]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:11 smithi144 ceph-mon[194833]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:11 smithi144 ceph-mon[194833]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:11 smithi144 ceph-mon[194833]: failed to encode map e451 with expected crc 2023-09-23T20:15:11.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:11 smithi144 ceph-mon[194833]: pgmap v1251: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 930 op/s; 47695/245781 objects degraded (19.405%); 9044/245781 objects misplaced (3.680%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T20:15:11.885 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:11.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:11.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:11.933 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:15:11.949 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:11.991 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:12.072 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:12.265 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:12.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:12.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:12.371 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:12.457 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:15:12.457 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:15:12.457 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T20:15:12.458 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:15:12.461 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:12.583 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:12.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:12.631 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:15:12.692 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:12.757 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4274519086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:12.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:12 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:12.910 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:12.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:12.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:12.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4274519086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:12.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:12 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:12.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4274519086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:12.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:12 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:13.021 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:13.067 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:13.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:13 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:13.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:13 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:15:13.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:13 smithi144 ceph-mon[194833]: osdmap e452: 8 total, 8 up, 6 in 2023-09-23T20:15:13.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:13 smithi144 ceph-mon[194833]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:13 smithi144 ceph-mon[194833]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:13 smithi144 ceph-mon[194833]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:13 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 47695/245781 objects degraded (19.405%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:13.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:13 smithi144 ceph-mon[194833]: pgmap v1253: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 47695/244641 objects degraded (19.496%); 9044/244641 objects misplaced (3.697%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:15:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:15:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:15:13] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-09-23T20:15:13.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:13.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:15:13.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[213871]: osdmap e452: 8 total, 8 up, 6 in 2023-09-23T20:15:13.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[213871]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[213871]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[213871]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 47695/245781 objects degraded (19.405%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:13.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[213871]: pgmap v1253: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 47695/244641 objects degraded (19.496%); 9044/244641 objects misplaced (3.697%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:15:13.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:13.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:15:13.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[221113]: osdmap e452: 8 total, 8 up, 6 in 2023-09-23T20:15:13.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[221113]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[221113]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[221113]: failed to encode map e452 with expected crc 2023-09-23T20:15:13.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 47695/245781 objects degraded (19.405%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:13.973 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:13 smithi135 ceph-mon[221113]: pgmap v1253: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 47695/244641 objects degraded (19.496%); 9044/244641 objects misplaced (3.697%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:15:14.333 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:14.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:14.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:14.439 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:14.443 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:14.516 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:14.661 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:14.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:14.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:14.766 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:14.824 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:15.303 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:15.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:15.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:15.383 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:15:15.407 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:15.479 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:15.722 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:15.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:15.773 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:15:15.810 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:15:15.835 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:15.849 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:15:15.887 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:15:15.914 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:15.927 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:15:15.962 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:15:16.009 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:15:16.048 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:15:16.065 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:15:16.065 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:15:16.065 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:15:16.065 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:15:16.065 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:15:16.065 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:15:16.065 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:15:16.066 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:15:16.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:16 smithi135 ceph-mon[221113]: pgmap v1254: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 47135/233151 objects degraded (20.217%); 8386/233151 objects misplaced (3.597%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:15:16.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/337885106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:16 smithi135 ceph-mon[213871]: pgmap v1254: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 47135/233151 objects degraded (20.217%); 8386/233151 objects misplaced (3.597%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:15:16.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/337885106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:16.310 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:15:16.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:16 smithi144 ceph-mon[194833]: pgmap v1254: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 47135/233151 objects degraded (20.217%); 8386/233151 objects misplaced (3.597%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:15:16.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/337885106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:16.836 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:16.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:16.941 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:16.975 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:17.613 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:17.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:17.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:17.683 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:17.725 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:17.769 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:17.883 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:17.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:17.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:18.000 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:18.072 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:18.176 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:18.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:18 smithi135 ceph-mon[213871]: pgmap v1255: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 47135/231591 objects degraded (20.353%); 8386/231591 objects misplaced (3.621%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:15:18.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:18 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 47135/231591 objects degraded (20.353%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:18.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:18 smithi135 ceph-mon[221113]: pgmap v1255: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 47135/231591 objects degraded (20.353%); 8386/231591 objects misplaced (3.621%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:15:18.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:18 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 47135/231591 objects degraded (20.353%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:18.321 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:18.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:18.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:18 smithi144 ceph-mon[194833]: pgmap v1255: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 47135/231591 objects degraded (20.353%); 8386/231591 objects misplaced (3.621%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:15:18.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:18 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 47135/231591 objects degraded (20.353%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:18.433 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:18.521 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:18.631 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:18.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:18.741 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:18.792 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:18.942 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:18.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:18.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:19.050 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:19.119 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:19.159 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:15:19.181 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2400050087' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:15:19.181 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2400050087' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:15:19.219 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:19.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:19.336 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:19.364 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:19.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2400050087' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:15:19.690 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:15:20.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[213871]: pgmap v1256: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 46583/220440 objects degraded (21.132%); 7683/220440 objects misplaced (3.485%); 10 MiB/s, 160 objects/s recovering 2023-09-23T20:15:20.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[213871]: failed to encode map e453 with expected crc 2023-09-23T20:15:20.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2400050087' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:15:20.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[213871]: osdmap e453: 8 total, 8 up, 6 in 2023-09-23T20:15:20.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[213871]: failed to encode map e453 with expected crc 2023-09-23T20:15:20.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[213871]: failed to encode map e453 with expected crc 2023-09-23T20:15:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[221113]: pgmap v1256: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 46583/220440 objects degraded (21.132%); 7683/220440 objects misplaced (3.485%); 10 MiB/s, 160 objects/s recovering 2023-09-23T20:15:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[221113]: failed to encode map e453 with expected crc 2023-09-23T20:15:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2400050087' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:15:20.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[221113]: osdmap e453: 8 total, 8 up, 6 in 2023-09-23T20:15:20.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[221113]: failed to encode map e453 with expected crc 2023-09-23T20:15:20.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:20 smithi135 ceph-mon[221113]: failed to encode map e453 with expected crc 2023-09-23T20:15:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:20 smithi144 ceph-mon[194833]: pgmap v1256: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 46583/220440 objects degraded (21.132%); 7683/220440 objects misplaced (3.485%); 10 MiB/s, 160 objects/s recovering 2023-09-23T20:15:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:20 smithi144 ceph-mon[194833]: failed to encode map e453 with expected crc 2023-09-23T20:15:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2400050087' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:15:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:20 smithi144 ceph-mon[194833]: osdmap e453: 8 total, 8 up, 6 in 2023-09-23T20:15:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:20 smithi144 ceph-mon[194833]: failed to encode map e453 with expected crc 2023-09-23T20:15:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:20 smithi144 ceph-mon[194833]: failed to encode map e453 with expected crc 2023-09-23T20:15:21.111 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:15:21.140 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:21.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:21.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:21.245 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:21.298 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:21.383 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[221113]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.383 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[221113]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.383 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[221113]: osdmap e454: 8 total, 8 up, 6 in 2023-09-23T20:15:21.383 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[221113]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[221113]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[221113]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[221113]: pgmap v1259: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 46583/219648 objects degraded (21.208%); 7683/219648 objects misplaced (3.498%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[213871]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[213871]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[213871]: osdmap e454: 8 total, 8 up, 6 in 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[213871]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[213871]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[213871]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.384 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:21 smithi135 ceph-mon[213871]: pgmap v1259: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 46583/219648 objects degraded (21.208%); 7683/219648 objects misplaced (3.498%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:15:21.385 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:21.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:21.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:21.493 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:21.582 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:21 smithi144 ceph-mon[194833]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:21 smithi144 ceph-mon[194833]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:21 smithi144 ceph-mon[194833]: osdmap e454: 8 total, 8 up, 6 in 2023-09-23T20:15:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:21 smithi144 ceph-mon[194833]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:21 smithi144 ceph-mon[194833]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:21 smithi144 ceph-mon[194833]: failed to encode map e454 with expected crc 2023-09-23T20:15:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:21 smithi144 ceph-mon[194833]: pgmap v1259: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 46583/219648 objects degraded (21.208%); 7683/219648 objects misplaced (3.498%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:15:21.815 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:21.831 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:21.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:21.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:21.935 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:21.985 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:22.013 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:15:22.013 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:15:22.084 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:22.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:22.198 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:22.268 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:22 smithi144 ceph-mon[194833]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.268 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:22 smithi144 ceph-mon[194833]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.268 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:22 smithi144 ceph-mon[194833]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.269 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:22 smithi144 ceph-mon[194833]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.269 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:22 smithi144 ceph-mon[194833]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.269 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:22 smithi144 ceph-mon[194833]: osdmap e455: 8 total, 8 up, 6 in 2023-09-23T20:15:22.269 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:22.278 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:22.372 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:22.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:22.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[213871]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[213871]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[213871]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[213871]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[213871]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[213871]: osdmap e455: 8 total, 8 up, 6 in 2023-09-23T20:15:22.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[221113]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[221113]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[221113]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[221113]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[221113]: failed to encode map e455 with expected crc 2023-09-23T20:15:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[221113]: osdmap e455: 8 total, 8 up, 6 in 2023-09-23T20:15:22.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:22.479 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:22.543 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:22.857 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:22.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:22.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:22.966 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:23.040 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:23.147 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:23.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:23.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:23.204 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:15:23.257 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:23.330 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:23 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2403113531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:15:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 46583/219648 objects degraded (21.208%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:23.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:23 smithi144 ceph-mon[194833]: pgmap v1261: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 46583/218442 objects degraded (21.325%); 7683/218442 objects misplaced (3.517%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:15:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:23 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2403113531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:15:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 46583/219648 objects degraded (21.208%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:23.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:23 smithi135 ceph-mon[213871]: pgmap v1261: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 46583/218442 objects degraded (21.325%); 7683/218442 objects misplaced (3.517%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:15:23.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:23 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2403113531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:15:23.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 46583/219648 objects degraded (21.208%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:23.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:23 smithi135 ceph-mon[221113]: pgmap v1261: 129 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 46583/218442 objects degraded (21.325%); 7683/218442 objects misplaced (3.517%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:15:23.779 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:23.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:23.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:23.884 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:23.933 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:15:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:15:23] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2023-09-23T20:15:24.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:24.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[221113]: failed to encode map e456 with expected crc 2023-09-23T20:15:24.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[221113]: failed to encode map e456 with expected crc 2023-09-23T20:15:24.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2403113531' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:15:24.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[221113]: osdmap e456: 8 total, 8 up, 6 in 2023-09-23T20:15:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[213871]: failed to encode map e456 with expected crc 2023-09-23T20:15:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[213871]: failed to encode map e456 with expected crc 2023-09-23T20:15:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2403113531' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:15:24.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:24 smithi135 ceph-mon[213871]: osdmap e456: 8 total, 8 up, 6 in 2023-09-23T20:15:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:24 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:24 smithi144 ceph-mon[194833]: failed to encode map e456 with expected crc 2023-09-23T20:15:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:24 smithi144 ceph-mon[194833]: failed to encode map e456 with expected crc 2023-09-23T20:15:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2403113531' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:15:24.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:24 smithi144 ceph-mon[194833]: osdmap e456: 8 total, 8 up, 6 in 2023-09-23T20:15:24.831 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:15:25.453 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:25 smithi135 ceph-mon[221113]: pgmap v1263: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 46026/207216 objects degraded (22.212%); 7011/207216 objects misplaced (3.383%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T20:15:25.453 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:25 smithi135 ceph-mon[213871]: pgmap v1263: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 46026/207216 objects degraded (22.212%); 7011/207216 objects misplaced (3.383%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T20:15:25.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:25 smithi144 ceph-mon[194833]: pgmap v1263: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 46026/207216 objects degraded (22.212%); 7011/207216 objects misplaced (3.383%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T20:15:25.686 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:25.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:25.764 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:25.794 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:25.856 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:25.963 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:25.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:25.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:26.073 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:26.147 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:26.249 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:26.291 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:26.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:26.396 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:26.437 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:15:26.464 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3874874398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:26.475 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:26.484 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:15:26.495 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:15:26.527 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:15:26.571 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:15:26.589 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:26.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:26.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:26.603 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:15:26.639 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:15:26.674 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:15:26.678 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:15:26.679 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:15:26.679 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:15:26.679 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:15:26.679 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:15:26.679 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:15:26.679 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:15:26.679 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:15:26.698 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:26.715 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3874874398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:26.774 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:26.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3874874398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:27.722 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:27.730 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:27 smithi135 ceph-mon[213871]: pgmap v1264: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 46026/205662 objects degraded (22.379%); 7011/205662 objects misplaced (3.409%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:15:27.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:27 smithi135 ceph-mon[221113]: pgmap v1264: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 46026/205662 objects degraded (22.379%); 7011/205662 objects misplaced (3.409%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:15:27.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:27.795 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:15:27.832 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:27 smithi144 ceph-mon[194833]: pgmap v1264: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 46026/205662 objects degraded (22.379%); 7011/205662 objects misplaced (3.409%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:15:27.917 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:28.653 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:28.660 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 46026/205662 objects degraded (22.379%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:28.661 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 46026/205662 objects degraded (22.379%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:28.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:28.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:28.763 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:28.809 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 46026/205662 objects degraded (22.379%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:28.812 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:28.923 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:28.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:29.033 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:29.104 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:29.199 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:29.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:29.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:29.304 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:29.332 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:29.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:29 smithi135 ceph-mon[221113]: pgmap v1265: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 45466/194628 objects degraded (23.360%); 6750/194628 objects misplaced (3.468%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:15:29.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:29 smithi135 ceph-mon[213871]: pgmap v1265: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 45466/194628 objects degraded (23.360%); 6750/194628 objects misplaced (3.468%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:15:29.751 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:29.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:29.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:29.866 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:29.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:29 smithi144 ceph-mon[194833]: pgmap v1265: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 45466/194628 objects degraded (23.360%); 6750/194628 objects misplaced (3.468%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:15:29.922 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:30.018 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:30.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:30.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:30.126 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:30.197 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:30.197 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:30.273 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:30.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:30.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:30.379 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:30.452 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:30.498 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:30 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:30.645 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:30 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:30.646 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:30.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:30 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:30.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:30.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:30.690 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:15:30.690 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:15:30.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:15:30.690 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:15:30.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116287 8.7 8:e86b327e:::benchmark_data_smithi135_460343_object32054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:30.976436+0000", 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 359.708280409, 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6110225419999999, 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116287 2023-09-23T20:15:30.691 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976436+0000", 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976436+0000", 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976434+0000", 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976442+0000", 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3610000000000003e-06 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976458+0000", 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6008999999999999e-05 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:30.976468+0000", 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0068000000000001e-05 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.047679+0000", 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.071211117000000004 2023-09-23T20:15:30.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.057547+0000", 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0098680380000000009 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.085877+0000", 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.028330208999999999 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.091492+0000", 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056143829999999997 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.418689+0000", 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.327197631 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.432118+0000", 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013428544000000001 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.456884+0000", 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024766474 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.498418+0000", 2023-09-23T20:15:30.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.041533931000000003 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.545058+0000", 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.046639997000000002 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.548122+0000", 2023-09-23T20:15:30.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003063891 2023-09-23T20:15:30.699 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.699 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.699 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.699 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.553892+0000", 2023-09-23T20:15:30.701 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0057700349999999997 2023-09-23T20:15:30.701 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.701 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.701 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.702 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.587458+0000", 2023-09-23T20:15:30.702 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.033566209999999999 2023-09-23T20:15:30.702 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.702 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.702 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.702 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.702 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.702 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116291 8.7 8:e6b5a36d:::benchmark_data_smithi135_460343_object32058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.703 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.051182+0000", 2023-09-23T20:15:30.703 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 359.63353472300003, 2023-09-23T20:15:30.703 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6927078039999999, 2023-09-23T20:15:30.703 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.703 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.703 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.703 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.704 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.704 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116291 2023-09-23T20:15:30.704 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.704 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.704 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.704 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.705 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051182+0000", 2023-09-23T20:15:30.705 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.705 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.705 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.705 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.705 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051182+0000", 2023-09-23T20:15:30.705 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.706 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.706 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.706 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.706 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051180+0000", 2023-09-23T20:15:30.706 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:15:30.706 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.707 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.707 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051189+0000", 2023-09-23T20:15:30.707 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9240000000000002e-06 2023-09-23T20:15:30.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.707 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051207+0000", 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7159e-05 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.051216+0000", 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3489999999999998e-06 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.708 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.058998+0000", 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007781769 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.065129+0000", 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061313779999999998 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.709 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.111839+0000", 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.046710086999999997 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.117249+0000", 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0054098159999999996 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.710 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.630036+0000", 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.512786883 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.637705+0000", 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0076694720000000001 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.653502+0000", 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015796528000000001 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.718486+0000", 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.064984564999999994 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.712 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.719651+0000", 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001165031 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.722210+0000", 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025589889999999998 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.713 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.726730+0000", 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004519866 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.743889+0000", 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017158965000000002 2023-09-23T20:15:30.714 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116311 8.f 8:f8d2a2d2:::benchmark_data_smithi135_460343_object32078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.295114+0000", 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 359.38960183, 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6782180090000001, 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.715 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116311 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295114+0000", 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.716 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295114+0000", 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295113+0000", 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.717 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295121+0000", 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0660000000000004e-06 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295136+0000", 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4980000000000001e-05 2023-09-23T20:15:30.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.295144+0000", 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7789999999999996e-06 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.336228+0000", 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.041084794000000001 2023-09-23T20:15:30.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.345483+0000", 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0092550489999999996 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.401553+0000", 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.056069359999999999 2023-09-23T20:15:30.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.411366+0000", 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0098127839999999997 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.914818+0000", 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5034523229999999 2023-09-23T20:15:30.721 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.921608+0000", 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067902980000000002 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.936336+0000", 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014727888999999999 2023-09-23T20:15:30.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.950371+0000", 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014034744 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.953642+0000", 2023-09-23T20:15:30.723 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003270911 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.955053+0000", 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014112020000000001 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.959874+0000", 2023-09-23T20:15:30.724 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048215749999999998 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.973332+0000", 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013457920999999999 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.725 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116312 8.7 8:e7a522c9:::benchmark_data_smithi135_460343_object32079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.297531+0000", 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 359.38718489500002, 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.711426772, 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.726 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116312 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297531+0000", 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.727 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297531+0000", 2023-09-23T20:15:30.728 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.728 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.728 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.728 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.728 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297531+0000", 2023-09-23T20:15:30.728 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:15:30.728 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297535+0000", 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4750000000000004e-06 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297545+0000", 2023-09-23T20:15:30.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7459999999999995e-06 2023-09-23T20:15:30.730 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.730 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.730 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.730 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.297550+0000", 2023-09-23T20:15:30.730 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7189999999999998e-06 2023-09-23T20:15:30.730 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.730 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.730 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.731 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.336588+0000", 2023-09-23T20:15:30.731 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.039037534999999998 2023-09-23T20:15:30.731 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.731 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.731 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.731 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.345661+0000", 2023-09-23T20:15:30.731 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0090730800000000007 2023-09-23T20:15:30.731 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.732 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.732 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.401924+0000", 2023-09-23T20:15:30.732 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.056263201999999998 2023-09-23T20:15:30.732 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.732 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.732 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.412620+0000", 2023-09-23T20:15:30.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010695377000000001 2023-09-23T20:15:30.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.733 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.733 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.917825+0000", 2023-09-23T20:15:30.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5052050750000001 2023-09-23T20:15:30.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.733 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.734 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.926015+0000", 2023-09-23T20:15:30.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081900619999999997 2023-09-23T20:15:30.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.734 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.734 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.939379+0000", 2023-09-23T20:15:30.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013364467 2023-09-23T20:15:30.735 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.735 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.735 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.954191+0000", 2023-09-23T20:15:30.735 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014811892 2023-09-23T20:15:30.735 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.735 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.736 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.957357+0000", 2023-09-23T20:15:30.736 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031655889999999999 2023-09-23T20:15:30.736 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.736 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.736 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.736 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.974900+0000", 2023-09-23T20:15:30.736 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.017543361 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.980566+0000", 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0056655630000000002 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.737 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.008958+0000", 2023-09-23T20:15:30.738 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.028392463999999999 2023-09-23T20:15:30.738 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.738 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.738 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.738 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.738 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.739 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116323 8.f 8:f7cc7664:::benchmark_data_smithi135_460343_object32090:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.739 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.357259+0000", 2023-09-23T20:15:30.739 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 359.32745773099998, 2023-09-23T20:15:30.739 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7112210409999999, 2023-09-23T20:15:30.739 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.739 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.739 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.740 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.740 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.740 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116323 2023-09-23T20:15:30.740 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.740 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.740 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.740 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357259+0000", 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357259+0000", 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357257+0000", 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357266+0000", 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0759999999999992e-06 2023-09-23T20:15:30.742 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357278+0000", 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2388999999999999e-05 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.357287+0000", 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4870000000000002e-06 2023-09-23T20:15:30.743 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.451249+0000", 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.093962813000000006 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.462359+0000", 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01111006 2023-09-23T20:15:30.744 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.521611+0000", 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.059251486999999999 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.536134+0000", 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014522765 2023-09-23T20:15:30.745 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.004103+0000", 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.467969313 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.015518+0000", 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011414565 2023-09-23T20:15:30.746 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.031900+0000", 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016382906999999999 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.051050+0000", 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019149817 2023-09-23T20:15:30.747 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.051647+0000", 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00059682099999999998 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.053566+0000", 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00191895 2023-09-23T20:15:30.748 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.056171+0000", 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002605357 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.068480+0000", 2023-09-23T20:15:30.749 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012308127 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116336 8.7 8:ee959f5b:::benchmark_data_smithi135_460343_object32103:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.535286+0000", 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 359.14943065699998, 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6772203409999999, 2023-09-23T20:15:30.750 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116336 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.751 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535286+0000", 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535286+0000", 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.752 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535284+0000", 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535297+0000", 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2571e-05 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.753 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535308+0000", 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0910999999999999e-05 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.535314+0000", 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0209999999999996e-06 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.754 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.541877+0000", 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0065639729999999999 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.547255+0000", 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053776240000000001 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.755 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.639621+0000", 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.092366243000000001 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.645009+0000", 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0053872670000000003 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.756 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.150126+0000", 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5051173279999999 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.155252+0000", 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051262410000000001 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.757 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.163894+0000", 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0086421810000000005 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.174763+0000", 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010868681 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.758 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.176261+0000", 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014980390000000001 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.197173+0000", 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.020911762 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.759 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.200121+0000", 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029481490000000002 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.212506+0000", 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012384904 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.760 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116353 8.7 8:ecc5e610:::benchmark_data_smithi135_460343_object32120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.633770+0000", 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 359.05094598599999, 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6240762310000001, 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.761 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.762 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.762 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.762 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116353 2023-09-23T20:15:30.762 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.762 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.762 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.762 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633770+0000", 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633770+0000", 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.763 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633769+0000", 2023-09-23T20:15:30.764 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.764 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.764 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.764 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633776+0000", 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5170000000000002e-06 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633790+0000", 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3749e-05 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.633797+0000", 2023-09-23T20:15:30.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9399999999999996e-06 2023-09-23T20:15:30.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.766 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.663793+0000", 2023-09-23T20:15:30.766 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029996364000000001 2023-09-23T20:15:30.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.670817+0000", 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007023334 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.751707+0000", 2023-09-23T20:15:30.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.080890225999999996 2023-09-23T20:15:30.768 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.768 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.768 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.768 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.758423+0000", 2023-09-23T20:15:30.768 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067167019999999997 2023-09-23T20:15:30.768 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.768 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.213064+0000", 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.454640854 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.217981+0000", 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049167359999999997 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.225569+0000", 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0075880000000000001 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.240287+0000", 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014717701999999999 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.240780+0000", 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00049317100000000004 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.242363+0000", 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015826379999999999 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.245271+0000", 2023-09-23T20:15:30.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029087150000000001 2023-09-23T20:15:30.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.257846+0000", 2023-09-23T20:15:30.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012575197999999999 2023-09-23T20:15:30.773 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.773 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.773 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.773 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.773 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116354 8.f 8:f582891d:::benchmark_data_smithi135_460343_object32121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.773 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:31.651719+0000", 2023-09-23T20:15:30.774 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 359.03299710099998, 2023-09-23T20:15:30.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.688988763, 2023-09-23T20:15:30.774 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.774 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.774 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.774 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.774 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.775 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116354 2023-09-23T20:15:30.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.775 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.775 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.775 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.775 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651719+0000", 2023-09-23T20:15:30.776 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.776 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.776 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651719+0000", 2023-09-23T20:15:30.776 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651718+0000", 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651726+0000", 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6769999999999995e-06 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651740+0000", 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4953999999999999e-05 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.651749+0000", 2023-09-23T20:15:30.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9050000000000003e-06 2023-09-23T20:15:30.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.664771+0000", 2023-09-23T20:15:30.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013021807999999999 2023-09-23T20:15:30.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.676017+0000", 2023-09-23T20:15:30.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011246143 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.759969+0000", 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.083951210999999998 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.782 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:31.771322+0000", 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011353439 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.283281+0000", 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5119590860000001 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.289058+0000", 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0057770820000000002 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.299557+0000", 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010498664 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.310230+0000", 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010673498 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.312280+0000", 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002049309 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.313009+0000", 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00072948999999999996 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.316874+0000", 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038645979999999999 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.786 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.340708+0000", 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.023834191000000001 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116396 8.f 8:f2fcea18:::benchmark_data_smithi135_460343_object32163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:32.088587+0000", 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 358.59612903999999, 2023-09-23T20:15:30.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6114060879999998, 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116396 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088587+0000", 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088587+0000", 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088586+0000", 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088594+0000", 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9899999999999997e-06 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.790 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088606+0000", 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2315e-05 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.088613+0000", 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6320000000000002e-06 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.227428+0000", 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13881549100000001 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.236699+0000", 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0092712660000000002 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.288248+0000", 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.051549141 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.295587+0000", 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073383440000000001 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.658757+0000", 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3631700179999999 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.663837+0000", 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050801659999999997 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.794 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.672583+0000", 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0087457950000000007 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.680624+0000", 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080414230000000007 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.684408+0000", 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003783818 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.685105+0000", 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00069654199999999997 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.796 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.687598+0000", 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024939239999999998 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.699993+0000", 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012394789 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.797 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:116398 8.f 8:f8620ac9:::benchmark_data_smithi135_460343_object32165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:32.098708+0000", 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 358.586008494, 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6899823980000002, 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.798 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 116398 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098708+0000", 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098708+0000", 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098706+0000", 2023-09-23T20:15:30.800 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.801 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098715+0000", 2023-09-23T20:15:30.801 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0599999999999997e-06 2023-09-23T20:15:30.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098731+0000", 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5438e-05 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.098739+0000", 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2470000000000008e-06 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.803 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.803 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.240960+0000", 2023-09-23T20:15:30.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14222137600000001 2023-09-23T20:15:30.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.803 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.803 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.249002+0000", 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080412609999999992 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.319967+0000", 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.070965500000000001 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:32.326060+0000", 2023-09-23T20:15:30.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0060927430000000003 2023-09-23T20:15:30.805 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.805 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.805 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.713791+0000", 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3877313149999999 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.718352+0000", 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045612430000000004 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.806 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.726422+0000", 2023-09-23T20:15:30.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0080694430000000008 2023-09-23T20:15:30.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.807 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.769597+0000", 2023-09-23T20:15:30.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.043174953000000002 2023-09-23T20:15:30.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.771434+0000", 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018374450000000001 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.773270+0000", 2023-09-23T20:15:30.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018357479999999999 2023-09-23T20:15:30.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.776449+0000", 2023-09-23T20:15:30.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003178934 2023-09-23T20:15:30.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:33.788690+0000", 2023-09-23T20:15:30.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01224129 2023-09-23T20:15:30.810 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.810 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.810 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.811 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:117890 8.f 8:f3976cab:::benchmark_data_smithi135_460343_object33657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:15:30.811 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:48.672824+0000", 2023-09-23T20:15:30.811 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 342.01189209400002, 2023-09-23T20:15:30.811 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.58877402, 2023-09-23T20:15:30.811 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.811 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.811 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.812 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.812 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.812 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117890 2023-09-23T20:15:30.812 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.812 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.812 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672824+0000", 2023-09-23T20:15:30.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672824+0000", 2023-09-23T20:15:30.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672823+0000", 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672830+0000", 2023-09-23T20:15:30.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8759999999999999e-06 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672841+0000", 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0763e-05 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.672847+0000", 2023-09-23T20:15:30.815 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.737e-06 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.674677+0000", 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001829458 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.683189+0000", 2023-09-23T20:15:30.816 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.008512759 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.738282+0000", 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.055092231999999998 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:48.745725+0000", 2023-09-23T20:15:30.817 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0074430210000000002 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.088007+0000", 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3422824659999999 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.093539+0000", 2023-09-23T20:15:30.818 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0055319289999999997 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.103612+0000", 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010072561000000001 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.220680+0000", 2023-09-23T20:15:30.819 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.117068696 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.222650+0000", 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001969518 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.820 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.247834+0000", 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.025184372 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.250538+0000", 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027036679999999998 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.821 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.261598+0000", 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011060199 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:117909 8.f 8:fab9314c:::benchmark_data_smithi135_460343_object33676:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:15:30.822 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:49.049290+0000", 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 341.63542604399998, 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5971690810000001, 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117909 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.823 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049290+0000", 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049290+0000", 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.824 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049288+0000", 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049300+0000", 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2026e-05 2023-09-23T20:15:30.825 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049320+0000", 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0078e-05 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.049330+0000", 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9110000000000005e-06 2023-09-23T20:15:30.826 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.170287+0000", 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.120956735 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.177100+0000", 2023-09-23T20:15:30.827 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0068132369999999998 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.220172+0000", 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.043072023000000001 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.232953+0000", 2023-09-23T20:15:30.828 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012781073 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.576010+0000", 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3430568809999999 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.580714+0000", 2023-09-23T20:15:30.829 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047042589999999997 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.590160+0000", 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0094457099999999995 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.830 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.630047+0000", 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.039886963999999997 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.630551+0000", 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050392899999999997 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.831 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.632109+0000", 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015582059999999999 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.634635+0000", 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025259560000000002 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.832 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.646459+0000", 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011824328 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:117927 8.f 8:f5e97549:::benchmark_data_smithi135_460343_object33694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:09:49.254223+0000", 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 341.43049313, 2023-09-23T20:15:30.833 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5708444940000001, 2023-09-23T20:15:30.834 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.834 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.834 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.834 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.834 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.834 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 117927 2023-09-23T20:15:30.834 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.834 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.835 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254223+0000", 2023-09-23T20:15:30.835 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.835 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.835 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.835 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.835 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254223+0000", 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254220+0000", 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999971 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.836 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254233+0000", 2023-09-23T20:15:30.837 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2813000000000001e-05 2023-09-23T20:15:30.837 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.837 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.837 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.837 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254253+0000", 2023-09-23T20:15:30.837 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0253000000000001e-05 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.254265+0000", 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1114999999999999e-05 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.838 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.288521+0000", 2023-09-23T20:15:30.839 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.034256712000000002 2023-09-23T20:15:30.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.839 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.839 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.839 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.301312+0000", 2023-09-23T20:15:30.839 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012790920000000001 2023-09-23T20:15:30.839 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.840 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.840 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.417009+0000", 2023-09-23T20:15:30.840 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.11569644599999999 2023-09-23T20:15:30.840 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.840 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.840 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.840 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:49.424745+0000", 2023-09-23T20:15:30.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0077367269999999997 2023-09-23T20:15:30.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.841 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.841 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.841 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.726652+0000", 2023-09-23T20:15:30.841 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.301907047 2023-09-23T20:15:30.841 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.735070+0000", 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0084176070000000006 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.755181+0000", 2023-09-23T20:15:30.842 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.020110801000000001 2023-09-23T20:15:30.843 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.843 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.843 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.843 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.810068+0000", 2023-09-23T20:15:30.843 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.054886783000000001 2023-09-23T20:15:30.843 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.843 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.844 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.844 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.810571+0000", 2023-09-23T20:15:30.844 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050297199999999997 2023-09-23T20:15:30.844 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.844 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.844 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.844 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.812309+0000", 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001738327 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.814876+0000", 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025667760000000002 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.845 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:09:50.825068+0000", 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010191891999999999 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123384 8.f 8:fe9c53da:::benchmark_data_smithi135_460343_object39151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 288.77869762300003, 2023-09-23T20:15:30.846 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4434314349999999, 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123384 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.847 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.848 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:15:30.848 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.848 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.848 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.848 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.848 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:15:30.849 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.849 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.849 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.849 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.849 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906017+0000", 2023-09-23T20:15:30.849 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.849 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.850 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.850 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.850 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906025+0000", 2023-09-23T20:15:30.850 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6260000000000003e-06 2023-09-23T20:15:30.850 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.850 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906041+0000", 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6183000000000001e-05 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906048+0000", 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7000000000000008e-06 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.851 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.969587+0000", 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.063539010000000007 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.974774+0000", 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051870179999999998 2023-09-23T20:15:30.852 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.005518+0000", 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030743594999999999 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.011676+0000", 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061576579999999999 2023-09-23T20:15:30.853 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.259997+0000", 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2483216180000001 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.274504+0000", 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014506939999999999 2023-09-23T20:15:30.854 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.290807+0000", 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016302543999999999 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.312744+0000", 2023-09-23T20:15:30.855 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021937076999999999 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.314532+0000", 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017877699999999999 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.336416+0000", 2023-09-23T20:15:30.856 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021884023999999998 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.338670+0000", 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022543989999999998 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.857 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.349450+0000", 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010779924 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123388 8.f 8:f5171e3d:::benchmark_data_smithi135_460343_object39155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:15:30.858 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 288.60610023200002, 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5028154900000001, 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123388 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.859 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.860 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078615+0000", 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078622+0000", 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.345e-06 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.861 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078636+0000", 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3645000000000001e-05 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078642+0000", 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5810000000000002e-06 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.862 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.092915+0000", 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014273083000000001 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.098087+0000", 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051715119999999996 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.863 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.864 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.137553+0000", 2023-09-23T20:15:30.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.039466342000000001 2023-09-23T20:15:30.864 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.864 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.864 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.864 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.146411+0000", 2023-09-23T20:15:30.864 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088579100000000001 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.513388+0000", 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.366976599 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.865 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.518018+0000", 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046298229999999999 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.527773+0000", 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097548610000000001 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.866 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.566237+0000", 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.038464921999999999 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.567647+0000", 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001409684 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.867 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.568272+0000", 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00062472500000000002 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.570874+0000", 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026022689999999999 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.868 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.581431+0000", 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010557383 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123394 8.f 8:ff01962c:::benchmark_data_smithi135_460343_object39161:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:15:30.869 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 288.48433786300001, 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.413050616, 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123394 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.870 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.871 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.871 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:15:30.871 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.871 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.871 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.871 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.871 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:15:30.871 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.872 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.872 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.872 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.872 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200376+0000", 2023-09-23T20:15:30.872 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:15:30.872 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.872 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.872 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200388+0000", 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2403e-05 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200413+0000", 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4596999999999998e-05 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.873 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200430+0000", 2023-09-23T20:15:30.874 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6908000000000001e-05 2023-09-23T20:15:30.874 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.874 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.874 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.874 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.221890+0000", 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021460679999999999 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.238665+0000", 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0167742 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.875 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.309438+0000", 2023-09-23T20:15:30.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.070773321 2023-09-23T20:15:30.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.876 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.876 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.876 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.319840+0000", 2023-09-23T20:15:30.876 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010402428 2023-09-23T20:15:30.876 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.545580+0000", 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.225739777 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.550259+0000", 2023-09-23T20:15:30.877 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046789290000000001 2023-09-23T20:15:30.878 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.878 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.878 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.878 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.558399+0000", 2023-09-23T20:15:30.878 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081401330000000008 2023-09-23T20:15:30.878 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.878 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.878 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.879 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.583234+0000", 2023-09-23T20:15:30.879 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024835052999999999 2023-09-23T20:15:30.879 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.879 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.879 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.879 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.585402+0000", 2023-09-23T20:15:30.879 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0021673310000000002 2023-09-23T20:15:30.879 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.586049+0000", 2023-09-23T20:15:30.880 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00064698200000000005 2023-09-23T20:15:30.880 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.880 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.880 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.880 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.591270+0000", 2023-09-23T20:15:30.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052211779999999999 2023-09-23T20:15:30.881 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.881 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.881 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.881 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.613429+0000", 2023-09-23T20:15:30.881 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.022159160000000001 2023-09-23T20:15:30.881 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.882 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.882 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.882 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.882 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.882 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123402 8.7 8:ec120430:::benchmark_data_smithi135_460343_object39169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:15:30.882 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:15:30.883 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 288.43158793499998, 2023-09-23T20:15:30.883 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4055715360000001, 2023-09-23T20:15:30.883 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.883 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.883 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.883 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.883 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.884 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123402 2023-09-23T20:15:30.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.884 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.884 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.884 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.884 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:15:30.884 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.884 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253126+0000", 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:15:30.885 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.886 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.886 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.886 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253138+0000", 2023-09-23T20:15:30.886 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1725e-05 2023-09-23T20:15:30.886 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.886 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.886 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.886 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253158+0000", 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0772999999999999e-05 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253169+0000", 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.022e-05 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.887 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.321269+0000", 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.068100777000000001 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.331911+0000", 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010641866999999999 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.888 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.411508+0000", 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.079596497000000002 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.422363+0000", 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010854744 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.889 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.612810+0000", 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1904476230000001 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.617814+0000", 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050038449999999998 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.890 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.627404+0000", 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0095902439999999995 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.643653+0000", 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016248987999999999 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.891 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.645238+0000", 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00158438 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.645689+0000", 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045135999999999999 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.892 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.647776+0000", 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020871879999999998 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.658700+0000", 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010923696 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.893 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123442 8.f 8:f6660517:::benchmark_data_smithi135_460343_object39209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 288.148531144, 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4673699600000001, 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.894 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123442 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.895 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536183+0000", 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.896 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536192+0000", 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1039999999999996e-06 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536207+0000", 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4834000000000001e-05 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.897 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536219+0000", 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1691999999999999e-05 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.615218+0000", 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.078999571000000005 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.898 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.620140+0000", 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049211439999999997 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.675848+0000", 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.055708879000000003 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.899 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.680973+0000", 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051250449999999999 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.828876+0000", 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1479029780000001 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.900 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.834877+0000", 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0060005409999999999 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.844577+0000", 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097000209999999996 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.901 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.858053+0000", 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013476228 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.859480+0000", 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001426407 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.902 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.991498+0000", 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13201818500000001 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.993799+0000", 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023010399999999999 2023-09-23T20:15:30.903 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.003555+0000", 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097562010000000008 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.904 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.905 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123449 8.f 8:f357d011:::benchmark_data_smithi135_460343_object39216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:15:30.905 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:15:30.905 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 288.11384663199999, 2023-09-23T20:15:30.905 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5039241049999998, 2023-09-23T20:15:30.905 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.905 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.905 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123449 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:15:30.906 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.907 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.907 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.907 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.907 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:15:30.907 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.907 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570867+0000", 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999971 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570879+0000", 2023-09-23T20:15:30.908 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2435e-05 2023-09-23T20:15:30.909 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.909 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.909 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.909 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570900+0000", 2023-09-23T20:15:30.909 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0509999999999998e-05 2023-09-23T20:15:30.909 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.909 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.909 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.910 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570919+0000", 2023-09-23T20:15:30.910 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8947999999999999e-05 2023-09-23T20:15:30.910 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.910 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.910 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.910 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.620845+0000", 2023-09-23T20:15:30.910 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.049925803999999997 2023-09-23T20:15:30.910 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.911 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.911 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.911 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.626417+0000", 2023-09-23T20:15:30.911 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0055726200000000004 2023-09-23T20:15:30.911 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.911 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.698693+0000", 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.072275866999999994 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.703708+0000", 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050152590000000002 2023-09-23T20:15:30.912 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.913 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.913 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.913 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.016373+0000", 2023-09-23T20:15:30.913 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.312664507 2023-09-23T20:15:30.913 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.913 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.913 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.914 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.020480+0000", 2023-09-23T20:15:30.914 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041066089999999998 2023-09-23T20:15:30.914 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.914 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.914 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.914 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.028636+0000", 2023-09-23T20:15:30.914 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081566039999999996 2023-09-23T20:15:30.915 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.915 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.915 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.915 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.041139+0000", 2023-09-23T20:15:30.915 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012503377 2023-09-23T20:15:30.915 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.042755+0000", 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016158360000000001 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.061415+0000", 2023-09-23T20:15:30.916 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018660039 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.063733+0000", 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023174549999999999 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.074794+0000", 2023-09-23T20:15:30.917 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011060876000000001 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123630 8.7 8:e7d430c0:::benchmark_data_smithi135_460343_object39397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 286.361340482, 2023-09-23T20:15:30.918 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4124915579999999, 2023-09-23T20:15:30.919 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:15:30.919 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:15:30.919 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:15:30.919 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:15:30.919 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:15:30.919 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123630 2023-09-23T20:15:30.920 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.920 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:15:30.920 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.920 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:15:30.920 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:15:30.920 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323374+0000", 2023-09-23T20:15:30.921 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323381+0000", 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8360000000000003e-06 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323393+0000", 2023-09-23T20:15:30.922 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.226e-05 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323402+0000", 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9330000000000007e-06 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.394929+0000", 2023-09-23T20:15:30.923 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.071526432000000001 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.399736+0000", 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048073309999999998 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.434854+0000", 2023-09-23T20:15:30.924 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.035117795 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.439795+0000", 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049409620000000001 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.537860+0000", 2023-09-23T20:15:30.925 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0980649309999999 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.542831+0000", 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004970833 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:15:30.926 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.551732+0000", 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0089008879999999992 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.719631+0000", 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16789942299999999 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:15:30.927 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.721555+0000", 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019236259999999999 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.723874+0000", 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023192669999999999 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:15:30.928 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.726072+0000", 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0021980229999999999 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.735867+0000", 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097953720000000001 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.929 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:15:30.930 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:15:30.930 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:15:30.930 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:30.996 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:31.111 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:31.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:31.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:31.221 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:31.287 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:31.466 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:31.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:31.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:31.524 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:15:31.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:31 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/597574705' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:15:31.549 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:31 smithi135 ceph-mon[213871]: pgmap v1266: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 45466/193836 objects degraded (23.456%); 6427/193836 objects misplaced (3.316%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:15:31.549 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:31 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/597574705' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:15:31.549 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:31 smithi135 ceph-mon[221113]: pgmap v1266: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 45466/193836 objects degraded (23.456%); 6427/193836 objects misplaced (3.316%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:15:31.576 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:15:31.577 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:31.608 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:31 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/597574705' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:15:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:31 smithi144 ceph-mon[194833]: pgmap v1266: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 45466/193836 objects degraded (23.456%); 6427/193836 objects misplaced (3.316%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:15:31.794 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:31.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:31.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:31.897 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:31.966 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:32.173 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:32.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:32.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:32.282 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:32.315 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[221113]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/597574705' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:15:32.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[221113]: osdmap e457: 8 total, 8 up, 6 in 2023-09-23T20:15:32.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[221113]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[221113]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[221113]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[213871]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/597574705' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:15:32.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[213871]: osdmap e457: 8 total, 8 up, 6 in 2023-09-23T20:15:32.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[213871]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[213871]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:32 smithi135 ceph-mon[213871]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:32 smithi144 ceph-mon[194833]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/597574705' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:15:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:32 smithi144 ceph-mon[194833]: osdmap e457: 8 total, 8 up, 6 in 2023-09-23T20:15:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:32 smithi144 ceph-mon[194833]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:32 smithi144 ceph-mon[194833]: failed to encode map e457 with expected crc 2023-09-23T20:15:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:32 smithi144 ceph-mon[194833]: failed to encode map e457 with expected crc 2023-09-23T20:15:33.509 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:15:33.509 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:33.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:33.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:33.615 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:33.625 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:15:33.625 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:15:33.734 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:15:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:15:33] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2023-09-23T20:15:33.734 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 45466/193836 objects degraded (23.456%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:33.735 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:33 smithi135 ceph-mon[221113]: pgmap v1268: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 45466/192723 objects degraded (23.591%); 6427/192723 objects misplaced (3.335%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:15:33.735 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 45466/193836 objects degraded (23.456%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:33.735 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:33 smithi135 ceph-mon[213871]: pgmap v1268: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 45466/192723 objects degraded (23.591%); 6427/192723 objects misplaced (3.335%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:15:33.747 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:33.821 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:33.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:33.837 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 45466/193836 objects degraded (23.456%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:15:33.837 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:33 smithi144 ceph-mon[194833]: pgmap v1268: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 45466/192723 objects degraded (23.591%); 6427/192723 objects misplaced (3.335%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:15:33.863 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:33.937 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:34.056 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:34.460 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:34.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:34.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:34.570 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:34.642 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:34.957 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:34.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:34.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:35.061 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:35.132 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:36.170 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:15:36.179 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:36.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:36.286 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:36.356 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:36 smithi144 ceph-mon[194833]: pgmap v1269: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 44986/182742 objects degraded (24.617%); 5837/182742 objects misplaced (3.194%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:15:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3748463025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:36.457 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[213871]: pgmap v1269: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 44986/182742 objects degraded (24.617%); 5837/182742 objects misplaced (3.194%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:15:36.457 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3748463025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:36.457 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:36.457 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:36.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:36.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[221113]: pgmap v1269: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 44986/182742 objects degraded (24.617%); 5837/182742 objects misplaced (3.194%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:15:36.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3748463025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:36.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:36.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:36.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:37.173 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:37.211 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:37.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:37.266 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:37.326 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:37 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:15:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:37 smithi144 ceph-mon[194833]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:37 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:15:37.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:37 smithi144 ceph-mon[194833]: osdmap e458: 8 total, 8 up, 6 in 2023-09-23T20:15:37.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:37 smithi144 ceph-mon[194833]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:37 smithi144 ceph-mon[194833]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:37 smithi144 ceph-mon[194833]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.387 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:37.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:15:37.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[221113]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.427 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:15:37.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[221113]: osdmap e458: 8 total, 8 up, 6 in 2023-09-23T20:15:37.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[221113]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[221113]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[221113]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.432 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:15:37.432 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[213871]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.432 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:15:37.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[213871]: osdmap e458: 8 total, 8 up, 6 in 2023-09-23T20:15:37.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[213871]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[213871]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:37 smithi135 ceph-mon[213871]: failed to encode map e458 with expected crc 2023-09-23T20:15:37.498 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:37.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:37.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:37.606 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:37.659 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:37.755 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:37.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:37.862 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:37.915 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:37.933 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:15:37.970 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:15:38.004 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:15:38.021 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:38.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:38.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:38.044 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:15:38.080 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:15:38.127 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:15:38.141 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:38.172 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:15:38.214 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:15:38.218 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:15:38.219 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:15:38.219 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:15:38.219 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:15:38.219 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:15:38.219 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:15:38.219 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:15:38.219 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:15:38.221 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:38.253 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[213871]: pgmap v1271: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 553 op/s; 44986/181344 objects degraded (24.807%); 5837/181344 objects misplaced (3.219%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:15:38.253 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:38.253 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:38.253 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 44986/181344 objects degraded (24.807%), 21 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:38.253 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/145694109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:38.256 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[221113]: pgmap v1271: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 553 op/s; 44986/181344 objects degraded (24.807%); 5837/181344 objects misplaced (3.219%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:15:38.256 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:38.256 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:38.257 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 44986/181344 objects degraded (24.807%), 21 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:38.257 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/145694109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:38.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:38 smithi144 ceph-mon[194833]: pgmap v1271: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 553 op/s; 44986/181344 objects degraded (24.807%); 5837/181344 objects misplaced (3.219%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:15:38.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:38.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:38.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 44986/181344 objects degraded (24.807%), 21 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:38.385 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/145694109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:39.219 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:15:39.738 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:39.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:39.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:39.843 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:39.890 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:39.999 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:40.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:40.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:40.109 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:40.137 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:40.181 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:40.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:40.286 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:40.341 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:40.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:40 smithi135 ceph-mon[221113]: pgmap v1272: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 943 op/s; 44488/171192 objects degraded (25.987%); 5581/171192 objects misplaced (3.260%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:15:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:40 smithi135 ceph-mon[213871]: pgmap v1272: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 943 op/s; 44488/171192 objects degraded (25.987%); 5581/171192 objects misplaced (3.260%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:15:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:40 smithi144 ceph-mon[194833]: pgmap v1272: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 943 op/s; 44488/171192 objects degraded (25.987%); 5581/171192 objects misplaced (3.260%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:15:41.220 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:41.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:41.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:41.329 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:41.385 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:41.436 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:41 smithi135 ceph-mon[221113]: pgmap v1273: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 806 op/s; 44488/170730 objects degraded (26.058%); 5254/170730 objects misplaced (3.077%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T20:15:41.444 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:41 smithi135 ceph-mon[213871]: pgmap v1273: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 806 op/s; 44488/170730 objects degraded (26.058%); 5254/170730 objects misplaced (3.077%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T20:15:41.497 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:41.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:41.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:41.509 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:41 smithi144 ceph-mon[194833]: pgmap v1273: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 806 op/s; 44488/170730 objects degraded (26.058%); 5254/170730 objects misplaced (3.077%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T20:15:41.610 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:41.679 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:41.692 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:41.913 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:41.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:42.023 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:42.094 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:42.185 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:42.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:42.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:42.262 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:15:42.262 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:15:42.296 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:42.341 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:15:42.344 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:42.352 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:15:42.612 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:42.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:42.726 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:42.789 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:42.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 44488/170730 objects degraded (26.058%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:42.917 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:42.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:42.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:42.932 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 44488/170730 objects degraded (26.058%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:42.932 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 44488/170730 objects degraded (26.058%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:43.019 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:43.099 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:43.189 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:43.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:43.246 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:15:43.298 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:43.380 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:43.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:43 smithi135 ceph-mon[213871]: from='client.56134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:15:43.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:43 smithi135 ceph-mon[213871]: pgmap v1274: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 958 op/s; 43938/163956 objects degraded (26.799%); 5122/163956 objects misplaced (3.124%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:15:43.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:15:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:15:43] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-09-23T20:15:43.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:43 smithi135 ceph-mon[221113]: from='client.56134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:15:43.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:43 smithi135 ceph-mon[221113]: pgmap v1274: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 958 op/s; 43938/163956 objects degraded (26.799%); 5122/163956 objects misplaced (3.124%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:15:44.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:43 smithi144 ceph-mon[194833]: from='client.56134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:15:44.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:43 smithi144 ceph-mon[194833]: pgmap v1274: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 958 op/s; 43938/163956 objects degraded (26.799%); 5122/163956 objects misplaced (3.124%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:15:45.242 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:45.246 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1273,"stamp":"2023-09-23T20:15:41.060500+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3732561325,"num_objects":56910,"num_object_clones":0,"num_object_copies":170730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":44488,"num_objects_misplaced":5254,"num_objects_unfound":0,"num_objects_dirty":56910,"num_whiteouts":0,"num_read":5996,"num_read_kb":13236,"num_write":151738,"num_write_kb":6659327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13594,"num_bytes_recovered":880671860,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":218951,"ondisk_log_size":218951,"up":387,"acting":366,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":391,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13881052,"kb_used_data":10064748,"kb_used_omap":25,"kb_used_meta":3815654,"kb_avail":548589860,"statfs":{"total":575970213888,"available":561756016640,"internally_reserved":0,"allocated":10306301952,"data_stored":10295426989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":3907230412},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-504758272,"num_objects":-7702,"num_object_clones":0,"num_object_copies":-23106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-978,"num_objects_misplaced":-1173,"num_objects_unfound":0,"num_objects_dirty":-7702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7702,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1155,"num_bytes_recovered":75694080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.549007"},"pg_stats":[{"pgid":"9.23","version":"458'5503","reported_seq":5592,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:37.978139+0000","last_change":"2023-09-23T20:14:36.092943+0000","last_active":"2023-09-23T20:15:37.978139+0000","last_peered":"2023-09-23T20:15:37.978139+0000","last_clean":"2023-09-23T20:14:33.760240+0000","last_became_active":"2023-09-23T20:14:36.076681+0000","last_became_peered":"2023-09-23T20:14:36.076681+0000","last_unstale":"2023-09-23T20:15:37.978139+0000","last_undegraded":"2023-09-23T20:14:35.911782+0000","last_fullsized":"2023-09-23T20:14:35.868390+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59916288,"num_objects":914,"num_object_clones":0,"num_object_copies":2742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2394,"num_write_kb":105872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":824},{"shards":"3,7","objects":-650}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.22","version":"458'5537","reported_seq":5626,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:36.234327+0000","last_change":"2023-09-23T20:14:36.006132+0000","last_active":"2023-09-23T20:15:36.234327+0000","last_peered":"2023-09-23T20:15:36.234327+0000","last_clean":"2023-09-23T20:14:33.601158+0000","last_became_active":"2023-09-23T20:14:35.991960+0000","last_became_peered":"2023-09-23T20:14:35.991960+0000","last_unstale":"2023-09-23T20:15:36.234327+0000","last_undegraded":"2023-09-23T20:14:35.848470+0000","last_fullsized":"2023-09-23T20:14:35.833465+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5537,"log_dups_size":0,"ondisk_log_size":5537,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60096512,"num_objects":917,"num_object_clones":0,"num_object_copies":2751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2401,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":802},{"shards":"2,3","objects":-627}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.21","version":"458'5437","reported_seq":5516,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:37.491834+0000","last_change":"2023-09-23T20:14:24.799466+0000","last_active":"2023-09-23T20:15:37.491834+0000","last_peered":"2023-09-23T20:15:37.491834+0000","last_clean":"2023-09-23T20:15:37.491834+0000","last_became_active":"2023-09-23T20:14:24.796974+0000","last_became_peered":"2023-09-23T20:14:24.796974+0000","last_unstale":"2023-09-23T20:15:37.491834+0000","last_undegraded":"2023-09-23T20:15:37.491834+0000","last_fullsized":"2023-09-23T20:15:37.491834+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5437,"log_dups_size":0,"ondisk_log_size":5437,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57966592,"num_objects":885,"num_object_clones":0,"num_object_copies":2655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2343,"num_write_kb":103264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.20","version":"457'5403","reported_seq":5490,"reported_epoch":457,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:35.968436+0000","last_change":"2023-09-23T20:14:36.109513+0000","last_active":"2023-09-23T20:15:35.968436+0000","last_peered":"2023-09-23T20:15:35.968436+0000","last_clean":"2023-09-23T20:14:33.769971+0000","last_became_active":"2023-09-23T20:14:36.106301+0000","last_became_peered":"2023-09-23T20:14:36.106301+0000","last_unstale":"2023-09-23T20:15:35.968436+0000","last_undegraded":"2023-09-23T20:14:35.845700+0000","last_fullsized":"2023-09-23T20:14:35.836337+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5403,"log_dups_size":0,"ondisk_log_size":5403,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60227584,"num_objects":919,"num_object_clones":0,"num_object_copies":2757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2321,"num_write_kb":103680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":844},{"shards":"0,2","objects":-626}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.10","version":"458'6319","reported_seq":12246,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:38.017028+0000","last_change":"2023-09-23T20:14:22.339078+0000","last_active":"2023-09-23T20:15:38.017028+0000","last_peered":"2023-09-23T20:15:38.017028+0000","last_clean":"2023-09-23T20:15:38.017028+0000","last_became_active":"2023-09-23T20:14:22.319507+0000","last_became_peered":"2023-09-23T20:14:22.319507+0000","last_unstale":"2023-09-23T20:15:38.017028+0000","last_undegraded":"2023-09-23T20:15:38.017028+0000","last_fullsized":"2023-09-23T20:15:38.017028+0000","mapping_epoch":434,"log_start":"411'751","ondisk_log_start":"411'751","created":418,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5568,"log_dups_size":0,"ondisk_log_size":5568,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114622464,"num_objects":1749,"num_object_clones":0,"num_object_copies":5247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4599,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1945,"num_bytes_recovered":127467520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":27,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224237+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:15:23.224237+0000","last_peered":"2023-09-23T20:15:23.224237+0000","last_clean":"2023-09-23T20:15:23.224237+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:15:23.224237+0000","last_undegraded":"2023-09-23T20:15:23.224237+0000","last_fullsized":"2023-09-23T20:15:23.224237+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":26,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.526566+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:15:31.526566+0000","last_peered":"2023-09-23T20:15:31.526566+0000","last_clean":"2023-09-23T20:15:31.526566+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:15:31.526566+0000","last_undegraded":"2023-09-23T20:15:31.526566+0000","last_fullsized":"2023-09-23T20:15:31.526566+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.13","version":"458'6450","reported_seq":11065,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:38.002434+0000","last_change":"2023-09-23T20:14:36.123690+0000","last_active":"2023-09-23T20:15:38.002434+0000","last_peered":"2023-09-23T20:15:38.002434+0000","last_clean":"2023-09-23T20:14:05.142062+0000","last_became_active":"2023-09-23T20:14:07.304693+0000","last_became_peered":"2023-09-23T20:14:07.304693+0000","last_unstale":"2023-09-23T20:15:38.002434+0000","last_undegraded":"2023-09-23T20:14:36.121893+0000","last_fullsized":"2023-09-23T20:14:07.197546+0000","mapping_epoch":429,"log_start":"411'1003","ondisk_log_start":"411'1003","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5447,"log_dups_size":0,"ondisk_log_size":5447,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111378432,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1424,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4718,"num_write_kb":205344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1511,"num_bytes_recovered":99024896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,0],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"0,7","objects":988},{"shards":"1,3,7","objects":712}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":203,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.516232+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:15:31.516232+0000","last_peered":"2023-09-23T20:15:31.516232+0000","last_clean":"2023-09-23T20:15:31.516232+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:15:31.516232+0000","last_undegraded":"2023-09-23T20:15:31.516232+0000","last_fullsized":"2023-09-23T20:15:31.516232+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.528460+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:15:31.528460+0000","last_peered":"2023-09-23T20:15:31.528460+0000","last_clean":"2023-09-23T20:15:31.528460+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:15:31.528460+0000","last_undegraded":"2023-09-23T20:15:31.528460+0000","last_fullsized":"2023-09-23T20:15:31.528460+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"458'6526","reported_seq":12987,"reported_epoch":458,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T20:15:38.259581+0000","last_change":"2023-09-23T20:14:36.234004+0000","last_active":"2023-09-23T20:15:38.259581+0000","last_peered":"2023-09-23T20:15:38.259581+0000","last_clean":"2023-09-23T20:14:33.790624+0000","last_became_active":"2023-09-23T20:14:36.220990+0000","last_became_peered":"2023-09-23T20:14:36.220990+0000","last_unstale":"2023-09-23T20:15:38.259581+0000","last_undegraded":"2023-09-23T20:15:38.259581+0000","last_fullsized":"2023-09-23T20:15:38.259581+0000","mapping_epoch":442,"log_start":"411'791","ondisk_log_start":"411'791","created":418,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5735,"log_dups_size":0,"ondisk_log_size":5735,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109314048,"num_objects":1668,"num_object_clones":0,"num_object_copies":5004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1668,"num_objects_unfound":0,"num_objects_dirty":1668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4776,"num_write_kb":206208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":139067392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1","3"],"object_location_counts":[{"shards":"0,1,5","objects":1668}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":26,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224661+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:15:23.224661+0000","last_peered":"2023-09-23T20:15:23.224661+0000","last_clean":"2023-09-23T20:15:23.224661+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:15:23.224661+0000","last_undegraded":"2023-09-23T20:15:23.224661+0000","last_fullsized":"2023-09-23T20:15:23.224661+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":413,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.532074+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:15:31.532074+0000","last_peered":"2023-09-23T20:15:31.532074+0000","last_clean":"2023-09-23T20:15:31.532074+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:15:31.532074+0000","last_undegraded":"2023-09-23T20:15:31.532074+0000","last_fullsized":"2023-09-23T20:15:31.532074+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1d","version":"458'7577","reported_seq":7656,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:37.978939+0000","last_change":"2023-09-23T20:14:36.076800+0000","last_active":"2023-09-23T20:15:37.978939+0000","last_peered":"2023-09-23T20:15:37.978939+0000","last_clean":"2023-09-23T20:14:23.775634+0000","last_became_active":"2023-09-23T20:14:36.070774+0000","last_became_peered":"2023-09-23T20:14:36.070774+0000","last_unstale":"2023-09-23T20:15:37.978939+0000","last_undegraded":"2023-09-23T20:14:35.892079+0000","last_fullsized":"2023-09-23T20:14:35.839513+0000","mapping_epoch":442,"log_start":"411'1040","ondisk_log_start":"411'1040","created":436,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6537,"log_dups_size":0,"ondisk_log_size":6537,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123928600,"num_objects":1892,"num_object_clones":0,"num_object_copies":5676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1892,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4788,"num_write_kb":213697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":1627},{"shards":"5,7","objects":-1183}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":384,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.531938+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:15:31.531938+0000","last_peered":"2023-09-23T20:15:31.531938+0000","last_clean":"2023-09-23T20:15:31.531938+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:15:31.531938+0000","last_undegraded":"2023-09-23T20:15:31.531938+0000","last_fullsized":"2023-09-23T20:15:31.531938+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":312,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.554513+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:15:31.554513+0000","last_peered":"2023-09-23T20:15:31.554513+0000","last_clean":"2023-09-23T20:15:31.554513+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:15:31.554513+0000","last_undegraded":"2023-09-23T20:15:31.554513+0000","last_fullsized":"2023-09-23T20:15:31.554513+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"458'7194","reported_seq":7263,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.438511+0000","last_change":"2023-09-23T20:14:24.810789+0000","last_active":"2023-09-23T20:15:36.438511+0000","last_peered":"2023-09-23T20:15:36.438511+0000","last_clean":"2023-09-23T20:15:36.438511+0000","last_became_active":"2023-09-23T20:14:24.807691+0000","last_became_peered":"2023-09-23T20:14:24.807691+0000","last_unstale":"2023-09-23T20:15:36.438511+0000","last_undegraded":"2023-09-23T20:15:36.438511+0000","last_fullsized":"2023-09-23T20:15:36.438511+0000","mapping_epoch":436,"log_start":"411'1215","ondisk_log_start":"411'1215","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115113984,"num_objects":1757,"num_object_clones":0,"num_object_copies":5271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4575,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":790,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224188+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:15:23.224188+0000","last_peered":"2023-09-23T20:15:23.224188+0000","last_clean":"2023-09-23T20:15:23.224188+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:15:23.224188+0000","last_undegraded":"2023-09-23T20:15:23.224188+0000","last_fullsized":"2023-09-23T20:15:23.224188+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":467,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.533657+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:15:31.533657+0000","last_peered":"2023-09-23T20:15:31.533657+0000","last_clean":"2023-09-23T20:15:31.533657+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:15:31.533657+0000","last_undegraded":"2023-09-23T20:15:31.533657+0000","last_fullsized":"2023-09-23T20:15:31.533657+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":322,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.554545+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:15:31.554545+0000","last_peered":"2023-09-23T20:15:31.554545+0000","last_clean":"2023-09-23T20:15:31.554545+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:15:31.554545+0000","last_undegraded":"2023-09-23T20:15:31.554545+0000","last_fullsized":"2023-09-23T20:15:31.554545+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1f","version":"458'7359","reported_seq":7437,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:38.382053+0000","last_change":"2023-09-23T20:14:36.232228+0000","last_active":"2023-09-23T20:15:38.382053+0000","last_peered":"2023-09-23T20:15:38.382053+0000","last_clean":"2023-09-23T20:14:33.842679+0000","last_became_active":"2023-09-23T20:14:36.217802+0000","last_became_peered":"2023-09-23T20:14:36.217802+0000","last_unstale":"2023-09-23T20:15:38.382053+0000","last_undegraded":"2023-09-23T20:14:36.129688+0000","last_fullsized":"2023-09-23T20:14:36.115967+0000","mapping_epoch":442,"log_start":"411'544","ondisk_log_start":"411'544","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6815,"log_dups_size":0,"ondisk_log_size":6815,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115933184,"num_objects":1769,"num_object_clones":0,"num_object_copies":5307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4673,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":1575},{"shards":"3,5","objects":-1258}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":151,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.525764+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:15:31.525764+0000","last_peered":"2023-09-23T20:15:31.525764+0000","last_clean":"2023-09-23T20:15:31.525764+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:15:31.525764+0000","last_undegraded":"2023-09-23T20:15:31.525764+0000","last_fullsized":"2023-09-23T20:15:31.525764+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":155,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224417+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:15:23.224417+0000","last_peered":"2023-09-23T20:15:23.224417+0000","last_clean":"2023-09-23T20:15:23.224417+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:15:23.224417+0000","last_undegraded":"2023-09-23T20:15:23.224417+0000","last_fullsized":"2023-09-23T20:15:23.224417+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.1e","version":"458'7411","reported_seq":13550,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:37.986511+0000","last_change":"2023-09-23T20:15:21.201333+0000","last_active":"2023-09-23T20:15:37.986511+0000","last_peered":"2023-09-23T20:15:37.986511+0000","last_clean":"2023-09-23T20:15:37.986511+0000","last_became_active":"2023-09-23T20:15:21.192111+0000","last_became_peered":"2023-09-23T20:15:21.192111+0000","last_unstale":"2023-09-23T20:15:37.986511+0000","last_undegraded":"2023-09-23T20:15:37.986511+0000","last_fullsized":"2023-09-23T20:15:37.986511+0000","mapping_epoch":454,"log_start":"411'899","ondisk_log_start":"411'899","created":436,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6512,"log_dups_size":0,"ondisk_log_size":6512,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116097024,"num_objects":1772,"num_object_clones":0,"num_object_copies":5316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4712,"num_write_kb":207456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3024,"num_bytes_recovered":198180864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":404,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.520872+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:15:31.520872+0000","last_peered":"2023-09-23T20:15:31.520872+0000","last_clean":"2023-09-23T20:15:31.520872+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:15:31.520872+0000","last_undegraded":"2023-09-23T20:15:31.520872+0000","last_fullsized":"2023-09-23T20:15:31.520872+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":335,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.532179+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:15:31.532179+0000","last_peered":"2023-09-23T20:15:31.532179+0000","last_clean":"2023-09-23T20:15:31.532179+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:15:31.532179+0000","last_undegraded":"2023-09-23T20:15:31.532179+0000","last_fullsized":"2023-09-23T20:15:31.532179+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"458'6504","reported_seq":6573,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:37.961212+0000","last_change":"2023-09-23T20:13:46.811423+0000","last_active":"2023-09-23T20:15:37.961212+0000","last_peered":"2023-09-23T20:15:37.961212+0000","last_clean":"2023-09-23T20:15:37.961212+0000","last_became_active":"2023-09-23T20:13:46.808266+0000","last_became_peered":"2023-09-23T20:13:46.808266+0000","last_unstale":"2023-09-23T20:15:37.961212+0000","last_undegraded":"2023-09-23T20:15:37.961212+0000","last_fullsized":"2023-09-23T20:15:37.961212+0000","mapping_epoch":418,"log_start":"411'1046","ondisk_log_start":"411'1046","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5458,"log_dups_size":0,"ondisk_log_size":5458,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115900416,"num_objects":1769,"num_object_clones":0,"num_object_copies":5307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4749,"num_write_kb":208544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":817,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.526182+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:15:31.526182+0000","last_peered":"2023-09-23T20:15:31.526182+0000","last_clean":"2023-09-23T20:15:31.526182+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:15:31.526182+0000","last_undegraded":"2023-09-23T20:15:31.526182+0000","last_fullsized":"2023-09-23T20:15:31.526182+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":151,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.534073+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:15:31.534073+0000","last_peered":"2023-09-23T20:15:31.534073+0000","last_clean":"2023-09-23T20:15:31.534073+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:15:31.534073+0000","last_undegraded":"2023-09-23T20:15:31.534073+0000","last_fullsized":"2023-09-23T20:15:31.534073+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":204,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.521042+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:15:31.521042+0000","last_peered":"2023-09-23T20:15:31.521042+0000","last_clean":"2023-09-23T20:15:31.521042+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:15:31.521042+0000","last_undegraded":"2023-09-23T20:15:31.521042+0000","last_fullsized":"2023-09-23T20:15:31.521042+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"458'6492","reported_seq":6561,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:38.348652+0000","last_change":"2023-09-23T20:13:46.824838+0000","last_active":"2023-09-23T20:15:38.348652+0000","last_peered":"2023-09-23T20:15:38.348652+0000","last_clean":"2023-09-23T20:15:38.348652+0000","last_became_active":"2023-09-23T20:13:46.820709+0000","last_became_peered":"2023-09-23T20:13:46.820709+0000","last_unstale":"2023-09-23T20:15:38.348652+0000","last_undegraded":"2023-09-23T20:15:38.348652+0000","last_fullsized":"2023-09-23T20:15:38.348652+0000","mapping_epoch":418,"log_start":"411'747","ondisk_log_start":"411'747","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115769344,"num_objects":1767,"num_object_clones":0,"num_object_copies":5301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4777,"num_write_kb":209376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":506,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.529190+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:15:31.529190+0000","last_peered":"2023-09-23T20:15:31.529190+0000","last_clean":"2023-09-23T20:15:31.529190+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:15:31.529190+0000","last_undegraded":"2023-09-23T20:15:31.529190+0000","last_fullsized":"2023-09-23T20:15:31.529190+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.528838+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:15:31.528838+0000","last_peered":"2023-09-23T20:15:31.528838+0000","last_clean":"2023-09-23T20:15:31.528838+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:15:31.528838+0000","last_undegraded":"2023-09-23T20:15:31.528838+0000","last_fullsized":"2023-09-23T20:15:31.528838+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":456,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.527001+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:15:31.527001+0000","last_peered":"2023-09-23T20:15:31.527001+0000","last_clean":"2023-09-23T20:15:31.527001+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:15:31.527001+0000","last_undegraded":"2023-09-23T20:15:31.527001+0000","last_fullsized":"2023-09-23T20:15:31.527001+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1b","version":"458'7259","reported_seq":7328,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:38.336524+0000","last_change":"2023-09-23T20:14:24.808547+0000","last_active":"2023-09-23T20:15:38.336524+0000","last_peered":"2023-09-23T20:15:38.336524+0000","last_clean":"2023-09-23T20:15:38.336524+0000","last_became_active":"2023-09-23T20:14:24.806384+0000","last_became_peered":"2023-09-23T20:14:24.806384+0000","last_unstale":"2023-09-23T20:15:38.336524+0000","last_undegraded":"2023-09-23T20:15:38.336524+0000","last_fullsized":"2023-09-23T20:15:38.336524+0000","mapping_epoch":436,"log_start":"411'409","ondisk_log_start":"411'409","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6850,"log_dups_size":0,"ondisk_log_size":6850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110329856,"num_objects":1684,"num_object_clones":0,"num_object_copies":5052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4648,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":567,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.517336+0000","last_change":"2023-09-23T20:14:37.887863+0000","last_active":"2023-09-23T20:15:31.517336+0000","last_peered":"2023-09-23T20:15:31.517336+0000","last_clean":"2023-09-23T20:15:31.517336+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:15:31.517336+0000","last_undegraded":"2023-09-23T20:15:31.517336+0000","last_fullsized":"2023-09-23T20:15:31.517336+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":145,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.524047+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:15:31.524047+0000","last_peered":"2023-09-23T20:15:31.524047+0000","last_clean":"2023-09-23T20:15:31.524047+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:15:31.524047+0000","last_undegraded":"2023-09-23T20:15:31.524047+0000","last_fullsized":"2023-09-23T20:15:31.524047+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":446,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.157291+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:15:36.157291+0000","last_peered":"2023-09-23T20:15:36.157291+0000","last_clean":"2023-09-23T20:15:36.157291+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:15:36.157291+0000","last_undegraded":"2023-09-23T20:15:36.157291+0000","last_fullsized":"2023-09-23T20:15:36.157291+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1a","version":"458'7365","reported_seq":7444,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:38.243893+0000","last_change":"2023-09-23T20:14:36.218516+0000","last_active":"2023-09-23T20:15:38.243893+0000","last_peered":"2023-09-23T20:15:38.243893+0000","last_clean":"2023-09-23T20:14:33.782041+0000","last_became_active":"2023-09-23T20:14:36.175134+0000","last_became_peered":"2023-09-23T20:14:36.175134+0000","last_unstale":"2023-09-23T20:15:38.243893+0000","last_undegraded":"2023-09-23T20:14:35.943267+0000","last_fullsized":"2023-09-23T20:14:35.848493+0000","mapping_epoch":442,"log_start":"411'433","ondisk_log_start":"411'433","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6932,"log_dups_size":0,"ondisk_log_size":6932,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114491392,"num_objects":1747,"num_object_clones":0,"num_object_copies":5241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4667,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":1500},{"shards":"5,7","objects":-1213}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":595,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.539592+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:15:31.539592+0000","last_peered":"2023-09-23T20:15:31.539592+0000","last_clean":"2023-09-23T20:15:31.539592+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:15:31.539592+0000","last_undegraded":"2023-09-23T20:15:31.539592+0000","last_fullsized":"2023-09-23T20:15:31.539592+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":101,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.523457+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:15:31.523457+0000","last_peered":"2023-09-23T20:15:31.523457+0000","last_clean":"2023-09-23T20:15:31.523457+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:15:31.523457+0000","last_undegraded":"2023-09-23T20:15:31.523457+0000","last_fullsized":"2023-09-23T20:15:31.523457+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":98,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.517213+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:15:31.517213+0000","last_peered":"2023-09-23T20:15:31.517213+0000","last_clean":"2023-09-23T20:15:31.517213+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:15:31.517213+0000","last_undegraded":"2023-09-23T20:15:31.517213+0000","last_fullsized":"2023-09-23T20:15:31.517213+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"458'6489","reported_seq":6559,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:38.384880+0000","last_change":"2023-09-23T20:13:46.790587+0000","last_active":"2023-09-23T20:15:38.384880+0000","last_peered":"2023-09-23T20:15:38.384880+0000","last_clean":"2023-09-23T20:15:38.384880+0000","last_became_active":"2023-09-23T20:13:46.788387+0000","last_became_peered":"2023-09-23T20:13:46.788387+0000","last_unstale":"2023-09-23T20:15:38.384880+0000","last_undegraded":"2023-09-23T20:15:38.384880+0000","last_fullsized":"2023-09-23T20:15:38.384880+0000","mapping_epoch":407,"log_start":"411'702","ondisk_log_start":"411'702","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115933184,"num_objects":1769,"num_object_clones":0,"num_object_copies":5307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4759,"num_write_kb":208896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":718,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.525437+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:15:31.525437+0000","last_peered":"2023-09-23T20:15:31.525437+0000","last_clean":"2023-09-23T20:15:31.525437+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:15:31.525437+0000","last_undegraded":"2023-09-23T20:15:31.525437+0000","last_fullsized":"2023-09-23T20:15:31.525437+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":149,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.525196+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:15:31.525196+0000","last_peered":"2023-09-23T20:15:31.525196+0000","last_clean":"2023-09-23T20:15:31.525196+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:15:31.525196+0000","last_undegraded":"2023-09-23T20:15:31.525196+0000","last_fullsized":"2023-09-23T20:15:31.525196+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":466,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.523533+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:15:31.523533+0000","last_peered":"2023-09-23T20:15:31.523533+0000","last_clean":"2023-09-23T20:15:31.523533+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:15:31.523533+0000","last_undegraded":"2023-09-23T20:15:31.523533+0000","last_fullsized":"2023-09-23T20:15:31.523533+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"458'6422","reported_seq":6500,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:37.939967+0000","last_change":"2023-09-23T20:14:36.032743+0000","last_active":"2023-09-23T20:15:37.939967+0000","last_peered":"2023-09-23T20:15:37.939967+0000","last_clean":"2023-09-23T20:14:33.800061+0000","last_became_active":"2023-09-23T20:14:36.025052+0000","last_became_peered":"2023-09-23T20:14:36.025052+0000","last_unstale":"2023-09-23T20:15:37.939967+0000","last_undegraded":"2023-09-23T20:14:35.860952+0000","last_fullsized":"2023-09-23T20:14:35.840056+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6422,"log_dups_size":0,"ondisk_log_size":6422,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113082368,"num_objects":1725,"num_object_clones":0,"num_object_copies":5175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3023,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4697,"num_write_kb":205536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1537},{"shards":"3,4","objects":-1298}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":805,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.516282+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:15:31.516282+0000","last_peered":"2023-09-23T20:15:31.516282+0000","last_clean":"2023-09-23T20:15:31.516282+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:15:31.516282+0000","last_undegraded":"2023-09-23T20:15:31.516282+0000","last_fullsized":"2023-09-23T20:15:31.516282+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":164,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.528532+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:15:31.528532+0000","last_peered":"2023-09-23T20:15:31.528532+0000","last_clean":"2023-09-23T20:15:31.528532+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:15:31.528532+0000","last_undegraded":"2023-09-23T20:15:31.528532+0000","last_fullsized":"2023-09-23T20:15:31.528532+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":458,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.515287+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:15:31.515287+0000","last_peered":"2023-09-23T20:15:31.515287+0000","last_clean":"2023-09-23T20:15:31.515287+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:15:31.515287+0000","last_undegraded":"2023-09-23T20:15:31.515287+0000","last_fullsized":"2023-09-23T20:15:31.515287+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.7","version":"458'6394","reported_seq":6464,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:38.353985+0000","last_change":"2023-09-23T20:13:46.802046+0000","last_active":"2023-09-23T20:15:38.353985+0000","last_peered":"2023-09-23T20:15:38.353985+0000","last_clean":"2023-09-23T20:15:38.353985+0000","last_became_active":"2023-09-23T20:13:46.800187+0000","last_became_peered":"2023-09-23T20:13:46.800187+0000","last_unstale":"2023-09-23T20:15:38.353985+0000","last_undegraded":"2023-09-23T20:15:38.353985+0000","last_fullsized":"2023-09-23T20:15:38.353985+0000","mapping_epoch":407,"log_start":"411'425","ondisk_log_start":"411'425","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5969,"log_dups_size":0,"ondisk_log_size":5969,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121110528,"num_objects":1848,"num_object_clones":0,"num_object_copies":5544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4684,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":530,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.528959+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:15:31.528959+0000","last_peered":"2023-09-23T20:15:31.528959+0000","last_clean":"2023-09-23T20:15:31.528959+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:15:31.528959+0000","last_undegraded":"2023-09-23T20:15:31.528959+0000","last_fullsized":"2023-09-23T20:15:31.528959+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":149,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.525622+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:15:31.525622+0000","last_peered":"2023-09-23T20:15:31.525622+0000","last_clean":"2023-09-23T20:15:31.525622+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:15:31.525622+0000","last_undegraded":"2023-09-23T20:15:31.525622+0000","last_fullsized":"2023-09-23T20:15:31.525622+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":46,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.528952+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:15:31.528952+0000","last_peered":"2023-09-23T20:15:31.528952+0000","last_clean":"2023-09-23T20:15:31.528952+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:15:31.528952+0000","last_undegraded":"2023-09-23T20:15:31.528952+0000","last_fullsized":"2023-09-23T20:15:31.528952+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"458'6690","reported_seq":6768,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:38.357166+0000","last_change":"2023-09-23T20:14:36.190976+0000","last_active":"2023-09-23T20:15:38.357166+0000","last_peered":"2023-09-23T20:15:38.357166+0000","last_clean":"2023-09-23T20:14:33.830609+0000","last_became_active":"2023-09-23T20:14:36.182918+0000","last_became_peered":"2023-09-23T20:14:36.182918+0000","last_unstale":"2023-09-23T20:15:38.357166+0000","last_undegraded":"2023-09-23T20:14:35.924455+0000","last_fullsized":"2023-09-23T20:14:35.847184+0000","mapping_epoch":442,"log_start":"411'1035","ondisk_log_start":"411'1035","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5655,"log_dups_size":0,"ondisk_log_size":5655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112459776,"num_objects":1716,"num_object_clones":0,"num_object_copies":5148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4908,"num_write_kb":211968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1601},{"shards":"2,5","objects":-1481}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":661,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.528265+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:15:31.528265+0000","last_peered":"2023-09-23T20:15:31.528265+0000","last_clean":"2023-09-23T20:15:31.528265+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:15:31.528265+0000","last_undegraded":"2023-09-23T20:15:31.528265+0000","last_fullsized":"2023-09-23T20:15:31.528265+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":506,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.529878+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:15:31.529878+0000","last_peered":"2023-09-23T20:15:31.529878+0000","last_clean":"2023-09-23T20:15:31.529878+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:15:31.529878+0000","last_undegraded":"2023-09-23T20:15:31.529878+0000","last_fullsized":"2023-09-23T20:15:31.529878+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":202,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.750434+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:15:31.750434+0000","last_peered":"2023-09-23T20:15:31.750434+0000","last_clean":"2023-09-23T20:15:31.750434+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:15:31.750434+0000","last_undegraded":"2023-09-23T20:15:31.750434+0000","last_fullsized":"2023-09-23T20:15:31.750434+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"458'5429","reported_seq":5509,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:37.831095+0000","last_change":"2023-09-23T20:14:24.782498+0000","last_active":"2023-09-23T20:15:37.831095+0000","last_peered":"2023-09-23T20:15:37.831095+0000","last_clean":"2023-09-23T20:15:37.831095+0000","last_became_active":"2023-09-23T20:14:24.780750+0000","last_became_peered":"2023-09-23T20:14:24.780750+0000","last_unstale":"2023-09-23T20:15:37.831095+0000","last_undegraded":"2023-09-23T20:15:37.831095+0000","last_fullsized":"2023-09-23T20:15:37.831095+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5429,"log_dups_size":0,"ondisk_log_size":5429,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58884096,"num_objects":898,"num_object_clones":0,"num_object_copies":2694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2334,"num_write_kb":103456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":840,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.523962+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:15:31.523962+0000","last_peered":"2023-09-23T20:15:31.523962+0000","last_clean":"2023-09-23T20:15:31.523962+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:15:31.523962+0000","last_undegraded":"2023-09-23T20:15:31.523962+0000","last_fullsized":"2023-09-23T20:15:31.523962+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":203,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.517482+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:15:31.517482+0000","last_peered":"2023-09-23T20:15:31.517482+0000","last_clean":"2023-09-23T20:15:31.517482+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:15:31.517482+0000","last_undegraded":"2023-09-23T20:15:31.517482+0000","last_fullsized":"2023-09-23T20:15:31.517482+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":26,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.520570+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:15:31.520570+0000","last_peered":"2023-09-23T20:15:31.520570+0000","last_clean":"2023-09-23T20:15:31.520570+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:15:31.520570+0000","last_undegraded":"2023-09-23T20:15:31.520570+0000","last_fullsized":"2023-09-23T20:15:31.520570+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"457'5434","reported_seq":5521,"reported_epoch":457,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:35.958778+0000","last_change":"2023-09-23T20:14:36.074365+0000","last_active":"2023-09-23T20:15:35.958778+0000","last_peered":"2023-09-23T20:15:35.958778+0000","last_clean":"2023-09-23T20:14:33.749328+0000","last_became_active":"2023-09-23T20:14:36.071685+0000","last_became_peered":"2023-09-23T20:14:36.071685+0000","last_unstale":"2023-09-23T20:15:35.958778+0000","last_undegraded":"2023-09-23T20:14:35.849009+0000","last_fullsized":"2023-09-23T20:14:35.834839+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60817408,"num_objects":928,"num_object_clones":0,"num_object_copies":2784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1547,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2352,"num_write_kb":104960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":835},{"shards":"0,2","objects":-619}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":805,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.523689+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:15:31.523689+0000","last_peered":"2023-09-23T20:15:31.523689+0000","last_clean":"2023-09-23T20:15:31.523689+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:15:31.523689+0000","last_undegraded":"2023-09-23T20:15:31.523689+0000","last_fullsized":"2023-09-23T20:15:31.523689+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":26,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.533283+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:15:31.533283+0000","last_peered":"2023-09-23T20:15:31.533283+0000","last_clean":"2023-09-23T20:15:31.533283+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:15:31.533283+0000","last_undegraded":"2023-09-23T20:15:31.533283+0000","last_fullsized":"2023-09-23T20:15:31.533283+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":357,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224771+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:15:23.224771+0000","last_peered":"2023-09-23T20:15:23.224771+0000","last_clean":"2023-09-23T20:15:23.224771+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:15:23.224771+0000","last_undegraded":"2023-09-23T20:15:23.224771+0000","last_fullsized":"2023-09-23T20:15:23.224771+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"458'5446","reported_seq":5536,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:37.971432+0000","last_change":"2023-09-23T20:14:36.109242+0000","last_active":"2023-09-23T20:15:37.971432+0000","last_peered":"2023-09-23T20:15:37.971432+0000","last_clean":"2023-09-23T20:14:33.735119+0000","last_became_active":"2023-09-23T20:14:36.106276+0000","last_became_peered":"2023-09-23T20:14:36.106276+0000","last_unstale":"2023-09-23T20:15:37.971432+0000","last_undegraded":"2023-09-23T20:14:35.867219+0000","last_fullsized":"2023-09-23T20:14:35.838359+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5446,"log_dups_size":0,"ondisk_log_size":5446,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61423616,"num_objects":937,"num_object_clones":0,"num_object_copies":2811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2337,"num_write_kb":104784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":798},{"shards":"3,7","objects":-561}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":519,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.532268+0000","last_change":"2023-09-23T20:14:37.892742+0000","last_active":"2023-09-23T20:15:31.532268+0000","last_peered":"2023-09-23T20:15:31.532268+0000","last_clean":"2023-09-23T20:15:31.532268+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:15:31.532268+0000","last_undegraded":"2023-09-23T20:15:31.532268+0000","last_fullsized":"2023-09-23T20:15:31.532268+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":348,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.223873+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:15:23.223873+0000","last_peered":"2023-09-23T20:15:23.223873+0000","last_clean":"2023-09-23T20:15:23.223873+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:15:23.223873+0000","last_undegraded":"2023-09-23T20:15:23.223873+0000","last_fullsized":"2023-09-23T20:15:23.223873+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":26,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.531000+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:15:31.531000+0000","last_peered":"2023-09-23T20:15:31.531000+0000","last_clean":"2023-09-23T20:15:31.531000+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:15:31.531000+0000","last_undegraded":"2023-09-23T20:15:31.531000+0000","last_fullsized":"2023-09-23T20:15:31.531000+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"458'5471","reported_seq":5561,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:36.368994+0000","last_change":"2023-09-23T20:14:35.994679+0000","last_active":"2023-09-23T20:15:36.368994+0000","last_peered":"2023-09-23T20:15:36.368994+0000","last_clean":"2023-09-23T20:14:33.798489+0000","last_became_active":"2023-09-23T20:14:35.986955+0000","last_became_peered":"2023-09-23T20:14:35.986955+0000","last_unstale":"2023-09-23T20:15:36.368994+0000","last_undegraded":"2023-09-23T20:14:35.846545+0000","last_fullsized":"2023-09-23T20:14:35.832866+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5471,"log_dups_size":0,"ondisk_log_size":5471,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64815104,"num_objects":989,"num_object_clones":0,"num_object_copies":2967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1542,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2335,"num_write_kb":106368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":869},{"shards":"2,3","objects":-553}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":796,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.540071+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:15:31.540071+0000","last_peered":"2023-09-23T20:15:31.540071+0000","last_clean":"2023-09-23T20:15:31.540071+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:15:31.540071+0000","last_undegraded":"2023-09-23T20:15:31.540071+0000","last_fullsized":"2023-09-23T20:15:31.540071+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":98,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.515338+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:15:31.515338+0000","last_peered":"2023-09-23T20:15:31.515338+0000","last_clean":"2023-09-23T20:15:31.515338+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:15:31.515338+0000","last_undegraded":"2023-09-23T20:15:31.515338+0000","last_fullsized":"2023-09-23T20:15:31.515338+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":202,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.517584+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:15:31.517584+0000","last_peered":"2023-09-23T20:15:31.517584+0000","last_clean":"2023-09-23T20:15:31.517584+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:15:31.517584+0000","last_undegraded":"2023-09-23T20:15:31.517584+0000","last_fullsized":"2023-09-23T20:15:31.517584+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"458'7395","reported_seq":7481,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:38.345495+0000","last_change":"2023-09-23T20:14:36.198248+0000","last_active":"2023-09-23T20:15:38.345495+0000","last_peered":"2023-09-23T20:15:38.345495+0000","last_clean":"2023-09-23T20:14:33.842789+0000","last_became_active":"2023-09-23T20:14:36.169565+0000","last_became_peered":"2023-09-23T20:14:36.169565+0000","last_unstale":"2023-09-23T20:15:38.345495+0000","last_undegraded":"2023-09-23T20:14:36.113647+0000","last_fullsized":"2023-09-23T20:14:35.847403+0000","mapping_epoch":442,"log_start":"411'563","ondisk_log_start":"411'563","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6832,"log_dups_size":0,"ondisk_log_size":6832,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113442816,"num_objects":1731,"num_object_clones":0,"num_object_copies":5193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4709,"num_write_kb":206080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":1538},{"shards":"3,5","objects":-1296}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":739,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.517335+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:15:31.517335+0000","last_peered":"2023-09-23T20:15:31.517335+0000","last_clean":"2023-09-23T20:15:31.517335+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:15:31.517335+0000","last_undegraded":"2023-09-23T20:15:31.517335+0000","last_fullsized":"2023-09-23T20:15:31.517335+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":448,"reported_epoch":457,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:31.520717+0000","last_change":"2023-09-23T20:14:35.895655+0000","last_active":"2023-09-23T20:15:31.520717+0000","last_peered":"2023-09-23T20:15:31.520717+0000","last_clean":"2023-09-23T20:07:25.847413+0000","last_became_active":"2023-09-23T20:14:35.895204+0000","last_became_peered":"2023-09-23T20:14:35.895204+0000","last_unstale":"2023-09-23T20:15:31.520717+0000","last_undegraded":"2023-09-23T20:14:35.838988+0000","last_fullsized":"2023-09-23T20:14:35.837779+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:17.785522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":458,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.526937+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:15:31.526937+0000","last_peered":"2023-09-23T20:15:31.526937+0000","last_clean":"2023-09-23T20:15:31.526937+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:15:31.526937+0000","last_undegraded":"2023-09-23T20:15:31.526937+0000","last_fullsized":"2023-09-23T20:15:31.526937+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":69,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.539985+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:15:31.539985+0000","last_peered":"2023-09-23T20:15:31.539985+0000","last_clean":"2023-09-23T20:15:31.539985+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:15:31.539985+0000","last_undegraded":"2023-09-23T20:15:31.539985+0000","last_fullsized":"2023-09-23T20:15:31.539985+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"458'6568","reported_seq":6638,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:37.967019+0000","last_change":"2023-09-23T20:13:46.785589+0000","last_active":"2023-09-23T20:15:37.967019+0000","last_peered":"2023-09-23T20:15:37.967019+0000","last_clean":"2023-09-23T20:15:37.967019+0000","last_became_active":"2023-09-23T20:13:46.781975+0000","last_became_peered":"2023-09-23T20:13:46.781975+0000","last_unstale":"2023-09-23T20:15:37.967019+0000","last_undegraded":"2023-09-23T20:15:37.967019+0000","last_fullsized":"2023-09-23T20:15:37.967019+0000","mapping_epoch":407,"log_start":"411'1168","ondisk_log_start":"411'1168","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5400,"log_dups_size":0,"ondisk_log_size":5400,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117080064,"num_objects":1786,"num_object_clones":0,"num_object_copies":5358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4812,"num_write_kb":211168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":801,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.526326+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:15:31.526326+0000","last_peered":"2023-09-23T20:15:31.526326+0000","last_clean":"2023-09-23T20:15:31.526326+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:15:31.526326+0000","last_undegraded":"2023-09-23T20:15:31.526326+0000","last_fullsized":"2023-09-23T20:15:31.526326+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1108,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.539688+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:15:31.539688+0000","last_peered":"2023-09-23T20:15:31.539688+0000","last_clean":"2023-09-23T20:15:31.539688+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:15:31.539688+0000","last_undegraded":"2023-09-23T20:15:31.539688+0000","last_fullsized":"2023-09-23T20:15:31.539688+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":26,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.531809+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:15:31.531809+0000","last_peered":"2023-09-23T20:15:31.531809+0000","last_clean":"2023-09-23T20:15:31.531809+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:15:31.531809+0000","last_undegraded":"2023-09-23T20:15:31.531809+0000","last_fullsized":"2023-09-23T20:15:31.531809+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":319,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224268+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:15:23.224268+0000","last_peered":"2023-09-23T20:15:23.224268+0000","last_clean":"2023-09-23T20:15:23.224268+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:15:23.224268+0000","last_undegraded":"2023-09-23T20:15:23.224268+0000","last_fullsized":"2023-09-23T20:15:23.224268+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"458'7390","reported_seq":7473,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:38.393974+0000","last_change":"2023-09-23T20:14:36.228252+0000","last_active":"2023-09-23T20:15:38.393974+0000","last_peered":"2023-09-23T20:15:38.393974+0000","last_clean":"2023-09-23T20:14:33.823351+0000","last_became_active":"2023-09-23T20:14:36.216536+0000","last_became_peered":"2023-09-23T20:14:36.216536+0000","last_unstale":"2023-09-23T20:15:38.393974+0000","last_undegraded":"2023-09-23T20:14:36.114735+0000","last_fullsized":"2023-09-23T20:14:35.945404+0000","mapping_epoch":442,"log_start":"411'629","ondisk_log_start":"411'629","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6761,"log_dups_size":0,"ondisk_log_size":6761,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117833728,"num_objects":1798,"num_object_clones":0,"num_object_copies":5394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3085,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4692,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":1638},{"shards":"5,7","objects":-1287}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":696,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.528873+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:15:31.528873+0000","last_peered":"2023-09-23T20:15:31.528873+0000","last_clean":"2023-09-23T20:15:31.528873+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:15:31.528873+0000","last_undegraded":"2023-09-23T20:15:31.528873+0000","last_fullsized":"2023-09-23T20:15:31.528873+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1095,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.223976+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:15:23.223976+0000","last_peered":"2023-09-23T20:15:23.223976+0000","last_clean":"2023-09-23T20:15:23.223976+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:15:23.223976+0000","last_undegraded":"2023-09-23T20:15:23.223976+0000","last_fullsized":"2023-09-23T20:15:23.223976+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":155,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.225247+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:15:23.225247+0000","last_peered":"2023-09-23T20:15:23.225247+0000","last_clean":"2023-09-23T20:15:23.225247+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:15:23.225247+0000","last_undegraded":"2023-09-23T20:15:23.225247+0000","last_fullsized":"2023-09-23T20:15:23.225247+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":123,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.532007+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:15:31.532007+0000","last_peered":"2023-09-23T20:15:31.532007+0000","last_clean":"2023-09-23T20:15:31.532007+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:15:31.532007+0000","last_undegraded":"2023-09-23T20:15:31.532007+0000","last_fullsized":"2023-09-23T20:15:31.532007+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"458'6387","reported_seq":6457,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:38.365919+0000","last_change":"2023-09-23T20:13:46.814034+0000","last_active":"2023-09-23T20:15:38.365919+0000","last_peered":"2023-09-23T20:15:38.365919+0000","last_clean":"2023-09-23T20:15:38.365919+0000","last_became_active":"2023-09-23T20:13:46.812300+0000","last_became_peered":"2023-09-23T20:13:46.812300+0000","last_unstale":"2023-09-23T20:15:38.365919+0000","last_undegraded":"2023-09-23T20:15:38.365919+0000","last_fullsized":"2023-09-23T20:15:38.365919+0000","mapping_epoch":407,"log_start":"411'499","ondisk_log_start":"411'499","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115310592,"num_objects":1759,"num_object_clones":0,"num_object_copies":5277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4671,"num_write_kb":205792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1366,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.532013+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:15:31.532013+0000","last_peered":"2023-09-23T20:15:31.532013+0000","last_clean":"2023-09-23T20:15:31.532013+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:15:31.532013+0000","last_undegraded":"2023-09-23T20:15:31.532013+0000","last_fullsized":"2023-09-23T20:15:31.532013+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":653,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.526977+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:15:31.526977+0000","last_peered":"2023-09-23T20:15:31.526977+0000","last_clean":"2023-09-23T20:15:31.526977+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:15:31.526977+0000","last_undegraded":"2023-09-23T20:15:31.526977+0000","last_fullsized":"2023-09-23T20:15:31.526977+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":323,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.539846+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:15:31.539846+0000","last_peered":"2023-09-23T20:15:31.539846+0000","last_clean":"2023-09-23T20:15:31.539846+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:15:31.539846+0000","last_undegraded":"2023-09-23T20:15:31.539846+0000","last_fullsized":"2023-09-23T20:15:31.539846+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":255,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.525506+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:15:31.525506+0000","last_peered":"2023-09-23T20:15:31.525506+0000","last_clean":"2023-09-23T20:15:31.525506+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:15:31.525506+0000","last_undegraded":"2023-09-23T20:15:31.525506+0000","last_fullsized":"2023-09-23T20:15:31.525506+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"458'7246","reported_seq":7316,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:38.322119+0000","last_change":"2023-09-23T20:14:24.789921+0000","last_active":"2023-09-23T20:15:38.322119+0000","last_peered":"2023-09-23T20:15:38.322119+0000","last_clean":"2023-09-23T20:15:38.322119+0000","last_became_active":"2023-09-23T20:14:24.787385+0000","last_became_peered":"2023-09-23T20:14:24.787385+0000","last_unstale":"2023-09-23T20:15:38.322119+0000","last_undegraded":"2023-09-23T20:15:38.322119+0000","last_fullsized":"2023-09-23T20:15:38.322119+0000","mapping_epoch":407,"log_start":"411'512","ondisk_log_start":"411'512","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6734,"log_dups_size":0,"ondisk_log_size":6734,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112230400,"num_objects":1712,"num_object_clones":0,"num_object_copies":5136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4634,"num_write_kb":203104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":519,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.532118+0000","last_change":"2023-09-23T20:14:37.894544+0000","last_active":"2023-09-23T20:15:31.532118+0000","last_peered":"2023-09-23T20:15:31.532118+0000","last_clean":"2023-09-23T20:15:31.532118+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:15:31.532118+0000","last_undegraded":"2023-09-23T20:15:31.532118+0000","last_fullsized":"2023-09-23T20:15:31.532118+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":904,"reported_epoch":457,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:31.526719+0000","last_change":"2023-09-23T20:14:35.882819+0000","last_active":"2023-09-23T20:15:31.526719+0000","last_peered":"2023-09-23T20:15:31.526719+0000","last_clean":"2023-09-23T20:14:31.826679+0000","last_became_active":"2023-09-23T20:14:35.882372+0000","last_became_peered":"2023-09-23T20:14:35.882372+0000","last_unstale":"2023-09-23T20:15:31.526719+0000","last_undegraded":"2023-09-23T20:14:35.846466+0000","last_fullsized":"2023-09-23T20:14:35.837669+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:41.083298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":403,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.515242+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:15:31.515242+0000","last_peered":"2023-09-23T20:15:31.515242+0000","last_clean":"2023-09-23T20:15:31.515242+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:15:31.515242+0000","last_undegraded":"2023-09-23T20:15:31.515242+0000","last_fullsized":"2023-09-23T20:15:31.515242+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":272,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.225485+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:15:23.225485+0000","last_peered":"2023-09-23T20:15:23.225485+0000","last_clean":"2023-09-23T20:15:23.225485+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:15:23.225485+0000","last_undegraded":"2023-09-23T20:15:23.225485+0000","last_fullsized":"2023-09-23T20:15:23.225485+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"458'7380","reported_seq":7459,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:37.945051+0000","last_change":"2023-09-23T20:14:36.189421+0000","last_active":"2023-09-23T20:15:37.945051+0000","last_peered":"2023-09-23T20:15:37.945051+0000","last_clean":"2023-09-23T20:14:33.834923+0000","last_became_active":"2023-09-23T20:14:36.182947+0000","last_became_peered":"2023-09-23T20:14:36.182947+0000","last_unstale":"2023-09-23T20:15:37.945051+0000","last_undegraded":"2023-09-23T20:14:35.901626+0000","last_fullsized":"2023-09-23T20:14:35.838743+0000","mapping_epoch":442,"log_start":"411'976","ondisk_log_start":"411'976","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6404,"log_dups_size":0,"ondisk_log_size":6404,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120487936,"num_objects":1838,"num_object_clones":0,"num_object_copies":5514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4680,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":1640},{"shards":"4,7","objects":-1224}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":630,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.750577+0000","last_change":"2023-09-23T20:13:57.748021+0000","last_active":"2023-09-23T20:15:31.750577+0000","last_peered":"2023-09-23T20:15:31.750577+0000","last_clean":"2023-09-23T20:15:31.750577+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:15:31.750577+0000","last_undegraded":"2023-09-23T20:15:31.750577+0000","last_fullsized":"2023-09-23T20:15:31.750577+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":158,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.525688+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:15:31.525688+0000","last_peered":"2023-09-23T20:15:31.525688+0000","last_clean":"2023-09-23T20:15:31.525688+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:15:31.525688+0000","last_undegraded":"2023-09-23T20:15:31.525688+0000","last_fullsized":"2023-09-23T20:15:31.525688+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":260,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224485+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:15:23.224485+0000","last_peered":"2023-09-23T20:15:23.224485+0000","last_clean":"2023-09-23T20:15:23.224485+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:15:23.224485+0000","last_undegraded":"2023-09-23T20:15:23.224485+0000","last_fullsized":"2023-09-23T20:15:23.224485+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":284,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.532550+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:15:31.532550+0000","last_peered":"2023-09-23T20:15:31.532550+0000","last_clean":"2023-09-23T20:15:31.532550+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:15:31.532550+0000","last_undegraded":"2023-09-23T20:15:31.532550+0000","last_fullsized":"2023-09-23T20:15:31.532550+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"458'7642","reported_seq":7721,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:37.939962+0000","last_change":"2023-09-23T20:14:36.072073+0000","last_active":"2023-09-23T20:15:37.939962+0000","last_peered":"2023-09-23T20:15:37.939962+0000","last_clean":"2023-09-23T20:14:33.811030+0000","last_became_active":"2023-09-23T20:14:36.065847+0000","last_became_peered":"2023-09-23T20:14:36.065847+0000","last_unstale":"2023-09-23T20:15:37.939962+0000","last_undegraded":"2023-09-23T20:14:35.915006+0000","last_fullsized":"2023-09-23T20:14:35.838163+0000","mapping_epoch":442,"log_start":"411'1335","ondisk_log_start":"411'1335","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120651776,"num_objects":1841,"num_object_clones":0,"num_object_copies":5523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3193,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4853,"num_write_kb":214208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":1687},{"shards":"5,7","objects":-1352}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":675,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224850+0000","last_change":"2023-09-23T20:14:25.790696+0000","last_active":"2023-09-23T20:15:23.224850+0000","last_peered":"2023-09-23T20:15:23.224850+0000","last_clean":"2023-09-23T20:15:23.224850+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:15:23.224850+0000","last_undegraded":"2023-09-23T20:15:23.224850+0000","last_fullsized":"2023-09-23T20:15:23.224850+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":300,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.524754+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:15:31.524754+0000","last_peered":"2023-09-23T20:15:31.524754+0000","last_clean":"2023-09-23T20:15:31.524754+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:15:31.524754+0000","last_undegraded":"2023-09-23T20:15:31.524754+0000","last_fullsized":"2023-09-23T20:15:31.524754+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":578,"reported_epoch":457,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:31.517636+0000","last_change":"2023-09-23T20:14:35.862576+0000","last_active":"2023-09-23T20:15:31.517636+0000","last_peered":"2023-09-23T20:15:31.517636+0000","last_clean":"2023-09-23T19:59:02.493120+0000","last_became_active":"2023-09-23T20:14:35.862051+0000","last_became_peered":"2023-09-23T20:14:35.862051+0000","last_unstale":"2023-09-23T20:15:31.517636+0000","last_undegraded":"2023-09-23T20:14:35.833833+0000","last_fullsized":"2023-09-23T20:14:35.832351+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:30.213257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":90,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.515348+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:15:31.515348+0000","last_peered":"2023-09-23T20:15:31.515348+0000","last_clean":"2023-09-23T20:15:31.515348+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:15:31.515348+0000","last_undegraded":"2023-09-23T20:15:31.515348+0000","last_fullsized":"2023-09-23T20:15:31.515348+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"458'7129","reported_seq":7200,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.521080+0000","last_change":"2023-09-23T20:14:25.775099+0000","last_active":"2023-09-23T20:15:36.521080+0000","last_peered":"2023-09-23T20:15:36.521080+0000","last_clean":"2023-09-23T20:15:36.521080+0000","last_became_active":"2023-09-23T20:14:24.777506+0000","last_became_peered":"2023-09-23T20:14:24.777506+0000","last_unstale":"2023-09-23T20:15:36.521080+0000","last_undegraded":"2023-09-23T20:15:36.521080+0000","last_fullsized":"2023-09-23T20:15:36.521080+0000","mapping_epoch":407,"log_start":"411'911","ondisk_log_start":"411'911","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6218,"log_dups_size":0,"ondisk_log_size":6218,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121864192,"num_objects":1859,"num_object_clones":0,"num_object_copies":5577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4509,"num_write_kb":203808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":783,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.516664+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:15:31.516664+0000","last_peered":"2023-09-23T20:15:31.516664+0000","last_clean":"2023-09-23T20:15:31.516664+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:15:31.516664+0000","last_undegraded":"2023-09-23T20:15:31.516664+0000","last_fullsized":"2023-09-23T20:15:31.516664+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":159,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.525118+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:15:31.525118+0000","last_peered":"2023-09-23T20:15:31.525118+0000","last_clean":"2023-09-23T20:15:31.525118+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:15:31.525118+0000","last_undegraded":"2023-09-23T20:15:31.525118+0000","last_fullsized":"2023-09-23T20:15:31.525118+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":30,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.524563+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:15:31.524563+0000","last_peered":"2023-09-23T20:15:31.524563+0000","last_clean":"2023-09-23T20:15:31.524563+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:15:31.524563+0000","last_undegraded":"2023-09-23T20:15:31.524563+0000","last_fullsized":"2023-09-23T20:15:31.524563+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":241,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.526469+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:15:31.526469+0000","last_peered":"2023-09-23T20:15:31.526469+0000","last_clean":"2023-09-23T20:15:31.526469+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:15:31.526469+0000","last_undegraded":"2023-09-23T20:15:31.526469+0000","last_fullsized":"2023-09-23T20:15:31.526469+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"458'6513","reported_seq":10731,"reported_epoch":458,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:15:37.950799+0000","last_change":"2023-09-23T20:14:35.970620+0000","last_active":"2023-09-23T20:15:37.950799+0000","last_peered":"2023-09-23T20:15:37.950799+0000","last_clean":"2023-09-23T20:14:33.799110+0000","last_became_active":"2023-09-23T20:14:35.967502+0000","last_became_peered":"2023-09-23T20:14:35.967502+0000","last_unstale":"2023-09-23T20:15:37.950799+0000","last_undegraded":"2023-09-23T20:15:37.950799+0000","last_fullsized":"2023-09-23T20:15:37.950799+0000","mapping_epoch":442,"log_start":"411'1110","ondisk_log_start":"411'1110","created":418,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5403,"log_dups_size":0,"ondisk_log_size":5403,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116129792,"num_objects":1772,"num_object_clones":0,"num_object_copies":5316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1772,"num_objects_unfound":0,"num_objects_dirty":1772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4768,"num_write_kb":209280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2059,"num_bytes_recovered":134938624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":1772}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":233,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.225346+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:15:23.225346+0000","last_peered":"2023-09-23T20:15:23.225346+0000","last_clean":"2023-09-23T20:15:23.225346+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:15:23.225346+0000","last_undegraded":"2023-09-23T20:15:23.225346+0000","last_fullsized":"2023-09-23T20:15:23.225346+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":233,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224560+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:15:23.224560+0000","last_peered":"2023-09-23T20:15:23.224560+0000","last_clean":"2023-09-23T20:15:23.224560+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:15:23.224560+0000","last_undegraded":"2023-09-23T20:15:23.224560+0000","last_fullsized":"2023-09-23T20:15:23.224560+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"458'6573","reported_seq":6650,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:38.384003+0000","last_change":"2023-09-23T20:14:36.202358+0000","last_active":"2023-09-23T20:15:38.384003+0000","last_peered":"2023-09-23T20:15:38.384003+0000","last_clean":"2023-09-23T20:14:33.783493+0000","last_became_active":"2023-09-23T20:14:36.193525+0000","last_became_peered":"2023-09-23T20:14:36.193525+0000","last_unstale":"2023-09-23T20:15:38.384003+0000","last_undegraded":"2023-09-23T20:14:36.034471+0000","last_fullsized":"2023-09-23T20:14:35.969590+0000","mapping_epoch":442,"log_start":"411'688","ondisk_log_start":"411'688","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5885,"log_dups_size":0,"ondisk_log_size":5885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122617856,"num_objects":1871,"num_object_clones":0,"num_object_copies":5613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4791,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1584},{"shards":"2,5","objects":-1173}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":475,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.529786+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:15:31.529786+0000","last_peered":"2023-09-23T20:15:31.529786+0000","last_clean":"2023-09-23T20:15:31.529786+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:15:31.529786+0000","last_undegraded":"2023-09-23T20:15:31.529786+0000","last_fullsized":"2023-09-23T20:15:31.529786+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":477,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.517557+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:15:31.517557+0000","last_peered":"2023-09-23T20:15:31.517557+0000","last_clean":"2023-09-23T20:15:31.517557+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:15:31.517557+0000","last_undegraded":"2023-09-23T20:15:31.517557+0000","last_fullsized":"2023-09-23T20:15:31.517557+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.17","version":"458'6452","reported_seq":6521,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:38.333377+0000","last_change":"2023-09-23T20:13:46.814598+0000","last_active":"2023-09-23T20:15:38.333377+0000","last_peered":"2023-09-23T20:15:38.333377+0000","last_clean":"2023-09-23T20:15:38.333377+0000","last_became_active":"2023-09-23T20:13:46.812685+0000","last_became_peered":"2023-09-23T20:13:46.812685+0000","last_unstale":"2023-09-23T20:15:38.333377+0000","last_undegraded":"2023-09-23T20:15:38.333377+0000","last_fullsized":"2023-09-23T20:15:38.333377+0000","mapping_epoch":418,"log_start":"411'781","ondisk_log_start":"411'781","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5671,"log_dups_size":0,"ondisk_log_size":5671,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4742,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":465,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.526866+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:15:31.526866+0000","last_peered":"2023-09-23T20:15:31.526866+0000","last_clean":"2023-09-23T20:15:31.526866+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:15:31.526866+0000","last_undegraded":"2023-09-23T20:15:31.526866+0000","last_fullsized":"2023-09-23T20:15:31.526866+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":454,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.523555+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:15:31.523555+0000","last_peered":"2023-09-23T20:15:31.523555+0000","last_clean":"2023-09-23T20:15:31.523555+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:15:31.523555+0000","last_undegraded":"2023-09-23T20:15:31.523555+0000","last_fullsized":"2023-09-23T20:15:31.523555+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"458'6462","reported_seq":9006,"reported_epoch":458,"state":"active+recovering+remapped","last_fresh":"2023-09-23T20:15:39.108692+0000","last_change":"2023-09-23T20:15:20.084166+0000","last_active":"2023-09-23T20:15:39.108692+0000","last_peered":"2023-09-23T20:15:39.108692+0000","last_clean":"2023-09-23T20:14:33.751783+0000","last_became_active":"2023-09-23T20:14:36.118350+0000","last_became_peered":"2023-09-23T20:14:36.118350+0000","last_unstale":"2023-09-23T20:15:39.108692+0000","last_undegraded":"2023-09-23T20:15:39.108692+0000","last_fullsized":"2023-09-23T20:15:39.108692+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6462,"log_dups_size":0,"ondisk_log_size":6462,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115572736,"num_objects":1764,"num_object_clones":0,"num_object_copies":5292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1814,"num_objects_unfound":0,"num_objects_dirty":1764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4738,"num_write_kb":208032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1233,"num_bytes_recovered":80805888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":-50}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":168,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.225616+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:15:23.225616+0000","last_peered":"2023-09-23T20:15:23.225616+0000","last_clean":"2023-09-23T20:15:23.225616+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:15:23.225616+0000","last_undegraded":"2023-09-23T20:15:23.225616+0000","last_fullsized":"2023-09-23T20:15:23.225616+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":316,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:26.053504+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:15:26.053504+0000","last_peered":"2023-09-23T20:15:26.053504+0000","last_clean":"2023-09-23T20:15:26.053504+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:15:26.053504+0000","last_undegraded":"2023-09-23T20:15:26.053504+0000","last_fullsized":"2023-09-23T20:15:26.053504+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"458'6408","reported_seq":10904,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:38.023625+0000","last_change":"2023-09-23T20:14:38.010024+0000","last_active":"2023-09-23T20:15:38.023625+0000","last_peered":"2023-09-23T20:15:38.023625+0000","last_clean":"2023-09-23T20:14:33.867233+0000","last_became_active":"2023-09-23T20:14:38.004874+0000","last_became_peered":"2023-09-23T20:14:38.004874+0000","last_unstale":"2023-09-23T20:15:38.023625+0000","last_undegraded":"2023-09-23T20:14:37.959307+0000","last_fullsized":"2023-09-23T20:14:37.876450+0000","mapping_epoch":444,"log_start":"411'908","ondisk_log_start":"411'908","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5500,"log_dups_size":0,"ondisk_log_size":5500,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113508352,"num_objects":1732,"num_object_clones":0,"num_object_copies":5196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3098,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4678,"num_write_kb":205120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1473,"num_bytes_recovered":96534528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":1549},{"shards":"3,7","objects":183}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":303,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:15:31.531846+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:15:31.531846+0000","last_peered":"2023-09-23T20:15:31.531846+0000","last_clean":"2023-09-23T20:15:31.531846+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:15:31.531846+0000","last_undegraded":"2023-09-23T20:15:31.531846+0000","last_fullsized":"2023-09-23T20:15:31.531846+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":26,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:15:23.224594+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:15:23.224594+0000","last_peered":"2023-09-23T20:15:23.224594+0000","last_clean":"2023-09-23T20:15:23.224594+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:15:23.224594+0000","last_undegraded":"2023-09-23T20:15:23.224594+0000","last_fullsized":"2023-09-23T20:15:23.224594+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":36,"stat_sum":{"num_bytes":3727753240,"num_objects":56882,"num_object_clones":0,"num_object_copies":170646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":44455,"num_objects_misplaced":5254,"num_objects_unfound":0,"num_objects_dirty":56882,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150824,"num_write_kb":6646529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13373,"num_bytes_recovered":876412928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13491515392,"data_stored":13491503176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":214536,"ondisk_log_size":214536,"up":108,"acting":90,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":178,"num_bytes_recovered":4194380,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234414,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2672772,"kb_used_data":2203324,"kb_used_omap":0,"kb_used_meta":469120,"kb_avail":91072380,"statfs":{"total":95995035648,"available":93258117120,"internally_reserved":0,"allocated":2256203776,"data_stored":2254802319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":480378880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:14:46 2023","interfaces":[{"interface":"back","average":{"1min":4.9560000000000004,"5min":2.1859999999999999,"15min":2.2610000000000001},"min":{"1min":0.25700000000000001,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":71.563000000000002,"5min":71.563000000000002,"15min":71.563000000000002},"last":2.069},{"interface":"front","average":{"1min":5.0620000000000003,"5min":2.3359999999999999,"15min":2.3650000000000002},"min":{"1min":0.29699999999999999,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":71.465000000000003,"5min":71.465000000000003,"15min":71.465000000000003},"last":4.5979999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:14:46 2023","interfaces":[{"interface":"back","average":{"1min":3.746,"5min":3.4460000000000002,"15min":3.7450000000000001},"min":{"1min":0.25600000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":47.197000000000003,"5min":91.335999999999999,"15min":104.16800000000001},"last":2.0960000000000001},{"interface":"front","average":{"1min":3.5539999999999998,"5min":3.952,"15min":4.0019999999999998},"min":{"1min":0.34100000000000003,"5min":0.249,"15min":0.223},"max":{"1min":47.436999999999998,"5min":91.516999999999996,"15min":104.003},"last":4.6429999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":3.6480000000000001,"15min":3.6880000000000002},"min":{"1min":0.33200000000000002,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":5.9340000000000002,"5min":67.760000000000005,"15min":105.842},"last":2.1200000000000001},{"interface":"front","average":{"1min":1.7470000000000001,"5min":3.9119999999999999,"15min":4.1360000000000001},"min":{"1min":0.35599999999999998,"5min":0.255,"15min":0.255},"max":{"1min":5.3840000000000003,"5min":67.488,"15min":105.66},"last":8.2880000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:14:46 2023","interfaces":[{"interface":"back","average":{"1min":3.5539999999999998,"5min":1.7290000000000001,"15min":3.931},"min":{"1min":0.36199999999999999,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":40.036000000000001,"5min":40.036000000000001,"15min":146.38800000000001},"last":2.1509999999999998},{"interface":"front","average":{"1min":3.641,"5min":1.722,"15min":3.9359999999999999},"min":{"1min":0.47699999999999998,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":38.838000000000001,"5min":38.838000000000001,"15min":146.851},"last":8.3369999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":0.79200000000000004,"15min":0.71399999999999997},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.22500000000000001},"max":{"1min":6.2569999999999997,"5min":6.2569999999999997,"15min":19.215},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.4009999999999998,"5min":0.98399999999999999,"15min":0.90700000000000003},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":10.882999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":2.028}]},{"osd":5,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.2450000000000001,"15min":1.105},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":14.069000000000001,"5min":14.069000000000001,"15min":35.488},"last":8.4440000000000008},{"interface":"front","average":{"1min":2.6469999999999998,"5min":1.103,"15min":0.97999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":13.43,"5min":13.43,"15min":34.896000000000001},"last":1.6679999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:14:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":0.90300000000000002,"15min":0.89500000000000002},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":5.718,"5min":5.718,"15min":41.902999999999999},"last":8.4120000000000008},{"interface":"front","average":{"1min":1.2410000000000001,"5min":0.72799999999999998,"15min":0.69399999999999995},"min":{"1min":0.371,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":5.9199999999999999,"5min":5.9199999999999999,"15min":6.3620000000000001},"last":0.377}]}]},{"osd":6,"up_from":0,"seq":1078036791455,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762613,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926150,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2309024,"kb_used_data":1567192,"kb_used_omap":8,"kb_used_meta":741815,"kb_avail":91436128,"statfs":{"total":95995035648,"available":93630595072,"internally_reserved":0,"allocated":1604804608,"data_stored":1602679329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":759619169},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.609,"15min":3.605},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":3.3780000000000001,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.1640000000000001,"15min":3.863},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.183},"max":{"1min":3.9470000000000001,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.314}]},{"osd":2,"last update":"Sat Sep 23 20:15:16 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.9359999999999999,"15min":5.0519999999999996},"min":{"1min":0.215,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":3.5920000000000001,"5min":160.542,"15min":270.51100000000002},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.5720000000000001,"5min":3.4990000000000001,"15min":4.9820000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":7.5220000000000002,"5min":160.75999999999999,"15min":269.69299999999998},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.292,"15min":3.9980000000000002},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.21299999999999999},"max":{"1min":3.6629999999999998,"5min":10.082000000000001,"15min":126.95399999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.5289999999999999,"15min":4.1779999999999999},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":4.9359999999999999,"5min":9.5440000000000005,"15min":127.294},"last":0.34499999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":1.4790000000000001,"15min":2.0499999999999998},"min":{"1min":0.33500000000000002,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":8.7360000000000007,"5min":9.6959999999999997,"15min":148.20599999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.734,"5min":1.3140000000000001,"15min":1.994},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.8230000000000004,"5min":8.4429999999999996,"15min":148.11000000000001},"last":0.66800000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.7609999999999999,"15min":1.8759999999999999},"min":{"1min":0.35999999999999999,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":6.8739999999999997,"5min":52.061,"15min":64.725999999999999},"last":5.5430000000000001},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.2450000000000001,"15min":1.899},"min":{"1min":0.312,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1500000000000004,"5min":51.741,"15min":51.741},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.117,"15min":1.5529999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":3.504,"5min":6.1660000000000004,"15min":15.744999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.2390000000000001,"15min":1.724},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":4.6580000000000004,"5min":8.3919999999999995,"15min":13.766999999999999},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.2969999999999999,"15min":1.7490000000000001},"min":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.7400000000000002,"5min":12.981,"15min":65.081999999999994},"last":0.65300000000000002},{"interface":"front","average":{"1min":2.5249999999999999,"5min":1.861,"15min":1.8620000000000001},"min":{"1min":0.442,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.2290000000000001,"5min":17.151,"15min":17.151},"last":0.70099999999999996}]}]},{"osd":2,"up_from":104,"seq":446676599088,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1673412,"kb_used_data":870332,"kb_used_omap":12,"kb_used_meta":803059,"kb_avail":92071740,"statfs":{"total":95995035648,"available":94281461760,"internally_reserved":0,"allocated":891219968,"data_stored":889058358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":822333438},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":2.0670000000000002,"15min":3.0339999999999998},"min":{"1min":0.253,"5min":0.221,"15min":0.192},"max":{"1min":5.923,"5min":65.382999999999996,"15min":76.251999999999995},"last":4.8780000000000001},{"interface":"front","average":{"1min":1.7230000000000001,"5min":1.994,"15min":3.081},"min":{"1min":0.26600000000000001,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":5.694,"5min":65.427000000000007,"15min":76.590000000000003},"last":6.6829999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.9020000000000001,"15min":3.8250000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":9.2590000000000003,"5min":70.453000000000003,"15min":94.768000000000001},"last":1.8799999999999999},{"interface":"front","average":{"1min":1.8149999999999999,"5min":2.7200000000000002,"15min":3.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.21099999999999999},"max":{"1min":11.981999999999999,"5min":69.825000000000003,"15min":96.808000000000007},"last":4.9169999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.5760000000000001,"15min":3.8730000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":9.2319999999999993,"5min":14.131,"15min":107.381},"last":1.843},{"interface":"front","average":{"1min":2.3940000000000001,"5min":1.5029999999999999,"15min":3.1469999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.927,"5min":14.927,"15min":106.301},"last":5.0179999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.272,"15min":1.964},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.7270000000000003,"5min":14.757999999999999,"15min":75.608000000000004},"last":4.9580000000000002},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.599,"15min":2.1779999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26700000000000002},"max":{"1min":9.1590000000000007,"5min":13.387,"15min":76.161000000000001},"last":1.8200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":3.1360000000000001,"15min":3.0310000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.956,"5min":135.202,"15min":172.505},"last":4.9889999999999999},{"interface":"front","average":{"1min":2.0350000000000001,"5min":1.966,"15min":2.7679999999999998},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32300000000000001},"max":{"1min":12.050000000000001,"5min":29.196999999999999,"15min":172.375},"last":2.7519999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:14:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.381,"15min":1.613},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.843,"5min":14.664,"15min":14.664},"last":5.0510000000000002},{"interface":"front","average":{"1min":1.8939999999999999,"5min":1.5409999999999999,"15min":1.8540000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.025,"5min":12.025,"15min":14.279999999999999},"last":2.7949999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.7849999999999999,"15min":1.9159999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2050000000000001,"5min":13.481999999999999,"15min":23.079999999999998},"last":1.788},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.3859999999999999,"15min":1.7609999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.7789999999999999,"5min":14.698,"15min":20.614999999999998},"last":5.0350000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435562,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2812860,"kb_used_data":1966260,"kb_used_omap":4,"kb_used_meta":846587,"kb_avail":90932292,"statfs":{"total":95995035648,"available":93114667008,"internally_reserved":0,"allocated":2013450240,"data_stored":2011053928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":866905197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:14:43 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.117,"15min":2.9780000000000002},"min":{"1min":0.20100000000000001,"5min":0.186,"15min":0.182},"max":{"1min":5.6230000000000002,"5min":7.6440000000000001,"15min":74.323999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.0910000000000002,"5min":1.3120000000000001,"15min":3.2639999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.19600000000000001},"max":{"1min":11.734,"5min":11.734,"15min":74.447000000000003},"last":0.54800000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:14:43 2023","interfaces":[{"interface":"back","average":{"1min":10.9,"5min":5.6120000000000001,"15min":4.6820000000000004},"min":{"1min":0.24399999999999999,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":183.87,"5min":183.87,"15min":183.87},"last":0.501},{"interface":"front","average":{"1min":11.297000000000001,"5min":5.6109999999999998,"15min":4.859},"min":{"1min":0.28199999999999997,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":185.84899999999999,"5min":185.84899999999999,"15min":185.84899999999999},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 20:14:43 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":4.0940000000000003,"15min":4.1980000000000004},"min":{"1min":0.25600000000000001,"5min":0.24199999999999999,"15min":0.23899999999999999},"max":{"1min":7.1349999999999998,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.7490000000000001,"5min":3.4550000000000001,"15min":3.8559999999999999},"min":{"1min":0.251,"5min":0.25,"15min":0.25},"max":{"1min":6.242,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.40000000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:14:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":1.004,"15min":2.8530000000000002},"min":{"1min":0.34200000000000003,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":7.4960000000000004,"5min":7.4960000000000004,"15min":109.66800000000001},"last":2.4809999999999999},{"interface":"front","average":{"1min":2.0230000000000001,"5min":1.3080000000000001,"15min":3.2330000000000001},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":11.815,"5min":11.815,"15min":110.295},"last":2.2709999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:14:43 2023","interfaces":[{"interface":"back","average":{"1min":11.939,"5min":3.3370000000000002,"15min":3.8969999999999998},"min":{"1min":0.40100000000000002,"5min":0.31,"15min":0.31},"max":{"1min":200.53200000000001,"5min":200.53200000000001,"15min":200.53200000000001},"last":2.1880000000000002},{"interface":"front","average":{"1min":12.055999999999999,"5min":3.2989999999999999,"15min":3.8879999999999999},"min":{"1min":0.45500000000000002,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":200.846,"5min":200.846,"15min":200.846},"last":4.4720000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:14:59 2023","interfaces":[{"interface":"back","average":{"1min":1.8280000000000001,"5min":1.329,"15min":3.0609999999999999},"min":{"1min":0.41599999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":6.774,"5min":6.774,"15min":82.281999999999996},"last":3.694},{"interface":"front","average":{"1min":1.5389999999999999,"5min":1.1339999999999999,"15min":3.1760000000000002},"min":{"1min":0.375,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":8.2629999999999999,"5min":8.2629999999999999,"15min":82.233000000000004},"last":3.6800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:14:43 2023","interfaces":[{"interface":"back","average":{"1min":8.5890000000000004,"5min":2.617,"15min":3.3530000000000002},"min":{"1min":0.38800000000000001,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":139.465,"5min":139.465,"15min":139.465},"last":2.5790000000000002},{"interface":"front","average":{"1min":9.0310000000000006,"5min":2.7090000000000001,"15min":3.323},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":141.56200000000001,"5min":141.56200000000001,"15min":141.56200000000001},"last":2.8570000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019324,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1467904,"kb_used_data":1047864,"kb_used_omap":0,"kb_used_meta":419968,"kb_avail":92277248,"statfs":{"total":95995035648,"available":94491901952,"internally_reserved":0,"allocated":1073012736,"data_stored":1071615363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":430047232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:25 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":1.2390000000000001,"15min":2.1720000000000002},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.9269999999999996,"5min":14.063000000000001,"15min":89.004999999999995},"last":1.6559999999999999},{"interface":"front","average":{"1min":1.831,"5min":1.268,"15min":2.169},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":10.289,"5min":14.048999999999999,"15min":89.024000000000001},"last":0.56200000000000006}]},{"osd":1,"last update":"Sat Sep 23 20:15:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":3.395,"15min":5.9109999999999996},"min":{"1min":0.27800000000000002,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":5.8259999999999996,"5min":138.47999999999999,"15min":161.613},"last":3.089},{"interface":"front","average":{"1min":2.6070000000000002,"5min":3.46,"15min":5.9889999999999999},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.217},"max":{"1min":10.368,"5min":138.429,"15min":162.017},"last":0.59399999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":3.7320000000000002,"15min":5.6970000000000001},"min":{"1min":0.30499999999999999,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.8570000000000002,"5min":144.916,"15min":182.96700000000001},"last":1.736},{"interface":"front","average":{"1min":2.4020000000000001,"5min":3.694,"15min":5.6100000000000003},"min":{"1min":0.40300000000000002,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":11.528,"5min":144.83099999999999,"15min":183.244},"last":0.63100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:15:31 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":3.0459999999999998,"15min":5.1020000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":15.707000000000001,"5min":100.361,"15min":184.37100000000001},"last":3.4780000000000002},{"interface":"front","average":{"1min":2.9700000000000002,"5min":3.3420000000000001,"15min":5.1900000000000004},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.28599999999999998},"max":{"1min":15.73,"5min":100.20699999999999,"15min":184.22300000000001},"last":2.8570000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:15:25 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":0.84099999999999997,"15min":1.177},"min":{"1min":0.26400000000000001,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":12.257999999999999,"5min":12.257999999999999,"15min":176.81899999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.1139999999999999,"5min":0.84199999999999997,"15min":1.2330000000000001},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":6.9770000000000003,"5min":6.9770000000000003,"15min":176.172},"last":2.5419999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:14:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.69199999999999995,"15min":0.57499999999999996},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":10.337999999999999,"5min":10.337999999999999,"15min":10.337999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.518,"5min":0.754,"15min":0.67400000000000004},"min":{"1min":0.39400000000000002,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":5.8769999999999998,"5min":5.8769999999999998,"15min":5.8769999999999998},"last":2.585}]},{"osd":7,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":0.82999999999999996,"15min":1.429},"min":{"1min":0.442,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.71,"5min":7.71,"15min":76.507000000000005},"last":2.665},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.88900000000000001,"15min":1.5169999999999999},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":5.8869999999999996,"5min":5.8869999999999996,"15min":76.462000000000003},"last":1.7090000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757688,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2945080,"kb_used_data":2409776,"kb_used_omap":0,"kb_used_meta":535104,"kb_avail":90800072,"statfs":{"total":95995035648,"available":92979273728,"internally_reserved":0,"allocated":2467610624,"data_stored":2466217692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":547946496},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:23 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":1.3100000000000001,"15min":1.8100000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.304,"5min":10.304,"15min":121.94799999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.2809999999999999,"15min":2.0489999999999999},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":8.1699999999999999,"5min":9.4250000000000007,"15min":121.964},"last":0.255}]},{"osd":1,"last update":"Sat Sep 23 20:14:49 2023","interfaces":[{"interface":"back","average":{"1min":12.496,"5min":5.5090000000000003,"15min":6.274},"min":{"1min":0.22900000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":150.16999999999999,"5min":150.16999999999999,"15min":266.197},"last":3.496},{"interface":"front","average":{"1min":10.928000000000001,"5min":5.1550000000000002,"15min":6.3109999999999999},"min":{"1min":0.253,"5min":0.24099999999999999,"15min":0.21199999999999999},"max":{"1min":150.12700000000001,"5min":150.12700000000001,"15min":266.14999999999998},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:15:05 2023","interfaces":[{"interface":"back","average":{"1min":2.8879999999999999,"5min":3.7519999999999998,"15min":4.3129999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.787000000000001,"5min":120.297,"15min":240.57300000000001},"last":2.2919999999999998},{"interface":"front","average":{"1min":2.919,"5min":3.8500000000000001,"15min":4.3810000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.81,"5min":120.325,"15min":240.464},"last":2.3239999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:14:49 2023","interfaces":[{"interface":"back","average":{"1min":9.6509999999999998,"5min":4.2999999999999998,"15min":4.9740000000000002},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":142.63800000000001,"5min":142.63800000000001,"15min":201.50899999999999},"last":0.621},{"interface":"front","average":{"1min":10.086,"5min":4.1550000000000002,"15min":4.8399999999999999},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":142.81399999999999,"5min":142.81399999999999,"15min":201.58099999999999},"last":0.56000000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:15:23 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":0.89600000000000002,"15min":1.345},"min":{"1min":0.25,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":8,"5min":8,"15min":191.78899999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.60499999999999998,"15min":1.2210000000000001},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":2.7160000000000002,"5min":3.2189999999999999,"15min":191.821},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:14:55 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":0.89600000000000002,"15min":0.74399999999999999},"min":{"1min":0.313,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":7.5250000000000004,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.60799999999999998,"15min":0.56599999999999995},"min":{"1min":0.39400000000000002,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":3.1219999999999999,"5min":3.1219999999999999,"15min":5.0119999999999996},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:15:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2360000000000002,"5min":1.1739999999999999,"15min":1.4139999999999999},"min":{"1min":0.33300000000000002,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":10.536,"5min":10.536,"15min":191.84299999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.395,"5min":0.999,"15min":1.413},"min":{"1min":0.40400000000000003,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":8.3260000000000005,"5min":8.3260000000000005,"15min":191.69},"last":2.7549999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1580400640,"data_stored":1580400640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3303276544,"data_stored":3303276544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":881197056,"data_stored":881197056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1990066176,"data_stored":1990066176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1066536960,"data_stored":1066532888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2436304896,"data_stored":2436300824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2233733120,"data_stored":2233729048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:15:45.255 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:15:45.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:45.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:45.358 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:45.366 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:15:45.585 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:45.708 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:45.711 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:45.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:45.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:45.813 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:45.881 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:45.965 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:45.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:45.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:46.077 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:46.129 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:46 smithi135 ceph-mon[213871]: pgmap v1275: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 43938/158193 objects degraded (27.775%); 4603/158193 objects misplaced (2.910%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:15:46.154 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:46.247 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:46.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:46.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:46.354 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:46.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:46 smithi144 ceph-mon[194833]: pgmap v1275: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 43938/158193 objects degraded (27.775%); 4603/158193 objects misplaced (2.910%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:15:46.422 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:46 smithi135 ceph-mon[221113]: pgmap v1275: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 43938/158193 objects degraded (27.775%); 4603/158193 objects misplaced (2.910%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:15:46.447 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:46.554 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:46.565 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:46.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:46.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:46.670 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:46.713 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:46.743 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:47.162 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:15:47.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3472445482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:47.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:47 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:47.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3472445482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:47.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:47 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3472445482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:47 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:15:48.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: pgmap v1276: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 43938/156507 objects degraded (28.074%); 4603/156507 objects misplaced (2.941%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: osdmap e459: 8 total, 8 up, 6 in 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 43938/156507 objects degraded (28.074%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: pgmap v1276: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 43938/156507 objects degraded (28.074%); 4603/156507 objects misplaced (2.941%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: osdmap e459: 8 total, 8 up, 6 in 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:48 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 43938/156507 objects degraded (28.074%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:48.502 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:48.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:48.607 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:48.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: pgmap v1276: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 43938/156507 objects degraded (28.074%); 4603/156507 objects misplaced (2.941%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:15:48.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:15:48.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:15:48.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: osdmap e459: 8 total, 8 up, 6 in 2023-09-23T20:15:48.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: failed to encode map e459 with expected crc 2023-09-23T20:15:48.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:48 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 43938/156507 objects degraded (28.074%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:48.670 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:49.369 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:49.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:49.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:49.473 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:49.539 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:50.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:50 smithi135 ceph-mon[213871]: pgmap v1278: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 43440/145743 objects degraded (29.806%); 4319/145743 objects misplaced (2.963%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:15:50.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:50 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:50.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:50 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:50.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:50 smithi135 ceph-mon[221113]: pgmap v1278: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 43440/145743 objects degraded (29.806%); 4319/145743 objects misplaced (2.963%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:15:50.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:50 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:50.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:50 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:50.626 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:50.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:50.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:50.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:50 smithi144 ceph-mon[194833]: pgmap v1278: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 43440/145743 objects degraded (29.806%); 4319/145743 objects misplaced (2.963%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:15:50.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:50 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:50.638 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:50 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:50.735 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:50.798 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:51.440 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:51.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:51.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:51.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:51.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:51.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:51.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:51.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:51.449 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:51.550 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:51.610 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:51.642 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:15:51.991 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:15:52.045 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:15:52.074 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:15:52.110 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:15:52.147 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:52.147 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:15:52.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:52.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:52.199 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:15:52.234 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:15:52.252 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:52.266 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:15:52.269 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:15:52.270 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:15:52.270 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:15:52.270 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:15:52.270 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:15:52.270 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:15:52.270 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:15:52.270 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:15:52.319 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:52 smithi135 ceph-mon[221113]: pgmap v1279: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 43440/145311 objects degraded (29.895%); 3909/145311 objects misplaced (2.690%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:15:52.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1421341434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:52.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:52 smithi135 ceph-mon[213871]: pgmap v1279: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 43440/145311 objects degraded (29.895%); 3909/145311 objects misplaced (2.690%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:15:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1421341434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:52.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:52 smithi144 ceph-mon[194833]: pgmap v1279: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 43440/145311 objects degraded (29.895%); 3909/145311 objects misplaced (2.690%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:15:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1421341434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:15:52.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:15:53.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:53.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 43440/145311 objects degraded (29.895%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:53.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:53 smithi135 ceph-mon[221113]: pgmap v1280: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 42910/138357 objects degraded (31.014%); 3777/138357 objects misplaced (2.730%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T20:15:53.197 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:15:53.210 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:53.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:53.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:53.248 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:15:53.316 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:53.392 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:53.459 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:15:53.459 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:15:53.459 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:15:53.459 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:15:53.459 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:15:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 43440/145311 objects degraded (29.895%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:53 smithi135 ceph-mon[213871]: pgmap v1280: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 42910/138357 objects degraded (31.014%); 3777/138357 objects misplaced (2.730%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T20:15:53.511 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:53.521 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:53.521 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 43440/145311 objects degraded (29.895%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:53.521 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:53 smithi144 ceph-mon[194833]: pgmap v1280: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 42910/138357 objects degraded (31.014%); 3777/138357 objects misplaced (2.730%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T20:15:53.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:53.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:53.622 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:15:53.676 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:53.972 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:15:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:15:53] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-09-23T20:15:54.292 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:15:54.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:54.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:15:54.405 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:54.470 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:54.518 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1280,"stamp":"2023-09-23T20:15:53.067993+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3025362349,"num_objects":46119,"num_object_clones":0,"num_object_copies":138357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":42910,"num_objects_misplaced":3777,"num_objects_unfound":0,"num_objects_dirty":46119,"num_whiteouts":0,"num_read":5996,"num_read_kb":13236,"num_write":162529,"num_write_kb":6659327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15184,"num_bytes_recovered":984874100,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":212520,"ondisk_log_size":212520,"up":387,"acting":366,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":391,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12280068,"kb_used_data":8397972,"kb_used_omap":25,"kb_used_meta":3881638,"kb_avail":550190844,"statfs":{"total":575970213888,"available":563395424256,"internally_reserved":0,"allocated":8599523328,"data_stored":8588612721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":3974798028},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-559218688,"num_objects":-8533,"num_object_clones":0,"num_object_copies":-25599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1028,"num_objects_misplaced":-1345,"num_objects_unfound":0,"num_objects_dirty":-8533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8533,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1315,"num_bytes_recovered":86179840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006787"},"pg_stats":[{"pgid":"9.23","version":"459'5723","reported_seq":5813,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:53.012218+0000","last_change":"2023-09-23T20:14:36.092943+0000","last_active":"2023-09-23T20:15:53.012218+0000","last_peered":"2023-09-23T20:15:53.012218+0000","last_clean":"2023-09-23T20:14:33.760240+0000","last_became_active":"2023-09-23T20:14:36.076681+0000","last_became_peered":"2023-09-23T20:14:36.076681+0000","last_unstale":"2023-09-23T20:15:53.012218+0000","last_undegraded":"2023-09-23T20:14:35.911782+0000","last_fullsized":"2023-09-23T20:14:35.868390+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45498368,"num_objects":694,"num_object_clones":0,"num_object_copies":2082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2614,"num_write_kb":105872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":604},{"shards":"3,7","objects":-870}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.22","version":"459'5720","reported_seq":5810,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:51.431520+0000","last_change":"2023-09-23T20:14:36.006132+0000","last_active":"2023-09-23T20:15:51.431520+0000","last_peered":"2023-09-23T20:15:51.431520+0000","last_clean":"2023-09-23T20:14:33.601158+0000","last_became_active":"2023-09-23T20:14:35.991960+0000","last_became_peered":"2023-09-23T20:14:35.991960+0000","last_unstale":"2023-09-23T20:15:51.431520+0000","last_undegraded":"2023-09-23T20:14:35.848470+0000","last_fullsized":"2023-09-23T20:14:35.833465+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5720,"log_dups_size":0,"ondisk_log_size":5720,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48103424,"num_objects":734,"num_object_clones":0,"num_object_copies":2202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2584,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":619},{"shards":"2,3","objects":-810}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.21","version":"459'5636","reported_seq":5716,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:52.886675+0000","last_change":"2023-09-23T20:14:24.799466+0000","last_active":"2023-09-23T20:15:52.886675+0000","last_peered":"2023-09-23T20:15:52.886675+0000","last_clean":"2023-09-23T20:15:52.886675+0000","last_became_active":"2023-09-23T20:14:24.796974+0000","last_became_peered":"2023-09-23T20:14:24.796974+0000","last_unstale":"2023-09-23T20:15:52.886675+0000","last_undegraded":"2023-09-23T20:15:52.886675+0000","last_fullsized":"2023-09-23T20:15:52.886675+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5636,"log_dups_size":0,"ondisk_log_size":5636,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44924928,"num_objects":686,"num_object_clones":0,"num_object_copies":2058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2542,"num_write_kb":103264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.20","version":"459'5634","reported_seq":5723,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:50.932460+0000","last_change":"2023-09-23T20:14:36.109513+0000","last_active":"2023-09-23T20:15:50.932460+0000","last_peered":"2023-09-23T20:15:50.932460+0000","last_clean":"2023-09-23T20:14:33.769971+0000","last_became_active":"2023-09-23T20:14:36.106301+0000","last_became_peered":"2023-09-23T20:14:36.106301+0000","last_unstale":"2023-09-23T20:15:50.932460+0000","last_undegraded":"2023-09-23T20:14:35.845700+0000","last_fullsized":"2023-09-23T20:14:35.836337+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5634,"log_dups_size":0,"ondisk_log_size":5634,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45088768,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2552,"num_write_kb":103680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":613},{"shards":"0,2","objects":-857}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.10","version":"459'6727","reported_seq":12656,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:52.826046+0000","last_change":"2023-09-23T20:14:22.339078+0000","last_active":"2023-09-23T20:15:52.826046+0000","last_peered":"2023-09-23T20:15:52.826046+0000","last_clean":"2023-09-23T20:15:52.826046+0000","last_became_active":"2023-09-23T20:14:22.319507+0000","last_became_peered":"2023-09-23T20:14:22.319507+0000","last_unstale":"2023-09-23T20:15:52.826046+0000","last_undegraded":"2023-09-23T20:15:52.826046+0000","last_fullsized":"2023-09-23T20:15:52.826046+0000","mapping_epoch":434,"log_start":"411'1560","ondisk_log_start":"411'1560","created":418,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5167,"log_dups_size":0,"ondisk_log_size":5167,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87883776,"num_objects":1341,"num_object_clones":0,"num_object_copies":4023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5007,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1945,"num_bytes_recovered":127467520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":29,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152371+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:15:36.152371+0000","last_peered":"2023-09-23T20:15:36.152371+0000","last_clean":"2023-09-23T20:15:36.152371+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:15:36.152371+0000","last_undegraded":"2023-09-23T20:15:36.152371+0000","last_fullsized":"2023-09-23T20:15:36.152371+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":27,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.150681+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:15:36.150681+0000","last_peered":"2023-09-23T20:15:36.150681+0000","last_clean":"2023-09-23T20:15:36.150681+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:15:36.150681+0000","last_undegraded":"2023-09-23T20:15:36.150681+0000","last_fullsized":"2023-09-23T20:15:36.150681+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.13","version":"459'6845","reported_seq":12646,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:53.034308+0000","last_change":"2023-09-23T20:14:36.123690+0000","last_active":"2023-09-23T20:15:53.034308+0000","last_peered":"2023-09-23T20:15:53.034308+0000","last_clean":"2023-09-23T20:14:05.142062+0000","last_became_active":"2023-09-23T20:14:07.304693+0000","last_became_peered":"2023-09-23T20:14:07.304693+0000","last_unstale":"2023-09-23T20:15:53.034308+0000","last_undegraded":"2023-09-23T20:14:36.121893+0000","last_fullsized":"2023-09-23T20:14:07.197546+0000","mapping_epoch":429,"log_start":"412'1769","ondisk_log_start":"412'1769","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5076,"log_dups_size":0,"ondisk_log_size":5076,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85491712,"num_objects":1305,"num_object_clones":0,"num_object_copies":3915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":634,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5113,"num_write_kb":205344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1906,"num_bytes_recovered":124911616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,0],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"0,7","objects":988},{"shards":"1,3,7","objects":317}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":204,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158539+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:15:36.158539+0000","last_peered":"2023-09-23T20:15:36.158539+0000","last_clean":"2023-09-23T20:15:36.158539+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:15:36.158539+0000","last_undegraded":"2023-09-23T20:15:36.158539+0000","last_fullsized":"2023-09-23T20:15:36.158539+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":167,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.157386+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:15:36.157386+0000","last_peered":"2023-09-23T20:15:36.157386+0000","last_clean":"2023-09-23T20:15:36.157386+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:15:36.157386+0000","last_undegraded":"2023-09-23T20:15:36.157386+0000","last_fullsized":"2023-09-23T20:15:36.157386+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"459'6788","reported_seq":13250,"reported_epoch":459,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T20:15:48.294421+0000","last_change":"2023-09-23T20:14:36.234004+0000","last_active":"2023-09-23T20:15:48.294421+0000","last_peered":"2023-09-23T20:15:48.294421+0000","last_clean":"2023-09-23T20:14:33.790624+0000","last_became_active":"2023-09-23T20:14:36.220990+0000","last_became_peered":"2023-09-23T20:14:36.220990+0000","last_unstale":"2023-09-23T20:15:48.294421+0000","last_undegraded":"2023-09-23T20:15:48.294421+0000","last_fullsized":"2023-09-23T20:15:48.294421+0000","mapping_epoch":442,"log_start":"411'1278","ondisk_log_start":"411'1278","created":418,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92143616,"num_objects":1406,"num_object_clones":0,"num_object_copies":4218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1406,"num_objects_unfound":0,"num_objects_dirty":1406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5038,"num_write_kb":206208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":139067392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1","3"],"object_location_counts":[{"shards":"0,1,5","objects":1406}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151782+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:15:36.151782+0000","last_peered":"2023-09-23T20:15:36.151782+0000","last_clean":"2023-09-23T20:15:36.151782+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:15:36.151782+0000","last_undegraded":"2023-09-23T20:15:36.151782+0000","last_fullsized":"2023-09-23T20:15:36.151782+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":414,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.160200+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:15:36.160200+0000","last_peered":"2023-09-23T20:15:36.160200+0000","last_clean":"2023-09-23T20:15:36.160200+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:15:36.160200+0000","last_undegraded":"2023-09-23T20:15:36.160200+0000","last_fullsized":"2023-09-23T20:15:36.160200+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1d","version":"459'7966","reported_seq":8046,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:52.895326+0000","last_change":"2023-09-23T20:14:36.076800+0000","last_active":"2023-09-23T20:15:52.895326+0000","last_peered":"2023-09-23T20:15:52.895326+0000","last_clean":"2023-09-23T20:14:23.775634+0000","last_became_active":"2023-09-23T20:14:36.070774+0000","last_became_peered":"2023-09-23T20:14:36.070774+0000","last_unstale":"2023-09-23T20:15:52.895326+0000","last_undegraded":"2023-09-23T20:14:35.892079+0000","last_fullsized":"2023-09-23T20:14:35.839513+0000","mapping_epoch":442,"log_start":"412'1822","ondisk_log_start":"412'1822","created":436,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98435096,"num_objects":1503,"num_object_clones":0,"num_object_copies":4509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1503,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5177,"num_write_kb":213697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":1238},{"shards":"5,7","objects":-1572}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":385,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.160069+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:15:36.160069+0000","last_peered":"2023-09-23T20:15:36.160069+0000","last_clean":"2023-09-23T20:15:36.160069+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:15:36.160069+0000","last_undegraded":"2023-09-23T20:15:36.160069+0000","last_fullsized":"2023-09-23T20:15:36.160069+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":313,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158137+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:15:36.158137+0000","last_peered":"2023-09-23T20:15:36.158137+0000","last_clean":"2023-09-23T20:15:36.158137+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:15:36.158137+0000","last_undegraded":"2023-09-23T20:15:36.158137+0000","last_fullsized":"2023-09-23T20:15:36.158137+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"459'7597","reported_seq":7667,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:51.526033+0000","last_change":"2023-09-23T20:14:24.810789+0000","last_active":"2023-09-23T20:15:51.526033+0000","last_peered":"2023-09-23T20:15:51.526033+0000","last_clean":"2023-09-23T20:15:51.526033+0000","last_became_active":"2023-09-23T20:14:24.807691+0000","last_became_peered":"2023-09-23T20:14:24.807691+0000","last_unstale":"2023-09-23T20:15:51.526033+0000","last_undegraded":"2023-09-23T20:15:51.526033+0000","last_fullsized":"2023-09-23T20:15:51.526033+0000","mapping_epoch":436,"log_start":"412'1997","ondisk_log_start":"412'1997","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5600,"log_dups_size":0,"ondisk_log_size":5600,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88702976,"num_objects":1354,"num_object_clones":0,"num_object_copies":4062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4978,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":792,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153471+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:15:36.153471+0000","last_peered":"2023-09-23T20:15:36.153471+0000","last_clean":"2023-09-23T20:15:36.153471+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:15:36.153471+0000","last_undegraded":"2023-09-23T20:15:36.153471+0000","last_fullsized":"2023-09-23T20:15:36.153471+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":468,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.148122+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:15:36.148122+0000","last_peered":"2023-09-23T20:15:36.148122+0000","last_clean":"2023-09-23T20:15:36.148122+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:15:36.148122+0000","last_undegraded":"2023-09-23T20:15:36.148122+0000","last_fullsized":"2023-09-23T20:15:36.148122+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":323,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158456+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:15:36.158456+0000","last_peered":"2023-09-23T20:15:36.158456+0000","last_clean":"2023-09-23T20:15:36.158456+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:15:36.158456+0000","last_undegraded":"2023-09-23T20:15:36.158456+0000","last_fullsized":"2023-09-23T20:15:36.158456+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1f","version":"459'7616","reported_seq":7695,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:48.372531+0000","last_change":"2023-09-23T20:14:36.232228+0000","last_active":"2023-09-23T20:15:48.372531+0000","last_peered":"2023-09-23T20:15:48.372531+0000","last_clean":"2023-09-23T20:14:33.842679+0000","last_became_active":"2023-09-23T20:14:36.217802+0000","last_became_peered":"2023-09-23T20:14:36.217802+0000","last_unstale":"2023-09-23T20:15:48.372531+0000","last_undegraded":"2023-09-23T20:14:36.129688+0000","last_fullsized":"2023-09-23T20:14:36.115967+0000","mapping_epoch":442,"log_start":"411'1059","ondisk_log_start":"411'1059","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6557,"log_dups_size":0,"ondisk_log_size":6557,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99090432,"num_objects":1512,"num_object_clones":0,"num_object_copies":4536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4930,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":1318},{"shards":"3,5","objects":-1515}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":152,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.166105+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:15:36.166105+0000","last_peered":"2023-09-23T20:15:36.166105+0000","last_clean":"2023-09-23T20:15:36.166105+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:15:36.166105+0000","last_undegraded":"2023-09-23T20:15:36.166105+0000","last_fullsized":"2023-09-23T20:15:36.166105+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152573+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:15:36.152573+0000","last_peered":"2023-09-23T20:15:36.152573+0000","last_clean":"2023-09-23T20:15:36.152573+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:15:36.152573+0000","last_undegraded":"2023-09-23T20:15:36.152573+0000","last_fullsized":"2023-09-23T20:15:36.152573+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.1e","version":"459'7798","reported_seq":13938,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:53.013278+0000","last_change":"2023-09-23T20:15:21.201333+0000","last_active":"2023-09-23T20:15:53.013278+0000","last_peered":"2023-09-23T20:15:53.013278+0000","last_clean":"2023-09-23T20:15:53.013278+0000","last_became_active":"2023-09-23T20:15:21.192111+0000","last_became_peered":"2023-09-23T20:15:21.192111+0000","last_unstale":"2023-09-23T20:15:53.013278+0000","last_undegraded":"2023-09-23T20:15:53.013278+0000","last_fullsized":"2023-09-23T20:15:53.013278+0000","mapping_epoch":454,"log_start":"412'1676","ondisk_log_start":"412'1676","created":436,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6122,"log_dups_size":0,"ondisk_log_size":6122,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90734592,"num_objects":1385,"num_object_clones":0,"num_object_copies":4155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5099,"num_write_kb":207456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3024,"num_bytes_recovered":198180864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":405,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153576+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:15:36.153576+0000","last_peered":"2023-09-23T20:15:36.153576+0000","last_clean":"2023-09-23T20:15:36.153576+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:15:36.153576+0000","last_undegraded":"2023-09-23T20:15:36.153576+0000","last_fullsized":"2023-09-23T20:15:36.153576+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":336,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152085+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:15:36.152085+0000","last_peered":"2023-09-23T20:15:36.152085+0000","last_clean":"2023-09-23T20:15:36.152085+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:15:36.152085+0000","last_undegraded":"2023-09-23T20:15:36.152085+0000","last_fullsized":"2023-09-23T20:15:36.152085+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"459'6934","reported_seq":7004,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:53.016609+0000","last_change":"2023-09-23T20:13:46.811423+0000","last_active":"2023-09-23T20:15:53.016609+0000","last_peered":"2023-09-23T20:15:53.016609+0000","last_clean":"2023-09-23T20:15:53.016609+0000","last_became_active":"2023-09-23T20:13:46.808266+0000","last_became_peered":"2023-09-23T20:13:46.808266+0000","last_unstale":"2023-09-23T20:15:53.016609+0000","last_undegraded":"2023-09-23T20:15:53.016609+0000","last_fullsized":"2023-09-23T20:15:53.016609+0000","mapping_epoch":418,"log_start":"412'1909","ondisk_log_start":"412'1909","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5025,"log_dups_size":0,"ondisk_log_size":5025,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87719936,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5179,"num_write_kb":208544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":818,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.142853+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:15:36.142853+0000","last_peered":"2023-09-23T20:15:36.142853+0000","last_clean":"2023-09-23T20:15:36.142853+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:15:36.142853+0000","last_undegraded":"2023-09-23T20:15:36.142853+0000","last_fullsized":"2023-09-23T20:15:36.142853+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":152,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.157108+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:15:36.157108+0000","last_peered":"2023-09-23T20:15:36.157108+0000","last_clean":"2023-09-23T20:15:36.157108+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:15:36.157108+0000","last_undegraded":"2023-09-23T20:15:36.157108+0000","last_fullsized":"2023-09-23T20:15:36.157108+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":205,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153738+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:15:36.153738+0000","last_peered":"2023-09-23T20:15:36.153738+0000","last_clean":"2023-09-23T20:15:36.153738+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:15:36.153738+0000","last_undegraded":"2023-09-23T20:15:36.153738+0000","last_fullsized":"2023-09-23T20:15:36.153738+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"459'6740","reported_seq":6810,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:48.215499+0000","last_change":"2023-09-23T20:13:46.824838+0000","last_active":"2023-09-23T20:15:48.215499+0000","last_peered":"2023-09-23T20:15:48.215499+0000","last_clean":"2023-09-23T20:15:48.215499+0000","last_became_active":"2023-09-23T20:13:46.820709+0000","last_became_peered":"2023-09-23T20:13:46.820709+0000","last_unstale":"2023-09-23T20:15:48.215499+0000","last_undegraded":"2023-09-23T20:15:48.215499+0000","last_fullsized":"2023-09-23T20:15:48.215499+0000","mapping_epoch":418,"log_start":"411'1247","ondisk_log_start":"411'1247","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5493,"log_dups_size":0,"ondisk_log_size":5493,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99516416,"num_objects":1519,"num_object_clones":0,"num_object_copies":4557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5025,"num_write_kb":209376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":507,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.143049+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:15:36.143049+0000","last_peered":"2023-09-23T20:15:36.143049+0000","last_clean":"2023-09-23T20:15:36.143049+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:15:36.143049+0000","last_undegraded":"2023-09-23T20:15:36.143049+0000","last_fullsized":"2023-09-23T20:15:36.143049+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":167,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.155383+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:15:36.155383+0000","last_peered":"2023-09-23T20:15:36.155383+0000","last_clean":"2023-09-23T20:15:36.155383+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:15:36.155383+0000","last_undegraded":"2023-09-23T20:15:36.155383+0000","last_fullsized":"2023-09-23T20:15:36.155383+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":457,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151209+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:15:36.151209+0000","last_peered":"2023-09-23T20:15:36.151209+0000","last_clean":"2023-09-23T20:15:36.151209+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:15:36.151209+0000","last_undegraded":"2023-09-23T20:15:36.151209+0000","last_fullsized":"2023-09-23T20:15:36.151209+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1b","version":"459'7517","reported_seq":7587,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:48.404176+0000","last_change":"2023-09-23T20:14:24.808547+0000","last_active":"2023-09-23T20:15:48.404176+0000","last_peered":"2023-09-23T20:15:48.404176+0000","last_clean":"2023-09-23T20:15:48.404176+0000","last_became_active":"2023-09-23T20:14:24.806384+0000","last_became_peered":"2023-09-23T20:14:24.806384+0000","last_unstale":"2023-09-23T20:15:48.404176+0000","last_undegraded":"2023-09-23T20:15:48.404176+0000","last_fullsized":"2023-09-23T20:15:48.404176+0000","mapping_epoch":436,"log_start":"411'887","ondisk_log_start":"411'887","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6630,"log_dups_size":0,"ondisk_log_size":6630,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93421568,"num_objects":1426,"num_object_clones":0,"num_object_copies":4278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4906,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":568,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.156826+0000","last_change":"2023-09-23T20:14:37.887863+0000","last_active":"2023-09-23T20:15:36.156826+0000","last_peered":"2023-09-23T20:15:36.156826+0000","last_clean":"2023-09-23T20:15:36.156826+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:15:36.156826+0000","last_undegraded":"2023-09-23T20:15:36.156826+0000","last_fullsized":"2023-09-23T20:15:36.156826+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":146,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.156645+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:15:36.156645+0000","last_peered":"2023-09-23T20:15:36.156645+0000","last_clean":"2023-09-23T20:15:36.156645+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:15:36.156645+0000","last_undegraded":"2023-09-23T20:15:36.156645+0000","last_fullsized":"2023-09-23T20:15:36.156645+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":447,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.555612+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:15:36.555612+0000","last_peered":"2023-09-23T20:15:36.555612+0000","last_clean":"2023-09-23T20:15:36.555612+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:15:36.555612+0000","last_undegraded":"2023-09-23T20:15:36.555612+0000","last_fullsized":"2023-09-23T20:15:36.555612+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1a","version":"459'7617","reported_seq":7697,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:48.362078+0000","last_change":"2023-09-23T20:14:36.218516+0000","last_active":"2023-09-23T20:15:48.362078+0000","last_peered":"2023-09-23T20:15:48.362078+0000","last_clean":"2023-09-23T20:14:33.782041+0000","last_became_active":"2023-09-23T20:14:36.175134+0000","last_became_peered":"2023-09-23T20:14:36.175134+0000","last_unstale":"2023-09-23T20:15:48.362078+0000","last_undegraded":"2023-09-23T20:14:35.943267+0000","last_fullsized":"2023-09-23T20:14:35.848493+0000","mapping_epoch":442,"log_start":"411'912","ondisk_log_start":"411'912","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6705,"log_dups_size":0,"ondisk_log_size":6705,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97976320,"num_objects":1495,"num_object_clones":0,"num_object_copies":4485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4919,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":1248},{"shards":"5,7","objects":-1465}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":596,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.150830+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:15:36.150830+0000","last_peered":"2023-09-23T20:15:36.150830+0000","last_clean":"2023-09-23T20:15:36.150830+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:15:36.150830+0000","last_undegraded":"2023-09-23T20:15:36.150830+0000","last_fullsized":"2023-09-23T20:15:36.150830+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":102,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.142851+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:15:36.142851+0000","last_peered":"2023-09-23T20:15:36.142851+0000","last_clean":"2023-09-23T20:15:36.142851+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:15:36.142851+0000","last_undegraded":"2023-09-23T20:15:36.142851+0000","last_fullsized":"2023-09-23T20:15:36.142851+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":99,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158501+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:15:36.158501+0000","last_peered":"2023-09-23T20:15:36.158501+0000","last_clean":"2023-09-23T20:15:36.158501+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:15:36.158501+0000","last_undegraded":"2023-09-23T20:15:36.158501+0000","last_fullsized":"2023-09-23T20:15:36.158501+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"459'6751","reported_seq":6822,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:48.382935+0000","last_change":"2023-09-23T20:13:46.790587+0000","last_active":"2023-09-23T20:15:48.382935+0000","last_peered":"2023-09-23T20:15:48.382935+0000","last_clean":"2023-09-23T20:15:48.382935+0000","last_became_active":"2023-09-23T20:13:46.788387+0000","last_became_peered":"2023-09-23T20:13:46.788387+0000","last_unstale":"2023-09-23T20:15:48.382935+0000","last_undegraded":"2023-09-23T20:15:48.382935+0000","last_fullsized":"2023-09-23T20:15:48.382935+0000","mapping_epoch":407,"log_start":"411'1197","ondisk_log_start":"411'1197","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5554,"log_dups_size":0,"ondisk_log_size":5554,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98762752,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5021,"num_write_kb":208896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":719,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.163288+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:15:36.163288+0000","last_peered":"2023-09-23T20:15:36.163288+0000","last_clean":"2023-09-23T20:15:36.163288+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:15:36.163288+0000","last_undegraded":"2023-09-23T20:15:36.163288+0000","last_fullsized":"2023-09-23T20:15:36.163288+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":150,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153339+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:15:36.153339+0000","last_peered":"2023-09-23T20:15:36.153339+0000","last_clean":"2023-09-23T20:15:36.153339+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:15:36.153339+0000","last_undegraded":"2023-09-23T20:15:36.153339+0000","last_fullsized":"2023-09-23T20:15:36.153339+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":467,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.142884+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:15:36.142884+0000","last_peered":"2023-09-23T20:15:36.142884+0000","last_clean":"2023-09-23T20:15:36.142884+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:15:36.142884+0000","last_undegraded":"2023-09-23T20:15:36.142884+0000","last_fullsized":"2023-09-23T20:15:36.142884+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"459'6777","reported_seq":6856,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:52.949053+0000","last_change":"2023-09-23T20:14:36.032743+0000","last_active":"2023-09-23T20:15:52.949053+0000","last_peered":"2023-09-23T20:15:52.949053+0000","last_clean":"2023-09-23T20:14:33.800061+0000","last_became_active":"2023-09-23T20:14:36.025052+0000","last_became_peered":"2023-09-23T20:14:36.025052+0000","last_unstale":"2023-09-23T20:15:52.949053+0000","last_undegraded":"2023-09-23T20:14:35.860952+0000","last_fullsized":"2023-09-23T20:14:35.840056+0000","mapping_epoch":442,"log_start":"411'229","ondisk_log_start":"411'229","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89817088,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3023,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5052,"num_write_kb":205536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1182},{"shards":"3,4","objects":-1653}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":806,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158062+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:15:36.158062+0000","last_peered":"2023-09-23T20:15:36.158062+0000","last_clean":"2023-09-23T20:15:36.158062+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:15:36.158062+0000","last_undegraded":"2023-09-23T20:15:36.158062+0000","last_fullsized":"2023-09-23T20:15:36.158062+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":165,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.157464+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:15:36.157464+0000","last_peered":"2023-09-23T20:15:36.157464+0000","last_clean":"2023-09-23T20:15:36.157464+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:15:36.157464+0000","last_undegraded":"2023-09-23T20:15:36.157464+0000","last_fullsized":"2023-09-23T20:15:36.157464+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":459,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158263+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:15:36.158263+0000","last_peered":"2023-09-23T20:15:36.158263+0000","last_clean":"2023-09-23T20:15:36.158263+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:15:36.158263+0000","last_undegraded":"2023-09-23T20:15:36.158263+0000","last_fullsized":"2023-09-23T20:15:36.158263+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.7","version":"459'6662","reported_seq":6733,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:48.346008+0000","last_change":"2023-09-23T20:13:46.802046+0000","last_active":"2023-09-23T20:15:48.346008+0000","last_peered":"2023-09-23T20:15:48.346008+0000","last_clean":"2023-09-23T20:15:48.346008+0000","last_became_active":"2023-09-23T20:13:46.800187+0000","last_became_peered":"2023-09-23T20:13:46.800187+0000","last_unstale":"2023-09-23T20:15:48.346008+0000","last_undegraded":"2023-09-23T20:15:48.346008+0000","last_fullsized":"2023-09-23T20:15:48.346008+0000","mapping_epoch":407,"log_start":"411'983","ondisk_log_start":"411'983","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5679,"log_dups_size":0,"ondisk_log_size":5679,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103546880,"num_objects":1580,"num_object_clones":0,"num_object_copies":4740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4952,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":531,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152576+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:15:36.152576+0000","last_peered":"2023-09-23T20:15:36.152576+0000","last_clean":"2023-09-23T20:15:36.152576+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:15:36.152576+0000","last_undegraded":"2023-09-23T20:15:36.152576+0000","last_fullsized":"2023-09-23T20:15:36.152576+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":150,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.163404+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:15:36.163404+0000","last_peered":"2023-09-23T20:15:36.163404+0000","last_clean":"2023-09-23T20:15:36.163404+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:15:36.163404+0000","last_undegraded":"2023-09-23T20:15:36.163404+0000","last_fullsized":"2023-09-23T20:15:36.163404+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":47,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.155612+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:15:36.155612+0000","last_peered":"2023-09-23T20:15:36.155612+0000","last_clean":"2023-09-23T20:15:36.155612+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:15:36.155612+0000","last_undegraded":"2023-09-23T20:15:36.155612+0000","last_fullsized":"2023-09-23T20:15:36.155612+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"459'6949","reported_seq":7028,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:48.273376+0000","last_change":"2023-09-23T20:14:36.190976+0000","last_active":"2023-09-23T20:15:48.273376+0000","last_peered":"2023-09-23T20:15:48.273376+0000","last_clean":"2023-09-23T20:14:33.830609+0000","last_became_active":"2023-09-23T20:14:36.182918+0000","last_became_peered":"2023-09-23T20:14:36.182918+0000","last_unstale":"2023-09-23T20:15:48.273376+0000","last_undegraded":"2023-09-23T20:14:35.924455+0000","last_fullsized":"2023-09-23T20:14:35.847184+0000","mapping_epoch":442,"log_start":"411'1540","ondisk_log_start":"411'1540","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5167,"num_write_kb":211968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1342},{"shards":"2,5","objects":-1740}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":662,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.157208+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:15:36.157208+0000","last_peered":"2023-09-23T20:15:36.157208+0000","last_clean":"2023-09-23T20:15:36.157208+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:15:36.157208+0000","last_undegraded":"2023-09-23T20:15:36.157208+0000","last_fullsized":"2023-09-23T20:15:36.157208+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":507,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.147405+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:15:36.147405+0000","last_peered":"2023-09-23T20:15:36.147405+0000","last_clean":"2023-09-23T20:15:36.147405+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:15:36.147405+0000","last_undegraded":"2023-09-23T20:15:36.147405+0000","last_fullsized":"2023-09-23T20:15:36.147405+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":203,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.142957+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:15:36.142957+0000","last_peered":"2023-09-23T20:15:36.142957+0000","last_clean":"2023-09-23T20:15:36.142957+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:15:36.142957+0000","last_undegraded":"2023-09-23T20:15:36.142957+0000","last_fullsized":"2023-09-23T20:15:36.142957+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"459'5645","reported_seq":5726,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:52.866840+0000","last_change":"2023-09-23T20:14:24.782498+0000","last_active":"2023-09-23T20:15:52.866840+0000","last_peered":"2023-09-23T20:15:52.866840+0000","last_clean":"2023-09-23T20:15:52.866840+0000","last_became_active":"2023-09-23T20:14:24.780750+0000","last_became_peered":"2023-09-23T20:14:24.780750+0000","last_unstale":"2023-09-23T20:15:52.866840+0000","last_undegraded":"2023-09-23T20:15:52.866840+0000","last_fullsized":"2023-09-23T20:15:52.866840+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44728320,"num_objects":682,"num_object_clones":0,"num_object_copies":2046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2550,"num_write_kb":103456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":841,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.156562+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:15:36.156562+0000","last_peered":"2023-09-23T20:15:36.156562+0000","last_clean":"2023-09-23T20:15:36.156562+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:15:36.156562+0000","last_undegraded":"2023-09-23T20:15:36.156562+0000","last_fullsized":"2023-09-23T20:15:36.156562+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":204,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.156985+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:15:36.156985+0000","last_peered":"2023-09-23T20:15:36.156985+0000","last_clean":"2023-09-23T20:15:36.156985+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:15:36.156985+0000","last_undegraded":"2023-09-23T20:15:36.156985+0000","last_fullsized":"2023-09-23T20:15:36.156985+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":27,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153452+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:15:36.153452+0000","last_peered":"2023-09-23T20:15:36.153452+0000","last_clean":"2023-09-23T20:15:36.153452+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:15:36.153452+0000","last_undegraded":"2023-09-23T20:15:36.153452+0000","last_fullsized":"2023-09-23T20:15:36.153452+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"459'5625","reported_seq":5714,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:50.954610+0000","last_change":"2023-09-23T20:14:36.074365+0000","last_active":"2023-09-23T20:15:50.954610+0000","last_peered":"2023-09-23T20:15:50.954610+0000","last_clean":"2023-09-23T20:14:33.749328+0000","last_became_active":"2023-09-23T20:14:36.071685+0000","last_became_peered":"2023-09-23T20:14:36.071685+0000","last_unstale":"2023-09-23T20:15:50.954610+0000","last_undegraded":"2023-09-23T20:14:35.849009+0000","last_fullsized":"2023-09-23T20:14:35.834839+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48300032,"num_objects":737,"num_object_clones":0,"num_object_copies":2211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1547,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2543,"num_write_kb":104960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":644},{"shards":"0,2","objects":-810}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":806,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.142966+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:15:36.142966+0000","last_peered":"2023-09-23T20:15:36.142966+0000","last_clean":"2023-09-23T20:15:36.142966+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:15:36.142966+0000","last_undegraded":"2023-09-23T20:15:36.142966+0000","last_fullsized":"2023-09-23T20:15:36.142966+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":27,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.144036+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:15:36.144036+0000","last_peered":"2023-09-23T20:15:36.144036+0000","last_clean":"2023-09-23T20:15:36.144036+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:15:36.144036+0000","last_undegraded":"2023-09-23T20:15:36.144036+0000","last_fullsized":"2023-09-23T20:15:36.144036+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":359,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152210+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:15:36.152210+0000","last_peered":"2023-09-23T20:15:36.152210+0000","last_clean":"2023-09-23T20:15:36.152210+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:15:36.152210+0000","last_undegraded":"2023-09-23T20:15:36.152210+0000","last_fullsized":"2023-09-23T20:15:36.152210+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"459'5635","reported_seq":5726,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:52.705792+0000","last_change":"2023-09-23T20:14:36.109242+0000","last_active":"2023-09-23T20:15:52.705792+0000","last_peered":"2023-09-23T20:15:52.705792+0000","last_clean":"2023-09-23T20:14:33.735119+0000","last_became_active":"2023-09-23T20:14:36.106276+0000","last_became_peered":"2023-09-23T20:14:36.106276+0000","last_unstale":"2023-09-23T20:15:52.705792+0000","last_undegraded":"2023-09-23T20:14:35.867219+0000","last_fullsized":"2023-09-23T20:14:35.838359+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5635,"log_dups_size":0,"ondisk_log_size":5635,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49037312,"num_objects":748,"num_object_clones":0,"num_object_copies":2244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2526,"num_write_kb":104784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":609},{"shards":"3,7","objects":-750}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":520,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152245+0000","last_change":"2023-09-23T20:14:37.892742+0000","last_active":"2023-09-23T20:15:36.152245+0000","last_peered":"2023-09-23T20:15:36.152245+0000","last_clean":"2023-09-23T20:15:36.152245+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:15:36.152245+0000","last_undegraded":"2023-09-23T20:15:36.152245+0000","last_fullsized":"2023-09-23T20:15:36.152245+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":350,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153658+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:15:36.153658+0000","last_peered":"2023-09-23T20:15:36.153658+0000","last_clean":"2023-09-23T20:15:36.153658+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:15:36.153658+0000","last_undegraded":"2023-09-23T20:15:36.153658+0000","last_fullsized":"2023-09-23T20:15:36.153658+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":27,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.144195+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:15:36.144195+0000","last_peered":"2023-09-23T20:15:36.144195+0000","last_clean":"2023-09-23T20:15:36.144195+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:15:36.144195+0000","last_undegraded":"2023-09-23T20:15:36.144195+0000","last_fullsized":"2023-09-23T20:15:36.144195+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"459'5675","reported_seq":5766,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:51.432034+0000","last_change":"2023-09-23T20:14:35.994679+0000","last_active":"2023-09-23T20:15:51.432034+0000","last_peered":"2023-09-23T20:15:51.432034+0000","last_clean":"2023-09-23T20:14:33.798489+0000","last_became_active":"2023-09-23T20:14:35.986955+0000","last_became_peered":"2023-09-23T20:14:35.986955+0000","last_unstale":"2023-09-23T20:15:51.432034+0000","last_undegraded":"2023-09-23T20:14:35.846545+0000","last_fullsized":"2023-09-23T20:14:35.832866+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5675,"log_dups_size":0,"ondisk_log_size":5675,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51445760,"num_objects":785,"num_object_clones":0,"num_object_copies":2355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1542,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2539,"num_write_kb":106368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":665},{"shards":"2,3","objects":-757}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":797,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151213+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:15:36.151213+0000","last_peered":"2023-09-23T20:15:36.151213+0000","last_clean":"2023-09-23T20:15:36.151213+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:15:36.151213+0000","last_undegraded":"2023-09-23T20:15:36.151213+0000","last_fullsized":"2023-09-23T20:15:36.151213+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":100,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:41.559015+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:15:41.559015+0000","last_peered":"2023-09-23T20:15:41.559015+0000","last_clean":"2023-09-23T20:15:41.559015+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:15:41.559015+0000","last_undegraded":"2023-09-23T20:15:41.559015+0000","last_fullsized":"2023-09-23T20:15:41.559015+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":203,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158646+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:15:36.158646+0000","last_peered":"2023-09-23T20:15:36.158646+0000","last_clean":"2023-09-23T20:15:36.158646+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:15:36.158646+0000","last_undegraded":"2023-09-23T20:15:36.158646+0000","last_fullsized":"2023-09-23T20:15:36.158646+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"459'7635","reported_seq":7722,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:48.368455+0000","last_change":"2023-09-23T20:14:36.198248+0000","last_active":"2023-09-23T20:15:48.368455+0000","last_peered":"2023-09-23T20:15:48.368455+0000","last_clean":"2023-09-23T20:14:33.842789+0000","last_became_active":"2023-09-23T20:14:36.169565+0000","last_became_peered":"2023-09-23T20:14:36.169565+0000","last_unstale":"2023-09-23T20:15:48.368455+0000","last_undegraded":"2023-09-23T20:14:36.113647+0000","last_fullsized":"2023-09-23T20:14:35.847403+0000","mapping_epoch":442,"log_start":"411'1043","ondisk_log_start":"411'1043","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6592,"log_dups_size":0,"ondisk_log_size":6592,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97714176,"num_objects":1491,"num_object_clones":0,"num_object_copies":4473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4949,"num_write_kb":206080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":1298},{"shards":"3,5","objects":-1536}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":741,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:41.559201+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:15:41.559201+0000","last_peered":"2023-09-23T20:15:41.559201+0000","last_clean":"2023-09-23T20:15:41.559201+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:15:41.559201+0000","last_undegraded":"2023-09-23T20:15:41.559201+0000","last_fullsized":"2023-09-23T20:15:41.559201+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":449,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:36.153577+0000","last_change":"2023-09-23T20:14:35.895655+0000","last_active":"2023-09-23T20:15:36.153577+0000","last_peered":"2023-09-23T20:15:36.153577+0000","last_clean":"2023-09-23T20:07:25.847413+0000","last_became_active":"2023-09-23T20:14:35.895204+0000","last_became_peered":"2023-09-23T20:14:35.895204+0000","last_unstale":"2023-09-23T20:15:36.153577+0000","last_undegraded":"2023-09-23T20:14:35.838988+0000","last_fullsized":"2023-09-23T20:14:35.837779+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:17.785522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":459,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151130+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:15:36.151130+0000","last_peered":"2023-09-23T20:15:36.151130+0000","last_clean":"2023-09-23T20:15:36.151130+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:15:36.151130+0000","last_undegraded":"2023-09-23T20:15:36.151130+0000","last_fullsized":"2023-09-23T20:15:36.151130+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151128+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:15:36.151128+0000","last_peered":"2023-09-23T20:15:36.151128+0000","last_clean":"2023-09-23T20:15:36.151128+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:15:36.151128+0000","last_undegraded":"2023-09-23T20:15:36.151128+0000","last_fullsized":"2023-09-23T20:15:36.151128+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"459'6982","reported_seq":7053,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:52.989532+0000","last_change":"2023-09-23T20:13:46.785589+0000","last_active":"2023-09-23T20:15:52.989532+0000","last_peered":"2023-09-23T20:15:52.989532+0000","last_clean":"2023-09-23T20:15:52.989532+0000","last_became_active":"2023-09-23T20:13:46.781975+0000","last_became_peered":"2023-09-23T20:13:46.781975+0000","last_unstale":"2023-09-23T20:15:52.989532+0000","last_undegraded":"2023-09-23T20:15:52.989532+0000","last_fullsized":"2023-09-23T20:15:52.989532+0000","mapping_epoch":407,"log_start":"412'1993","ondisk_log_start":"412'1993","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4989,"log_dups_size":0,"ondisk_log_size":4989,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89948160,"num_objects":1372,"num_object_clones":0,"num_object_copies":4116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5226,"num_write_kb":211168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":802,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.142996+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:15:36.142996+0000","last_peered":"2023-09-23T20:15:36.142996+0000","last_clean":"2023-09-23T20:15:36.142996+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:15:36.142996+0000","last_undegraded":"2023-09-23T20:15:36.142996+0000","last_fullsized":"2023-09-23T20:15:36.142996+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1109,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.150921+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:15:36.150921+0000","last_peered":"2023-09-23T20:15:36.150921+0000","last_clean":"2023-09-23T20:15:36.150921+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:15:36.150921+0000","last_undegraded":"2023-09-23T20:15:36.150921+0000","last_fullsized":"2023-09-23T20:15:36.150921+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":27,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151535+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:15:36.151535+0000","last_peered":"2023-09-23T20:15:36.151535+0000","last_clean":"2023-09-23T20:15:36.151535+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:15:36.151535+0000","last_undegraded":"2023-09-23T20:15:36.151535+0000","last_fullsized":"2023-09-23T20:15:36.151535+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":321,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153556+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:15:36.153556+0000","last_peered":"2023-09-23T20:15:36.153556+0000","last_clean":"2023-09-23T20:15:36.153556+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:15:36.153556+0000","last_undegraded":"2023-09-23T20:15:36.153556+0000","last_fullsized":"2023-09-23T20:15:36.153556+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"459'7665","reported_seq":7749,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:48.336527+0000","last_change":"2023-09-23T20:14:36.228252+0000","last_active":"2023-09-23T20:15:48.336527+0000","last_peered":"2023-09-23T20:15:48.336527+0000","last_clean":"2023-09-23T20:14:33.823351+0000","last_became_active":"2023-09-23T20:14:36.216536+0000","last_became_peered":"2023-09-23T20:14:36.216536+0000","last_unstale":"2023-09-23T20:15:48.336527+0000","last_undegraded":"2023-09-23T20:14:36.114735+0000","last_fullsized":"2023-09-23T20:14:35.945404+0000","mapping_epoch":442,"log_start":"411'1212","ondisk_log_start":"411'1212","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6453,"log_dups_size":0,"ondisk_log_size":6453,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99811328,"num_objects":1523,"num_object_clones":0,"num_object_copies":4569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3085,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4967,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":1363},{"shards":"5,7","objects":-1562}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":697,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.155458+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:15:36.155458+0000","last_peered":"2023-09-23T20:15:36.155458+0000","last_clean":"2023-09-23T20:15:36.155458+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:15:36.155458+0000","last_undegraded":"2023-09-23T20:15:36.155458+0000","last_fullsized":"2023-09-23T20:15:36.155458+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1097,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153750+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:15:36.153750+0000","last_peered":"2023-09-23T20:15:36.153750+0000","last_clean":"2023-09-23T20:15:36.153750+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:15:36.153750+0000","last_undegraded":"2023-09-23T20:15:36.153750+0000","last_fullsized":"2023-09-23T20:15:36.153750+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151870+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:15:36.151870+0000","last_peered":"2023-09-23T20:15:36.151870+0000","last_clean":"2023-09-23T20:15:36.151870+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:15:36.151870+0000","last_undegraded":"2023-09-23T20:15:36.151870+0000","last_fullsized":"2023-09-23T20:15:36.151870+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":124,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.160132+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:15:36.160132+0000","last_peered":"2023-09-23T20:15:36.160132+0000","last_clean":"2023-09-23T20:15:36.160132+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:15:36.160132+0000","last_undegraded":"2023-09-23T20:15:36.160132+0000","last_fullsized":"2023-09-23T20:15:36.160132+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"459'6651","reported_seq":6722,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:48.338280+0000","last_change":"2023-09-23T20:13:46.814034+0000","last_active":"2023-09-23T20:15:48.338280+0000","last_peered":"2023-09-23T20:15:48.338280+0000","last_clean":"2023-09-23T20:15:48.338280+0000","last_became_active":"2023-09-23T20:13:46.812300+0000","last_became_peered":"2023-09-23T20:13:46.812300+0000","last_unstale":"2023-09-23T20:15:48.338280+0000","last_undegraded":"2023-09-23T20:15:48.338280+0000","last_fullsized":"2023-09-23T20:15:48.338280+0000","mapping_epoch":407,"log_start":"411'1046","ondisk_log_start":"411'1046","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98009088,"num_objects":1495,"num_object_clones":0,"num_object_copies":4485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4935,"num_write_kb":205792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1367,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151835+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:15:36.151835+0000","last_peered":"2023-09-23T20:15:36.151835+0000","last_clean":"2023-09-23T20:15:36.151835+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:15:36.151835+0000","last_undegraded":"2023-09-23T20:15:36.151835+0000","last_fullsized":"2023-09-23T20:15:36.151835+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":654,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152032+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:15:36.152032+0000","last_peered":"2023-09-23T20:15:36.152032+0000","last_clean":"2023-09-23T20:15:36.152032+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:15:36.152032+0000","last_undegraded":"2023-09-23T20:15:36.152032+0000","last_fullsized":"2023-09-23T20:15:36.152032+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":324,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151057+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:15:36.151057+0000","last_peered":"2023-09-23T20:15:36.151057+0000","last_clean":"2023-09-23T20:15:36.151057+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:15:36.151057+0000","last_undegraded":"2023-09-23T20:15:36.151057+0000","last_fullsized":"2023-09-23T20:15:36.151057+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":256,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.163281+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:15:36.163281+0000","last_peered":"2023-09-23T20:15:36.163281+0000","last_clean":"2023-09-23T20:15:36.163281+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:15:36.163281+0000","last_undegraded":"2023-09-23T20:15:36.163281+0000","last_fullsized":"2023-09-23T20:15:36.163281+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"459'7495","reported_seq":7566,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:48.315124+0000","last_change":"2023-09-23T20:14:24.789921+0000","last_active":"2023-09-23T20:15:48.315124+0000","last_peered":"2023-09-23T20:15:48.315124+0000","last_clean":"2023-09-23T20:15:48.315124+0000","last_became_active":"2023-09-23T20:14:24.787385+0000","last_became_peered":"2023-09-23T20:14:24.787385+0000","last_unstale":"2023-09-23T20:15:48.315124+0000","last_undegraded":"2023-09-23T20:15:48.315124+0000","last_fullsized":"2023-09-23T20:15:48.315124+0000","mapping_epoch":407,"log_start":"411'1045","ondisk_log_start":"411'1045","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6450,"log_dups_size":0,"ondisk_log_size":6450,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95911936,"num_objects":1463,"num_object_clones":0,"num_object_copies":4389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4883,"num_write_kb":203104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":520,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152007+0000","last_change":"2023-09-23T20:14:37.894544+0000","last_active":"2023-09-23T20:15:36.152007+0000","last_peered":"2023-09-23T20:15:36.152007+0000","last_clean":"2023-09-23T20:15:36.152007+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:15:36.152007+0000","last_undegraded":"2023-09-23T20:15:36.152007+0000","last_fullsized":"2023-09-23T20:15:36.152007+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":905,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:36.151710+0000","last_change":"2023-09-23T20:14:35.882819+0000","last_active":"2023-09-23T20:15:36.151710+0000","last_peered":"2023-09-23T20:15:36.151710+0000","last_clean":"2023-09-23T20:14:31.826679+0000","last_became_active":"2023-09-23T20:14:35.882372+0000","last_became_peered":"2023-09-23T20:14:35.882372+0000","last_unstale":"2023-09-23T20:15:36.151710+0000","last_undegraded":"2023-09-23T20:14:35.846466+0000","last_fullsized":"2023-09-23T20:14:35.837669+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:41.083298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":404,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158186+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:15:36.158186+0000","last_peered":"2023-09-23T20:15:36.158186+0000","last_clean":"2023-09-23T20:15:36.158186+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:15:36.158186+0000","last_undegraded":"2023-09-23T20:15:36.158186+0000","last_fullsized":"2023-09-23T20:15:36.158186+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":274,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152056+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:15:36.152056+0000","last_peered":"2023-09-23T20:15:36.152056+0000","last_clean":"2023-09-23T20:15:36.152056+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:15:36.152056+0000","last_undegraded":"2023-09-23T20:15:36.152056+0000","last_fullsized":"2023-09-23T20:15:36.152056+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"459'7800","reported_seq":7880,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:52.985051+0000","last_change":"2023-09-23T20:14:36.189421+0000","last_active":"2023-09-23T20:15:52.985051+0000","last_peered":"2023-09-23T20:15:52.985051+0000","last_clean":"2023-09-23T20:14:33.834923+0000","last_became_active":"2023-09-23T20:14:36.182947+0000","last_became_peered":"2023-09-23T20:14:36.182947+0000","last_unstale":"2023-09-23T20:15:52.985051+0000","last_undegraded":"2023-09-23T20:14:35.901626+0000","last_fullsized":"2023-09-23T20:14:35.838743+0000","mapping_epoch":442,"log_start":"412'1825","ondisk_log_start":"412'1825","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92962816,"num_objects":1418,"num_object_clones":0,"num_object_copies":4254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5100,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":1220},{"shards":"4,7","objects":-1644}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":631,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.143127+0000","last_change":"2023-09-23T20:13:57.748021+0000","last_active":"2023-09-23T20:15:36.143127+0000","last_peered":"2023-09-23T20:15:36.143127+0000","last_clean":"2023-09-23T20:15:36.143127+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:15:36.143127+0000","last_undegraded":"2023-09-23T20:15:36.143127+0000","last_fullsized":"2023-09-23T20:15:36.143127+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":159,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.163483+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:15:36.163483+0000","last_peered":"2023-09-23T20:15:36.163483+0000","last_clean":"2023-09-23T20:15:36.163483+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:15:36.163483+0000","last_undegraded":"2023-09-23T20:15:36.163483+0000","last_fullsized":"2023-09-23T20:15:36.163483+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":262,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153784+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:15:36.153784+0000","last_peered":"2023-09-23T20:15:36.153784+0000","last_clean":"2023-09-23T20:15:36.153784+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:15:36.153784+0000","last_undegraded":"2023-09-23T20:15:36.153784+0000","last_fullsized":"2023-09-23T20:15:36.153784+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":285,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151674+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:15:36.151674+0000","last_peered":"2023-09-23T20:15:36.151674+0000","last_clean":"2023-09-23T20:15:36.151674+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:15:36.151674+0000","last_undegraded":"2023-09-23T20:15:36.151674+0000","last_fullsized":"2023-09-23T20:15:36.151674+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"459'8084","reported_seq":8164,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:52.987010+0000","last_change":"2023-09-23T20:14:36.072073+0000","last_active":"2023-09-23T20:15:52.987010+0000","last_peered":"2023-09-23T20:15:52.987010+0000","last_clean":"2023-09-23T20:14:33.811030+0000","last_became_active":"2023-09-23T20:14:36.065847+0000","last_became_peered":"2023-09-23T20:14:36.065847+0000","last_unstale":"2023-09-23T20:15:52.987010+0000","last_undegraded":"2023-09-23T20:14:35.915006+0000","last_fullsized":"2023-09-23T20:14:35.838163+0000","mapping_epoch":442,"log_start":"412'2242","ondisk_log_start":"412'2242","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5842,"log_dups_size":0,"ondisk_log_size":5842,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91684864,"num_objects":1399,"num_object_clones":0,"num_object_copies":4197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3193,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5295,"num_write_kb":214208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":1245},{"shards":"5,7","objects":-1794}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":677,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152223+0000","last_change":"2023-09-23T20:14:25.790696+0000","last_active":"2023-09-23T20:15:36.152223+0000","last_peered":"2023-09-23T20:15:36.152223+0000","last_clean":"2023-09-23T20:15:36.152223+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:15:36.152223+0000","last_undegraded":"2023-09-23T20:15:36.152223+0000","last_fullsized":"2023-09-23T20:15:36.152223+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":301,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153079+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:15:36.153079+0000","last_peered":"2023-09-23T20:15:36.153079+0000","last_clean":"2023-09-23T20:15:36.153079+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:15:36.153079+0000","last_undegraded":"2023-09-23T20:15:36.153079+0000","last_fullsized":"2023-09-23T20:15:36.153079+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":579,"reported_epoch":458,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:36.156195+0000","last_change":"2023-09-23T20:14:35.862576+0000","last_active":"2023-09-23T20:15:36.156195+0000","last_peered":"2023-09-23T20:15:36.156195+0000","last_clean":"2023-09-23T19:59:02.493120+0000","last_became_active":"2023-09-23T20:14:35.862051+0000","last_became_peered":"2023-09-23T20:14:35.862051+0000","last_unstale":"2023-09-23T20:15:36.156195+0000","last_undegraded":"2023-09-23T20:14:35.833833+0000","last_fullsized":"2023-09-23T20:14:35.832351+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:30.213257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.157426+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:15:36.157426+0000","last_peered":"2023-09-23T20:15:36.157426+0000","last_clean":"2023-09-23T20:15:36.157426+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:15:36.157426+0000","last_undegraded":"2023-09-23T20:15:36.157426+0000","last_fullsized":"2023-09-23T20:15:36.157426+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"459'7523","reported_seq":7595,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:51.497514+0000","last_change":"2023-09-23T20:14:25.775099+0000","last_active":"2023-09-23T20:15:51.497514+0000","last_peered":"2023-09-23T20:15:51.497514+0000","last_clean":"2023-09-23T20:15:51.497514+0000","last_became_active":"2023-09-23T20:14:24.777506+0000","last_became_peered":"2023-09-23T20:14:24.777506+0000","last_unstale":"2023-09-23T20:15:51.497514+0000","last_undegraded":"2023-09-23T20:15:51.497514+0000","last_fullsized":"2023-09-23T20:15:51.497514+0000","mapping_epoch":407,"log_start":"412'1736","ondisk_log_start":"412'1736","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96043008,"num_objects":1465,"num_object_clones":0,"num_object_copies":4395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4903,"num_write_kb":203808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":784,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.158431+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:15:36.158431+0000","last_peered":"2023-09-23T20:15:36.158431+0000","last_clean":"2023-09-23T20:15:36.158431+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:15:36.158431+0000","last_undegraded":"2023-09-23T20:15:36.158431+0000","last_fullsized":"2023-09-23T20:15:36.158431+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":160,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153274+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:15:36.153274+0000","last_peered":"2023-09-23T20:15:36.153274+0000","last_clean":"2023-09-23T20:15:36.153274+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:15:36.153274+0000","last_undegraded":"2023-09-23T20:15:36.153274+0000","last_fullsized":"2023-09-23T20:15:36.153274+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":31,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153577+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:15:36.153577+0000","last_peered":"2023-09-23T20:15:36.153577+0000","last_clean":"2023-09-23T20:15:36.153577+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:15:36.153577+0000","last_undegraded":"2023-09-23T20:15:36.153577+0000","last_fullsized":"2023-09-23T20:15:36.153577+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":242,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.143111+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:15:36.143111+0000","last_peered":"2023-09-23T20:15:36.143111+0000","last_clean":"2023-09-23T20:15:36.143111+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:15:36.143111+0000","last_undegraded":"2023-09-23T20:15:36.143111+0000","last_fullsized":"2023-09-23T20:15:36.143111+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"459'6926","reported_seq":11145,"reported_epoch":459,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:15:52.910306+0000","last_change":"2023-09-23T20:14:35.970620+0000","last_active":"2023-09-23T20:15:52.910306+0000","last_peered":"2023-09-23T20:15:52.910306+0000","last_clean":"2023-09-23T20:14:33.799110+0000","last_became_active":"2023-09-23T20:14:35.967502+0000","last_became_peered":"2023-09-23T20:14:35.967502+0000","last_unstale":"2023-09-23T20:15:52.910306+0000","last_undegraded":"2023-09-23T20:15:52.910306+0000","last_fullsized":"2023-09-23T20:15:52.910306+0000","mapping_epoch":442,"log_start":"412'1905","ondisk_log_start":"412'1905","created":418,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5021,"log_dups_size":0,"ondisk_log_size":5021,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89063424,"num_objects":1359,"num_object_clones":0,"num_object_copies":4077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1359,"num_objects_unfound":0,"num_objects_dirty":1359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5181,"num_write_kb":209280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2059,"num_bytes_recovered":134938624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":1359}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":235,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.154000+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:15:36.154000+0000","last_peered":"2023-09-23T20:15:36.154000+0000","last_clean":"2023-09-23T20:15:36.154000+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:15:36.154000+0000","last_undegraded":"2023-09-23T20:15:36.154000+0000","last_fullsized":"2023-09-23T20:15:36.154000+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":235,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153863+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:15:36.153863+0000","last_peered":"2023-09-23T20:15:36.153863+0000","last_clean":"2023-09-23T20:15:36.153863+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:15:36.153863+0000","last_undegraded":"2023-09-23T20:15:36.153863+0000","last_fullsized":"2023-09-23T20:15:36.153863+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"459'6854","reported_seq":6932,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:48.385047+0000","last_change":"2023-09-23T20:14:36.202358+0000","last_active":"2023-09-23T20:15:48.385047+0000","last_peered":"2023-09-23T20:15:48.385047+0000","last_clean":"2023-09-23T20:14:33.783493+0000","last_became_active":"2023-09-23T20:14:36.193525+0000","last_became_peered":"2023-09-23T20:14:36.193525+0000","last_unstale":"2023-09-23T20:15:48.385047+0000","last_undegraded":"2023-09-23T20:14:36.034471+0000","last_fullsized":"2023-09-23T20:14:35.969590+0000","mapping_epoch":442,"log_start":"411'1287","ondisk_log_start":"411'1287","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5567,"log_dups_size":0,"ondisk_log_size":5567,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104202240,"num_objects":1590,"num_object_clones":0,"num_object_copies":4770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5072,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1303},{"shards":"2,5","objects":-1454}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":476,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.147325+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:15:36.147325+0000","last_peered":"2023-09-23T20:15:36.147325+0000","last_clean":"2023-09-23T20:15:36.147325+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:15:36.147325+0000","last_undegraded":"2023-09-23T20:15:36.147325+0000","last_fullsized":"2023-09-23T20:15:36.147325+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":478,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.157066+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:15:36.157066+0000","last_peered":"2023-09-23T20:15:36.157066+0000","last_clean":"2023-09-23T20:15:36.157066+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:15:36.157066+0000","last_undegraded":"2023-09-23T20:15:36.157066+0000","last_fullsized":"2023-09-23T20:15:36.157066+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.17","version":"459'6708","reported_seq":6777,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:15:48.310027+0000","last_change":"2023-09-23T20:13:46.814598+0000","last_active":"2023-09-23T20:15:48.310027+0000","last_peered":"2023-09-23T20:15:48.310027+0000","last_clean":"2023-09-23T20:15:48.310027+0000","last_became_active":"2023-09-23T20:13:46.812685+0000","last_became_peered":"2023-09-23T20:13:46.812685+0000","last_unstale":"2023-09-23T20:15:48.310027+0000","last_undegraded":"2023-09-23T20:15:48.310027+0000","last_fullsized":"2023-09-23T20:15:48.310027+0000","mapping_epoch":418,"log_start":"411'1228","ondisk_log_start":"411'1228","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94633984,"num_objects":1444,"num_object_clones":0,"num_object_copies":4332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4998,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":466,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151051+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:15:36.151051+0000","last_peered":"2023-09-23T20:15:36.151051+0000","last_clean":"2023-09-23T20:15:36.151051+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:15:36.151051+0000","last_undegraded":"2023-09-23T20:15:36.151051+0000","last_fullsized":"2023-09-23T20:15:36.151051+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":455,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.168962+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:15:36.168962+0000","last_peered":"2023-09-23T20:15:36.168962+0000","last_clean":"2023-09-23T20:15:36.168962+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:15:36.168962+0000","last_undegraded":"2023-09-23T20:15:36.168962+0000","last_fullsized":"2023-09-23T20:15:36.168962+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"459'6746","reported_seq":10895,"reported_epoch":459,"state":"active+recovering+remapped","last_fresh":"2023-09-23T20:15:49.106881+0000","last_change":"2023-09-23T20:15:20.084166+0000","last_active":"2023-09-23T20:15:49.106881+0000","last_peered":"2023-09-23T20:15:49.106881+0000","last_clean":"2023-09-23T20:14:33.751783+0000","last_became_active":"2023-09-23T20:14:36.118350+0000","last_became_peered":"2023-09-23T20:14:36.118350+0000","last_unstale":"2023-09-23T20:15:49.106881+0000","last_undegraded":"2023-09-23T20:15:49.106881+0000","last_fullsized":"2023-09-23T20:15:49.106881+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6746,"log_dups_size":0,"ondisk_log_size":6746,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96960512,"num_objects":1480,"num_object_clones":0,"num_object_copies":4440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1012,"num_objects_unfound":0,"num_objects_dirty":1480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5022,"num_write_kb":208032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2035,"num_bytes_recovered":133365760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":468}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":170,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.152212+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:15:36.152212+0000","last_peered":"2023-09-23T20:15:36.152212+0000","last_clean":"2023-09-23T20:15:36.152212+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:15:36.152212+0000","last_undegraded":"2023-09-23T20:15:36.152212+0000","last_fullsized":"2023-09-23T20:15:36.152212+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":318,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.153815+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:15:36.153815+0000","last_peered":"2023-09-23T20:15:36.153815+0000","last_clean":"2023-09-23T20:15:36.153815+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:15:36.153815+0000","last_undegraded":"2023-09-23T20:15:36.153815+0000","last_fullsized":"2023-09-23T20:15:36.153815+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"459'6801","reported_seq":12480,"reported_epoch":459,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:15:53.022045+0000","last_change":"2023-09-23T20:14:38.010024+0000","last_active":"2023-09-23T20:15:53.022045+0000","last_peered":"2023-09-23T20:15:53.022045+0000","last_clean":"2023-09-23T20:14:33.867233+0000","last_became_active":"2023-09-23T20:14:38.004874+0000","last_became_peered":"2023-09-23T20:14:38.004874+0000","last_unstale":"2023-09-23T20:15:53.022045+0000","last_undegraded":"2023-09-23T20:14:37.959307+0000","last_fullsized":"2023-09-23T20:14:37.876450+0000","mapping_epoch":444,"log_start":"412'1715","ondisk_log_start":"412'1715","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87752704,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2310,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5071,"num_write_kb":205120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1866,"num_bytes_recovered":122290176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,4,5,7","objects":2},{"shards":"0,5,7","objects":1154},{"shards":"3,7","objects":183}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":304,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.159999+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:15:36.159999+0000","last_peered":"2023-09-23T20:15:36.159999+0000","last_clean":"2023-09-23T20:15:36.159999+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:15:36.159999+0000","last_undegraded":"2023-09-23T20:15:36.159999+0000","last_fullsized":"2023-09-23T20:15:36.159999+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:15:36.151700+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:15:36.151700+0000","last_peered":"2023-09-23T20:15:36.151700+0000","last_clean":"2023-09-23T20:15:36.151700+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:15:36.151700+0000","last_undegraded":"2023-09-23T20:15:36.151700+0000","last_fullsized":"2023-09-23T20:15:36.151700+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":36,"stat_sum":{"num_bytes":3020554264,"num_objects":46091,"num_object_clones":0,"num_object_copies":138273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":42877,"num_objects_misplaced":3777,"num_objects_unfound":0,"num_objects_dirty":46091,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161615,"num_write_kb":6646529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14963,"num_bytes_recovered":980615168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11730038784,"data_stored":11730026568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208105,"ondisk_log_size":208105,"up":108,"acting":90,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":178,"num_bytes_recovered":4194380,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234417,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2201872,"kb_used_data":1713160,"kb_used_omap":0,"kb_used_meta":488704,"kb_avail":91543280,"statfs":{"total":95995035648,"available":93740318720,"internally_reserved":0,"allocated":1754275840,"data_stored":1752870876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":500432896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":11.353,"5min":3.855,"15min":2.8380000000000001},"min":{"1min":0.219,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":149.25999999999999,"5min":149.25999999999999,"15min":149.25999999999999},"last":0.26800000000000002},{"interface":"front","average":{"1min":12.917,"5min":4.3019999999999996,"15min":3.0609999999999999},"min":{"1min":0.248,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":149.11799999999999,"5min":149.11799999999999,"15min":149.11799999999999},"last":0.69999999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":3.2250000000000001,"15min":3.5779999999999998},"min":{"1min":0.29699999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.0599999999999996,"5min":91.335999999999999,"15min":104.16800000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.972,"5min":3.738,"15min":3.8370000000000002},"min":{"1min":0.29599999999999999,"5min":0.249,"15min":0.223},"max":{"1min":6.8609999999999998,"5min":91.516999999999996,"15min":104.003},"last":0.73799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":3.6480000000000001,"15min":3.6880000000000002},"min":{"1min":0.33200000000000002,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":5.9340000000000002,"5min":67.760000000000005,"15min":105.842},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.7470000000000001,"5min":3.9119999999999999,"15min":4.1360000000000001},"min":{"1min":0.35599999999999998,"5min":0.255,"15min":0.255},"max":{"1min":5.3840000000000003,"5min":67.488,"15min":105.66},"last":1.3}]},{"osd":3,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":14.476000000000001,"5min":4.5030000000000001,"15min":4.6440000000000001},"min":{"1min":0.33400000000000002,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":185.17099999999999,"5min":185.17099999999999,"15min":185.17099999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":15.061999999999999,"5min":4.593,"15min":4.649},"min":{"1min":0.63,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":184.99100000000001,"5min":184.99100000000001,"15min":184.99100000000001},"last":0.76500000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":0.79200000000000004,"15min":0.71399999999999997},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.22500000000000001},"max":{"1min":6.2569999999999997,"5min":6.2569999999999997,"15min":19.215},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.4009999999999998,"5min":0.98399999999999999,"15min":0.90700000000000003},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":10.882999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.2450000000000001,"15min":1.105},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":14.069000000000001,"5min":14.069000000000001,"15min":35.488},"last":0.81899999999999995},{"interface":"front","average":{"1min":2.6469999999999998,"5min":1.103,"15min":0.97999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":13.43,"5min":13.43,"15min":34.896000000000001},"last":0.78100000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":1.234,"15min":0.99199999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":8.4120000000000008,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.522,"5min":0.93700000000000006,"15min":0.754},"min":{"1min":0.36699999999999999,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":7.5659999999999998,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.748}]}]},{"osd":6,"up_from":0,"seq":1078036791457,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762615,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926152,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2139820,"kb_used_data":1381476,"kb_used_omap":8,"kb_used_meta":758199,"kb_avail":91605332,"statfs":{"total":95995035648,"available":93803859968,"internally_reserved":0,"allocated":1414631424,"data_stored":1412502638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":776396385},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.609,"15min":3.605},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":3.3780000000000001,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.1640000000000001,"15min":3.863},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.183},"max":{"1min":3.9470000000000001,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:15:16 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.9359999999999999,"15min":5.0519999999999996},"min":{"1min":0.215,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":3.5920000000000001,"5min":160.542,"15min":270.51100000000002},"last":0.502},{"interface":"front","average":{"1min":1.5720000000000001,"5min":3.4990000000000001,"15min":4.9820000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":7.5220000000000002,"5min":160.75999999999999,"15min":269.69299999999998},"last":0.34799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.292,"15min":3.9980000000000002},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.21299999999999999},"max":{"1min":3.6629999999999998,"5min":10.082000000000001,"15min":126.95399999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.5289999999999999,"15min":4.1779999999999999},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":4.9359999999999999,"5min":9.5440000000000005,"15min":127.294},"last":0.51700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":1.4790000000000001,"15min":2.0499999999999998},"min":{"1min":0.33500000000000002,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":8.7360000000000007,"5min":9.6959999999999997,"15min":148.20599999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.734,"5min":1.3140000000000001,"15min":1.994},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.8230000000000004,"5min":8.4429999999999996,"15min":148.11000000000001},"last":2.7509999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.7609999999999999,"15min":1.8759999999999999},"min":{"1min":0.35999999999999999,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":6.8739999999999997,"5min":52.061,"15min":64.725999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.2450000000000001,"15min":1.899},"min":{"1min":0.312,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1500000000000004,"5min":51.741,"15min":51.741},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.117,"15min":1.5529999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":3.504,"5min":6.1660000000000004,"15min":15.744999999999999},"last":2.6720000000000002},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.2390000000000001,"15min":1.724},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":4.6580000000000004,"5min":8.3919999999999995,"15min":13.766999999999999},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.2969999999999999,"15min":1.7490000000000001},"min":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.7400000000000002,"5min":12.981,"15min":65.081999999999994},"last":0.69599999999999995},{"interface":"front","average":{"1min":2.5249999999999999,"5min":1.861,"15min":1.8620000000000001},"min":{"1min":0.442,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.2290000000000001,"5min":17.151,"15min":17.151},"last":6.1609999999999996}]}]},{"osd":2,"up_from":104,"seq":446676599091,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514012,"kb_used_data":715028,"kb_used_omap":12,"kb_used_meta":798963,"kb_avail":92231140,"statfs":{"total":95995035648,"available":94444687360,"internally_reserved":0,"allocated":732188672,"data_stored":730020048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":818139134},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":2.0670000000000002,"15min":3.0339999999999998},"min":{"1min":0.253,"5min":0.221,"15min":0.192},"max":{"1min":5.923,"5min":65.382999999999996,"15min":76.251999999999995},"last":0.17399999999999999},{"interface":"front","average":{"1min":1.7230000000000001,"5min":1.994,"15min":3.081},"min":{"1min":0.26600000000000001,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":5.694,"5min":65.427000000000007,"15min":76.590000000000003},"last":0.19900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.9020000000000001,"15min":3.8250000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":9.2590000000000003,"5min":70.453000000000003,"15min":94.768000000000001},"last":0.23400000000000001},{"interface":"front","average":{"1min":1.8149999999999999,"5min":2.7200000000000002,"15min":3.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.21099999999999999},"max":{"1min":11.981999999999999,"5min":69.825000000000003,"15min":96.808000000000007},"last":0.222}]},{"osd":3,"last update":"Sat Sep 23 20:15:40 2023","interfaces":[{"interface":"back","average":{"1min":7.5960000000000001,"5min":2.9870000000000001,"15min":4.1130000000000004},"min":{"1min":0.222,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":132.82499999999999,"5min":132.82499999999999,"15min":132.82499999999999},"last":126.345},{"interface":"front","average":{"1min":8.1519999999999992,"5min":2.9900000000000002,"15min":3.411},"min":{"1min":0.23400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":130.97300000000001,"5min":130.97300000000001,"15min":130.97300000000001},"last":126.40300000000001}]},{"osd":4,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.272,"15min":1.964},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.7270000000000003,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.599,"15min":2.1779999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26700000000000002},"max":{"1min":9.1590000000000007,"5min":13.387,"15min":76.161000000000001},"last":0.372}]},{"osd":5,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":3.1360000000000001,"15min":3.0310000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.956,"5min":135.202,"15min":172.505},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.0350000000000001,"5min":1.966,"15min":2.7679999999999998},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32300000000000001},"max":{"1min":12.050000000000001,"5min":29.196999999999999,"15min":172.375},"last":0.68999999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:14:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.381,"15min":1.613},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.843,"5min":14.664,"15min":14.664},"last":1.5229999999999999},{"interface":"front","average":{"1min":1.8939999999999999,"5min":1.5409999999999999,"15min":1.8540000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.025,"5min":12.025,"15min":14.279999999999999},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.7849999999999999,"15min":1.9159999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2050000000000001,"5min":13.481999999999999,"15min":23.079999999999998},"last":3.7189999999999999},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.3859999999999999,"15min":1.7609999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.7789999999999999,"5min":14.698,"15min":20.614999999999998},"last":3.7320000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435565,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2459676,"kb_used_data":1604820,"kb_used_omap":4,"kb_used_meta":854779,"kb_avail":91285476,"statfs":{"total":95995035648,"available":93476327424,"internally_reserved":0,"allocated":1643335680,"data_stored":1640924766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":875293805},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5940000000000003,"5min":1.8560000000000001,"15min":3.0859999999999999},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":76.632000000000005,"5min":76.632000000000005,"15min":76.632000000000005},"last":0.76800000000000002},{"interface":"front","average":{"1min":5.4580000000000002,"5min":2.266,"15min":3.4159999999999999},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.19600000000000001},"max":{"1min":81.429000000000002,"5min":81.429000000000002,"15min":81.429000000000002},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":5.0510000000000002,"15min":4.6260000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":9,"5min":183.87,"15min":183.87},"last":1.149},{"interface":"front","average":{"1min":1.2230000000000001,"5min":4.9740000000000002,"15min":4.758},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":5.5810000000000004,"5min":185.84899999999999,"15min":185.84899999999999},"last":0.28100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":4.2610000000000001,"15min":4.109},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":9.0289999999999999,"5min":163.83500000000001,"15min":163.83500000000001},"last":1.1779999999999999},{"interface":"front","average":{"1min":2.2679999999999998,"5min":3.6669999999999998,"15min":3.819},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":16.550999999999998,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.1120000000000001,"15min":2.786},"min":{"1min":0.32400000000000001,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":5.21,"5min":7.4960000000000004,"15min":109.66800000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6619999999999999,"15min":3.2469999999999999},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":9.6470000000000002,"5min":11.815,"15min":110.295},"last":2.9820000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":3.6349999999999998,"15min":3.9129999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6799999999999997,"5min":200.53200000000001,"15min":200.53200000000001},"last":3.0270000000000001},{"interface":"front","average":{"1min":3.411,"5min":3.7599999999999998,"15min":3.9769999999999999},"min":{"1min":0.48799999999999999,"5min":0.34799999999999998,"15min":0.26900000000000002},"max":{"1min":9.9429999999999996,"5min":200.846,"15min":200.846},"last":1.127}]},{"osd":6,"last update":"Sat Sep 23 20:14:59 2023","interfaces":[{"interface":"back","average":{"1min":1.8280000000000001,"5min":1.329,"15min":3.0609999999999999},"min":{"1min":0.41599999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":6.774,"5min":6.774,"15min":82.281999999999996},"last":5.306},{"interface":"front","average":{"1min":1.5389999999999999,"5min":1.1339999999999999,"15min":3.1760000000000002},"min":{"1min":0.375,"5min":0.36699999999999999,"15min":0.33400000000000002},"max":{"1min":8.2629999999999999,"5min":8.2629999999999999,"15min":82.233000000000004},"last":0.35199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.9470000000000001,"15min":3.3969999999999998},"min":{"1min":0.42799999999999999,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":9.0600000000000005,"5min":139.465,"15min":139.465},"last":1.9670000000000001},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.8540000000000001,"15min":3.27},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":5.3010000000000002,"5min":141.56200000000001,"15min":141.56200000000001},"last":0.47399999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019327,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1273612,"kb_used_data":842180,"kb_used_omap":0,"kb_used_meta":431424,"kb_avail":92471540,"statfs":{"total":95995035648,"available":94690856960,"internally_reserved":0,"allocated":862392320,"data_stored":860991440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":441778176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:25 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":1.2390000000000001,"15min":2.1720000000000002},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.9269999999999996,"5min":14.063000000000001,"15min":89.004999999999995},"last":4.9980000000000002},{"interface":"front","average":{"1min":1.831,"5min":1.268,"15min":2.169},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":10.289,"5min":14.048999999999999,"15min":89.024000000000001},"last":5.0099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:15:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":3.395,"15min":5.9109999999999996},"min":{"1min":0.27800000000000002,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":5.8259999999999996,"5min":138.47999999999999,"15min":161.613},"last":14.763},{"interface":"front","average":{"1min":2.6070000000000002,"5min":3.46,"15min":5.9889999999999999},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.217},"max":{"1min":10.368,"5min":138.429,"15min":162.017},"last":13.368}]},{"osd":2,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":3.7320000000000002,"15min":5.6970000000000001},"min":{"1min":0.30499999999999999,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.8570000000000002,"5min":144.916,"15min":182.96700000000001},"last":1.452},{"interface":"front","average":{"1min":2.4020000000000001,"5min":3.694,"15min":5.6100000000000003},"min":{"1min":0.40300000000000002,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":11.528,"5min":144.83099999999999,"15min":183.244},"last":1.419}]},{"osd":3,"last update":"Sat Sep 23 20:15:31 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":3.0459999999999998,"15min":5.1020000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":15.707000000000001,"5min":100.361,"15min":184.37100000000001},"last":2.2610000000000001},{"interface":"front","average":{"1min":2.9700000000000002,"5min":3.3420000000000001,"15min":5.1900000000000004},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.28599999999999998},"max":{"1min":15.73,"5min":100.20699999999999,"15min":184.22300000000001},"last":2.1080000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:15:25 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":0.84099999999999997,"15min":1.177},"min":{"1min":0.26400000000000001,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":12.257999999999999,"5min":12.257999999999999,"15min":176.81899999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":2.1139999999999999,"5min":0.84199999999999997,"15min":1.2330000000000001},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":6.9770000000000003,"5min":6.9770000000000003,"15min":176.172},"last":0.33300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":0.85899999999999999,"15min":0.627},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":8.5660000000000007,"5min":10.337999999999999,"15min":10.337999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.3050000000000002,"5min":1.119,"15min":0.79500000000000004},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":11.348000000000001,"5min":11.348000000000001,"15min":11.348000000000001},"last":0.41199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":0.82999999999999996,"15min":1.429},"min":{"1min":0.442,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.71,"5min":7.71,"15min":76.507000000000005},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.88900000000000001,"15min":1.5169999999999999},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":5.8869999999999996,"5min":5.8869999999999996,"15min":76.462000000000003},"last":0.39900000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757690,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2691076,"kb_used_data":2141308,"kb_used_omap":0,"kb_used_meta":549568,"kb_avail":91054076,"statfs":{"total":95995035648,"available":93239373824,"internally_reserved":0,"allocated":2192699392,"data_stored":2191302953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":562757632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:23 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":1.3100000000000001,"15min":1.8100000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.304,"5min":10.304,"15min":121.94799999999999},"last":0.97999999999999998},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.2809999999999999,"15min":2.0489999999999999},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":8.1699999999999999,"5min":9.4250000000000007,"15min":121.964},"last":3.0459999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:14:49 2023","interfaces":[{"interface":"back","average":{"1min":12.496,"5min":5.5090000000000003,"15min":6.274},"min":{"1min":0.22900000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":150.16999999999999,"5min":150.16999999999999,"15min":266.197},"last":3.9289999999999998},{"interface":"front","average":{"1min":10.928000000000001,"5min":5.1550000000000002,"15min":6.3109999999999999},"min":{"1min":0.253,"5min":0.24099999999999999,"15min":0.21199999999999999},"max":{"1min":150.12700000000001,"5min":150.12700000000001,"15min":266.14999999999998},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:15:05 2023","interfaces":[{"interface":"back","average":{"1min":2.8879999999999999,"5min":3.7519999999999998,"15min":4.3129999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.787000000000001,"5min":120.297,"15min":240.57300000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":2.919,"5min":3.8500000000000001,"15min":4.3810000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.81,"5min":120.325,"15min":240.464},"last":1.0640000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:14:49 2023","interfaces":[{"interface":"back","average":{"1min":9.6509999999999998,"5min":4.2999999999999998,"15min":4.9740000000000002},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":142.63800000000001,"5min":142.63800000000001,"15min":201.50899999999999},"last":5.524},{"interface":"front","average":{"1min":10.086,"5min":4.1550000000000002,"15min":4.8399999999999999},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":142.81399999999999,"5min":142.81399999999999,"15min":201.58099999999999},"last":5.7969999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:15:23 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":0.89600000000000002,"15min":1.345},"min":{"1min":0.25,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":8,"5min":8,"15min":191.78899999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.60499999999999998,"15min":1.2210000000000001},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":2.7160000000000002,"5min":3.2189999999999999,"15min":191.821},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:14:55 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":0.89600000000000002,"15min":0.74399999999999999},"min":{"1min":0.313,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":7.5250000000000004,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":1.8180000000000001},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.60799999999999998,"15min":0.56599999999999995},"min":{"1min":0.39400000000000002,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":3.1219999999999999,"5min":3.1219999999999999,"15min":5.0119999999999996},"last":0.60199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:15:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2360000000000002,"5min":1.1739999999999999,"15min":1.4139999999999999},"min":{"1min":0.33300000000000002,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":10.536,"5min":10.536,"15min":191.84299999999999},"last":0.94099999999999995},{"interface":"front","average":{"1min":2.395,"5min":0.999,"15min":1.413},"min":{"1min":0.40400000000000003,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":8.3260000000000005,"5min":8.3260000000000005,"15min":191.69},"last":0.90300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1390477312,"data_stored":1390477312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3249537024,"data_stored":3249537024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":723910656,"data_stored":723910656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1620639744,"data_stored":1620639744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":856363008,"data_stored":856358936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2159808512,"data_stored":2159804440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1729302528,"data_stored":1729298456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:15:54.526 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:15:54.718 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:54.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:54.822 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:54.846 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:54 smithi135 ceph-mon[221113]: from='client.56140 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:15:54.847 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:54 smithi135 ceph-mon[213871]: from='client.56140 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:15:54.849 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:55.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:54 smithi144 ceph-mon[194833]: from='client.56140 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:15:55.278 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:55.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:55.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:55.351 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:55.381 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:55.388 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:15:55.438 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:55.566 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:55.669 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:55.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:55.784 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:15:55.855 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:55.939 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:15:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:55.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:15:55.955 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:55 smithi135 ceph-mon[213871]: pgmap v1281: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 42910/132075 objects degraded (32.489%); 3196/132075 objects misplaced (2.420%); 5.9 MiB/s, 134 objects/s recovering 2023-09-23T20:15:55.959 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:55 smithi135 ceph-mon[221113]: pgmap v1281: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 42910/132075 objects degraded (32.489%); 3196/132075 objects misplaced (2.420%); 5.9 MiB/s, 134 objects/s recovering 2023-09-23T20:15:56.056 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:15:56.126 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:56.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:55 smithi144 ceph-mon[194833]: pgmap v1281: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 42910/132075 objects degraded (32.489%); 3196/132075 objects misplaced (2.420%); 5.9 MiB/s, 134 objects/s recovering 2023-09-23T20:15:56.309 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:15:56.798 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:15:56.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:56.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:15:56.903 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:15:56.957 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:57.063 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:15:57.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:57.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:15:57.174 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:15:57.223 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:57.276 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:15:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:57.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:15:57.381 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:57.459 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:57.558 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:57.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:57.669 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:15:57.739 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:57.742 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:57.901 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:15:57.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:57.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:15:58.006 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:58.062 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:58.314 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:58 smithi135 ceph-mon[221113]: pgmap v1282: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 871 op/s; 42910/130362 objects degraded (32.916%); 3196/130362 objects misplaced (2.452%); 5.9 MiB/s, 134 objects/s recovering 2023-09-23T20:15:58.315 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:58 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 42910/130362 objects degraded (32.916%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:58.315 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:58 smithi135 ceph-mon[213871]: pgmap v1282: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 871 op/s; 42910/130362 objects degraded (32.916%); 3196/130362 objects misplaced (2.452%); 5.9 MiB/s, 134 objects/s recovering 2023-09-23T20:15:58.316 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:58 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 42910/130362 objects degraded (32.916%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:58.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:58 smithi144 ceph-mon[194833]: pgmap v1282: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 871 op/s; 42910/130362 objects degraded (32.916%); 3196/130362 objects misplaced (2.452%); 5.9 MiB/s, 134 objects/s recovering 2023-09-23T20:15:58.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:58 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 42910/130362 objects degraded (32.916%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:15:58.562 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:58.575 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:58.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:58.690 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:15:58.749 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:58.858 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:15:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:58.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:15:58.972 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:58.994 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:15:58.994 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 42910/130362 objects degraded (32.916%), 21 pgs degraded, 21 pgs undersized","count":42},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1728,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":459,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":6,"osd_in_since":1695500073,"num_remapped_pgs":24},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":105},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":21},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait+remapped","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":43454,"data_bytes":2850708909,"bytes_used":12149133312,"bytes_avail":563821080576,"bytes_total":575970213888,"degraded_objects":42910,"degraded_total":130362,"degraded_ratio":0.32916033813534618,"misplaced_objects":3196,"misplaced_total":130362,"misplaced_ratio":0.024516346788174471,"recovering_objects_per_sec":134,"recovering_bytes_per_sec":6170530,"recovering_keys_per_sec":0,"num_objects_recovered":1346,"num_bytes_recovered":61734912,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":871},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-23T20:06:56.817144+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"fb5142fa-2e5d-413d-a713-d719d0723614":{"message":"Global Recovery Event (110s)\n [======================......] (remaining: 25s)","progress":0.8139534592628479,"add_to_ceph_s":true}}} 2023-09-23T20:15:59.027 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:59.117 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:15:59.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:15:59.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:15:59.158 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:15:59.222 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:15:59.288 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3172164317' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:15:59.288 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/774713196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:59.288 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:15:59 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:59.288 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3172164317' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:15:59.289 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/774713196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:59.289 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:15:59 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:59.301 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:15:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3172164317' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:15:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/774713196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:15:59.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:15:59 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:00.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[221113]: pgmap v1283: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 42380/119373 objects degraded (35.502%); 2937/119373 objects misplaced (2.460%); 6.3 MiB/s, 135 objects/s recovering 2023-09-23T20:16:00.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:00.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[221113]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[221113]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:00.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[221113]: osdmap e460: 8 total, 8 up, 6 in 2023-09-23T20:16:00.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[221113]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.405 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[221113]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[213871]: pgmap v1283: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 42380/119373 objects degraded (35.502%); 2937/119373 objects misplaced (2.460%); 6.3 MiB/s, 135 objects/s recovering 2023-09-23T20:16:00.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:00.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[213871]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[213871]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:00.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[213871]: osdmap e460: 8 total, 8 up, 6 in 2023-09-23T20:16:00.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[213871]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.406 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:00 smithi135 ceph-mon[213871]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.458 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:00.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:00.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:00.574 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:00.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:00 smithi144 ceph-mon[194833]: pgmap v1283: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 42380/119373 objects degraded (35.502%); 2937/119373 objects misplaced (2.460%); 6.3 MiB/s, 135 objects/s recovering 2023-09-23T20:16:00.602 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:00 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:00.602 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:00 smithi144 ceph-mon[194833]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.602 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:00 smithi144 ceph-mon[194833]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.602 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:00 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:00.602 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:00 smithi144 ceph-mon[194833]: osdmap e460: 8 total, 8 up, 6 in 2023-09-23T20:16:00.602 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:00 smithi144 ceph-mon[194833]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.602 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:00 smithi144 ceph-mon[194833]: failed to encode map e460 with expected crc 2023-09-23T20:16:00.603 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:00.644 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:00.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:00.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:00.749 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:00.820 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:00.899 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:00.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:00.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:01.006 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:01.058 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:01.143 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:16:01.144 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:16:01.340 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:01.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:01.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:01.449 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:01.512 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:01.576 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:01.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:01.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:01.682 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:01.712 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:01.764 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:01.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:01.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:01.869 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:01.897 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:02.270 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:16:02.270 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:16:02.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52207 8.6 8:6fd8f8f3:::benchmark_data_smithi135_460343_object52206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 542.141836528, 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1234665740000001, 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.271 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52207 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.272 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909077+0000", 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909076+0000", 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909102+0000", 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.605e-05 2023-09-23T20:16:02.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909495+0000", 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00039314200000000002 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909506+0000", 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0699999999999999e-05 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910549+0000", 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001043034 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910557+0000", 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.816e-06 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914406+0000", 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0038485909999999 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914418+0000", 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2016e-05 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.031329+0000", 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1169108190000001 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.032544+0000", 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001214705 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.277 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52216 8.6 8:60b8198d:::benchmark_data_smithi135_460343_object52215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 542.14139547499997, 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1446887769999998, 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.278 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52216 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909518+0000", 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909517+0000", 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909548+0000", 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0108999999999999e-05 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909557+0000", 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.842e-06 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.909563+0000", 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0759999999999999e-06 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910589+0000", 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010254730000000001 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.910596+0000", 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4970000000000004e-06 2023-09-23T20:16:02.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914445+0000", 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.003848268 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914452+0000", 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7100000000000007e-06 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.033864+0000", 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1194116679999999 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.034445+0000", 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00058121900000000001 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:02.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.034641+0000", 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019598099999999999 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050209+0000", 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.015567925 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050294+0000", 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5099000000000006e-05 2023-09-23T20:16:02.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.052593+0000", 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022989210000000002 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.052660+0000", 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6950000000000001e-05 2023-09-23T20:16:02.286 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054207+0000", 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001546658 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52226 8.6 8:64c4735f:::benchmark_data_smithi135_460343_object52225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:02.287 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 542.13581739000006, 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1418229260000001, 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52226 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915096+0000", 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915095+0000", 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.915130+0000", 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.536e-05 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.918885+0000", 2023-09-23T20:16:02.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0037552050000000002 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.918902+0000", 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6691e-05 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.919727+0000", 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00082474700000000002 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.919738+0000", 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1235e-05 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914477+0000", 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.99473878900000001 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914486+0000", 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0350000000000007e-06 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036650+0000", 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.122164451 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036775+0000", 2023-09-23T20:16:02.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012457700000000001 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.036925+0000", 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000150464 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050301+0000", 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013375705999999999 2023-09-23T20:16:02.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054287+0000", 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0039856179999999998 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055326+0000", 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001039744 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.295 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055373+0000", 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6264999999999998e-05 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.056919+0000", 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001546551 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.296 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52231 8.6 8:64b25628:::benchmark_data_smithi135_460343_object52230:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 542.12265951500001, 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.130363241, 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.297 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52231 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928254+0000", 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928273+0000", 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9911e-05 2023-09-23T20:16:02.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928281+0000", 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2910000000000002e-06 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928286+0000", 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3159999999999996e-06 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928328+0000", 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1780000000000003e-05 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.928337+0000", 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5899999999999997e-06 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914511+0000", 2023-09-23T20:16:02.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.98617375299999999 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914519+0000", 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5680000000000002e-06 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.038973+0000", 2023-09-23T20:16:02.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.12445419 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039130+0000", 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000156861 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.039293+0000", 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000163535 2023-09-23T20:16:02.303 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.305 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050307+0000", 2023-09-23T20:16:02.305 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011013761 2023-09-23T20:16:02.305 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.305 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.305 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.305 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054370+0000", 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0040626919999999997 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.057016+0000", 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026464700000000002 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.306 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.057055+0000", 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8605000000000001e-05 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058617+0000", 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0015624199999999999 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.307 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52236 8.6 8:693d9ca7:::benchmark_data_smithi135_460343_object52235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 542.11269309099998, 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1218329040000001, 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52236 2023-09-23T20:16:02.308 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.309 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938220+0000", 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938241+0000", 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1342999999999999e-05 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.310 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938249+0000", 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7800000000000001e-06 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.938253+0000", 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.777e-06 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.311 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.939183+0000", 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00093006800000000004 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.939192+0000", 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2129999999999997e-06 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914544+0000", 2023-09-23T20:16:02.312 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.97535231300000003 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914552+0000", 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.5190000000000003e-06 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041404+0000", 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.126851673 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.313 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041553+0000", 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000148877 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.041722+0000", 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016971799999999999 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.314 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050312+0000", 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0085901160000000001 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054415+0000", 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041022569999999998 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058671+0000", 2023-09-23T20:16:02.315 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0042560890000000002 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.058717+0000", 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6471000000000003e-05 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060053+0000", 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001336222 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.316 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52242 8.6 8:663215dc:::benchmark_data_smithi135_460343_object52241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 542.10589029400001, 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.116376196, 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.317 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52242 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.318 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945023+0000", 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945022+0000", 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.319 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945047+0000", 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5188e-05 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945066+0000", 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8719000000000002e-05 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945074+0000", 2023-09-23T20:16:02.320 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0460000000000006e-06 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945123+0000", 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9542000000000001e-05 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.945133+0000", 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6490000000000003e-06 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.321 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914577+0000", 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96944391900000004 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.914584+0000", 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4880000000000001e-06 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.322 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043641+0000", 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.129056426 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043776+0000", 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0001348 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.323 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.043935+0000", 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015974000000000001 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050322+0000", 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0063863770000000004 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.324 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.054682+0000", 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00436075 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060087+0000", 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005404252 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.060126+0000", 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9467e-05 2023-09-23T20:16:02.325 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061399+0000", 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001273281 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:52249 8.6 8:6b568471:::benchmark_data_smithi135_460343_object52248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:02.326 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 542.10134105400005, 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1131886460000002, 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 52249 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.327 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949573+0000", 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.328 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949571+0000", 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949593+0000", 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.1430999999999999e-05 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.329 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.330 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949607+0000", 2023-09-23T20:16:02.330 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4381e-05 2023-09-23T20:16:02.330 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.330 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.330 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.330 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949614+0000", 2023-09-23T20:16:02.330 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0330000000000003e-06 2023-09-23T20:16:02.330 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949662+0000", 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7685000000000002e-05 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:06:59.949675+0000", 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2575000000000001e-05 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.331 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.915769+0000", 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.96609494600000001 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:00.915781+0000", 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1364e-05 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.332 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.047967+0000", 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1321858119999999 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.048151+0000", 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000184138 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.048355+0000", 2023-09-23T20:16:02.333 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020468600000000001 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.050329+0000", 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019734399999999999 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.055290+0000", 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049610740000000002 2023-09-23T20:16:02.334 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061446+0000", 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061564150000000002 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.061507+0000", 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0739999999999998e-05 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.335 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:02.062761+0000", 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012539999999999999 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65642 8.7 8:e801740d:::benchmark_data_smithi135_460343_object65641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 523.12480798499996, 2023-09-23T20:16:02.336 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.760197558, 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65642 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:16:02.337 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926106+0000", 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926104+0000", 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:02.338 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926155+0000", 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1158e-05 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926172+0000", 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.664e-05 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.339 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926183+0000", 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0896999999999999e-05 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926311+0000", 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012874499999999999 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.340 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926319+0000", 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.294e-06 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927738+0000", 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001418891 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.341 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927746+0000", 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5960000000000004e-06 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.684466+0000", 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.756720061 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.686303+0000", 2023-09-23T20:16:02.342 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001836935 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65643 8.3 8:c7d581fe:::benchmark_data_smithi135_460343_object65642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 523.12471111599996, 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7775143610000002, 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.343 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65643 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.344 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926202+0000", 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.345 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926223+0000", 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0891999999999999e-05 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926230+0000", 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3799999999999996e-06 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.346 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926236+0000", 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2179999999999996e-06 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926302+0000", 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5423e-05 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.347 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926309+0000", 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.3889999999999999e-06 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927982+0000", 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016733589999999999 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.927990+0000", 2023-09-23T20:16:02.348 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8369999999999997e-06 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.701891+0000", 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.773900823 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.703717+0000", 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018257550000000001 2023-09-23T20:16:02.349 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65645 8.3 8:cf69bd2e:::benchmark_data_smithi135_460343_object65644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 523.12466724000001, 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7804623400000001, 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:16:02.350 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65645 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.351 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926246+0000", 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.352 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926264+0000", 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8465e-05 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926271+0000", 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.849e-06 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.353 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926334+0000", 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2448999999999998e-05 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926375+0000", 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1554999999999998e-05 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926381+0000", 2023-09-23T20:16:02.354 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.083e-06 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.928014+0000", 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001632362 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.928020+0000", 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4629999999999998e-06 2023-09-23T20:16:02.355 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.706613+0000", 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7785930159999999 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.706709+0000", 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5585e-05 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.356 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65667 8.a 8:5e718162:::benchmark_data_smithi135_460343_object65666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 523.12457209800004, 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.820842632, 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.357 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65667 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.358 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926341+0000", 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.359 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926366+0000", 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4946000000000001e-05 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926374+0000", 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0150000000000003e-06 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.360 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.926380+0000", 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.981e-06 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.933090+0000", 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0067101180000000002 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:18.933100+0000", 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0087e-05 2023-09-23T20:16:02.361 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704577+0000", 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7714773450000001 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704719+0000", 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000141182 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.362 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.704870+0000", 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015157699999999999 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713627+0000", 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0087563789999999999 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.363 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713675+0000", 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7994000000000002e-05 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.743266+0000", 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029591681000000002 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.364 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.743323+0000", 2023-09-23T20:16:02.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7122e-05 2023-09-23T20:16:02.365 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.365 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.365 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.365 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747184+0000", 2023-09-23T20:16:02.365 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038607680000000001 2023-09-23T20:16:02.365 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:65682 8.a 8:51fe7b50:::benchmark_data_smithi135_460343_object65681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 522.91713012800005, 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.61738943, 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.366 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 65682 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.367 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133783+0000", 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133782+0000", 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.368 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133803+0000", 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0800000000000001e-05 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133815+0000", 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2177999999999999e-05 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133821+0000", 2023-09-23T20:16:02.369 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0170000000000002e-06 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133873+0000", 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2039000000000001e-05 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:19.133885+0000", 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1158000000000001e-05 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.370 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710505+0000", 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.576620546 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710683+0000", 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017801700000000001 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.371 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.710887+0000", 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020342900000000001 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713666+0000", 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0027793280000000002 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.372 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.713740+0000", 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4468000000000006e-05 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747234+0000", 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.033493756 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.747286+0000", 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.185e-05 2023-09-23T20:16:02.373 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:07:20.751173+0000", 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0038869239999999999 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.374 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86869 8.f 8:fd41b8fd:::benchmark_data_smithi135_460343_object2636:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 477.30878358199999, 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0711724490000001, 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86869 2023-09-23T20:16:02.375 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742130+0000", 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.376 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742129+0000", 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742133+0000", 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5920000000000002e-06 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.377 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742144+0000", 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0358e-05 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742153+0000", 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1579999999999994e-06 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.378 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742207+0000", 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4622e-05 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742216+0000", 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3820000000000004e-06 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744303+0000", 2023-09-23T20:16:02.379 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020874449999999998 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744310+0000", 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9330000000000004e-06 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.767416+0000", 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.023105787 2023-09-23T20:16:02.380 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.768079+0000", 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00066287699999999998 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.799565+0000", 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.031486449999999999 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.381 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.800979+0000", 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001414092 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.801070+0000", 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0818999999999999e-05 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.382 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803811+0000", 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002740592 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804527+0000", 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00071653999999999999 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.811597+0000", 2023-09-23T20:16:02.383 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0070696429999999996 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.811634+0000", 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7245e-05 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.813302+0000", 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016681619999999999 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.384 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86884 8.f 8:f6bbdc58:::benchmark_data_smithi135_460343_object2651:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 477.30874605100001, 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1026705059999999, 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.385 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86884 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.386 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742168+0000", 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742167+0000", 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742170+0000", 2023-09-23T20:16:02.387 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5610000000000001e-06 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742176+0000", 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0070000000000003e-06 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742180+0000", 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.651e-06 2023-09-23T20:16:02.388 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742295+0000", 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011517499999999999 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742308+0000", 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2274e-05 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.389 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744328+0000", 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002019982 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744334+0000", 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2779999999999999e-06 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.390 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803248+0000", 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0589140269999999 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803789+0000", 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00054099499999999995 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817686+0000", 2023-09-23T20:16:02.391 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013896531 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817787+0000", 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000101207 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.817881+0000", 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4238000000000001e-05 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.392 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.832086+0000", 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014205493 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843262+0000", 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011175433 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.393 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843318+0000", 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6292000000000002e-05 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843373+0000", 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4737000000000001e-05 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.394 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844838+0000", 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001465084 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86888 8.f 8:ff8ee601:::benchmark_data_smithi135_460343_object2655:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 477.30870403099999, 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1038876470000001, 2023-09-23T20:16:02.395 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86888 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.396 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742210+0000", 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742209+0000", 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:16:02.397 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742214+0000", 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8230000000000002e-06 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742260+0000", 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5964000000000002e-05 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.398 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742266+0000", 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6209999999999998e-06 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742385+0000", 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011896900000000001 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:16:02.399 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742393+0000", 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4590000000000001e-06 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744351+0000", 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019579710000000002 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.400 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744358+0000", 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.803866+0000", 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0595081719999999 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:02.401 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804470+0000", 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060409500000000002 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823610+0000", 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019139976999999999 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.402 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823691+0000", 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0525999999999994e-05 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.823816+0000", 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000125029 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.403 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.837821+0000", 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014005521999999999 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844894+0000", 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.007073368 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844944+0000", 2023-09-23T20:16:02.404 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9799999999999998e-05 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.845000+0000", 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5933999999999997e-05 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846097+0000", 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001096999 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.405 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86947 8.f 8:fc570a5a:::benchmark_data_smithi135_460343_object2714:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 477.30862506199998, 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.10541584, 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.406 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86947 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.407 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742289+0000", 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742288+0000", 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742292+0000", 2023-09-23T20:16:02.408 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4179999999999999e-06 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742299+0000", 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0110000000000003e-06 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742304+0000", 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4710000000000001e-06 2023-09-23T20:16:02.409 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742429+0000", 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012482199999999999 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742435+0000", 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5150000000000003e-06 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.410 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744369+0000", 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001934203 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744375+0000", 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.9190000000000004e-06 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.411 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.804569+0000", 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0601939 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805072+0000", 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050289400000000004 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.412 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829232+0000", 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024160198000000001 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829317+0000", 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4447e-05 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:16:02.413 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.829401+0000", 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4468999999999994e-05 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.843205+0000", 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013804293 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.414 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846168+0000", 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029630279999999999 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846218+0000", 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9574999999999999e-05 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.415 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846273+0000", 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5058e-05 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847704+0000", 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0014312109999999999 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.416 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:86957 8.f 8:fdc750af:::benchmark_data_smithi135_460343_object2724:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 477.30860506099998, 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.106922379, 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 86957 2023-09-23T20:16:02.417 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742309+0000", 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.418 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742308+0000", 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742311+0000", 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5840000000000002e-06 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.419 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742316+0000", 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.746e-06 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742320+0000", 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1649999999999999e-06 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.420 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742500+0000", 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000180003 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742506+0000", 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.263e-06 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744392+0000", 2023-09-23T20:16:02.421 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001886284 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744398+0000", 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1149999999999999e-06 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805115+0000", 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0607168629999999 2023-09-23T20:16:02.422 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805787+0000", 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00067139499999999998 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.834880+0000", 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.029093853999999999 2023-09-23T20:16:02.423 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.835004+0000", 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123545 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.835169+0000", 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000164506 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.424 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.844858+0000", 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0096890859999999995 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847759+0000", 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0029018820000000002 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.425 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847800+0000", 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0590999999999998e-05 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.847847+0000", 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6615e-05 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.426 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849231+0000", 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0013841960000000001 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:87007 8.f 8:ff8f150f:::benchmark_data_smithi135_460343_object2774:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 477.30859053099999, 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.108736019, 2023-09-23T20:16:02.427 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.429 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.429 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.429 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.429 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.429 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 87007 2023-09-23T20:16:02.429 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.429 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.429 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.430 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742323+0000", 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742325+0000", 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.824e-06 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.431 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742329+0000", 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7099999999999998e-06 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742333+0000", 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8589999999999998e-06 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.432 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742551+0000", 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00021761999999999999 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.742559+0000", 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.2020000000000004e-06 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744418+0000", 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001859468 2023-09-23T20:16:02.433 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:04.744426+0000", 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0770000000000002e-06 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.805831+0000", 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0614059330000001 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.434 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.806364+0000", 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00053292300000000003 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840788+0000", 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.034423435000000002 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.435 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840892+0000", 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010409299999999999 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.840986+0000", 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3930000000000004e-05 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.436 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.846116+0000", 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005129761 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849292+0000", 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031762359999999998 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849356+0000", 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4146000000000006e-05 2023-09-23T20:16:02.437 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.849403+0000", 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7151999999999998e-05 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:08:05.851059+0000", 2023-09-23T20:16:02.438 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001655903 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:125547 8.f 8:f06e005a:::benchmark_data_smithi135_460343_object41314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:11:02.542323+0000", 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 299.50859091500001, 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.61188724500000002, 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.439 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 125547 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542323+0000", 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.440 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542323+0000", 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542322+0000", 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.441 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542328+0000", 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6019999999999996e-06 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542338+0000", 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.008e-05 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.442 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.542344+0000", 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1589999999999998e-06 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.829850+0000", 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28750529699999999 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.840856+0000", 2023-09-23T20:16:02.443 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011006592000000001 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.927237+0000", 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.086380278000000005 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.930065+0000", 2023-09-23T20:16:02.444 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028284019999999998 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.934871+0000", 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004806388 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:02.980254+0000", 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.045382458000000001 2023-09-23T20:16:02.445 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:03.075878+0000", 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.095624656000000002 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:03.148645+0000", 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.072766843999999997 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.446 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:03.149741+0000", 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001095256 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:11:03.154210+0000", 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0044692960000000002 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.447 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.36186.0:97538 9.20 9:0761484d:::benchmark_data_smithi135_549536_object97537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:14:33.874503+0000", 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 88.176410883000003, 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2478944040000002, 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.36186", 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:02.448 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 97538 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:33.874503+0000", 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:33.874503+0000", 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:16:02.449 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:33.874501+0000", 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:33.874528+0000", 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6959000000000001e-05 2023-09-23T20:16:02.450 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:33.874548+0000", 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9854999999999999e-05 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:33.874560+0000", 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2313999999999999e-05 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.451 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:33.874635+0000", 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4999000000000005e-05 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:33.874648+0000", 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3337e-05 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.452 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:34.826815+0000", 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.95216648000000004 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:34.826828+0000", 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3447e-05 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:36.108322+0000", 2023-09-23T20:16:02.453 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.281494111 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:36.108493+0000", 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000170915 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:36.108649+0000", 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015574799999999999 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.454 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:36.110015+0000", 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001365745 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:36.110607+0000", 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00059229900000000002 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.455 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:36.121062+0000", 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010454913999999999 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:36.121129+0000", 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6497999999999999e-05 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:14:36.122397+0000", 2023-09-23T20:16:02.456 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012685050000000001 2023-09-23T20:16:02.457 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.457 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.457 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.457 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:16:02.457 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:16:02.457 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:16:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[221113]: pgmap v1285: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 878 op/s; 42380/118947 objects degraded (35.629%); 2455/118947 objects misplaced (2.064%); 3.4 MiB/s, 143 objects/s recovering 2023-09-23T20:16:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[221113]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[221113]: osdmap e461: 8 total, 8 up, 6 in 2023-09-23T20:16:02.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[221113]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[221113]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[221113]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[213871]: pgmap v1285: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 878 op/s; 42380/118947 objects degraded (35.629%); 2455/118947 objects misplaced (2.064%); 3.4 MiB/s, 143 objects/s recovering 2023-09-23T20:16:02.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[213871]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[213871]: osdmap e461: 8 total, 8 up, 6 in 2023-09-23T20:16:02.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[213871]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[213871]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:02 smithi135 ceph-mon[213871]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.627 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:02.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:02 smithi144 ceph-mon[194833]: pgmap v1285: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 878 op/s; 42380/118947 objects degraded (35.629%); 2455/118947 objects misplaced (2.064%); 3.4 MiB/s, 143 objects/s recovering 2023-09-23T20:16:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:02 smithi144 ceph-mon[194833]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:02 smithi144 ceph-mon[194833]: osdmap e461: 8 total, 8 up, 6 in 2023-09-23T20:16:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:02 smithi144 ceph-mon[194833]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:02 smithi144 ceph-mon[194833]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:02 smithi144 ceph-mon[194833]: failed to encode map e461 with expected crc 2023-09-23T20:16:02.732 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:02.781 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:03.189 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:16:03.449 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:03.454 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[213871]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.454 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[213871]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[213871]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[213871]: osdmap e462: 8 total, 8 up, 6 in 2023-09-23T20:16:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[213871]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[213871]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 42380/118947 objects degraded (35.629%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:03.455 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[213871]: pgmap v1288: 129 pgs: 1 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 964 op/s; 40488/112995 objects degraded (35.832%); 3528/112995 objects misplaced (3.122%); 7.9 MiB/s, 209 objects/s recovering 2023-09-23T20:16:03.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[221113]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[221113]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[221113]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[221113]: osdmap e462: 8 total, 8 up, 6 in 2023-09-23T20:16:03.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[221113]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[221113]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 42380/118947 objects degraded (35.629%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:03.456 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:03 smithi135 ceph-mon[221113]: pgmap v1288: 129 pgs: 1 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 964 op/s; 40488/112995 objects degraded (35.832%); 3528/112995 objects misplaced (3.122%); 7.9 MiB/s, 209 objects/s recovering 2023-09-23T20:16:03.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:03.557 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:03.578 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:16:03.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:03 smithi144 ceph-mon[194833]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:03 smithi144 ceph-mon[194833]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:03 smithi144 ceph-mon[194833]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:03 smithi144 ceph-mon[194833]: osdmap e462: 8 total, 8 up, 6 in 2023-09-23T20:16:03.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:03 smithi144 ceph-mon[194833]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:03 smithi144 ceph-mon[194833]: failed to encode map e462 with expected crc 2023-09-23T20:16:03.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:03 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 42380/118947 objects degraded (35.629%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:03.619 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:03 smithi144 ceph-mon[194833]: pgmap v1288: 129 pgs: 1 peering, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 964 op/s; 40488/112995 objects degraded (35.832%); 3528/112995 objects misplaced (3.122%); 7.9 MiB/s, 209 objects/s recovering 2023-09-23T20:16:03.623 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:03.664 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:03.713 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:03.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:16:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:16:03] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-09-23T20:16:03.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:03.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:03.825 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:03.884 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:03.993 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:04.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:04.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:04.105 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:04.128 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:16:04.159 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:16:04.168 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:04.190 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:16:04.221 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:16:04.251 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:16:04.261 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:04.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:04.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:04.293 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:04.304 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:16:04.360 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:16:04.381 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:04.402 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:16:04.406 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:16:04.406 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:16:04.406 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:16:04.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:16:04.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:16:04.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:16:04.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:16:04.407 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:16:04.449 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:04.974 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2515746633' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:04.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2515746633' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:05.138 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2515746633' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:05.757 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:05.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:05.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:05.862 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:05.912 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:06.034 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:06.067 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:05 smithi135 ceph-mon[221113]: pgmap v1289: 129 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 40488/107739 objects degraded (37.580%); 3302/107739 objects misplaced (3.065%); 5.2 MiB/s, 182 objects/s recovering 2023-09-23T20:16:06.067 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:05 smithi135 ceph-mon[213871]: pgmap v1289: 129 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 40488/107739 objects degraded (37.580%); 3302/107739 objects misplaced (3.065%); 5.2 MiB/s, 182 objects/s recovering 2023-09-23T20:16:06.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:05 smithi144 ceph-mon[194833]: pgmap v1289: 129 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 40488/107739 objects degraded (37.580%); 3302/107739 objects misplaced (3.065%); 5.2 MiB/s, 182 objects/s recovering 2023-09-23T20:16:06.133 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:06.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:06.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:06.241 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:06.324 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:06.677 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:06.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:06.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:06.786 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:06.827 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:07.342 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:07.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:07.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:07.451 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:07.516 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:07.616 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:07.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:07.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:07.727 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:07.791 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:07.905 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:07.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:07.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:08.020 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:16:08.023 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:08.023 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:16:08.057 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:08.077 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:16:08.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:08.119 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:08.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:08.220 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:08.281 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:08.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:08 smithi135 ceph-mon[221113]: pgmap v1290: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 40400/106305 objects degraded (38.004%); 2876/106305 objects misplaced (2.705%); 7.3 MiB/s, 142 objects/s recovering 2023-09-23T20:16:08.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:08.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:08 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 40400/106305 objects degraded (38.004%), 20 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:08.349 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:08 smithi135 ceph-mon[213871]: pgmap v1290: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 40400/106305 objects degraded (38.004%); 2876/106305 objects misplaced (2.705%); 7.3 MiB/s, 142 objects/s recovering 2023-09-23T20:16:08.349 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:08.349 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:08 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 40400/106305 objects degraded (38.004%), 20 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:08 smithi144 ceph-mon[194833]: pgmap v1290: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 40400/106305 objects degraded (38.004%); 2876/106305 objects misplaced (2.705%); 7.3 MiB/s, 142 objects/s recovering 2023-09-23T20:16:08.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:08 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 40400/106305 objects degraded (38.004%), 20 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:08.686 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:08.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:08.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:08.791 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:08.817 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:09.345 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:09.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:09.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:09.401 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:09 smithi135 ceph-mon[213871]: from='client.56161 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:09.401 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:09 smithi135 ceph-mon[221113]: from='client.56161 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:09.420 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1290,"stamp":"2023-09-23T20:16:07.073354+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2325175725,"num_objects":35435,"num_object_clones":0,"num_object_copies":106305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40400,"num_objects_misplaced":2876,"num_objects_unfound":0,"num_objects_dirty":35435,"num_whiteouts":0,"num_read":5996,"num_read_kb":13236,"num_write":173213,"num_write_kb":6659327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17485,"num_bytes_recovered":1063976052,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203197,"ondisk_log_size":203197,"up":387,"acting":367,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":391,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10708468,"kb_used_data":6741124,"kb_used_omap":25,"kb_used_meta":3967270,"kb_avail":551762444,"statfs":{"total":575970213888,"available":565004742656,"internally_reserved":0,"allocated":6902910976,"data_stored":6891913899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":4062485196},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,1,4,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-276168704,"num_objects":-4214,"num_object_clones":0,"num_object_copies":-12642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1980,"num_objects_misplaced":421,"num_objects_unfound":0,"num_objects_dirty":-4214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4214,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1130,"num_bytes_recovered":60424192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.925683"},"pg_stats":[{"pgid":"9.23","version":"462'5867","reported_seq":5960,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:02.901963+0000","last_change":"2023-09-23T20:14:36.092943+0000","last_active":"2023-09-23T20:16:02.901963+0000","last_peered":"2023-09-23T20:16:02.901963+0000","last_clean":"2023-09-23T20:14:33.760240+0000","last_became_active":"2023-09-23T20:14:36.076681+0000","last_became_peered":"2023-09-23T20:14:36.076681+0000","last_unstale":"2023-09-23T20:16:02.901963+0000","last_undegraded":"2023-09-23T20:14:35.911782+0000","last_fullsized":"2023-09-23T20:14:35.868390+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36061184,"num_objects":550,"num_object_clones":0,"num_object_copies":1650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2758,"num_write_kb":105872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":460},{"shards":"3,7","objects":-1014}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.22","version":"462'5910","reported_seq":6003,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:06.512558+0000","last_change":"2023-09-23T20:14:36.006132+0000","last_active":"2023-09-23T20:16:06.512558+0000","last_peered":"2023-09-23T20:16:06.512558+0000","last_clean":"2023-09-23T20:14:33.601158+0000","last_became_active":"2023-09-23T20:14:35.991960+0000","last_became_peered":"2023-09-23T20:14:35.991960+0000","last_unstale":"2023-09-23T20:16:06.512558+0000","last_undegraded":"2023-09-23T20:14:35.848470+0000","last_fullsized":"2023-09-23T20:14:35.833465+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35651584,"num_objects":544,"num_object_clones":0,"num_object_copies":1632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2774,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":429},{"shards":"2,3","objects":-1000}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.21","version":"462'5744","reported_seq":5827,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.684008+0000","last_change":"2023-09-23T20:14:24.799466+0000","last_active":"2023-09-23T20:16:02.684008+0000","last_peered":"2023-09-23T20:16:02.684008+0000","last_clean":"2023-09-23T20:16:02.684008+0000","last_became_active":"2023-09-23T20:14:24.796974+0000","last_became_peered":"2023-09-23T20:14:24.796974+0000","last_unstale":"2023-09-23T20:16:02.684008+0000","last_undegraded":"2023-09-23T20:16:02.684008+0000","last_fullsized":"2023-09-23T20:16:02.684008+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37847040,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2650,"num_write_kb":103264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.20","version":"462'5810","reported_seq":5902,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:05.676870+0000","last_change":"2023-09-23T20:14:36.109513+0000","last_active":"2023-09-23T20:16:05.676870+0000","last_peered":"2023-09-23T20:16:05.676870+0000","last_clean":"2023-09-23T20:14:33.769971+0000","last_became_active":"2023-09-23T20:14:36.106301+0000","last_became_peered":"2023-09-23T20:14:36.106301+0000","last_unstale":"2023-09-23T20:16:05.676870+0000","last_undegraded":"2023-09-23T20:14:35.845700+0000","last_fullsized":"2023-09-23T20:14:35.836337+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5810,"log_dups_size":0,"ondisk_log_size":5810,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33554432,"num_objects":512,"num_object_clones":0,"num_object_copies":1536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2728,"num_write_kb":103680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":437},{"shards":"0,2","objects":-1033}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.10","version":"462'6973","reported_seq":12905,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.004949+0000","last_change":"2023-09-23T20:14:22.339078+0000","last_active":"2023-09-23T20:16:03.004949+0000","last_peered":"2023-09-23T20:16:03.004949+0000","last_clean":"2023-09-23T20:16:03.004949+0000","last_became_active":"2023-09-23T20:14:22.319507+0000","last_became_peered":"2023-09-23T20:14:22.319507+0000","last_unstale":"2023-09-23T20:16:03.004949+0000","last_undegraded":"2023-09-23T20:16:03.004949+0000","last_fullsized":"2023-09-23T20:16:03.004949+0000","mapping_epoch":434,"log_start":"412'2054","ondisk_log_start":"412'2054","created":418,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4919,"log_dups_size":0,"ondisk_log_size":4919,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71761920,"num_objects":1095,"num_object_clones":0,"num_object_copies":3285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5253,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1945,"num_bytes_recovered":127467520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":32,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.182108+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:16:01.182108+0000","last_peered":"2023-09-23T20:16:01.182108+0000","last_clean":"2023-09-23T20:16:01.182108+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:16:01.182108+0000","last_undegraded":"2023-09-23T20:16:01.182108+0000","last_fullsized":"2023-09-23T20:16:01.182108+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":30,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.199876+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:16:01.199876+0000","last_peered":"2023-09-23T20:16:01.199876+0000","last_clean":"2023-09-23T20:16:01.199876+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:16:01.199876+0000","last_undegraded":"2023-09-23T20:16:01.199876+0000","last_fullsized":"2023-09-23T20:16:01.199876+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.13","version":"462'7066","reported_seq":13533,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.014720+0000","last_change":"2023-09-23T20:14:36.123690+0000","last_active":"2023-09-23T20:16:03.014720+0000","last_peered":"2023-09-23T20:16:03.014720+0000","last_clean":"2023-09-23T20:14:05.142062+0000","last_became_active":"2023-09-23T20:14:07.304693+0000","last_became_peered":"2023-09-23T20:14:07.304693+0000","last_unstale":"2023-09-23T20:16:03.014720+0000","last_undegraded":"2023-09-23T20:14:36.121893+0000","last_fullsized":"2023-09-23T20:14:07.197546+0000","mapping_epoch":429,"log_start":"412'2220","ondisk_log_start":"412'2220","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4846,"log_dups_size":0,"ondisk_log_size":4846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71008256,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":192,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5334,"num_write_kb":205344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2127,"num_bytes_recovered":139395072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,0],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"0,7","objects":988},{"shards":"1,3,7","objects":96}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":207,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188547+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:16:01.188547+0000","last_peered":"2023-09-23T20:16:01.188547+0000","last_clean":"2023-09-23T20:16:01.188547+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:16:01.188547+0000","last_undegraded":"2023-09-23T20:16:01.188547+0000","last_fullsized":"2023-09-23T20:16:01.188547+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":170,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191523+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:16:01.191523+0000","last_peered":"2023-09-23T20:16:01.191523+0000","last_clean":"2023-09-23T20:16:01.191523+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:16:01.191523+0000","last_undegraded":"2023-09-23T20:16:01.191523+0000","last_fullsized":"2023-09-23T20:16:01.191523+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"462'7192","reported_seq":13657,"reported_epoch":462,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T20:16:03.305569+0000","last_change":"2023-09-23T20:14:36.234004+0000","last_active":"2023-09-23T20:16:03.305569+0000","last_peered":"2023-09-23T20:16:03.305569+0000","last_clean":"2023-09-23T20:14:33.790624+0000","last_became_active":"2023-09-23T20:14:36.220990+0000","last_became_peered":"2023-09-23T20:14:36.220990+0000","last_unstale":"2023-09-23T20:16:03.305569+0000","last_undegraded":"2023-09-23T20:16:03.305569+0000","last_fullsized":"2023-09-23T20:16:03.305569+0000","mapping_epoch":442,"log_start":"412'2098","ondisk_log_start":"412'2098","created":418,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5094,"log_dups_size":0,"ondisk_log_size":5094,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65667072,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1002,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5442,"num_write_kb":206208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":139067392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1","3"],"object_location_counts":[{"shards":"0,1,5","objects":1002}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":31,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.180742+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:16:01.180742+0000","last_peered":"2023-09-23T20:16:01.180742+0000","last_clean":"2023-09-23T20:16:01.180742+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:16:01.180742+0000","last_undegraded":"2023-09-23T20:16:01.180742+0000","last_fullsized":"2023-09-23T20:16:01.180742+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":417,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.211795+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:16:01.211795+0000","last_peered":"2023-09-23T20:16:01.211795+0000","last_clean":"2023-09-23T20:16:01.211795+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:16:01.211795+0000","last_undegraded":"2023-09-23T20:16:01.211795+0000","last_fullsized":"2023-09-23T20:16:01.211795+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1d","version":"462'8237","reported_seq":8320,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.023819+0000","last_change":"2023-09-23T20:14:36.076800+0000","last_active":"2023-09-23T20:16:03.023819+0000","last_peered":"2023-09-23T20:16:03.023819+0000","last_clean":"2023-09-23T20:14:23.775634+0000","last_became_active":"2023-09-23T20:14:36.070774+0000","last_became_peered":"2023-09-23T20:14:36.070774+0000","last_unstale":"2023-09-23T20:16:03.023819+0000","last_undegraded":"2023-09-23T20:14:35.892079+0000","last_fullsized":"2023-09-23T20:14:35.839513+0000","mapping_epoch":442,"log_start":"412'2358","ondisk_log_start":"412'2358","created":436,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5879,"log_dups_size":0,"ondisk_log_size":5879,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80674840,"num_objects":1232,"num_object_clones":0,"num_object_copies":3696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1232,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5448,"num_write_kb":213697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":967},{"shards":"5,7","objects":-1843}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":388,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.211653+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:16:01.211653+0000","last_peered":"2023-09-23T20:16:01.211653+0000","last_clean":"2023-09-23T20:16:01.211653+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:16:01.211653+0000","last_undegraded":"2023-09-23T20:16:01.211653+0000","last_fullsized":"2023-09-23T20:16:01.211653+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":316,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.183620+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:16:01.183620+0000","last_peered":"2023-09-23T20:16:01.183620+0000","last_clean":"2023-09-23T20:16:01.183620+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:16:01.183620+0000","last_undegraded":"2023-09-23T20:16:01.183620+0000","last_fullsized":"2023-09-23T20:16:01.183620+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"462'7994","reported_seq":8067,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:06.502277+0000","last_change":"2023-09-23T20:14:24.810789+0000","last_active":"2023-09-23T20:16:06.502277+0000","last_peered":"2023-09-23T20:16:06.502277+0000","last_clean":"2023-09-23T20:16:06.502277+0000","last_became_active":"2023-09-23T20:14:24.807691+0000","last_became_peered":"2023-09-23T20:14:24.807691+0000","last_unstale":"2023-09-23T20:16:06.502277+0000","last_undegraded":"2023-09-23T20:16:06.502277+0000","last_fullsized":"2023-09-23T20:16:06.502277+0000","mapping_epoch":436,"log_start":"413'2795","ondisk_log_start":"413'2795","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5199,"log_dups_size":0,"ondisk_log_size":5199,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62685184,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5375,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":795,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.181302+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:16:01.181302+0000","last_peered":"2023-09-23T20:16:01.181302+0000","last_clean":"2023-09-23T20:16:01.181302+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:16:01.181302+0000","last_undegraded":"2023-09-23T20:16:01.181302+0000","last_fullsized":"2023-09-23T20:16:01.181302+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":471,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.204165+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:16:01.204165+0000","last_peered":"2023-09-23T20:16:01.204165+0000","last_clean":"2023-09-23T20:16:01.204165+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:16:01.204165+0000","last_undegraded":"2023-09-23T20:16:01.204165+0000","last_fullsized":"2023-09-23T20:16:01.204165+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":326,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188428+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:16:01.188428+0000","last_peered":"2023-09-23T20:16:01.188428+0000","last_clean":"2023-09-23T20:16:01.188428+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:16:01.188428+0000","last_undegraded":"2023-09-23T20:16:01.188428+0000","last_fullsized":"2023-09-23T20:16:01.188428+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1f","version":"462'8021","reported_seq":8103,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.337258+0000","last_change":"2023-09-23T20:14:36.232228+0000","last_active":"2023-09-23T20:16:03.337258+0000","last_peered":"2023-09-23T20:16:03.337258+0000","last_clean":"2023-09-23T20:14:33.842679+0000","last_became_active":"2023-09-23T20:14:36.217802+0000","last_became_peered":"2023-09-23T20:14:36.217802+0000","last_unstale":"2023-09-23T20:16:03.337258+0000","last_undegraded":"2023-09-23T20:14:36.129688+0000","last_fullsized":"2023-09-23T20:14:36.115967+0000","mapping_epoch":442,"log_start":"412'1893","ondisk_log_start":"412'1893","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6128,"log_dups_size":0,"ondisk_log_size":6128,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72548352,"num_objects":1107,"num_object_clones":0,"num_object_copies":3321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5335,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":913},{"shards":"3,5","objects":-1920}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":155,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191906+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:16:01.191906+0000","last_peered":"2023-09-23T20:16:01.191906+0000","last_clean":"2023-09-23T20:16:01.191906+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:16:01.191906+0000","last_undegraded":"2023-09-23T20:16:01.191906+0000","last_fullsized":"2023-09-23T20:16:01.191906+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":160,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.182294+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:16:01.182294+0000","last_peered":"2023-09-23T20:16:01.182294+0000","last_clean":"2023-09-23T20:16:01.182294+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:16:01.182294+0000","last_undegraded":"2023-09-23T20:16:01.182294+0000","last_fullsized":"2023-09-23T20:16:01.182294+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.1e","version":"462'8029","reported_seq":14172,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.039188+0000","last_change":"2023-09-23T20:15:21.201333+0000","last_active":"2023-09-23T20:16:03.039188+0000","last_peered":"2023-09-23T20:16:03.039188+0000","last_clean":"2023-09-23T20:16:03.039188+0000","last_became_active":"2023-09-23T20:15:21.192111+0000","last_became_peered":"2023-09-23T20:15:21.192111+0000","last_unstale":"2023-09-23T20:16:03.039188+0000","last_undegraded":"2023-09-23T20:16:03.039188+0000","last_fullsized":"2023-09-23T20:16:03.039188+0000","mapping_epoch":454,"log_start":"412'2126","ondisk_log_start":"412'2126","created":436,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5903,"log_dups_size":0,"ondisk_log_size":5903,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75595776,"num_objects":1154,"num_object_clones":0,"num_object_copies":3462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5330,"num_write_kb":207456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3024,"num_bytes_recovered":198180864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":408,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.189056+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:16:01.189056+0000","last_peered":"2023-09-23T20:16:01.189056+0000","last_clean":"2023-09-23T20:16:01.189056+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:16:01.189056+0000","last_undegraded":"2023-09-23T20:16:01.189056+0000","last_fullsized":"2023-09-23T20:16:01.189056+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":339,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.199627+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:16:01.199627+0000","last_peered":"2023-09-23T20:16:01.199627+0000","last_clean":"2023-09-23T20:16:01.199627+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:16:01.199627+0000","last_undegraded":"2023-09-23T20:16:01.199627+0000","last_fullsized":"2023-09-23T20:16:01.199627+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"462'7181","reported_seq":7254,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.022492+0000","last_change":"2023-09-23T20:13:46.811423+0000","last_active":"2023-09-23T20:16:03.022492+0000","last_peered":"2023-09-23T20:16:03.022492+0000","last_clean":"2023-09-23T20:16:03.022492+0000","last_became_active":"2023-09-23T20:13:46.808266+0000","last_became_peered":"2023-09-23T20:13:46.808266+0000","last_unstale":"2023-09-23T20:16:03.022492+0000","last_undegraded":"2023-09-23T20:16:03.022492+0000","last_fullsized":"2023-09-23T20:16:03.022492+0000","mapping_epoch":418,"log_start":"412'2419","ondisk_log_start":"412'2419","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4762,"log_dups_size":0,"ondisk_log_size":4762,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71532544,"num_objects":1092,"num_object_clones":0,"num_object_copies":3276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5426,"num_write_kb":208544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":821,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.198033+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:16:01.198033+0000","last_peered":"2023-09-23T20:16:01.198033+0000","last_clean":"2023-09-23T20:16:01.198033+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:16:01.198033+0000","last_undegraded":"2023-09-23T20:16:01.198033+0000","last_fullsized":"2023-09-23T20:16:01.198033+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":155,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.189913+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:16:01.189913+0000","last_peered":"2023-09-23T20:16:01.189913+0000","last_clean":"2023-09-23T20:16:01.189913+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:16:01.189913+0000","last_undegraded":"2023-09-23T20:16:01.189913+0000","last_fullsized":"2023-09-23T20:16:01.189913+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":208,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.189271+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:16:01.189271+0000","last_peered":"2023-09-23T20:16:01.189271+0000","last_clean":"2023-09-23T20:16:01.189271+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:16:01.189271+0000","last_undegraded":"2023-09-23T20:16:01.189271+0000","last_fullsized":"2023-09-23T20:16:01.189271+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"462'7157","reported_seq":7230,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.237195+0000","last_change":"2023-09-23T20:13:46.824838+0000","last_active":"2023-09-23T20:16:03.237195+0000","last_peered":"2023-09-23T20:16:03.237195+0000","last_clean":"2023-09-23T20:16:03.237195+0000","last_became_active":"2023-09-23T20:13:46.820709+0000","last_became_peered":"2023-09-23T20:13:46.820709+0000","last_unstale":"2023-09-23T20:16:03.237195+0000","last_undegraded":"2023-09-23T20:16:03.237195+0000","last_fullsized":"2023-09-23T20:16:03.237195+0000","mapping_epoch":418,"log_start":"412'2051","ondisk_log_start":"412'2051","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5106,"log_dups_size":0,"ondisk_log_size":5106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72187904,"num_objects":1102,"num_object_clones":0,"num_object_copies":3306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5442,"num_write_kb":209376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":510,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.207686+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:16:01.207686+0000","last_peered":"2023-09-23T20:16:01.207686+0000","last_clean":"2023-09-23T20:16:01.207686+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:16:01.207686+0000","last_undegraded":"2023-09-23T20:16:01.207686+0000","last_fullsized":"2023-09-23T20:16:01.207686+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188423+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:16:01.188423+0000","last_peered":"2023-09-23T20:16:01.188423+0000","last_clean":"2023-09-23T20:16:01.188423+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:16:01.188423+0000","last_undegraded":"2023-09-23T20:16:01.188423+0000","last_fullsized":"2023-09-23T20:16:01.188423+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":460,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.200284+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:16:01.200284+0000","last_peered":"2023-09-23T20:16:01.200284+0000","last_clean":"2023-09-23T20:16:01.200284+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:16:01.200284+0000","last_undegraded":"2023-09-23T20:16:01.200284+0000","last_fullsized":"2023-09-23T20:16:01.200284+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1b","version":"462'7850","reported_seq":7923,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.381984+0000","last_change":"2023-09-23T20:14:24.808547+0000","last_active":"2023-09-23T20:16:03.381984+0000","last_peered":"2023-09-23T20:16:03.381984+0000","last_clean":"2023-09-23T20:16:03.381984+0000","last_became_active":"2023-09-23T20:14:24.806384+0000","last_became_peered":"2023-09-23T20:14:24.806384+0000","last_unstale":"2023-09-23T20:16:03.381984+0000","last_undegraded":"2023-09-23T20:16:03.381984+0000","last_fullsized":"2023-09-23T20:16:03.381984+0000","mapping_epoch":436,"log_start":"412'1574","ondisk_log_start":"412'1574","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6276,"log_dups_size":0,"ondisk_log_size":6276,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71598080,"num_objects":1093,"num_object_clones":0,"num_object_copies":3279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5239,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":571,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188113+0000","last_change":"2023-09-23T20:14:37.887863+0000","last_active":"2023-09-23T20:16:01.188113+0000","last_peered":"2023-09-23T20:16:01.188113+0000","last_clean":"2023-09-23T20:16:01.188113+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:16:01.188113+0000","last_undegraded":"2023-09-23T20:16:01.188113+0000","last_fullsized":"2023-09-23T20:16:01.188113+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":149,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.190976+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:16:01.190976+0000","last_peered":"2023-09-23T20:16:01.190976+0000","last_clean":"2023-09-23T20:16:01.190976+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:16:01.190976+0000","last_undegraded":"2023-09-23T20:16:01.190976+0000","last_fullsized":"2023-09-23T20:16:01.190976+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":450,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.183674+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:16:01.183674+0000","last_peered":"2023-09-23T20:16:01.183674+0000","last_clean":"2023-09-23T20:16:01.183674+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:16:01.183674+0000","last_undegraded":"2023-09-23T20:16:01.183674+0000","last_fullsized":"2023-09-23T20:16:01.183674+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1a","version":"462'7982","reported_seq":8065,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.380052+0000","last_change":"2023-09-23T20:14:36.218516+0000","last_active":"2023-09-23T20:16:03.380052+0000","last_peered":"2023-09-23T20:16:03.380052+0000","last_clean":"2023-09-23T20:14:33.782041+0000","last_became_active":"2023-09-23T20:14:36.175134+0000","last_became_peered":"2023-09-23T20:14:36.175134+0000","last_unstale":"2023-09-23T20:16:03.380052+0000","last_undegraded":"2023-09-23T20:14:35.943267+0000","last_fullsized":"2023-09-23T20:14:35.848493+0000","mapping_epoch":442,"log_start":"412'1644","ondisk_log_start":"412'1644","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6338,"log_dups_size":0,"ondisk_log_size":6338,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74055680,"num_objects":1130,"num_object_clones":0,"num_object_copies":3390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5284,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":883},{"shards":"5,7","objects":-1830}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":599,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.202849+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:16:01.202849+0000","last_peered":"2023-09-23T20:16:01.202849+0000","last_clean":"2023-09-23T20:16:01.202849+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:16:01.202849+0000","last_undegraded":"2023-09-23T20:16:01.202849+0000","last_fullsized":"2023-09-23T20:16:01.202849+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":105,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.207408+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:16:01.207408+0000","last_peered":"2023-09-23T20:16:01.207408+0000","last_clean":"2023-09-23T20:16:01.207408+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:16:01.207408+0000","last_undegraded":"2023-09-23T20:16:01.207408+0000","last_fullsized":"2023-09-23T20:16:01.207408+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":102,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191334+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:16:01.191334+0000","last_peered":"2023-09-23T20:16:01.191334+0000","last_clean":"2023-09-23T20:16:01.191334+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:16:01.191334+0000","last_undegraded":"2023-09-23T20:16:01.191334+0000","last_fullsized":"2023-09-23T20:16:01.191334+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"462'7141","reported_seq":7215,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.362851+0000","last_change":"2023-09-23T20:13:46.790587+0000","last_active":"2023-09-23T20:16:03.362851+0000","last_peered":"2023-09-23T20:16:03.362851+0000","last_clean":"2023-09-23T20:16:03.362851+0000","last_became_active":"2023-09-23T20:13:46.788387+0000","last_became_peered":"2023-09-23T20:13:46.788387+0000","last_unstale":"2023-09-23T20:16:03.362851+0000","last_undegraded":"2023-09-23T20:16:03.362851+0000","last_fullsized":"2023-09-23T20:16:03.362851+0000","mapping_epoch":407,"log_start":"412'1969","ondisk_log_start":"412'1969","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5172,"log_dups_size":0,"ondisk_log_size":5172,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73203712,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5411,"num_write_kb":208896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":722,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191534+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:16:01.191534+0000","last_peered":"2023-09-23T20:16:01.191534+0000","last_clean":"2023-09-23T20:16:01.191534+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:16:01.191534+0000","last_undegraded":"2023-09-23T20:16:01.191534+0000","last_fullsized":"2023-09-23T20:16:01.191534+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":153,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.192678+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:16:01.192678+0000","last_peered":"2023-09-23T20:16:01.192678+0000","last_clean":"2023-09-23T20:16:01.192678+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:16:01.192678+0000","last_undegraded":"2023-09-23T20:16:01.192678+0000","last_fullsized":"2023-09-23T20:16:01.192678+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":470,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.207504+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:16:01.207504+0000","last_peered":"2023-09-23T20:16:01.207504+0000","last_clean":"2023-09-23T20:16:01.207504+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:16:01.207504+0000","last_undegraded":"2023-09-23T20:16:01.207504+0000","last_fullsized":"2023-09-23T20:16:01.207504+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"462'7039","reported_seq":7514,"reported_epoch":462,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.024502+0000","last_change":"2023-09-23T20:16:00.310118+0000","last_active":"2023-09-23T20:16:03.024502+0000","last_peered":"2023-09-23T20:16:03.024502+0000","last_clean":"2023-09-23T20:14:33.800061+0000","last_became_active":"2023-09-23T20:14:36.025052+0000","last_became_peered":"2023-09-23T20:14:36.025052+0000","last_unstale":"2023-09-23T20:16:03.024502+0000","last_undegraded":"2023-09-23T20:14:35.860952+0000","last_fullsized":"2023-09-23T20:14:35.840056+0000","mapping_epoch":442,"log_start":"411'755","ondisk_log_start":"411'755","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6284,"log_dups_size":0,"ondisk_log_size":6284,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72646656,"num_objects":1108,"num_object_clones":0,"num_object_copies":3324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":760,"num_objects_unfound":0,"num_objects_dirty":1108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5314,"num_write_kb":205536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":196,"num_bytes_recovered":12845056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":760},{"shards":"3,4","objects":-1719}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":809,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188852+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:16:01.188852+0000","last_peered":"2023-09-23T20:16:01.188852+0000","last_clean":"2023-09-23T20:16:01.188852+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:16:01.188852+0000","last_undegraded":"2023-09-23T20:16:01.188852+0000","last_fullsized":"2023-09-23T20:16:01.188852+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":168,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191608+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:16:01.191608+0000","last_peered":"2023-09-23T20:16:01.191608+0000","last_clean":"2023-09-23T20:16:01.191608+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:16:01.191608+0000","last_undegraded":"2023-09-23T20:16:01.191608+0000","last_fullsized":"2023-09-23T20:16:01.191608+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":462,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.183423+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:16:01.183423+0000","last_peered":"2023-09-23T20:16:01.183423+0000","last_clean":"2023-09-23T20:16:01.183423+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:16:01.183423+0000","last_undegraded":"2023-09-23T20:16:01.183423+0000","last_fullsized":"2023-09-23T20:16:01.183423+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.7","version":"462'7096","reported_seq":7170,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.349708+0000","last_change":"2023-09-23T20:13:46.802046+0000","last_active":"2023-09-23T20:16:03.349708+0000","last_peered":"2023-09-23T20:16:03.349708+0000","last_clean":"2023-09-23T20:16:03.349708+0000","last_became_active":"2023-09-23T20:13:46.800187+0000","last_became_peered":"2023-09-23T20:13:46.800187+0000","last_unstale":"2023-09-23T20:16:03.349708+0000","last_undegraded":"2023-09-23T20:16:03.349708+0000","last_fullsized":"2023-09-23T20:16:03.349708+0000","mapping_epoch":407,"log_start":"412'1901","ondisk_log_start":"412'1901","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5195,"log_dups_size":0,"ondisk_log_size":5195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75104256,"num_objects":1146,"num_object_clones":0,"num_object_copies":3438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5386,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":534,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.209357+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:16:01.209357+0000","last_peered":"2023-09-23T20:16:01.209357+0000","last_clean":"2023-09-23T20:16:01.209357+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:16:01.209357+0000","last_undegraded":"2023-09-23T20:16:01.209357+0000","last_fullsized":"2023-09-23T20:16:01.209357+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":153,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191740+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:16:01.191740+0000","last_peered":"2023-09-23T20:16:01.191740+0000","last_clean":"2023-09-23T20:16:01.191740+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:16:01.191740+0000","last_undegraded":"2023-09-23T20:16:01.191740+0000","last_fullsized":"2023-09-23T20:16:01.191740+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":50,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188554+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:16:01.188554+0000","last_peered":"2023-09-23T20:16:01.188554+0000","last_clean":"2023-09-23T20:16:01.188554+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:16:01.188554+0000","last_undegraded":"2023-09-23T20:16:01.188554+0000","last_fullsized":"2023-09-23T20:16:01.188554+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"462'7342","reported_seq":7424,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.343425+0000","last_change":"2023-09-23T20:14:36.190976+0000","last_active":"2023-09-23T20:16:03.343425+0000","last_peered":"2023-09-23T20:16:03.343425+0000","last_clean":"2023-09-23T20:14:33.830609+0000","last_became_active":"2023-09-23T20:14:36.182918+0000","last_became_peered":"2023-09-23T20:14:36.182918+0000","last_unstale":"2023-09-23T20:16:03.343425+0000","last_undegraded":"2023-09-23T20:14:35.924455+0000","last_fullsized":"2023-09-23T20:14:35.847184+0000","mapping_epoch":442,"log_start":"412'2283","ondisk_log_start":"412'2283","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5059,"log_dups_size":0,"ondisk_log_size":5059,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69730304,"num_objects":1064,"num_object_clones":0,"num_object_copies":3192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5560,"num_write_kb":211968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":949},{"shards":"2,5","objects":-2133}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":665,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191331+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:16:01.191331+0000","last_peered":"2023-09-23T20:16:01.191331+0000","last_clean":"2023-09-23T20:16:01.191331+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:16:01.191331+0000","last_undegraded":"2023-09-23T20:16:01.191331+0000","last_fullsized":"2023-09-23T20:16:01.191331+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":510,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.200921+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:16:01.200921+0000","last_peered":"2023-09-23T20:16:01.200921+0000","last_clean":"2023-09-23T20:16:01.200921+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:16:01.200921+0000","last_undegraded":"2023-09-23T20:16:01.200921+0000","last_fullsized":"2023-09-23T20:16:01.200921+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":206,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.193437+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:16:01.193437+0000","last_peered":"2023-09-23T20:16:01.193437+0000","last_clean":"2023-09-23T20:16:01.193437+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:16:01.193437+0000","last_undegraded":"2023-09-23T20:16:01.193437+0000","last_fullsized":"2023-09-23T20:16:01.193437+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"462'5785","reported_seq":5869,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.814902+0000","last_change":"2023-09-23T20:14:24.782498+0000","last_active":"2023-09-23T20:16:02.814902+0000","last_peered":"2023-09-23T20:16:02.814902+0000","last_clean":"2023-09-23T20:16:02.814902+0000","last_became_active":"2023-09-23T20:14:24.780750+0000","last_became_peered":"2023-09-23T20:14:24.780750+0000","last_unstale":"2023-09-23T20:16:02.814902+0000","last_undegraded":"2023-09-23T20:16:02.814902+0000","last_fullsized":"2023-09-23T20:16:02.814902+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5785,"log_dups_size":0,"ondisk_log_size":5785,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35553280,"num_objects":542,"num_object_clones":0,"num_object_copies":1626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2690,"num_write_kb":103456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":844,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.190904+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:16:01.190904+0000","last_peered":"2023-09-23T20:16:01.190904+0000","last_clean":"2023-09-23T20:16:01.190904+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:16:01.190904+0000","last_undegraded":"2023-09-23T20:16:01.190904+0000","last_fullsized":"2023-09-23T20:16:01.190904+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":207,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188216+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:16:01.188216+0000","last_peered":"2023-09-23T20:16:01.188216+0000","last_clean":"2023-09-23T20:16:01.188216+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:16:01.188216+0000","last_undegraded":"2023-09-23T20:16:01.188216+0000","last_fullsized":"2023-09-23T20:16:01.188216+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":30,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188682+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:16:01.188682+0000","last_peered":"2023-09-23T20:16:01.188682+0000","last_clean":"2023-09-23T20:16:01.188682+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:16:01.188682+0000","last_undegraded":"2023-09-23T20:16:01.188682+0000","last_fullsized":"2023-09-23T20:16:01.188682+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"462'5830","reported_seq":5922,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:06.103805+0000","last_change":"2023-09-23T20:14:36.074365+0000","last_active":"2023-09-23T20:16:06.103805+0000","last_peered":"2023-09-23T20:16:06.103805+0000","last_clean":"2023-09-23T20:14:33.749328+0000","last_became_active":"2023-09-23T20:14:36.071685+0000","last_became_peered":"2023-09-23T20:14:36.071685+0000","last_unstale":"2023-09-23T20:16:06.103805+0000","last_undegraded":"2023-09-23T20:14:35.849009+0000","last_fullsized":"2023-09-23T20:14:35.834839+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5830,"log_dups_size":0,"ondisk_log_size":5830,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34865152,"num_objects":532,"num_object_clones":0,"num_object_copies":1596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1547,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2748,"num_write_kb":104960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":439},{"shards":"0,2","objects":-1015}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":809,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.207590+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:16:01.207590+0000","last_peered":"2023-09-23T20:16:01.207590+0000","last_clean":"2023-09-23T20:16:01.207590+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:16:01.207590+0000","last_undegraded":"2023-09-23T20:16:01.207590+0000","last_fullsized":"2023-09-23T20:16:01.207590+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":30,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.198094+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:16:01.198094+0000","last_peered":"2023-09-23T20:16:01.198094+0000","last_clean":"2023-09-23T20:16:01.198094+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:16:01.198094+0000","last_undegraded":"2023-09-23T20:16:01.198094+0000","last_fullsized":"2023-09-23T20:16:01.198094+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":362,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.181806+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:16:01.181806+0000","last_peered":"2023-09-23T20:16:01.181806+0000","last_clean":"2023-09-23T20:16:01.181806+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:16:01.181806+0000","last_undegraded":"2023-09-23T20:16:01.181806+0000","last_fullsized":"2023-09-23T20:16:01.181806+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"462'5757","reported_seq":5851,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:02.805024+0000","last_change":"2023-09-23T20:14:36.109242+0000","last_active":"2023-09-23T20:16:02.805024+0000","last_peered":"2023-09-23T20:16:02.805024+0000","last_clean":"2023-09-23T20:14:33.735119+0000","last_became_active":"2023-09-23T20:14:36.106276+0000","last_became_peered":"2023-09-23T20:14:36.106276+0000","last_unstale":"2023-09-23T20:16:02.805024+0000","last_undegraded":"2023-09-23T20:14:35.867219+0000","last_fullsized":"2023-09-23T20:14:35.838359+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41041920,"num_objects":626,"num_object_clones":0,"num_object_copies":1878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2648,"num_write_kb":104784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":487},{"shards":"3,7","objects":-872}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":523,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.199761+0000","last_change":"2023-09-23T20:14:37.892742+0000","last_active":"2023-09-23T20:16:01.199761+0000","last_peered":"2023-09-23T20:16:01.199761+0000","last_clean":"2023-09-23T20:16:01.199761+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:16:01.199761+0000","last_undegraded":"2023-09-23T20:16:01.199761+0000","last_fullsized":"2023-09-23T20:16:01.199761+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":353,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.180965+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:16:01.180965+0000","last_peered":"2023-09-23T20:16:01.180965+0000","last_clean":"2023-09-23T20:16:01.180965+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:16:01.180965+0000","last_undegraded":"2023-09-23T20:16:01.180965+0000","last_fullsized":"2023-09-23T20:16:01.180965+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":30,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.200056+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:16:01.200056+0000","last_peered":"2023-09-23T20:16:01.200056+0000","last_clean":"2023-09-23T20:16:01.200056+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:16:01.200056+0000","last_undegraded":"2023-09-23T20:16:01.200056+0000","last_fullsized":"2023-09-23T20:16:01.200056+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"462'5895","reported_seq":7224,"reported_epoch":462,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:06.563798+0000","last_change":"2023-09-23T20:16:00.307885+0000","last_active":"2023-09-23T20:16:06.563798+0000","last_peered":"2023-09-23T20:16:06.563798+0000","last_clean":"2023-09-23T20:14:33.798489+0000","last_became_active":"2023-09-23T20:14:35.986955+0000","last_became_peered":"2023-09-23T20:14:35.986955+0000","last_unstale":"2023-09-23T20:16:06.563798+0000","last_undegraded":"2023-09-23T20:14:35.846545+0000","last_fullsized":"2023-09-23T20:14:35.832866+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37027840,"num_objects":565,"num_object_clones":0,"num_object_copies":1695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2759,"num_write_kb":106368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":617,"num_bytes_recovered":34668544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":-360}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":800,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.203249+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:16:01.203249+0000","last_peered":"2023-09-23T20:16:01.203249+0000","last_clean":"2023-09-23T20:16:01.203249+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:16:01.203249+0000","last_undegraded":"2023-09-23T20:16:01.203249+0000","last_fullsized":"2023-09-23T20:16:01.203249+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":103,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188530+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:16:01.188530+0000","last_peered":"2023-09-23T20:16:01.188530+0000","last_clean":"2023-09-23T20:16:01.188530+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:16:01.188530+0000","last_undegraded":"2023-09-23T20:16:01.188530+0000","last_fullsized":"2023-09-23T20:16:01.188530+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":206,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191677+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:16:01.191677+0000","last_peered":"2023-09-23T20:16:01.191677+0000","last_clean":"2023-09-23T20:16:01.191677+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:16:01.191677+0000","last_undegraded":"2023-09-23T20:16:01.191677+0000","last_fullsized":"2023-09-23T20:16:01.191677+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"462'8022","reported_seq":8112,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.391191+0000","last_change":"2023-09-23T20:14:36.198248+0000","last_active":"2023-09-23T20:16:03.391191+0000","last_peered":"2023-09-23T20:16:03.391191+0000","last_clean":"2023-09-23T20:14:33.842789+0000","last_became_active":"2023-09-23T20:14:36.169565+0000","last_became_peered":"2023-09-23T20:14:36.169565+0000","last_unstale":"2023-09-23T20:16:03.391191+0000","last_undegraded":"2023-09-23T20:14:36.113647+0000","last_fullsized":"2023-09-23T20:14:35.847403+0000","mapping_epoch":442,"log_start":"412'1787","ondisk_log_start":"412'1787","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6235,"log_dups_size":0,"ondisk_log_size":6235,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72351744,"num_objects":1104,"num_object_clones":0,"num_object_copies":3312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5336,"num_write_kb":206080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":911},{"shards":"3,5","objects":-1923}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":744,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191433+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:16:01.191433+0000","last_peered":"2023-09-23T20:16:01.191433+0000","last_clean":"2023-09-23T20:16:01.191433+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:16:01.191433+0000","last_undegraded":"2023-09-23T20:16:01.191433+0000","last_fullsized":"2023-09-23T20:16:01.191433+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":452,"reported_epoch":461,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:01.188858+0000","last_change":"2023-09-23T20:14:35.895655+0000","last_active":"2023-09-23T20:16:01.188858+0000","last_peered":"2023-09-23T20:16:01.188858+0000","last_clean":"2023-09-23T20:07:25.847413+0000","last_became_active":"2023-09-23T20:14:35.895204+0000","last_became_peered":"2023-09-23T20:14:35.895204+0000","last_unstale":"2023-09-23T20:16:01.188858+0000","last_undegraded":"2023-09-23T20:14:35.838988+0000","last_fullsized":"2023-09-23T20:14:35.837779+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:17.785522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":462,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.200222+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:16:01.200222+0000","last_peered":"2023-09-23T20:16:01.200222+0000","last_clean":"2023-09-23T20:16:01.200222+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:16:01.200222+0000","last_undegraded":"2023-09-23T20:16:01.200222+0000","last_fullsized":"2023-09-23T20:16:01.200222+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.203173+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:16:01.203173+0000","last_peered":"2023-09-23T20:16:01.203173+0000","last_clean":"2023-09-23T20:16:01.203173+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:16:01.203173+0000","last_undegraded":"2023-09-23T20:16:01.203173+0000","last_fullsized":"2023-09-23T20:16:01.203173+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"462'7242","reported_seq":7316,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.003220+0000","last_change":"2023-09-23T20:13:46.785589+0000","last_active":"2023-09-23T20:16:03.003220+0000","last_peered":"2023-09-23T20:16:03.003220+0000","last_clean":"2023-09-23T20:16:03.003220+0000","last_became_active":"2023-09-23T20:13:46.781975+0000","last_became_peered":"2023-09-23T20:13:46.781975+0000","last_unstale":"2023-09-23T20:16:03.003220+0000","last_undegraded":"2023-09-23T20:16:03.003220+0000","last_fullsized":"2023-09-23T20:16:03.003220+0000","mapping_epoch":407,"log_start":"412'2509","ondisk_log_start":"412'2509","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4733,"log_dups_size":0,"ondisk_log_size":4733,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72908800,"num_objects":1112,"num_object_clones":0,"num_object_copies":3336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5486,"num_write_kb":211168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":805,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.198192+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:16:01.198192+0000","last_peered":"2023-09-23T20:16:01.198192+0000","last_clean":"2023-09-23T20:16:01.198192+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:16:01.198192+0000","last_undegraded":"2023-09-23T20:16:01.198192+0000","last_fullsized":"2023-09-23T20:16:01.198192+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1112,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.202945+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:16:01.202945+0000","last_peered":"2023-09-23T20:16:01.202945+0000","last_clean":"2023-09-23T20:16:01.202945+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:16:01.202945+0000","last_undegraded":"2023-09-23T20:16:01.202945+0000","last_fullsized":"2023-09-23T20:16:01.202945+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":30,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.198776+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:16:01.198776+0000","last_peered":"2023-09-23T20:16:01.198776+0000","last_clean":"2023-09-23T20:16:01.198776+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:16:01.198776+0000","last_undegraded":"2023-09-23T20:16:01.198776+0000","last_fullsized":"2023-09-23T20:16:01.198776+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":324,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.181372+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:16:01.181372+0000","last_peered":"2023-09-23T20:16:01.181372+0000","last_clean":"2023-09-23T20:16:01.181372+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:16:01.181372+0000","last_undegraded":"2023-09-23T20:16:01.181372+0000","last_fullsized":"2023-09-23T20:16:01.181372+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"462'8071","reported_seq":8158,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.376790+0000","last_change":"2023-09-23T20:14:36.228252+0000","last_active":"2023-09-23T20:16:03.376790+0000","last_peered":"2023-09-23T20:16:03.376790+0000","last_clean":"2023-09-23T20:14:33.823351+0000","last_became_active":"2023-09-23T20:14:36.216536+0000","last_became_peered":"2023-09-23T20:14:36.216536+0000","last_unstale":"2023-09-23T20:16:03.376790+0000","last_undegraded":"2023-09-23T20:14:36.114735+0000","last_fullsized":"2023-09-23T20:14:35.945404+0000","mapping_epoch":442,"log_start":"412'2044","ondisk_log_start":"412'2044","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73203712,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3085,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5373,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":957},{"shards":"5,7","objects":-1968}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":700,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188466+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:16:01.188466+0000","last_peered":"2023-09-23T20:16:01.188466+0000","last_clean":"2023-09-23T20:16:01.188466+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:16:01.188466+0000","last_undegraded":"2023-09-23T20:16:01.188466+0000","last_fullsized":"2023-09-23T20:16:01.188466+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1100,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.181042+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:16:01.181042+0000","last_peered":"2023-09-23T20:16:01.181042+0000","last_clean":"2023-09-23T20:16:01.181042+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:16:01.181042+0000","last_undegraded":"2023-09-23T20:16:01.181042+0000","last_fullsized":"2023-09-23T20:16:01.181042+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":160,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.180828+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:16:01.180828+0000","last_peered":"2023-09-23T20:16:01.180828+0000","last_clean":"2023-09-23T20:16:01.180828+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:16:01.180828+0000","last_undegraded":"2023-09-23T20:16:01.180828+0000","last_fullsized":"2023-09-23T20:16:01.180828+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.211725+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:16:01.211725+0000","last_peered":"2023-09-23T20:16:01.211725+0000","last_clean":"2023-09-23T20:16:01.211725+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:16:01.211725+0000","last_undegraded":"2023-09-23T20:16:01.211725+0000","last_fullsized":"2023-09-23T20:16:01.211725+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"462'7046","reported_seq":7120,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.380657+0000","last_change":"2023-09-23T20:13:46.814034+0000","last_active":"2023-09-23T20:16:03.380657+0000","last_peered":"2023-09-23T20:16:03.380657+0000","last_clean":"2023-09-23T20:16:03.380657+0000","last_became_active":"2023-09-23T20:13:46.812300+0000","last_became_peered":"2023-09-23T20:13:46.812300+0000","last_unstale":"2023-09-23T20:16:03.380657+0000","last_undegraded":"2023-09-23T20:16:03.380657+0000","last_fullsized":"2023-09-23T20:16:03.380657+0000","mapping_epoch":407,"log_start":"412'1830","ondisk_log_start":"412'1830","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5216,"log_dups_size":0,"ondisk_log_size":5216,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72122368,"num_objects":1100,"num_object_clones":0,"num_object_copies":3300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5330,"num_write_kb":205792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1370,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.199398+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:16:01.199398+0000","last_peered":"2023-09-23T20:16:01.199398+0000","last_clean":"2023-09-23T20:16:01.199398+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:16:01.199398+0000","last_undegraded":"2023-09-23T20:16:01.199398+0000","last_fullsized":"2023-09-23T20:16:01.199398+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":657,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188547+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:16:01.188547+0000","last_peered":"2023-09-23T20:16:01.188547+0000","last_clean":"2023-09-23T20:16:01.188547+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:16:01.188547+0000","last_undegraded":"2023-09-23T20:16:01.188547+0000","last_fullsized":"2023-09-23T20:16:01.188547+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":327,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.203099+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:16:01.203099+0000","last_peered":"2023-09-23T20:16:01.203099+0000","last_clean":"2023-09-23T20:16:01.203099+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:16:01.203099+0000","last_undegraded":"2023-09-23T20:16:01.203099+0000","last_fullsized":"2023-09-23T20:16:01.203099+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":259,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191609+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:16:01.191609+0000","last_peered":"2023-09-23T20:16:01.191609+0000","last_clean":"2023-09-23T20:16:01.191609+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:16:01.191609+0000","last_undegraded":"2023-09-23T20:16:01.191609+0000","last_fullsized":"2023-09-23T20:16:01.191609+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"462'7865","reported_seq":7939,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.230906+0000","last_change":"2023-09-23T20:14:24.789921+0000","last_active":"2023-09-23T20:16:03.230906+0000","last_peered":"2023-09-23T20:16:03.230906+0000","last_clean":"2023-09-23T20:16:03.230906+0000","last_became_active":"2023-09-23T20:14:24.787385+0000","last_became_peered":"2023-09-23T20:14:24.787385+0000","last_unstale":"2023-09-23T20:16:03.230906+0000","last_undegraded":"2023-09-23T20:16:03.230906+0000","last_fullsized":"2023-09-23T20:16:03.230906+0000","mapping_epoch":407,"log_start":"412'1760","ondisk_log_start":"412'1760","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6105,"log_dups_size":0,"ondisk_log_size":6105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71663616,"num_objects":1093,"num_object_clones":0,"num_object_copies":3279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5253,"num_write_kb":203104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":523,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.199555+0000","last_change":"2023-09-23T20:14:37.894544+0000","last_active":"2023-09-23T20:16:01.199555+0000","last_peered":"2023-09-23T20:16:01.199555+0000","last_clean":"2023-09-23T20:16:01.199555+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:16:01.199555+0000","last_undegraded":"2023-09-23T20:16:01.199555+0000","last_fullsized":"2023-09-23T20:16:01.199555+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":908,"reported_epoch":461,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:01.188194+0000","last_change":"2023-09-23T20:14:35.882819+0000","last_active":"2023-09-23T20:16:01.188194+0000","last_peered":"2023-09-23T20:16:01.188194+0000","last_clean":"2023-09-23T20:14:31.826679+0000","last_became_active":"2023-09-23T20:14:35.882372+0000","last_became_peered":"2023-09-23T20:14:35.882372+0000","last_unstale":"2023-09-23T20:16:01.188194+0000","last_undegraded":"2023-09-23T20:14:35.846466+0000","last_fullsized":"2023-09-23T20:14:35.837669+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:41.083298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":408,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.182614+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:16:02.182614+0000","last_peered":"2023-09-23T20:16:02.182614+0000","last_clean":"2023-09-23T20:16:02.182614+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:16:02.182614+0000","last_undegraded":"2023-09-23T20:16:02.182614+0000","last_fullsized":"2023-09-23T20:16:02.182614+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":277,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.186080+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:16:01.186080+0000","last_peered":"2023-09-23T20:16:01.186080+0000","last_clean":"2023-09-23T20:16:01.186080+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:16:01.186080+0000","last_undegraded":"2023-09-23T20:16:01.186080+0000","last_fullsized":"2023-09-23T20:16:01.186080+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"462'8072","reported_seq":8155,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.003215+0000","last_change":"2023-09-23T20:14:36.189421+0000","last_active":"2023-09-23T20:16:03.003215+0000","last_peered":"2023-09-23T20:16:03.003215+0000","last_clean":"2023-09-23T20:14:33.834923+0000","last_became_active":"2023-09-23T20:14:36.182947+0000","last_became_peered":"2023-09-23T20:14:36.182947+0000","last_unstale":"2023-09-23T20:16:03.003215+0000","last_undegraded":"2023-09-23T20:14:35.901626+0000","last_fullsized":"2023-09-23T20:14:35.838743+0000","mapping_epoch":442,"log_start":"412'2371","ondisk_log_start":"412'2371","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75137024,"num_objects":1146,"num_object_clones":0,"num_object_copies":3438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5372,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":948},{"shards":"4,7","objects":-1916}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":634,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.193670+0000","last_change":"2023-09-23T20:13:57.748021+0000","last_active":"2023-09-23T20:16:01.193670+0000","last_peered":"2023-09-23T20:16:01.193670+0000","last_clean":"2023-09-23T20:16:01.193670+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:16:01.193670+0000","last_undegraded":"2023-09-23T20:16:01.193670+0000","last_fullsized":"2023-09-23T20:16:01.193670+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":162,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.191816+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:16:01.191816+0000","last_peered":"2023-09-23T20:16:01.191816+0000","last_clean":"2023-09-23T20:16:01.191816+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:16:01.191816+0000","last_undegraded":"2023-09-23T20:16:01.191816+0000","last_fullsized":"2023-09-23T20:16:01.191816+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":265,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.181558+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:16:01.181558+0000","last_peered":"2023-09-23T20:16:01.181558+0000","last_clean":"2023-09-23T20:16:01.181558+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:16:01.181558+0000","last_undegraded":"2023-09-23T20:16:01.181558+0000","last_fullsized":"2023-09-23T20:16:01.181558+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":288,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.199236+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:16:01.199236+0000","last_peered":"2023-09-23T20:16:01.199236+0000","last_clean":"2023-09-23T20:16:01.199236+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:16:01.199236+0000","last_undegraded":"2023-09-23T20:16:01.199236+0000","last_fullsized":"2023-09-23T20:16:01.199236+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"462'8348","reported_seq":8431,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:02.943625+0000","last_change":"2023-09-23T20:14:36.072073+0000","last_active":"2023-09-23T20:16:02.943625+0000","last_peered":"2023-09-23T20:16:02.943625+0000","last_clean":"2023-09-23T20:14:33.811030+0000","last_became_active":"2023-09-23T20:14:36.065847+0000","last_became_peered":"2023-09-23T20:14:36.065847+0000","last_unstale":"2023-09-23T20:16:02.943625+0000","last_undegraded":"2023-09-23T20:14:35.915006+0000","last_fullsized":"2023-09-23T20:14:35.838163+0000","mapping_epoch":442,"log_start":"413'2754","ondisk_log_start":"413'2754","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5594,"log_dups_size":0,"ondisk_log_size":5594,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74383360,"num_objects":1135,"num_object_clones":0,"num_object_copies":3405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3193,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5559,"num_write_kb":214208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":981},{"shards":"5,7","objects":-2058}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":681,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.238304+0000","last_change":"2023-09-23T20:14:25.790696+0000","last_active":"2023-09-23T20:16:02.238304+0000","last_peered":"2023-09-23T20:16:02.238304+0000","last_clean":"2023-09-23T20:16:02.238304+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:16:02.238304+0000","last_undegraded":"2023-09-23T20:16:02.238304+0000","last_fullsized":"2023-09-23T20:16:02.238304+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":304,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.192407+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:16:01.192407+0000","last_peered":"2023-09-23T20:16:01.192407+0000","last_clean":"2023-09-23T20:16:01.192407+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:16:01.192407+0000","last_undegraded":"2023-09-23T20:16:01.192407+0000","last_fullsized":"2023-09-23T20:16:01.192407+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":619,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.235622+0000","last_change":"2023-09-23T20:16:02.235622+0000","last_active":"2023-09-23T20:16:02.235622+0000","last_peered":"2023-09-23T20:16:02.235622+0000","last_clean":"2023-09-23T20:16:02.235622+0000","last_became_active":"2023-09-23T20:16:02.234933+0000","last_became_peered":"2023-09-23T20:16:02.234933+0000","last_unstale":"2023-09-23T20:16:02.235622+0000","last_undegraded":"2023-09-23T20:16:02.235622+0000","last_fullsized":"2023-09-23T20:16:02.235622+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:21.855632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":94,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.183108+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:16:01.183108+0000","last_peered":"2023-09-23T20:16:01.183108+0000","last_clean":"2023-09-23T20:16:01.183108+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:16:01.183108+0000","last_undegraded":"2023-09-23T20:16:01.183108+0000","last_fullsized":"2023-09-23T20:16:01.183108+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"462'7876","reported_seq":7951,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:06.502279+0000","last_change":"2023-09-23T20:14:25.775099+0000","last_active":"2023-09-23T20:16:06.502279+0000","last_peered":"2023-09-23T20:16:06.502279+0000","last_clean":"2023-09-23T20:16:06.502279+0000","last_became_active":"2023-09-23T20:14:24.777506+0000","last_became_peered":"2023-09-23T20:14:24.777506+0000","last_unstale":"2023-09-23T20:16:06.502279+0000","last_undegraded":"2023-09-23T20:16:06.502279+0000","last_fullsized":"2023-09-23T20:16:06.502279+0000","mapping_epoch":407,"log_start":"413'2475","ondisk_log_start":"413'2475","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72908800,"num_objects":1112,"num_object_clones":0,"num_object_copies":3336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5256,"num_write_kb":203808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":787,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.189277+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:16:01.189277+0000","last_peered":"2023-09-23T20:16:01.189277+0000","last_clean":"2023-09-23T20:16:01.189277+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:16:01.189277+0000","last_undegraded":"2023-09-23T20:16:01.189277+0000","last_fullsized":"2023-09-23T20:16:01.189277+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":163,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.192592+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:16:01.192592+0000","last_peered":"2023-09-23T20:16:01.192592+0000","last_clean":"2023-09-23T20:16:01.192592+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:16:01.192592+0000","last_undegraded":"2023-09-23T20:16:01.192592+0000","last_fullsized":"2023-09-23T20:16:01.192592+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":34,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188975+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:16:01.188975+0000","last_peered":"2023-09-23T20:16:01.188975+0000","last_clean":"2023-09-23T20:16:01.188975+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:16:01.188975+0000","last_undegraded":"2023-09-23T20:16:01.188975+0000","last_fullsized":"2023-09-23T20:16:01.188975+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":245,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.198316+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:16:01.198316+0000","last_peered":"2023-09-23T20:16:01.198316+0000","last_clean":"2023-09-23T20:16:01.198316+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:16:01.198316+0000","last_undegraded":"2023-09-23T20:16:01.198316+0000","last_fullsized":"2023-09-23T20:16:01.198316+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"462'7171","reported_seq":11393,"reported_epoch":462,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:16:02.912047+0000","last_change":"2023-09-23T20:14:35.970620+0000","last_active":"2023-09-23T20:16:02.912047+0000","last_peered":"2023-09-23T20:16:02.912047+0000","last_clean":"2023-09-23T20:14:33.799110+0000","last_became_active":"2023-09-23T20:14:35.967502+0000","last_became_peered":"2023-09-23T20:14:35.967502+0000","last_unstale":"2023-09-23T20:16:02.912047+0000","last_undegraded":"2023-09-23T20:16:02.912047+0000","last_fullsized":"2023-09-23T20:16:02.912047+0000","mapping_epoch":442,"log_start":"412'2375","ondisk_log_start":"412'2375","created":418,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4796,"log_dups_size":0,"ondisk_log_size":4796,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73007104,"num_objects":1114,"num_object_clones":0,"num_object_copies":3342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1114,"num_objects_unfound":0,"num_objects_dirty":1114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5426,"num_write_kb":209280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2059,"num_bytes_recovered":134938624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":1114}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":238,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.184216+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:16:01.184216+0000","last_peered":"2023-09-23T20:16:01.184216+0000","last_clean":"2023-09-23T20:16:01.184216+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:16:01.184216+0000","last_undegraded":"2023-09-23T20:16:01.184216+0000","last_fullsized":"2023-09-23T20:16:01.184216+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":239,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.181623+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:16:01.181623+0000","last_peered":"2023-09-23T20:16:01.181623+0000","last_clean":"2023-09-23T20:16:01.181623+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:16:01.181623+0000","last_undegraded":"2023-09-23T20:16:01.181623+0000","last_fullsized":"2023-09-23T20:16:01.181623+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"462'7234","reported_seq":7315,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.390473+0000","last_change":"2023-09-23T20:14:36.202358+0000","last_active":"2023-09-23T20:16:03.390473+0000","last_peered":"2023-09-23T20:16:03.390473+0000","last_clean":"2023-09-23T20:14:33.783493+0000","last_became_active":"2023-09-23T20:14:36.193525+0000","last_became_peered":"2023-09-23T20:14:36.193525+0000","last_unstale":"2023-09-23T20:16:03.390473+0000","last_undegraded":"2023-09-23T20:14:36.034471+0000","last_fullsized":"2023-09-23T20:14:35.969590+0000","mapping_epoch":442,"log_start":"412'2075","ondisk_log_start":"412'2075","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5159,"log_dups_size":0,"ondisk_log_size":5159,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79298560,"num_objects":1210,"num_object_clones":0,"num_object_copies":3630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5452,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":923},{"shards":"2,5","objects":-1834}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":479,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.200839+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:16:01.200839+0000","last_peered":"2023-09-23T20:16:01.200839+0000","last_clean":"2023-09-23T20:16:01.200839+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:16:01.200839+0000","last_undegraded":"2023-09-23T20:16:01.200839+0000","last_fullsized":"2023-09-23T20:16:01.200839+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":481,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.188215+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:16:01.188215+0000","last_peered":"2023-09-23T20:16:01.188215+0000","last_clean":"2023-09-23T20:16:01.188215+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:16:01.188215+0000","last_undegraded":"2023-09-23T20:16:01.188215+0000","last_fullsized":"2023-09-23T20:16:01.188215+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.17","version":"462'7115","reported_seq":7188,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.281423+0000","last_change":"2023-09-23T20:13:46.814598+0000","last_active":"2023-09-23T20:16:03.281423+0000","last_peered":"2023-09-23T20:16:03.281423+0000","last_clean":"2023-09-23T20:16:03.281423+0000","last_became_active":"2023-09-23T20:13:46.812685+0000","last_became_peered":"2023-09-23T20:13:46.812685+0000","last_unstale":"2023-09-23T20:16:03.281423+0000","last_undegraded":"2023-09-23T20:16:03.281423+0000","last_fullsized":"2023-09-23T20:16:03.281423+0000","mapping_epoch":418,"log_start":"412'2022","ondisk_log_start":"412'2022","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5093,"log_dups_size":0,"ondisk_log_size":5093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67960832,"num_objects":1037,"num_object_clones":0,"num_object_copies":3111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5405,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":469,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.200152+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:16:01.200152+0000","last_peered":"2023-09-23T20:16:01.200152+0000","last_clean":"2023-09-23T20:16:01.200152+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:16:01.200152+0000","last_undegraded":"2023-09-23T20:16:01.200152+0000","last_fullsized":"2023-09-23T20:16:01.200152+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":458,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.200200+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:16:01.200200+0000","last_peered":"2023-09-23T20:16:01.200200+0000","last_clean":"2023-09-23T20:16:01.200200+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:16:01.200200+0000","last_undegraded":"2023-09-23T20:16:01.200200+0000","last_fullsized":"2023-09-23T20:16:01.200200+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"462'7128","reported_seq":13315,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:03.371923+0000","last_change":"2023-09-23T20:16:02.232381+0000","last_active":"2023-09-23T20:16:03.371923+0000","last_peered":"2023-09-23T20:16:03.371923+0000","last_clean":"2023-09-23T20:16:03.371923+0000","last_became_active":"2023-09-23T20:16:02.185816+0000","last_became_peered":"2023-09-23T20:16:02.185816+0000","last_unstale":"2023-09-23T20:16:03.371923+0000","last_undegraded":"2023-09-23T20:16:03.371923+0000","last_fullsized":"2023-09-23T20:16:03.371923+0000","mapping_epoch":461,"log_start":"412'2003","ondisk_log_start":"412'2003","created":418,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5125,"log_dups_size":0,"ondisk_log_size":5125,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71925760,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5404,"num_write_kb":208032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3047,"num_bytes_recovered":134610944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":173,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.185781+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:16:01.185781+0000","last_peered":"2023-09-23T20:16:01.185781+0000","last_clean":"2023-09-23T20:16:01.185781+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:16:01.185781+0000","last_undegraded":"2023-09-23T20:16:01.185781+0000","last_fullsized":"2023-09-23T20:16:01.185781+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":321,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.181115+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:16:01.181115+0000","last_peered":"2023-09-23T20:16:01.181115+0000","last_clean":"2023-09-23T20:16:01.181115+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:16:01.181115+0000","last_undegraded":"2023-09-23T20:16:01.181115+0000","last_fullsized":"2023-09-23T20:16:01.181115+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"462'7043","reported_seq":13448,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:03.043835+0000","last_change":"2023-09-23T20:14:38.010024+0000","last_active":"2023-09-23T20:16:03.043835+0000","last_peered":"2023-09-23T20:16:03.043835+0000","last_clean":"2023-09-23T20:14:33.867233+0000","last_became_active":"2023-09-23T20:14:38.004874+0000","last_became_peered":"2023-09-23T20:14:38.004874+0000","last_unstale":"2023-09-23T20:16:03.043835+0000","last_undegraded":"2023-09-23T20:14:37.959307+0000","last_fullsized":"2023-09-23T20:14:37.876450+0000","mapping_epoch":444,"log_start":"412'2204","ondisk_log_start":"412'2204","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4839,"log_dups_size":0,"ondisk_log_size":4839,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1828,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5313,"num_write_kb":205120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2108,"num_bytes_recovered":138149888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":914},{"shards":"3,7","objects":183}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":307,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.211579+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:16:01.211579+0000","last_peered":"2023-09-23T20:16:01.211579+0000","last_clean":"2023-09-23T20:16:01.211579+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:16:01.211579+0000","last_undegraded":"2023-09-23T20:16:01.211579+0000","last_fullsized":"2023-09-23T20:16:01.211579+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-09-23T20:16:01.180678+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:16:01.180678+0000","last_peered":"2023-09-23T20:16:01.180678+0000","last_clean":"2023-09-23T20:16:01.180678+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:16:01.180678+0000","last_undegraded":"2023-09-23T20:16:01.180678+0000","last_fullsized":"2023-09-23T20:16:01.180678+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":36,"stat_sum":{"num_bytes":2320367640,"num_objects":35407,"num_object_clones":0,"num_object_copies":106221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40380,"num_objects_misplaced":2876,"num_objects_unfound":0,"num_objects_dirty":35407,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172299,"num_write_kb":6646529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17251,"num_bytes_recovered":1059717120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9951129600,"data_stored":9951117384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":198782,"ondisk_log_size":198782,"up":108,"acting":90,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":191,"num_bytes_recovered":4194380,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234419,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1918836,"kb_used_data":1422444,"kb_used_omap":0,"kb_used_meta":496384,"kb_avail":91826316,"statfs":{"total":95995035648,"available":94030147584,"internally_reserved":0,"allocated":1456582656,"data_stored":1455167047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":508297216},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":11.353,"5min":3.855,"15min":2.8380000000000001},"min":{"1min":0.219,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":149.25999999999999,"5min":149.25999999999999,"15min":149.25999999999999},"last":2.3570000000000002},{"interface":"front","average":{"1min":12.917,"5min":4.3019999999999996,"15min":3.0609999999999999},"min":{"1min":0.248,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":149.11799999999999,"5min":149.11799999999999,"15min":149.11799999999999},"last":0.999}]},{"osd":1,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":3.2250000000000001,"15min":3.5779999999999998},"min":{"1min":0.29699999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.0599999999999996,"5min":91.335999999999999,"15min":104.16800000000001},"last":2.3050000000000002},{"interface":"front","average":{"1min":1.972,"5min":3.738,"15min":3.8370000000000002},"min":{"1min":0.29599999999999999,"5min":0.249,"15min":0.223},"max":{"1min":6.8609999999999998,"5min":91.516999999999996,"15min":104.003},"last":4.7270000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":3.3969999999999998,"15min":3.7629999999999999},"min":{"1min":0.32700000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":7.3600000000000003,"5min":67.760000000000005,"15min":105.842},"last":2.2669999999999999},{"interface":"front","average":{"1min":2.633,"5min":3.738,"15min":4.1950000000000003},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":8.6669999999999998,"5min":67.488,"15min":105.66},"last":0.40999999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":14.476000000000001,"5min":4.5030000000000001,"15min":4.6440000000000001},"min":{"1min":0.33400000000000002,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":185.17099999999999,"5min":185.17099999999999,"15min":185.17099999999999},"last":6.4379999999999997},{"interface":"front","average":{"1min":15.061999999999999,"5min":4.593,"15min":4.649},"min":{"1min":0.63,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":184.99100000000001,"5min":184.99100000000001,"15min":184.99100000000001},"last":0.45800000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":0.79200000000000004,"15min":0.71399999999999997},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.22500000000000001},"max":{"1min":6.2569999999999997,"5min":6.2569999999999997,"15min":19.215},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.4009999999999998,"5min":0.98399999999999999,"15min":0.90700000000000003},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":10.882999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":2.3319999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.2450000000000001,"15min":1.105},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":14.069000000000001,"5min":14.069000000000001,"15min":35.488},"last":3.2200000000000002},{"interface":"front","average":{"1min":2.6469999999999998,"5min":1.103,"15min":0.97999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":13.43,"5min":13.43,"15min":34.896000000000001},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":1.234,"15min":0.99199999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":8.4120000000000008,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.522,"5min":0.93700000000000006,"15min":0.754},"min":{"1min":0.36699999999999999,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":7.5659999999999998,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.497}]}]},{"osd":6,"up_from":0,"seq":1078036791460,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762618,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926155,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1884692,"kb_used_data":1120716,"kb_used_omap":8,"kb_used_meta":763959,"kb_avail":91860460,"statfs":{"total":95995035648,"available":94065111040,"internally_reserved":0,"allocated":1147613184,"data_stored":1145469975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":782294625},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.609,"15min":3.605},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":3.3780000000000001,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.1640000000000001,"15min":3.863},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.183},"max":{"1min":3.9470000000000001,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.45200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:15:16 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.9359999999999999,"15min":5.0519999999999996},"min":{"1min":0.215,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":3.5920000000000001,"5min":160.542,"15min":270.51100000000002},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.5720000000000001,"5min":3.4990000000000001,"15min":4.9820000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":7.5220000000000002,"5min":160.75999999999999,"15min":269.69299999999998},"last":2.0659999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.292,"15min":3.9980000000000002},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.21299999999999999},"max":{"1min":3.6629999999999998,"5min":10.082000000000001,"15min":126.95399999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.5289999999999999,"15min":4.1779999999999999},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":4.9359999999999999,"5min":9.5440000000000005,"15min":127.294},"last":0.54000000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":1.4790000000000001,"15min":2.0499999999999998},"min":{"1min":0.33500000000000002,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":8.7360000000000007,"5min":9.6959999999999997,"15min":148.20599999999999},"last":1.96},{"interface":"front","average":{"1min":1.734,"5min":1.3140000000000001,"15min":1.994},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.8230000000000004,"5min":8.4429999999999996,"15min":148.11000000000001},"last":105.282}]},{"osd":5,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.7609999999999999,"15min":1.8759999999999999},"min":{"1min":0.35999999999999999,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":6.8739999999999997,"5min":52.061,"15min":64.725999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.2450000000000001,"15min":1.899},"min":{"1min":0.312,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1500000000000004,"5min":51.741,"15min":51.741},"last":1.984}]},{"osd":6,"last update":"Sat Sep 23 20:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.117,"15min":1.5529999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":3.504,"5min":6.1660000000000004,"15min":15.744999999999999},"last":2.012},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.2390000000000001,"15min":1.724},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":4.6580000000000004,"5min":8.3919999999999995,"15min":13.766999999999999},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.2969999999999999,"15min":1.7490000000000001},"min":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.7400000000000002,"5min":12.981,"15min":65.081999999999994},"last":1.4610000000000001},{"interface":"front","average":{"1min":2.5249999999999999,"5min":1.861,"15min":1.8620000000000001},"min":{"1min":0.442,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.2290000000000001,"5min":17.151,"15min":17.151},"last":2.048}]}]},{"osd":2,"up_from":104,"seq":446676599094,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1397704,"kb_used_data":568384,"kb_used_omap":12,"kb_used_meta":829299,"kb_avail":92347448,"statfs":{"total":95995035648,"available":94563786752,"internally_reserved":0,"allocated":582025216,"data_stored":579838433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":849203198},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":2.0670000000000002,"15min":3.0339999999999998},"min":{"1min":0.253,"5min":0.221,"15min":0.192},"max":{"1min":5.923,"5min":65.382999999999996,"15min":76.251999999999995},"last":2.8610000000000002},{"interface":"front","average":{"1min":1.7230000000000001,"5min":1.994,"15min":3.081},"min":{"1min":0.26600000000000001,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":5.694,"5min":65.427000000000007,"15min":76.590000000000003},"last":2.7170000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.9020000000000001,"15min":3.8250000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":9.2590000000000003,"5min":70.453000000000003,"15min":94.768000000000001},"last":2.5640000000000001},{"interface":"front","average":{"1min":1.8149999999999999,"5min":2.7200000000000002,"15min":3.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.21099999999999999},"max":{"1min":11.981999999999999,"5min":69.825000000000003,"15min":96.808000000000007},"last":3}]},{"osd":3,"last update":"Sat Sep 23 20:15:40 2023","interfaces":[{"interface":"back","average":{"1min":7.5960000000000001,"5min":2.9870000000000001,"15min":4.1130000000000004},"min":{"1min":0.222,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":132.82499999999999,"5min":132.82499999999999,"15min":132.82499999999999},"last":3.1150000000000002},{"interface":"front","average":{"1min":8.1519999999999992,"5min":2.9900000000000002,"15min":3.411},"min":{"1min":0.23400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":130.97300000000001,"5min":130.97300000000001,"15min":130.97300000000001},"last":2.9500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.272,"15min":1.964},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.7270000000000003,"5min":14.757999999999999,"15min":75.608000000000004},"last":2.915},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.599,"15min":2.1779999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26700000000000002},"max":{"1min":9.1590000000000007,"5min":13.387,"15min":76.161000000000001},"last":2.5270000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":3.1360000000000001,"15min":3.0310000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.956,"5min":135.202,"15min":172.505},"last":2.976},{"interface":"front","average":{"1min":2.0350000000000001,"5min":1.966,"15min":2.7679999999999998},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32300000000000001},"max":{"1min":12.050000000000001,"5min":29.196999999999999,"15min":172.375},"last":5.1260000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.4930000000000001,"15min":1.5920000000000001},"min":{"1min":0.435,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.0510000000000002,"5min":14.664,"15min":14.664},"last":3.024},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.758,"15min":1.8400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.2649999999999997,"5min":12.025,"15min":14.279999999999999},"last":2.7440000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.7849999999999999,"15min":1.9159999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2050000000000001,"5min":13.481999999999999,"15min":23.079999999999998},"last":2.5910000000000002},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.3859999999999999,"15min":1.7609999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.7789999999999999,"5min":14.698,"15min":20.614999999999998},"last":3.8460000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435568,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2112844,"kb_used_data":1248196,"kb_used_omap":4,"kb_used_meta":864635,"kb_avail":91632308,"statfs":{"total":95995035648,"available":93831483392,"internally_reserved":0,"allocated":1278152704,"data_stored":1275719837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":885386349},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5940000000000003,"5min":1.8560000000000001,"15min":3.0859999999999999},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":76.632000000000005,"5min":76.632000000000005,"15min":76.632000000000005},"last":0.53700000000000003},{"interface":"front","average":{"1min":5.4580000000000002,"5min":2.266,"15min":3.4159999999999999},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.19600000000000001},"max":{"1min":81.429000000000002,"5min":81.429000000000002,"15min":81.429000000000002},"last":0.20499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":5.0510000000000002,"15min":4.6260000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":9,"5min":183.87,"15min":183.87},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.2230000000000001,"5min":4.9740000000000002,"15min":4.758},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":5.5810000000000004,"5min":185.84899999999999,"15min":185.84899999999999},"last":5.3499999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":4.2610000000000001,"15min":4.109},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":9.0289999999999999,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.375},{"interface":"front","average":{"1min":2.2679999999999998,"5min":3.6669999999999998,"15min":3.819},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":16.550999999999998,"5min":163.92099999999999,"15min":163.92099999999999},"last":4.9889999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.1120000000000001,"15min":2.786},"min":{"1min":0.32400000000000001,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":5.21,"5min":7.4960000000000004,"15min":109.66800000000001},"last":5.016},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6619999999999999,"15min":3.2469999999999999},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":9.6470000000000002,"5min":11.815,"15min":110.295},"last":2.9180000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":3.6349999999999998,"15min":3.9129999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6799999999999997,"5min":200.53200000000001,"15min":200.53200000000001},"last":2.887},{"interface":"front","average":{"1min":3.411,"5min":3.7599999999999998,"15min":3.9769999999999999},"min":{"1min":0.48799999999999999,"5min":0.34799999999999998,"15min":0.26900000000000002},"max":{"1min":9.9429999999999996,"5min":200.846,"15min":200.846},"last":4.3529999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.6539999999999999,"15min":2.8660000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.32300000000000001},"max":{"1min":8.9610000000000003,"5min":8.9610000000000003,"15min":82.281999999999996},"last":3.5409999999999999},{"interface":"front","average":{"1min":1.641,"5min":1.2669999999999999,"15min":2.9020000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":5.8959999999999999,"5min":8.2629999999999999,"15min":82.233000000000004},"last":4.9249999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.9470000000000001,"15min":3.3969999999999998},"min":{"1min":0.42799999999999999,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":9.0600000000000005,"5min":139.465,"15min":139.465},"last":2.3620000000000001},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.8540000000000001,"15min":3.27},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":5.3010000000000002,"5min":141.56200000000001,"15min":141.56200000000001},"last":1.6879999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019329,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1162672,"kb_used_data":714152,"kb_used_omap":0,"kb_used_meta":448512,"kb_avail":92582480,"statfs":{"total":95995035648,"available":94804459520,"internally_reserved":0,"allocated":731291648,"data_stored":729880123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":459276288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:25 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":1.2390000000000001,"15min":2.1720000000000002},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.9269999999999996,"5min":14.063000000000001,"15min":89.004999999999995},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.831,"5min":1.268,"15min":2.169},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":10.289,"5min":14.048999999999999,"15min":89.024000000000001},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:15:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":3.395,"15min":5.9109999999999996},"min":{"1min":0.27800000000000002,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":5.8259999999999996,"5min":138.47999999999999,"15min":161.613},"last":1.2949999999999999},{"interface":"front","average":{"1min":2.6070000000000002,"5min":3.46,"15min":5.9889999999999999},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.217},"max":{"1min":10.368,"5min":138.429,"15min":162.017},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":3.7320000000000002,"15min":5.6970000000000001},"min":{"1min":0.30499999999999999,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.8570000000000002,"5min":144.916,"15min":182.96700000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.4020000000000001,"5min":3.694,"15min":5.6100000000000003},"min":{"1min":0.40300000000000002,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":11.528,"5min":144.83099999999999,"15min":183.244},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:15:31 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":3.0459999999999998,"15min":5.1020000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":15.707000000000001,"5min":100.361,"15min":184.37100000000001},"last":2.411},{"interface":"front","average":{"1min":2.9700000000000002,"5min":3.3420000000000001,"15min":5.1900000000000004},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.28599999999999998},"max":{"1min":15.73,"5min":100.20699999999999,"15min":184.22300000000001},"last":1.4430000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:15:25 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":0.84099999999999997,"15min":1.177},"min":{"1min":0.26400000000000001,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":12.257999999999999,"5min":12.257999999999999,"15min":176.81899999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.1139999999999999,"5min":0.84199999999999997,"15min":1.2330000000000001},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":6.9770000000000003,"5min":6.9770000000000003,"15min":176.172},"last":0.745}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":0.85899999999999999,"15min":0.627},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":8.5660000000000007,"5min":10.337999999999999,"15min":10.337999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.3050000000000002,"5min":1.119,"15min":0.79500000000000004},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":11.348000000000001,"5min":11.348000000000001,"15min":11.348000000000001},"last":0.43099999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":0.82999999999999996,"15min":1.429},"min":{"1min":0.442,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.71,"5min":7.71,"15min":76.507000000000005},"last":0.77000000000000002},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.88900000000000001,"15min":1.5169999999999999},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":5.8869999999999996,"5min":5.8869999999999996,"15min":76.462000000000003},"last":0.73499999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757693,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2231720,"kb_used_data":1667232,"kb_used_omap":0,"kb_used_meta":564480,"kb_avail":91513432,"statfs":{"total":95995035648,"available":93709754368,"internally_reserved":0,"allocated":1707245568,"data_stored":1705838484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":578027520},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:23 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":1.3100000000000001,"15min":1.8100000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.304,"5min":10.304,"15min":121.94799999999999},"last":0.94199999999999995},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.2809999999999999,"15min":2.0489999999999999},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":8.1699999999999999,"5min":9.4250000000000007,"15min":121.964},"last":0.79600000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:15:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":5.0540000000000003,"15min":5.3209999999999997},"min":{"1min":0.22,"5min":0.22,"15min":0.21299999999999999},"max":{"1min":7.383,"5min":150.16999999999999,"15min":171.68299999999999},"last":1.8600000000000001},{"interface":"front","average":{"1min":2.6749999999999998,"5min":4.7960000000000003,"15min":5.4020000000000001},"min":{"1min":0.32100000000000001,"5min":0.24099999999999999,"15min":0.21199999999999999},"max":{"1min":7.3099999999999996,"5min":150.12700000000001,"15min":171.64099999999999},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:15:05 2023","interfaces":[{"interface":"back","average":{"1min":2.8879999999999999,"5min":3.7519999999999998,"15min":4.3129999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.787000000000001,"5min":120.297,"15min":240.57300000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":2.919,"5min":3.8500000000000001,"15min":4.3810000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.81,"5min":120.325,"15min":240.464},"last":0.995}]},{"osd":3,"last update":"Sat Sep 23 20:15:49 2023","interfaces":[{"interface":"back","average":{"1min":10.634,"5min":6.1920000000000002,"15min":5.556},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":195.84700000000001,"5min":195.84700000000001,"15min":201.50899999999999},"last":0.376},{"interface":"front","average":{"1min":10.66,"5min":6.0990000000000002,"15min":5.4089999999999998},"min":{"1min":0.39500000000000002,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":195.87899999999999,"5min":195.87899999999999,"15min":201.58099999999999},"last":1.2170000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:15:23 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":0.89600000000000002,"15min":1.345},"min":{"1min":0.25,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":8,"5min":8,"15min":191.78899999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.60499999999999998,"15min":1.2210000000000001},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":2.7160000000000002,"5min":3.2189999999999999,"15min":191.821},"last":0.40699999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.1870000000000001,"15min":0.83099999999999996},"min":{"1min":0.33300000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":5.4000000000000004,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.65400000000000003,"15min":0.58199999999999996},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":3.3180000000000001,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":0.52800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:15:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2360000000000002,"5min":1.1739999999999999,"15min":1.4139999999999999},"min":{"1min":0.33300000000000002,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":10.536,"5min":10.536,"15min":191.84299999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":2.395,"5min":0.999,"15min":1.413},"min":{"1min":0.40400000000000003,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":8.3260000000000005,"5min":8.3260000000000005,"15min":191.69},"last":0.439}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1122304000,"data_stored":1122304000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3164930048,"data_stored":3164930048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":574357504,"data_stored":574357504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1256325120,"data_stored":1256325120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":725618688,"data_stored":725614616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1672548352,"data_stored":1672544280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1435045888,"data_stored":1435041816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:16:09.453 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:09.532 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:09 smithi144 ceph-mon[194833]: from='client.56161 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:09.638 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:09.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:09.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:09.745 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:09.758 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:09.813 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:09.910 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:09.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:09.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:10.020 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:10.078 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:10.154 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:10 smithi135 ceph-mon[213871]: pgmap v1291: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1k op/s; 39968/94917 objects degraded (42.108%); 2319/94917 objects misplaced (2.443%); 11 MiB/s, 203 objects/s recovering 2023-09-23T20:16:10.154 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/990579092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:10.154 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:10.163 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:16:10.188 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:10.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:10.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:10.225 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:16:10.225 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:16:10.301 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:10.375 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:10.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:10 smithi144 ceph-mon[194833]: pgmap v1291: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1k op/s; 39968/94917 objects degraded (42.108%); 2319/94917 objects misplaced (2.443%); 11 MiB/s, 203 objects/s recovering 2023-09-23T20:16:10.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/990579092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:10.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:10.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:10 smithi135 ceph-mon[221113]: pgmap v1291: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1k op/s; 39968/94917 objects degraded (42.108%); 2319/94917 objects misplaced (2.443%); 11 MiB/s, 203 objects/s recovering 2023-09-23T20:16:10.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/990579092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:10.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:11.418 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:11.419 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[213871]: failed to encode map e463 with expected crc 2023-09-23T20:16:11.419 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:16:11.419 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[213871]: osdmap e463: 8 total, 8 up, 6 in 2023-09-23T20:16:11.419 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[213871]: failed to encode map e463 with expected crc 2023-09-23T20:16:11.419 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[213871]: failed to encode map e463 with expected crc 2023-09-23T20:16:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[221113]: failed to encode map e463 with expected crc 2023-09-23T20:16:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:16:11.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[221113]: osdmap e463: 8 total, 8 up, 6 in 2023-09-23T20:16:11.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[221113]: failed to encode map e463 with expected crc 2023-09-23T20:16:11.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:11 smithi135 ceph-mon[221113]: failed to encode map e463 with expected crc 2023-09-23T20:16:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:11 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:11 smithi144 ceph-mon[194833]: failed to encode map e463 with expected crc 2023-09-23T20:16:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:11 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:16:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:11 smithi144 ceph-mon[194833]: osdmap e463: 8 total, 8 up, 6 in 2023-09-23T20:16:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:11 smithi144 ceph-mon[194833]: failed to encode map e463 with expected crc 2023-09-23T20:16:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:11 smithi144 ceph-mon[194833]: failed to encode map e463 with expected crc 2023-09-23T20:16:12.168 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:12.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:12.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:12.276 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:12.301 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:12.358 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:12.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:12.429 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:16:12.465 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:12 smithi135 ceph-mon[221113]: pgmap v1293: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 39968/94917 objects degraded (42.108%); 2319/94917 objects misplaced (2.443%); 9.3 MiB/s, 170 objects/s recovering 2023-09-23T20:16:12.465 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:12 smithi135 ceph-mon[213871]: pgmap v1293: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 39968/94917 objects degraded (42.108%); 2319/94917 objects misplaced (2.443%); 9.3 MiB/s, 170 objects/s recovering 2023-09-23T20:16:12.465 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:12.498 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:16:12.536 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:12.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:12 smithi144 ceph-mon[194833]: pgmap v1293: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 39968/94917 objects degraded (42.108%); 2319/94917 objects misplaced (2.443%); 9.3 MiB/s, 170 objects/s recovering 2023-09-23T20:16:12.716 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:12.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:12.830 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:12.909 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:12.995 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:13.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:13.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:13.102 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:13.166 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:13.311 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:13.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:13.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:13.331 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:13.432 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:13.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:13 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 39968/94917 objects degraded (42.108%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:13.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:13 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 39968/94917 objects degraded (42.108%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:13.487 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:16:13.496 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:13 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 39968/94917 objects degraded (42.108%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:13.506 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:13.513 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:16:13.546 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:16:13.549 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:13.581 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:16:13.609 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:13.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:13.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:13.619 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:16:13.659 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:16:13.707 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:16:13.717 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:13.742 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:16:13.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:16:13.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:16:13.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:16:13.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:16:13.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:16:13.757 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:16:13.758 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:16:13.758 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:16:13.773 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:13.856 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:16:13.882 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:13.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:13.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:13.955 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:16:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:16:13] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-09-23T20:16:13.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:14.088 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:14.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:14 smithi135 ceph-mon[221113]: pgmap v1294: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 824 op/s; 39200/88245 objects degraded (44.422%); 1897/88245 objects misplaced (2.150%); 8.6 MiB/s, 204 objects/s recovering 2023-09-23T20:16:14.224 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3178719464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:14.225 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:14 smithi135 ceph-mon[213871]: pgmap v1294: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 824 op/s; 39200/88245 objects degraded (44.422%); 1897/88245 objects misplaced (2.150%); 8.6 MiB/s, 204 objects/s recovering 2023-09-23T20:16:14.226 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3178719464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:14 smithi144 ceph-mon[194833]: pgmap v1294: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 824 op/s; 39200/88245 objects degraded (44.422%); 1897/88245 objects misplaced (2.150%); 8.6 MiB/s, 204 objects/s recovering 2023-09-23T20:16:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3178719464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:15.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:15 smithi135 ceph-mon[221113]: pgmap v1295: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 836 op/s; 39200/82629 objects degraded (47.441%); 1785/82629 objects misplaced (2.160%); 8.6 MiB/s, 193 objects/s recovering 2023-09-23T20:16:15.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:15 smithi135 ceph-mon[213871]: pgmap v1295: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 836 op/s; 39200/82629 objects degraded (47.441%); 1785/82629 objects misplaced (2.160%); 8.6 MiB/s, 193 objects/s recovering 2023-09-23T20:16:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:15 smithi144 ceph-mon[194833]: pgmap v1295: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 836 op/s; 39200/82629 objects degraded (47.441%); 1785/82629 objects misplaced (2.160%); 8.6 MiB/s, 193 objects/s recovering 2023-09-23T20:16:15.833 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:15.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:15.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:15.945 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:15.997 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:16.139 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:16.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:16.251 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:16.309 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:16.346 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:16.423 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:16.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:16.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:16.533 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:16.605 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:16.700 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:16.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:16.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:16.818 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:16.878 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:16.968 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:16.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:16.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:16.981 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:16:16.981 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:16:16.982 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:16:16.982 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:16:16.982 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:16:17.078 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:17.141 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:17.184 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:16:17.201 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:16:17.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:17 smithi144 ceph-mon[194833]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:17 smithi144 ceph-mon[194833]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:17 smithi144 ceph-mon[194833]: osdmap e464: 8 total, 8 up, 6 in 2023-09-23T20:16:17.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:17 smithi144 ceph-mon[194833]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:17 smithi144 ceph-mon[194833]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:17 smithi144 ceph-mon[194833]: pgmap v1297: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 38457/81081 objects degraded (47.430%); 2073/81081 objects misplaced (2.557%); 3.6 MiB/s, 172 objects/s recovering 2023-09-23T20:16:17.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:17 smithi144 ceph-mon[194833]: from='client.46289 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:17.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[221113]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[221113]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[221113]: osdmap e464: 8 total, 8 up, 6 in 2023-09-23T20:16:17.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[221113]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[221113]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[221113]: pgmap v1297: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 38457/81081 objects degraded (47.430%); 2073/81081 objects misplaced (2.557%); 3.6 MiB/s, 172 objects/s recovering 2023-09-23T20:16:17.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[221113]: from='client.46289 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:17.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[213871]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[213871]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[213871]: osdmap e464: 8 total, 8 up, 6 in 2023-09-23T20:16:17.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[213871]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[213871]: failed to encode map e464 with expected crc 2023-09-23T20:16:17.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[213871]: pgmap v1297: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 38457/81081 objects degraded (47.430%); 2073/81081 objects misplaced (2.557%); 3.6 MiB/s, 172 objects/s recovering 2023-09-23T20:16:17.974 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:17 smithi135 ceph-mon[213871]: from='client.46289 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:18.230 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:16:18.404 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:18.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:18.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:18.509 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:18.538 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:18.815 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:16:18.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[221113]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[221113]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[221113]: osdmap e465: 8 total, 8 up, 6 in 2023-09-23T20:16:18.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[221113]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[221113]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[221113]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[221113]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 38457/81081 objects degraded (47.430%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[213871]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[213871]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[213871]: osdmap e465: 8 total, 8 up, 6 in 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[213871]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[213871]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[213871]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.817 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[213871]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.817 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:18 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 38457/81081 objects degraded (47.430%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:18.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:18 smithi144 ceph-mon[194833]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:18 smithi144 ceph-mon[194833]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:18 smithi144 ceph-mon[194833]: osdmap e465: 8 total, 8 up, 6 in 2023-09-23T20:16:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:18 smithi144 ceph-mon[194833]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:18 smithi144 ceph-mon[194833]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:18 smithi144 ceph-mon[194833]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:18 smithi144 ceph-mon[194833]: failed to encode map e465 with expected crc 2023-09-23T20:16:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:18 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 38457/81081 objects degraded (47.430%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:19.171 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:19.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:19.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:19.278 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:19.303 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:19.505 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:19.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:19.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:19.610 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:19 smithi144 ceph-mon[194833]: pgmap v1299: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 37977/70260 objects degraded (54.052%); 1662/70260 objects misplaced (2.365%); 6.0 MiB/s, 235 objects/s recovering 2023-09-23T20:16:19.616 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:19.675 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:19.720 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:19.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:19.779 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1297,"stamp":"2023-09-23T20:16:17.079967+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1774149037,"num_objects":27027,"num_object_clones":0,"num_object_copies":81081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38457,"num_objects_misplaced":2073,"num_objects_unfound":0,"num_objects_dirty":27027,"num_whiteouts":0,"num_read":5996,"num_read_kb":13236,"num_write":181621,"num_write_kb":6659327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19366,"num_bytes_recovered":1126628468,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":196861,"ondisk_log_size":196861,"up":387,"acting":367,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":391,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9412284,"kb_used_data":5450636,"kb_used_omap":25,"kb_used_meta":3961126,"kb_avail":553058628,"statfs":{"total":575970213888,"available":566332035072,"internally_reserved":0,"allocated":5581451264,"data_stored":5570432773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":4056193740},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-302252032,"num_objects":-4612,"num_object_clones":0,"num_object_copies":-13836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1511,"num_objects_misplaced":-246,"num_objects_unfound":0,"num_objects_dirty":-4612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4612,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1381,"num_bytes_recovered":29884416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005845"},"pg_stats":[{"pgid":"9.23","version":"463'5992","reported_seq":6086,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:12.937377+0000","last_change":"2023-09-23T20:14:36.092943+0000","last_active":"2023-09-23T20:16:12.937377+0000","last_peered":"2023-09-23T20:16:12.937377+0000","last_clean":"2023-09-23T20:14:33.760240+0000","last_became_active":"2023-09-23T20:14:36.076681+0000","last_became_peered":"2023-09-23T20:14:36.076681+0000","last_unstale":"2023-09-23T20:16:12.937377+0000","last_undegraded":"2023-09-23T20:14:35.911782+0000","last_fullsized":"2023-09-23T20:14:35.868390+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5992,"log_dups_size":0,"ondisk_log_size":5992,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27869184,"num_objects":425,"num_object_clones":0,"num_object_copies":1275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2883,"num_write_kb":105872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":335},{"shards":"3,7","objects":-1139}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.22","version":"463'6049","reported_seq":6148,"reported_epoch":463,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:16.566148+0000","last_change":"2023-09-23T20:16:16.542393+0000","last_active":"2023-09-23T20:16:16.566148+0000","last_peered":"2023-09-23T20:16:16.566148+0000","last_clean":"2023-09-23T20:14:33.601158+0000","last_became_active":"2023-09-23T20:14:35.991960+0000","last_became_peered":"2023-09-23T20:14:35.991960+0000","last_unstale":"2023-09-23T20:16:16.566148+0000","last_undegraded":"2023-09-23T20:14:35.848470+0000","last_fullsized":"2023-09-23T20:14:35.833465+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6049,"log_dups_size":0,"ondisk_log_size":6049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1254,"num_objects_misplaced":288,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2913,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":288},{"shards":"2,3","objects":-1137}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.21","version":"463'5859","reported_seq":5943,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:12.820606+0000","last_change":"2023-09-23T20:14:24.799466+0000","last_active":"2023-09-23T20:16:12.820606+0000","last_peered":"2023-09-23T20:16:12.820606+0000","last_clean":"2023-09-23T20:16:12.820606+0000","last_became_active":"2023-09-23T20:14:24.796974+0000","last_became_peered":"2023-09-23T20:14:24.796974+0000","last_unstale":"2023-09-23T20:16:12.820606+0000","last_undegraded":"2023-09-23T20:16:12.820606+0000","last_fullsized":"2023-09-23T20:16:12.820606+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5859,"log_dups_size":0,"ondisk_log_size":5859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30310400,"num_objects":463,"num_object_clones":0,"num_object_copies":1389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2765,"num_write_kb":103264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.20","version":"463'5946","reported_seq":6039,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:15.877736+0000","last_change":"2023-09-23T20:14:36.109513+0000","last_active":"2023-09-23T20:16:15.877736+0000","last_peered":"2023-09-23T20:16:15.877736+0000","last_clean":"2023-09-23T20:14:33.769971+0000","last_became_active":"2023-09-23T20:14:36.106301+0000","last_became_peered":"2023-09-23T20:14:36.106301+0000","last_unstale":"2023-09-23T20:16:15.877736+0000","last_undegraded":"2023-09-23T20:14:35.845700+0000","last_fullsized":"2023-09-23T20:14:35.836337+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24641536,"num_objects":376,"num_object_clones":0,"num_object_copies":1128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2864,"num_write_kb":103680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":301},{"shards":"0,2","objects":-1169}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.10","version":"463'7221","reported_seq":13154,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.027063+0000","last_change":"2023-09-23T20:14:22.339078+0000","last_active":"2023-09-23T20:16:13.027063+0000","last_peered":"2023-09-23T20:16:13.027063+0000","last_clean":"2023-09-23T20:16:13.027063+0000","last_became_active":"2023-09-23T20:14:22.319507+0000","last_became_peered":"2023-09-23T20:14:22.319507+0000","last_unstale":"2023-09-23T20:16:13.027063+0000","last_undegraded":"2023-09-23T20:16:13.027063+0000","last_fullsized":"2023-09-23T20:16:13.027063+0000","mapping_epoch":434,"log_start":"413'2571","ondisk_log_start":"413'2571","created":418,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4650,"log_dups_size":0,"ondisk_log_size":4650,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55508992,"num_objects":847,"num_object_clones":0,"num_object_copies":2541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5501,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1945,"num_bytes_recovered":127467520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":33,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.237790+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:16:02.237790+0000","last_peered":"2023-09-23T20:16:02.237790+0000","last_clean":"2023-09-23T20:16:02.237790+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:16:02.237790+0000","last_undegraded":"2023-09-23T20:16:02.237790+0000","last_fullsized":"2023-09-23T20:16:02.237790+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186135+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:16:02.186135+0000","last_peered":"2023-09-23T20:16:02.186135+0000","last_clean":"2023-09-23T20:16:02.186135+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:16:02.186135+0000","last_undegraded":"2023-09-23T20:16:02.186135+0000","last_fullsized":"2023-09-23T20:16:02.186135+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.13","version":"463'7324","reported_seq":14080,"reported_epoch":463,"state":"active+recovery_wait+undersized+remapped","last_fresh":"2023-09-23T20:16:13.032501+0000","last_change":"2023-09-23T20:16:06.703336+0000","last_active":"2023-09-23T20:16:13.032501+0000","last_peered":"2023-09-23T20:16:13.032501+0000","last_clean":"2023-09-23T20:14:05.142062+0000","last_became_active":"2023-09-23T20:14:07.304693+0000","last_became_peered":"2023-09-23T20:14:07.304693+0000","last_unstale":"2023-09-23T20:16:13.032501+0000","last_undegraded":"2023-09-23T20:16:13.032501+0000","last_fullsized":"2023-09-23T20:14:07.197546+0000","mapping_epoch":429,"log_start":"413'2711","ondisk_log_start":"413'2711","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4613,"log_dups_size":0,"ondisk_log_size":4613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54099968,"num_objects":826,"num_object_clones":0,"num_object_copies":2478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5592,"num_write_kb":205344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2223,"num_bytes_recovered":145686528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,0],"avail_no_missing":["7","0","1","3","5"],"object_location_counts":[{"shards":"0,7","objects":826}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183024+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:16:02.183024+0000","last_peered":"2023-09-23T20:16:02.183024+0000","last_clean":"2023-09-23T20:16:02.183024+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:16:02.183024+0000","last_undegraded":"2023-09-23T20:16:02.183024+0000","last_fullsized":"2023-09-23T20:16:02.183024+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":171,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.193129+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:16:02.193129+0000","last_peered":"2023-09-23T20:16:02.193129+0000","last_clean":"2023-09-23T20:16:02.193129+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:16:02.193129+0000","last_undegraded":"2023-09-23T20:16:02.193129+0000","last_fullsized":"2023-09-23T20:16:02.193129+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"463'7435","reported_seq":13901,"reported_epoch":463,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T20:16:13.392063+0000","last_change":"2023-09-23T20:14:36.234004+0000","last_active":"2023-09-23T20:16:13.392063+0000","last_peered":"2023-09-23T20:16:13.392063+0000","last_clean":"2023-09-23T20:14:33.790624+0000","last_became_active":"2023-09-23T20:14:36.220990+0000","last_became_peered":"2023-09-23T20:14:36.220990+0000","last_unstale":"2023-09-23T20:16:13.392063+0000","last_undegraded":"2023-09-23T20:16:13.392063+0000","last_fullsized":"2023-09-23T20:16:13.392063+0000","mapping_epoch":442,"log_start":"413'2574","ondisk_log_start":"413'2574","created":418,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4861,"log_dups_size":0,"ondisk_log_size":4861,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49741824,"num_objects":759,"num_object_clones":0,"num_object_copies":2277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":759,"num_objects_unfound":0,"num_objects_dirty":759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5685,"num_write_kb":206208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":139067392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1","3"],"object_location_counts":[{"shards":"0,1,5","objects":759}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":32,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.236123+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:16:02.236123+0000","last_peered":"2023-09-23T20:16:02.236123+0000","last_clean":"2023-09-23T20:16:02.236123+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:16:02.236123+0000","last_undegraded":"2023-09-23T20:16:02.236123+0000","last_fullsized":"2023-09-23T20:16:02.236123+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":418,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.189373+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:16:02.189373+0000","last_peered":"2023-09-23T20:16:02.189373+0000","last_clean":"2023-09-23T20:16:02.189373+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:16:02.189373+0000","last_undegraded":"2023-09-23T20:16:02.189373+0000","last_fullsized":"2023-09-23T20:16:02.189373+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1d","version":"463'8518","reported_seq":8602,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.007950+0000","last_change":"2023-09-23T20:14:36.076800+0000","last_active":"2023-09-23T20:16:13.007950+0000","last_peered":"2023-09-23T20:16:13.007950+0000","last_clean":"2023-09-23T20:14:23.775634+0000","last_became_active":"2023-09-23T20:14:36.070774+0000","last_became_peered":"2023-09-23T20:14:36.070774+0000","last_unstale":"2023-09-23T20:16:13.007950+0000","last_undegraded":"2023-09-23T20:14:35.892079+0000","last_fullsized":"2023-09-23T20:14:35.839513+0000","mapping_epoch":442,"log_start":"413'2953","ondisk_log_start":"413'2953","created":436,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5565,"log_dups_size":0,"ondisk_log_size":5565,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62259224,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5729,"num_write_kb":213697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":686},{"shards":"5,7","objects":-2124}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":389,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.189240+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:16:02.189240+0000","last_peered":"2023-09-23T20:16:02.189240+0000","last_clean":"2023-09-23T20:16:02.189240+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:16:02.189240+0000","last_undegraded":"2023-09-23T20:16:02.189240+0000","last_fullsized":"2023-09-23T20:16:02.189240+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":317,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.182680+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:16:02.182680+0000","last_peered":"2023-09-23T20:16:02.182680+0000","last_clean":"2023-09-23T20:16:02.182680+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:16:02.182680+0000","last_undegraded":"2023-09-23T20:16:02.182680+0000","last_fullsized":"2023-09-23T20:16:02.182680+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"463'8239","reported_seq":8313,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:16.486384+0000","last_change":"2023-09-23T20:14:24.810789+0000","last_active":"2023-09-23T20:16:16.486384+0000","last_peered":"2023-09-23T20:16:16.486384+0000","last_clean":"2023-09-23T20:16:16.486384+0000","last_became_active":"2023-09-23T20:14:24.807691+0000","last_became_peered":"2023-09-23T20:14:24.807691+0000","last_unstale":"2023-09-23T20:16:16.486384+0000","last_undegraded":"2023-09-23T20:16:16.486384+0000","last_fullsized":"2023-09-23T20:16:16.486384+0000","mapping_epoch":436,"log_start":"417'3296","ondisk_log_start":"417'3296","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46628864,"num_objects":712,"num_object_clones":0,"num_object_copies":2136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5620,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":796,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.238040+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:16:02.238040+0000","last_peered":"2023-09-23T20:16:02.238040+0000","last_clean":"2023-09-23T20:16:02.238040+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:16:02.238040+0000","last_undegraded":"2023-09-23T20:16:02.238040+0000","last_fullsized":"2023-09-23T20:16:02.238040+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":472,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.188197+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:16:02.188197+0000","last_peered":"2023-09-23T20:16:02.188197+0000","last_clean":"2023-09-23T20:16:02.188197+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:16:02.188197+0000","last_undegraded":"2023-09-23T20:16:02.188197+0000","last_fullsized":"2023-09-23T20:16:02.188197+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":327,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.182930+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:16:02.182930+0000","last_peered":"2023-09-23T20:16:02.182930+0000","last_clean":"2023-09-23T20:16:02.182930+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:16:02.182930+0000","last_undegraded":"2023-09-23T20:16:02.182930+0000","last_fullsized":"2023-09-23T20:16:02.182930+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1f","version":"463'8274","reported_seq":8357,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.342102+0000","last_change":"2023-09-23T20:14:36.232228+0000","last_active":"2023-09-23T20:16:13.342102+0000","last_peered":"2023-09-23T20:16:13.342102+0000","last_clean":"2023-09-23T20:14:33.842679+0000","last_became_active":"2023-09-23T20:14:36.217802+0000","last_became_peered":"2023-09-23T20:14:36.217802+0000","last_unstale":"2023-09-23T20:16:13.342102+0000","last_undegraded":"2023-09-23T20:14:36.129688+0000","last_fullsized":"2023-09-23T20:14:36.115967+0000","mapping_epoch":442,"log_start":"412'2415","ondisk_log_start":"412'2415","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5859,"log_dups_size":0,"ondisk_log_size":5859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55967744,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5588,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":660},{"shards":"3,5","objects":-2173}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":156,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.193273+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:16:02.193273+0000","last_peered":"2023-09-23T20:16:02.193273+0000","last_clean":"2023-09-23T20:16:02.193273+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:16:02.193273+0000","last_undegraded":"2023-09-23T20:16:02.193273+0000","last_fullsized":"2023-09-23T20:16:02.193273+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":161,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.237887+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:16:02.237887+0000","last_peered":"2023-09-23T20:16:02.237887+0000","last_clean":"2023-09-23T20:16:02.237887+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:16:02.237887+0000","last_undegraded":"2023-09-23T20:16:02.237887+0000","last_fullsized":"2023-09-23T20:16:02.237887+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.1e","version":"463'8296","reported_seq":14440,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:12.980787+0000","last_change":"2023-09-23T20:15:21.201333+0000","last_active":"2023-09-23T20:16:12.980787+0000","last_peered":"2023-09-23T20:16:12.980787+0000","last_clean":"2023-09-23T20:16:12.980787+0000","last_became_active":"2023-09-23T20:15:21.192111+0000","last_became_peered":"2023-09-23T20:15:21.192111+0000","last_unstale":"2023-09-23T20:16:12.980787+0000","last_undegraded":"2023-09-23T20:16:12.980787+0000","last_fullsized":"2023-09-23T20:16:12.980787+0000","mapping_epoch":454,"log_start":"413'2657","ondisk_log_start":"413'2657","created":436,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5639,"log_dups_size":0,"ondisk_log_size":5639,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58097664,"num_objects":887,"num_object_clones":0,"num_object_copies":2661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5597,"num_write_kb":207456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3024,"num_bytes_recovered":198180864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":409,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.194932+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:16:02.194932+0000","last_peered":"2023-09-23T20:16:02.194932+0000","last_clean":"2023-09-23T20:16:02.194932+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:16:02.194932+0000","last_undegraded":"2023-09-23T20:16:02.194932+0000","last_fullsized":"2023-09-23T20:16:02.194932+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":340,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186528+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:16:02.186528+0000","last_peered":"2023-09-23T20:16:02.186528+0000","last_clean":"2023-09-23T20:16:02.186528+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:16:02.186528+0000","last_undegraded":"2023-09-23T20:16:02.186528+0000","last_fullsized":"2023-09-23T20:16:02.186528+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"463'7428","reported_seq":7502,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.008636+0000","last_change":"2023-09-23T20:13:46.811423+0000","last_active":"2023-09-23T20:16:13.008636+0000","last_peered":"2023-09-23T20:16:13.008636+0000","last_clean":"2023-09-23T20:16:13.008636+0000","last_became_active":"2023-09-23T20:13:46.808266+0000","last_became_peered":"2023-09-23T20:13:46.808266+0000","last_unstale":"2023-09-23T20:16:13.008636+0000","last_undegraded":"2023-09-23T20:16:13.008636+0000","last_fullsized":"2023-09-23T20:16:13.008636+0000","mapping_epoch":418,"log_start":"413'2894","ondisk_log_start":"413'2894","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4534,"log_dups_size":0,"ondisk_log_size":4534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55345152,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5673,"num_write_kb":208544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":822,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.184531+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:16:02.184531+0000","last_peered":"2023-09-23T20:16:02.184531+0000","last_clean":"2023-09-23T20:16:02.184531+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:16:02.184531+0000","last_undegraded":"2023-09-23T20:16:02.184531+0000","last_fullsized":"2023-09-23T20:16:02.184531+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":156,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.194253+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:16:02.194253+0000","last_peered":"2023-09-23T20:16:02.194253+0000","last_clean":"2023-09-23T20:16:02.194253+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:16:02.194253+0000","last_undegraded":"2023-09-23T20:16:02.194253+0000","last_fullsized":"2023-09-23T20:16:02.194253+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":209,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.192486+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:16:02.192486+0000","last_peered":"2023-09-23T20:16:02.192486+0000","last_clean":"2023-09-23T20:16:02.192486+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:16:02.192486+0000","last_undegraded":"2023-09-23T20:16:02.192486+0000","last_fullsized":"2023-09-23T20:16:02.192486+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"463'7417","reported_seq":7491,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.413291+0000","last_change":"2023-09-23T20:13:46.824838+0000","last_active":"2023-09-23T20:16:13.413291+0000","last_peered":"2023-09-23T20:16:13.413291+0000","last_clean":"2023-09-23T20:16:13.413291+0000","last_became_active":"2023-09-23T20:13:46.820709+0000","last_became_peered":"2023-09-23T20:13:46.820709+0000","last_unstale":"2023-09-23T20:16:13.413291+0000","last_undegraded":"2023-09-23T20:16:13.413291+0000","last_fullsized":"2023-09-23T20:16:13.413291+0000","mapping_epoch":418,"log_start":"413'2566","ondisk_log_start":"413'2566","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4851,"log_dups_size":0,"ondisk_log_size":4851,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55148544,"num_objects":842,"num_object_clones":0,"num_object_copies":2526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5702,"num_write_kb":209376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":511,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183786+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:16:02.183786+0000","last_peered":"2023-09-23T20:16:02.183786+0000","last_clean":"2023-09-23T20:16:02.183786+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:16:02.183786+0000","last_undegraded":"2023-09-23T20:16:02.183786+0000","last_fullsized":"2023-09-23T20:16:02.183786+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":171,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.191304+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:16:02.191304+0000","last_peered":"2023-09-23T20:16:02.191304+0000","last_clean":"2023-09-23T20:16:02.191304+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:16:02.191304+0000","last_undegraded":"2023-09-23T20:16:02.191304+0000","last_fullsized":"2023-09-23T20:16:02.191304+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":461,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186665+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:16:02.186665+0000","last_peered":"2023-09-23T20:16:02.186665+0000","last_clean":"2023-09-23T20:16:02.186665+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:16:02.186665+0000","last_undegraded":"2023-09-23T20:16:02.186665+0000","last_fullsized":"2023-09-23T20:16:02.186665+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1b","version":"463'8091","reported_seq":8165,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.342231+0000","last_change":"2023-09-23T20:14:24.808547+0000","last_active":"2023-09-23T20:16:13.342231+0000","last_peered":"2023-09-23T20:16:13.342231+0000","last_clean":"2023-09-23T20:16:13.342231+0000","last_became_active":"2023-09-23T20:14:24.806384+0000","last_became_peered":"2023-09-23T20:14:24.806384+0000","last_unstale":"2023-09-23T20:16:13.342231+0000","last_undegraded":"2023-09-23T20:16:13.342231+0000","last_fullsized":"2023-09-23T20:16:13.342231+0000","mapping_epoch":436,"log_start":"412'2083","ondisk_log_start":"412'2083","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6008,"log_dups_size":0,"ondisk_log_size":6008,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55803904,"num_objects":852,"num_object_clones":0,"num_object_copies":2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5480,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":572,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.182943+0000","last_change":"2023-09-23T20:14:37.887863+0000","last_active":"2023-09-23T20:16:02.182943+0000","last_peered":"2023-09-23T20:16:02.182943+0000","last_clean":"2023-09-23T20:16:02.182943+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:16:02.182943+0000","last_undegraded":"2023-09-23T20:16:02.182943+0000","last_fullsized":"2023-09-23T20:16:02.182943+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":150,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.191701+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:16:02.191701+0000","last_peered":"2023-09-23T20:16:02.191701+0000","last_clean":"2023-09-23T20:16:02.191701+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:16:02.191701+0000","last_undegraded":"2023-09-23T20:16:02.191701+0000","last_fullsized":"2023-09-23T20:16:02.191701+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":451,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183507+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:16:02.183507+0000","last_peered":"2023-09-23T20:16:02.183507+0000","last_clean":"2023-09-23T20:16:02.183507+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:16:02.183507+0000","last_undegraded":"2023-09-23T20:16:02.183507+0000","last_fullsized":"2023-09-23T20:16:02.183507+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1a","version":"463'8240","reported_seq":8324,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.360115+0000","last_change":"2023-09-23T20:14:36.218516+0000","last_active":"2023-09-23T20:16:13.360115+0000","last_peered":"2023-09-23T20:16:13.360115+0000","last_clean":"2023-09-23T20:14:33.782041+0000","last_became_active":"2023-09-23T20:14:36.175134+0000","last_became_peered":"2023-09-23T20:14:36.175134+0000","last_unstale":"2023-09-23T20:16:13.360115+0000","last_undegraded":"2023-09-23T20:14:35.943267+0000","last_fullsized":"2023-09-23T20:14:35.848493+0000","mapping_epoch":442,"log_start":"412'2144","ondisk_log_start":"412'2144","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6096,"log_dups_size":0,"ondisk_log_size":6096,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57147392,"num_objects":872,"num_object_clones":0,"num_object_copies":2616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5542,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":625},{"shards":"5,7","objects":-2088}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":600,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186206+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:16:02.186206+0000","last_peered":"2023-09-23T20:16:02.186206+0000","last_clean":"2023-09-23T20:16:02.186206+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:16:02.186206+0000","last_undegraded":"2023-09-23T20:16:02.186206+0000","last_fullsized":"2023-09-23T20:16:02.186206+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":106,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183634+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:16:02.183634+0000","last_peered":"2023-09-23T20:16:02.183634+0000","last_clean":"2023-09-23T20:16:02.183634+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:16:02.183634+0000","last_undegraded":"2023-09-23T20:16:02.183634+0000","last_fullsized":"2023-09-23T20:16:02.183634+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":104,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:10.162248+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:16:10.162248+0000","last_peered":"2023-09-23T20:16:10.162248+0000","last_clean":"2023-09-23T20:16:10.162248+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:16:10.162248+0000","last_undegraded":"2023-09-23T20:16:10.162248+0000","last_fullsized":"2023-09-23T20:16:10.162248+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"463'7438","reported_seq":7513,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.358358+0000","last_change":"2023-09-23T20:13:46.790587+0000","last_active":"2023-09-23T20:16:13.358358+0000","last_peered":"2023-09-23T20:16:13.358358+0000","last_clean":"2023-09-23T20:16:13.358358+0000","last_became_active":"2023-09-23T20:13:46.788387+0000","last_became_peered":"2023-09-23T20:13:46.788387+0000","last_unstale":"2023-09-23T20:16:13.358358+0000","last_undegraded":"2023-09-23T20:16:13.358358+0000","last_fullsized":"2023-09-23T20:16:13.358358+0000","mapping_epoch":407,"log_start":"413'2567","ondisk_log_start":"413'2567","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4871,"log_dups_size":0,"ondisk_log_size":4871,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53739520,"num_objects":820,"num_object_clones":0,"num_object_copies":2460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5708,"num_write_kb":208896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":723,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.192878+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:16:02.192878+0000","last_peered":"2023-09-23T20:16:02.192878+0000","last_clean":"2023-09-23T20:16:02.192878+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:16:02.192878+0000","last_undegraded":"2023-09-23T20:16:02.192878+0000","last_fullsized":"2023-09-23T20:16:02.192878+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":154,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.195326+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:16:02.195326+0000","last_peered":"2023-09-23T20:16:02.195326+0000","last_clean":"2023-09-23T20:16:02.195326+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:16:02.195326+0000","last_undegraded":"2023-09-23T20:16:02.195326+0000","last_fullsized":"2023-09-23T20:16:02.195326+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":471,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183673+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:16:02.183673+0000","last_peered":"2023-09-23T20:16:02.183673+0000","last_clean":"2023-09-23T20:16:02.183673+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:16:02.183673+0000","last_undegraded":"2023-09-23T20:16:02.183673+0000","last_fullsized":"2023-09-23T20:16:02.183673+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"463'7305","reported_seq":8961,"reported_epoch":463,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.033552+0000","last_change":"2023-09-23T20:16:00.310118+0000","last_active":"2023-09-23T20:16:13.033552+0000","last_peered":"2023-09-23T20:16:13.033552+0000","last_clean":"2023-09-23T20:14:33.800061+0000","last_became_active":"2023-09-23T20:14:36.025052+0000","last_became_peered":"2023-09-23T20:14:36.025052+0000","last_unstale":"2023-09-23T20:16:13.033552+0000","last_undegraded":"2023-09-23T20:14:35.860952+0000","last_fullsized":"2023-09-23T20:14:35.840056+0000","mapping_epoch":442,"log_start":"411'1322","ondisk_log_start":"411'1322","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5983,"log_dups_size":0,"ondisk_log_size":5983,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55214080,"num_objects":842,"num_object_clones":0,"num_object_copies":2526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":170,"num_objects_unfound":0,"num_objects_dirty":842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5580,"num_write_kb":205536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":786,"num_bytes_recovered":51511296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":170},{"shards":"3,4","objects":-1395}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":810,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183473+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:16:02.183473+0000","last_peered":"2023-09-23T20:16:02.183473+0000","last_clean":"2023-09-23T20:16:02.183473+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:16:02.183473+0000","last_undegraded":"2023-09-23T20:16:02.183473+0000","last_fullsized":"2023-09-23T20:16:02.183473+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":169,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.193206+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:16:02.193206+0000","last_peered":"2023-09-23T20:16:02.193206+0000","last_clean":"2023-09-23T20:16:02.193206+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:16:02.193206+0000","last_undegraded":"2023-09-23T20:16:02.193206+0000","last_fullsized":"2023-09-23T20:16:02.193206+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":463,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.182621+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:16:02.182621+0000","last_peered":"2023-09-23T20:16:02.182621+0000","last_clean":"2023-09-23T20:16:02.182621+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:16:02.182621+0000","last_undegraded":"2023-09-23T20:16:02.182621+0000","last_fullsized":"2023-09-23T20:16:02.182621+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.7","version":"463'7355","reported_seq":7430,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.409653+0000","last_change":"2023-09-23T20:13:46.802046+0000","last_active":"2023-09-23T20:16:13.409653+0000","last_peered":"2023-09-23T20:16:13.409653+0000","last_clean":"2023-09-23T20:16:13.409653+0000","last_became_active":"2023-09-23T20:13:46.800187+0000","last_became_peered":"2023-09-23T20:13:46.800187+0000","last_unstale":"2023-09-23T20:16:13.409653+0000","last_undegraded":"2023-09-23T20:16:13.409653+0000","last_fullsized":"2023-09-23T20:16:13.409653+0000","mapping_epoch":407,"log_start":"413'2480","ondisk_log_start":"413'2480","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4875,"log_dups_size":0,"ondisk_log_size":4875,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58130432,"num_objects":887,"num_object_clones":0,"num_object_copies":2661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5645,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":535,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.188521+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:16:02.188521+0000","last_peered":"2023-09-23T20:16:02.188521+0000","last_clean":"2023-09-23T20:16:02.188521+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:16:02.188521+0000","last_undegraded":"2023-09-23T20:16:02.188521+0000","last_fullsized":"2023-09-23T20:16:02.188521+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":154,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.193104+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:16:02.193104+0000","last_peered":"2023-09-23T20:16:02.193104+0000","last_clean":"2023-09-23T20:16:02.193104+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:16:02.193104+0000","last_undegraded":"2023-09-23T20:16:02.193104+0000","last_fullsized":"2023-09-23T20:16:02.193104+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":51,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.191478+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:16:02.191478+0000","last_peered":"2023-09-23T20:16:02.191478+0000","last_clean":"2023-09-23T20:16:02.191478+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:16:02.191478+0000","last_undegraded":"2023-09-23T20:16:02.191478+0000","last_fullsized":"2023-09-23T20:16:02.191478+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"463'7633","reported_seq":7715,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.319697+0000","last_change":"2023-09-23T20:14:36.190976+0000","last_active":"2023-09-23T20:16:13.319697+0000","last_peered":"2023-09-23T20:16:13.319697+0000","last_clean":"2023-09-23T20:14:33.830609+0000","last_became_active":"2023-09-23T20:14:36.182918+0000","last_became_peered":"2023-09-23T20:14:36.182918+0000","last_unstale":"2023-09-23T20:16:13.319697+0000","last_undegraded":"2023-09-23T20:14:35.924455+0000","last_fullsized":"2023-09-23T20:14:35.847184+0000","mapping_epoch":442,"log_start":"413'2845","ondisk_log_start":"413'2845","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4788,"log_dups_size":0,"ondisk_log_size":4788,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50659328,"num_objects":773,"num_object_clones":0,"num_object_copies":2319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5851,"num_write_kb":211968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":658},{"shards":"2,5","objects":-2424}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":666,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.192945+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:16:02.192945+0000","last_peered":"2023-09-23T20:16:02.192945+0000","last_clean":"2023-09-23T20:16:02.192945+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:16:02.192945+0000","last_undegraded":"2023-09-23T20:16:02.192945+0000","last_fullsized":"2023-09-23T20:16:02.192945+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":511,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.232162+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:16:02.232162+0000","last_peered":"2023-09-23T20:16:02.232162+0000","last_clean":"2023-09-23T20:16:02.232162+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:16:02.232162+0000","last_undegraded":"2023-09-23T20:16:02.232162+0000","last_fullsized":"2023-09-23T20:16:02.232162+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":207,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.234203+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:16:02.234203+0000","last_peered":"2023-09-23T20:16:02.234203+0000","last_clean":"2023-09-23T20:16:02.234203+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:16:02.234203+0000","last_undegraded":"2023-09-23T20:16:02.234203+0000","last_fullsized":"2023-09-23T20:16:02.234203+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"463'5918","reported_seq":6003,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:12.950556+0000","last_change":"2023-09-23T20:14:24.782498+0000","last_active":"2023-09-23T20:16:12.950556+0000","last_peered":"2023-09-23T20:16:12.950556+0000","last_clean":"2023-09-23T20:16:12.950556+0000","last_became_active":"2023-09-23T20:14:24.780750+0000","last_became_peered":"2023-09-23T20:14:24.780750+0000","last_unstale":"2023-09-23T20:16:12.950556+0000","last_undegraded":"2023-09-23T20:16:12.950556+0000","last_fullsized":"2023-09-23T20:16:12.950556+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26836992,"num_objects":409,"num_object_clones":0,"num_object_copies":1227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2823,"num_write_kb":103456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":845,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.191634+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:16:02.191634+0000","last_peered":"2023-09-23T20:16:02.191634+0000","last_clean":"2023-09-23T20:16:02.191634+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:16:02.191634+0000","last_undegraded":"2023-09-23T20:16:02.191634+0000","last_fullsized":"2023-09-23T20:16:02.191634+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":209,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:10.161424+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:16:10.161424+0000","last_peered":"2023-09-23T20:16:10.161424+0000","last_clean":"2023-09-23T20:16:10.161424+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:16:10.161424+0000","last_undegraded":"2023-09-23T20:16:10.161424+0000","last_fullsized":"2023-09-23T20:16:10.161424+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.191154+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:16:02.191154+0000","last_peered":"2023-09-23T20:16:02.191154+0000","last_clean":"2023-09-23T20:16:02.191154+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:16:02.191154+0000","last_undegraded":"2023-09-23T20:16:02.191154+0000","last_fullsized":"2023-09-23T20:16:02.191154+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"463'5966","reported_seq":6059,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:16.041894+0000","last_change":"2023-09-23T20:14:36.074365+0000","last_active":"2023-09-23T20:16:16.041894+0000","last_peered":"2023-09-23T20:16:16.041894+0000","last_clean":"2023-09-23T20:14:33.749328+0000","last_became_active":"2023-09-23T20:14:36.071685+0000","last_became_peered":"2023-09-23T20:14:36.071685+0000","last_unstale":"2023-09-23T20:16:16.041894+0000","last_undegraded":"2023-09-23T20:14:35.849009+0000","last_fullsized":"2023-09-23T20:14:35.834839+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5966,"log_dups_size":0,"ondisk_log_size":5966,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25952256,"num_objects":396,"num_object_clones":0,"num_object_copies":1188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1547,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2884,"num_write_kb":104960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":303},{"shards":"0,2","objects":-1151}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":810,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183724+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:16:02.183724+0000","last_peered":"2023-09-23T20:16:02.183724+0000","last_clean":"2023-09-23T20:16:02.183724+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:16:02.183724+0000","last_undegraded":"2023-09-23T20:16:02.183724+0000","last_fullsized":"2023-09-23T20:16:02.183724+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":31,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183630+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:16:02.183630+0000","last_peered":"2023-09-23T20:16:02.183630+0000","last_clean":"2023-09-23T20:16:02.183630+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:16:02.183630+0000","last_undegraded":"2023-09-23T20:16:02.183630+0000","last_fullsized":"2023-09-23T20:16:02.183630+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":364,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:10.175705+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:16:10.175705+0000","last_peered":"2023-09-23T20:16:10.175705+0000","last_clean":"2023-09-23T20:16:10.175705+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:16:10.175705+0000","last_undegraded":"2023-09-23T20:16:10.175705+0000","last_fullsized":"2023-09-23T20:16:10.175705+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"463'5912","reported_seq":6007,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.031126+0000","last_change":"2023-09-23T20:14:36.109242+0000","last_active":"2023-09-23T20:16:13.031126+0000","last_peered":"2023-09-23T20:16:13.031126+0000","last_clean":"2023-09-23T20:14:33.735119+0000","last_became_active":"2023-09-23T20:14:36.106276+0000","last_became_peered":"2023-09-23T20:14:36.106276+0000","last_unstale":"2023-09-23T20:16:13.031126+0000","last_undegraded":"2023-09-23T20:14:35.867219+0000","last_fullsized":"2023-09-23T20:14:35.838359+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5912,"log_dups_size":0,"ondisk_log_size":5912,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30883840,"num_objects":471,"num_object_clones":0,"num_object_copies":1413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2803,"num_write_kb":104784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":332},{"shards":"3,7","objects":-1027}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":524,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186644+0000","last_change":"2023-09-23T20:14:37.892742+0000","last_active":"2023-09-23T20:16:02.186644+0000","last_peered":"2023-09-23T20:16:02.186644+0000","last_clean":"2023-09-23T20:16:02.186644+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:16:02.186644+0000","last_undegraded":"2023-09-23T20:16:02.186644+0000","last_fullsized":"2023-09-23T20:16:02.186644+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":355,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:10.173612+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:16:10.173612+0000","last_peered":"2023-09-23T20:16:10.173612+0000","last_clean":"2023-09-23T20:16:10.173612+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:16:10.173612+0000","last_undegraded":"2023-09-23T20:16:10.173612+0000","last_fullsized":"2023-09-23T20:16:10.173612+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":31,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186327+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:16:02.186327+0000","last_peered":"2023-09-23T20:16:02.186327+0000","last_clean":"2023-09-23T20:16:02.186327+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:16:02.186327+0000","last_undegraded":"2023-09-23T20:16:02.186327+0000","last_fullsized":"2023-09-23T20:16:02.186327+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"463'6028","reported_seq":9209,"reported_epoch":463,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:16:16.536039+0000","last_change":"2023-09-23T20:16:16.536039+0000","last_active":"2023-09-23T20:16:16.536039+0000","last_peered":"2023-09-23T20:16:16.536039+0000","last_clean":"2023-09-23T20:14:33.798489+0000","last_became_active":"2023-09-23T20:14:35.986955+0000","last_became_peered":"2023-09-23T20:14:35.986955+0000","last_unstale":"2023-09-23T20:16:16.536039+0000","last_undegraded":"2023-09-23T20:16:16.536039+0000","last_fullsized":"2023-09-23T20:14:35.832866+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6028,"log_dups_size":0,"ondisk_log_size":6028,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28311552,"num_objects":432,"num_object_clones":0,"num_object_copies":1296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2892,"num_write_kb":106368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1542,"num_bytes_recovered":34668544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2","7"],"object_location_counts":[{"shards":"2,3","objects":432}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":801,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.188134+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:16:02.188134+0000","last_peered":"2023-09-23T20:16:02.188134+0000","last_clean":"2023-09-23T20:16:02.188134+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:16:02.188134+0000","last_undegraded":"2023-09-23T20:16:02.188134+0000","last_fullsized":"2023-09-23T20:16:02.188134+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":104,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.182441+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:16:02.182441+0000","last_peered":"2023-09-23T20:16:02.182441+0000","last_clean":"2023-09-23T20:16:02.182441+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:16:02.182441+0000","last_undegraded":"2023-09-23T20:16:02.182441+0000","last_fullsized":"2023-09-23T20:16:02.182441+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":207,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183386+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:16:02.183386+0000","last_peered":"2023-09-23T20:16:02.183386+0000","last_clean":"2023-09-23T20:16:02.183386+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:16:02.183386+0000","last_undegraded":"2023-09-23T20:16:02.183386+0000","last_fullsized":"2023-09-23T20:16:02.183386+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"463'8265","reported_seq":8356,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.371353+0000","last_change":"2023-09-23T20:14:36.198248+0000","last_active":"2023-09-23T20:16:13.371353+0000","last_peered":"2023-09-23T20:16:13.371353+0000","last_clean":"2023-09-23T20:14:33.842789+0000","last_became_active":"2023-09-23T20:14:36.169565+0000","last_became_peered":"2023-09-23T20:14:36.169565+0000","last_unstale":"2023-09-23T20:16:13.371353+0000","last_undegraded":"2023-09-23T20:14:36.113647+0000","last_fullsized":"2023-09-23T20:14:35.847403+0000","mapping_epoch":442,"log_start":"412'2259","ondisk_log_start":"412'2259","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56426496,"num_objects":861,"num_object_clones":0,"num_object_copies":2583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5579,"num_write_kb":206080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":668},{"shards":"3,5","objects":-2166}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":745,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183083+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:16:02.183083+0000","last_peered":"2023-09-23T20:16:02.183083+0000","last_clean":"2023-09-23T20:16:02.183083+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:16:02.183083+0000","last_undegraded":"2023-09-23T20:16:02.183083+0000","last_fullsized":"2023-09-23T20:16:02.183083+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":453,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:02.191292+0000","last_change":"2023-09-23T20:14:35.895655+0000","last_active":"2023-09-23T20:16:02.191292+0000","last_peered":"2023-09-23T20:16:02.191292+0000","last_clean":"2023-09-23T20:07:25.847413+0000","last_became_active":"2023-09-23T20:14:35.895204+0000","last_became_peered":"2023-09-23T20:14:35.895204+0000","last_unstale":"2023-09-23T20:16:02.191292+0000","last_undegraded":"2023-09-23T20:14:35.838988+0000","last_fullsized":"2023-09-23T20:14:35.837779+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:17.785522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":463,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186590+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:16:02.186590+0000","last_peered":"2023-09-23T20:16:02.186590+0000","last_clean":"2023-09-23T20:16:02.186590+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:16:02.186590+0000","last_undegraded":"2023-09-23T20:16:02.186590+0000","last_fullsized":"2023-09-23T20:16:02.186590+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":74,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.188051+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:16:02.188051+0000","last_peered":"2023-09-23T20:16:02.188051+0000","last_clean":"2023-09-23T20:16:02.188051+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:16:02.188051+0000","last_undegraded":"2023-09-23T20:16:02.188051+0000","last_fullsized":"2023-09-23T20:16:02.188051+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"463'7534","reported_seq":7609,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.044001+0000","last_change":"2023-09-23T20:13:46.785589+0000","last_active":"2023-09-23T20:16:13.044001+0000","last_peered":"2023-09-23T20:16:13.044001+0000","last_clean":"2023-09-23T20:16:13.044001+0000","last_became_active":"2023-09-23T20:13:46.781975+0000","last_became_peered":"2023-09-23T20:13:46.781975+0000","last_unstale":"2023-09-23T20:16:13.044001+0000","last_undegraded":"2023-09-23T20:16:13.044001+0000","last_fullsized":"2023-09-23T20:16:13.044001+0000","mapping_epoch":407,"log_start":"415'3107","ondisk_log_start":"415'3107","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4427,"log_dups_size":0,"ondisk_log_size":4427,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53772288,"num_objects":820,"num_object_clones":0,"num_object_copies":2460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5778,"num_write_kb":211168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":806,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.184661+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:16:02.184661+0000","last_peered":"2023-09-23T20:16:02.184661+0000","last_clean":"2023-09-23T20:16:02.184661+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:16:02.184661+0000","last_undegraded":"2023-09-23T20:16:02.184661+0000","last_fullsized":"2023-09-23T20:16:02.184661+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1113,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186307+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:16:02.186307+0000","last_peered":"2023-09-23T20:16:02.186307+0000","last_clean":"2023-09-23T20:16:02.186307+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:16:02.186307+0000","last_undegraded":"2023-09-23T20:16:02.186307+0000","last_fullsized":"2023-09-23T20:16:02.186307+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186140+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:16:02.186140+0000","last_peered":"2023-09-23T20:16:02.186140+0000","last_clean":"2023-09-23T20:16:02.186140+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:16:02.186140+0000","last_undegraded":"2023-09-23T20:16:02.186140+0000","last_fullsized":"2023-09-23T20:16:02.186140+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":325,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.238075+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:16:02.238075+0000","last_peered":"2023-09-23T20:16:02.238075+0000","last_clean":"2023-09-23T20:16:02.238075+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:16:02.238075+0000","last_undegraded":"2023-09-23T20:16:02.238075+0000","last_fullsized":"2023-09-23T20:16:02.238075+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"463'8343","reported_seq":8431,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.406718+0000","last_change":"2023-09-23T20:14:36.228252+0000","last_active":"2023-09-23T20:16:13.406718+0000","last_peered":"2023-09-23T20:16:13.406718+0000","last_clean":"2023-09-23T20:14:33.823351+0000","last_became_active":"2023-09-23T20:14:36.216536+0000","last_became_peered":"2023-09-23T20:14:36.216536+0000","last_unstale":"2023-09-23T20:16:13.406718+0000","last_undegraded":"2023-09-23T20:14:36.114735+0000","last_fullsized":"2023-09-23T20:14:35.945404+0000","mapping_epoch":442,"log_start":"413'2571","ondisk_log_start":"413'2571","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5772,"log_dups_size":0,"ondisk_log_size":5772,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3085,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5645,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":685},{"shards":"5,7","objects":-2240}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":701,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.191390+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:16:02.191390+0000","last_peered":"2023-09-23T20:16:02.191390+0000","last_clean":"2023-09-23T20:16:02.191390+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:16:02.191390+0000","last_undegraded":"2023-09-23T20:16:02.191390+0000","last_fullsized":"2023-09-23T20:16:02.191390+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1101,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.236221+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:16:02.236221+0000","last_peered":"2023-09-23T20:16:02.236221+0000","last_clean":"2023-09-23T20:16:02.236221+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:16:02.236221+0000","last_undegraded":"2023-09-23T20:16:02.236221+0000","last_fullsized":"2023-09-23T20:16:02.236221+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":161,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.236475+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:16:02.236475+0000","last_peered":"2023-09-23T20:16:02.236475+0000","last_clean":"2023-09-23T20:16:02.236475+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:16:02.236475+0000","last_undegraded":"2023-09-23T20:16:02.236475+0000","last_fullsized":"2023-09-23T20:16:02.236475+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":128,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.189303+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:16:02.189303+0000","last_peered":"2023-09-23T20:16:02.189303+0000","last_clean":"2023-09-23T20:16:02.189303+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:16:02.189303+0000","last_undegraded":"2023-09-23T20:16:02.189303+0000","last_fullsized":"2023-09-23T20:16:02.189303+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"463'7294","reported_seq":7369,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.413213+0000","last_change":"2023-09-23T20:13:46.814034+0000","last_active":"2023-09-23T20:16:13.413213+0000","last_peered":"2023-09-23T20:16:13.413213+0000","last_clean":"2023-09-23T20:16:13.413213+0000","last_became_active":"2023-09-23T20:13:46.812300+0000","last_became_peered":"2023-09-23T20:13:46.812300+0000","last_unstale":"2023-09-23T20:16:13.413213+0000","last_undegraded":"2023-09-23T20:16:13.413213+0000","last_fullsized":"2023-09-23T20:16:13.413213+0000","mapping_epoch":407,"log_start":"412'2354","ondisk_log_start":"412'2354","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4940,"log_dups_size":0,"ondisk_log_size":4940,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55869440,"num_objects":852,"num_object_clones":0,"num_object_copies":2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5578,"num_write_kb":205792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1371,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186365+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:16:02.186365+0000","last_peered":"2023-09-23T20:16:02.186365+0000","last_clean":"2023-09-23T20:16:02.186365+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:16:02.186365+0000","last_undegraded":"2023-09-23T20:16:02.186365+0000","last_fullsized":"2023-09-23T20:16:02.186365+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":658,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.190933+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:16:02.190933+0000","last_peered":"2023-09-23T20:16:02.190933+0000","last_clean":"2023-09-23T20:16:02.190933+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:16:02.190933+0000","last_undegraded":"2023-09-23T20:16:02.190933+0000","last_fullsized":"2023-09-23T20:16:02.190933+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":328,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.187966+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:16:02.187966+0000","last_peered":"2023-09-23T20:16:02.187966+0000","last_clean":"2023-09-23T20:16:02.187966+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:16:02.187966+0000","last_undegraded":"2023-09-23T20:16:02.187966+0000","last_fullsized":"2023-09-23T20:16:02.187966+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":260,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.192962+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:16:02.192962+0000","last_peered":"2023-09-23T20:16:02.192962+0000","last_clean":"2023-09-23T20:16:02.192962+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:16:02.192962+0000","last_undegraded":"2023-09-23T20:16:02.192962+0000","last_fullsized":"2023-09-23T20:16:02.192962+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"463'8105","reported_seq":8180,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.231012+0000","last_change":"2023-09-23T20:14:24.789921+0000","last_active":"2023-09-23T20:16:13.231012+0000","last_peered":"2023-09-23T20:16:13.231012+0000","last_clean":"2023-09-23T20:16:13.231012+0000","last_became_active":"2023-09-23T20:14:24.787385+0000","last_became_peered":"2023-09-23T20:14:24.787385+0000","last_unstale":"2023-09-23T20:16:13.231012+0000","last_undegraded":"2023-09-23T20:16:13.231012+0000","last_fullsized":"2023-09-23T20:16:13.231012+0000","mapping_epoch":407,"log_start":"412'2212","ondisk_log_start":"412'2212","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5893,"log_dups_size":0,"ondisk_log_size":5893,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55934976,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5493,"num_write_kb":203104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":524,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186487+0000","last_change":"2023-09-23T20:14:37.894544+0000","last_active":"2023-09-23T20:16:02.186487+0000","last_peered":"2023-09-23T20:16:02.186487+0000","last_clean":"2023-09-23T20:16:02.186487+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:16:02.186487+0000","last_undegraded":"2023-09-23T20:16:02.186487+0000","last_fullsized":"2023-09-23T20:16:02.186487+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":909,"reported_epoch":462,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:02.190700+0000","last_change":"2023-09-23T20:14:35.882819+0000","last_active":"2023-09-23T20:16:02.190700+0000","last_peered":"2023-09-23T20:16:02.190700+0000","last_clean":"2023-09-23T20:14:31.826679+0000","last_became_active":"2023-09-23T20:14:35.882372+0000","last_became_peered":"2023-09-23T20:14:35.882372+0000","last_unstale":"2023-09-23T20:16:02.190700+0000","last_undegraded":"2023-09-23T20:14:35.846466+0000","last_fullsized":"2023-09-23T20:14:35.837669+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:41.083298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":409,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:06.568366+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:16:06.568366+0000","last_peered":"2023-09-23T20:16:06.568366+0000","last_clean":"2023-09-23T20:16:06.568366+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:16:06.568366+0000","last_undegraded":"2023-09-23T20:16:06.568366+0000","last_fullsized":"2023-09-23T20:16:06.568366+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":278,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.238156+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:16:02.238156+0000","last_peered":"2023-09-23T20:16:02.238156+0000","last_clean":"2023-09-23T20:16:02.238156+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:16:02.238156+0000","last_undegraded":"2023-09-23T20:16:02.238156+0000","last_fullsized":"2023-09-23T20:16:02.238156+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"463'8348","reported_seq":8432,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:12.976914+0000","last_change":"2023-09-23T20:14:36.189421+0000","last_active":"2023-09-23T20:16:12.976914+0000","last_peered":"2023-09-23T20:16:12.976914+0000","last_clean":"2023-09-23T20:14:33.834923+0000","last_became_active":"2023-09-23T20:14:36.182947+0000","last_became_peered":"2023-09-23T20:14:36.182947+0000","last_unstale":"2023-09-23T20:16:12.976914+0000","last_undegraded":"2023-09-23T20:14:35.901626+0000","last_fullsized":"2023-09-23T20:14:35.838743+0000","mapping_epoch":442,"log_start":"414'2932","ondisk_log_start":"414'2932","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5416,"log_dups_size":0,"ondisk_log_size":5416,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57049088,"num_objects":870,"num_object_clones":0,"num_object_copies":2610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5648,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":672},{"shards":"4,7","objects":-2192}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":636,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:08.175008+0000","last_change":"2023-09-23T20:13:57.748021+0000","last_active":"2023-09-23T20:16:08.175008+0000","last_peered":"2023-09-23T20:16:08.175008+0000","last_clean":"2023-09-23T20:16:08.175008+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:16:08.175008+0000","last_undegraded":"2023-09-23T20:16:08.175008+0000","last_fullsized":"2023-09-23T20:16:08.175008+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":163,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.193187+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:16:02.193187+0000","last_peered":"2023-09-23T20:16:02.193187+0000","last_clean":"2023-09-23T20:16:02.193187+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:16:02.193187+0000","last_undegraded":"2023-09-23T20:16:02.193187+0000","last_fullsized":"2023-09-23T20:16:02.193187+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":266,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.238215+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:16:02.238215+0000","last_peered":"2023-09-23T20:16:02.238215+0000","last_clean":"2023-09-23T20:16:02.238215+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:16:02.238215+0000","last_undegraded":"2023-09-23T20:16:02.238215+0000","last_fullsized":"2023-09-23T20:16:02.238215+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":289,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186225+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:16:02.186225+0000","last_peered":"2023-09-23T20:16:02.186225+0000","last_clean":"2023-09-23T20:16:02.186225+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:16:02.186225+0000","last_undegraded":"2023-09-23T20:16:02.186225+0000","last_fullsized":"2023-09-23T20:16:02.186225+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"463'8608","reported_seq":8692,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.009397+0000","last_change":"2023-09-23T20:14:36.072073+0000","last_active":"2023-09-23T20:16:13.009397+0000","last_peered":"2023-09-23T20:16:13.009397+0000","last_clean":"2023-09-23T20:14:33.811030+0000","last_became_active":"2023-09-23T20:14:36.065847+0000","last_became_peered":"2023-09-23T20:14:36.065847+0000","last_unstale":"2023-09-23T20:16:13.009397+0000","last_undegraded":"2023-09-23T20:14:35.915006+0000","last_fullsized":"2023-09-23T20:14:35.838163+0000","mapping_epoch":442,"log_start":"415'3223","ondisk_log_start":"415'3223","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5385,"log_dups_size":0,"ondisk_log_size":5385,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57344000,"num_objects":875,"num_object_clones":0,"num_object_copies":2625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3193,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5819,"num_write_kb":214208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":721},{"shards":"5,7","objects":-2318}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":682,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:06.139784+0000","last_change":"2023-09-23T20:14:25.790696+0000","last_active":"2023-09-23T20:16:06.139784+0000","last_peered":"2023-09-23T20:16:06.139784+0000","last_clean":"2023-09-23T20:16:06.139784+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:16:06.139784+0000","last_undegraded":"2023-09-23T20:16:06.139784+0000","last_fullsized":"2023-09-23T20:16:06.139784+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":305,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.195172+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:16:02.195172+0000","last_peered":"2023-09-23T20:16:02.195172+0000","last_clean":"2023-09-23T20:16:02.195172+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:16:02.195172+0000","last_undegraded":"2023-09-23T20:16:02.195172+0000","last_fullsized":"2023-09-23T20:16:02.195172+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":620,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.240156+0000","last_change":"2023-09-23T20:16:02.235622+0000","last_active":"2023-09-23T20:16:02.240156+0000","last_peered":"2023-09-23T20:16:02.240156+0000","last_clean":"2023-09-23T20:16:02.240156+0000","last_became_active":"2023-09-23T20:16:02.234933+0000","last_became_peered":"2023-09-23T20:16:02.234933+0000","last_unstale":"2023-09-23T20:16:02.240156+0000","last_undegraded":"2023-09-23T20:16:02.240156+0000","last_fullsized":"2023-09-23T20:16:02.240156+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:21.855632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":95,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.182447+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:16:02.182447+0000","last_peered":"2023-09-23T20:16:02.182447+0000","last_clean":"2023-09-23T20:16:02.182447+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:16:02.182447+0000","last_undegraded":"2023-09-23T20:16:02.182447+0000","last_fullsized":"2023-09-23T20:16:02.182447+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"463'8147","reported_seq":8223,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:16.533672+0000","last_change":"2023-09-23T20:14:25.775099+0000","last_active":"2023-09-23T20:16:16.533672+0000","last_peered":"2023-09-23T20:16:16.533672+0000","last_clean":"2023-09-23T20:16:16.533672+0000","last_became_active":"2023-09-23T20:14:24.777506+0000","last_became_peered":"2023-09-23T20:14:24.777506+0000","last_unstale":"2023-09-23T20:16:16.533672+0000","last_undegraded":"2023-09-23T20:16:16.533672+0000","last_fullsized":"2023-09-23T20:16:16.533672+0000","mapping_epoch":407,"log_start":"415'3010","ondisk_log_start":"415'3010","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5137,"log_dups_size":0,"ondisk_log_size":5137,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55148544,"num_objects":841,"num_object_clones":0,"num_object_copies":2523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5527,"num_write_kb":203808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":788,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183723+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:16:02.183723+0000","last_peered":"2023-09-23T20:16:02.183723+0000","last_clean":"2023-09-23T20:16:02.183723+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:16:02.183723+0000","last_undegraded":"2023-09-23T20:16:02.183723+0000","last_fullsized":"2023-09-23T20:16:02.183723+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":164,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.195237+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:16:02.195237+0000","last_peered":"2023-09-23T20:16:02.195237+0000","last_clean":"2023-09-23T20:16:02.195237+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:16:02.195237+0000","last_undegraded":"2023-09-23T20:16:02.195237+0000","last_fullsized":"2023-09-23T20:16:02.195237+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":35,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.191262+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:16:02.191262+0000","last_peered":"2023-09-23T20:16:02.191262+0000","last_clean":"2023-09-23T20:16:02.191262+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:16:02.191262+0000","last_undegraded":"2023-09-23T20:16:02.191262+0000","last_fullsized":"2023-09-23T20:16:02.191262+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":246,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.184774+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:16:02.184774+0000","last_peered":"2023-09-23T20:16:02.184774+0000","last_clean":"2023-09-23T20:16:02.184774+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:16:02.184774+0000","last_undegraded":"2023-09-23T20:16:02.184774+0000","last_fullsized":"2023-09-23T20:16:02.184774+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"463'7429","reported_seq":11652,"reported_epoch":463,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:16:12.881193+0000","last_change":"2023-09-23T20:14:35.970620+0000","last_active":"2023-09-23T20:16:12.881193+0000","last_peered":"2023-09-23T20:16:12.881193+0000","last_clean":"2023-09-23T20:14:33.799110+0000","last_became_active":"2023-09-23T20:14:35.967502+0000","last_became_peered":"2023-09-23T20:14:35.967502+0000","last_unstale":"2023-09-23T20:16:12.881193+0000","last_undegraded":"2023-09-23T20:16:12.881193+0000","last_fullsized":"2023-09-23T20:16:12.881193+0000","mapping_epoch":442,"log_start":"413'2923","ondisk_log_start":"413'2923","created":418,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4506,"log_dups_size":0,"ondisk_log_size":4506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56098816,"num_objects":856,"num_object_clones":0,"num_object_copies":2568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":856,"num_objects_unfound":0,"num_objects_dirty":856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5684,"num_write_kb":209280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2059,"num_bytes_recovered":134938624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":856}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":240,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:10.175203+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:16:10.175203+0000","last_peered":"2023-09-23T20:16:10.175203+0000","last_clean":"2023-09-23T20:16:10.175203+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:16:10.175203+0000","last_undegraded":"2023-09-23T20:16:10.175203+0000","last_fullsized":"2023-09-23T20:16:10.175203+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":240,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.238196+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:16:02.238196+0000","last_peered":"2023-09-23T20:16:02.238196+0000","last_clean":"2023-09-23T20:16:02.238196+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:16:02.238196+0000","last_undegraded":"2023-09-23T20:16:02.238196+0000","last_fullsized":"2023-09-23T20:16:02.238196+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"463'7507","reported_seq":7589,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.387875+0000","last_change":"2023-09-23T20:14:36.202358+0000","last_active":"2023-09-23T20:16:13.387875+0000","last_peered":"2023-09-23T20:16:13.387875+0000","last_clean":"2023-09-23T20:14:33.783493+0000","last_became_active":"2023-09-23T20:14:36.193525+0000","last_became_peered":"2023-09-23T20:14:36.193525+0000","last_unstale":"2023-09-23T20:16:13.387875+0000","last_undegraded":"2023-09-23T20:14:36.034471+0000","last_fullsized":"2023-09-23T20:14:35.969590+0000","mapping_epoch":442,"log_start":"413'2659","ondisk_log_start":"413'2659","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61407232,"num_objects":937,"num_object_clones":0,"num_object_copies":2811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5725,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":650},{"shards":"2,5","objects":-2107}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":480,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.232161+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:16:02.232161+0000","last_peered":"2023-09-23T20:16:02.232161+0000","last_clean":"2023-09-23T20:16:02.232161+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:16:02.232161+0000","last_undegraded":"2023-09-23T20:16:02.232161+0000","last_fullsized":"2023-09-23T20:16:02.232161+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":482,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.183214+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:16:02.183214+0000","last_peered":"2023-09-23T20:16:02.183214+0000","last_clean":"2023-09-23T20:16:02.183214+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:16:02.183214+0000","last_undegraded":"2023-09-23T20:16:02.183214+0000","last_fullsized":"2023-09-23T20:16:02.183214+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.17","version":"463'7379","reported_seq":7453,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.386041+0000","last_change":"2023-09-23T20:13:46.814598+0000","last_active":"2023-09-23T20:16:13.386041+0000","last_peered":"2023-09-23T20:16:13.386041+0000","last_clean":"2023-09-23T20:16:13.386041+0000","last_became_active":"2023-09-23T20:13:46.812685+0000","last_became_peered":"2023-09-23T20:13:46.812685+0000","last_unstale":"2023-09-23T20:16:13.386041+0000","last_undegraded":"2023-09-23T20:16:13.386041+0000","last_fullsized":"2023-09-23T20:16:13.386041+0000","mapping_epoch":418,"log_start":"413'2500","ondisk_log_start":"413'2500","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4879,"log_dups_size":0,"ondisk_log_size":4879,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50659328,"num_objects":773,"num_object_clones":0,"num_object_copies":2319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5669,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":470,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.186314+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:16:02.186314+0000","last_peered":"2023-09-23T20:16:02.186314+0000","last_clean":"2023-09-23T20:16:02.186314+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:16:02.186314+0000","last_undegraded":"2023-09-23T20:16:02.186314+0000","last_fullsized":"2023-09-23T20:16:02.186314+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":459,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.181951+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:16:02.181951+0000","last_peered":"2023-09-23T20:16:02.181951+0000","last_clean":"2023-09-23T20:16:02.181951+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:16:02.181951+0000","last_undegraded":"2023-09-23T20:16:02.181951+0000","last_fullsized":"2023-09-23T20:16:02.181951+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"463'7385","reported_seq":13573,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:13.426487+0000","last_change":"2023-09-23T20:16:02.232381+0000","last_active":"2023-09-23T20:16:13.426487+0000","last_peered":"2023-09-23T20:16:13.426487+0000","last_clean":"2023-09-23T20:16:13.426487+0000","last_became_active":"2023-09-23T20:16:02.185816+0000","last_became_peered":"2023-09-23T20:16:02.185816+0000","last_unstale":"2023-09-23T20:16:13.426487+0000","last_undegraded":"2023-09-23T20:16:13.426487+0000","last_fullsized":"2023-09-23T20:16:13.426487+0000","mapping_epoch":461,"log_start":"413'2529","ondisk_log_start":"413'2529","created":418,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4856,"log_dups_size":0,"ondisk_log_size":4856,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55083008,"num_objects":841,"num_object_clones":0,"num_object_copies":2523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5661,"num_write_kb":208032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3047,"num_bytes_recovered":134610944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":175,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T20:16:10.175360+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:16:10.175360+0000","last_peered":"2023-09-23T20:16:10.175360+0000","last_clean":"2023-09-23T20:16:10.175360+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:16:10.175360+0000","last_undegraded":"2023-09-23T20:16:10.175360+0000","last_fullsized":"2023-09-23T20:16:10.175360+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":322,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.236296+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:16:02.236296+0000","last_peered":"2023-09-23T20:16:02.236296+0000","last_clean":"2023-09-23T20:16:02.236296+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:16:02.236296+0000","last_undegraded":"2023-09-23T20:16:02.236296+0000","last_fullsized":"2023-09-23T20:16:02.236296+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"463'7311","reported_seq":14521,"reported_epoch":463,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:13.009874+0000","last_change":"2023-09-23T20:14:38.010024+0000","last_active":"2023-09-23T20:16:13.009874+0000","last_peered":"2023-09-23T20:16:13.009874+0000","last_clean":"2023-09-23T20:14:33.867233+0000","last_became_active":"2023-09-23T20:14:38.004874+0000","last_became_peered":"2023-09-23T20:14:38.004874+0000","last_unstale":"2023-09-23T20:16:13.009874+0000","last_undegraded":"2023-09-23T20:14:37.959307+0000","last_fullsized":"2023-09-23T20:14:37.876450+0000","mapping_epoch":444,"log_start":"413'2736","ondisk_log_start":"413'2736","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4575,"log_dups_size":0,"ondisk_log_size":4575,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54329344,"num_objects":829,"num_object_clones":0,"num_object_copies":2487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5581,"num_write_kb":205120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2376,"num_bytes_recovered":155713536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":646},{"shards":"3,7","objects":183}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":308,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.189171+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:16:02.189171+0000","last_peered":"2023-09-23T20:16:02.189171+0000","last_clean":"2023-09-23T20:16:02.189171+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:16:02.189171+0000","last_undegraded":"2023-09-23T20:16:02.189171+0000","last_fullsized":"2023-09-23T20:16:02.189171+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":32,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T20:16:02.236043+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:16:02.236043+0000","last_peered":"2023-09-23T20:16:02.236043+0000","last_clean":"2023-09-23T20:16:02.236043+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:16:02.236043+0000","last_undegraded":"2023-09-23T20:16:02.236043+0000","last_fullsized":"2023-09-23T20:16:02.236043+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":36,"stat_sum":{"num_bytes":1769340952,"num_objects":26999,"num_object_clones":0,"num_object_copies":80997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38437,"num_objects_misplaced":2073,"num_objects_unfound":0,"num_objects_dirty":26999,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":180707,"num_write_kb":6646529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19132,"num_bytes_recovered":1122369536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8561242112,"data_stored":8561229896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":192446,"ondisk_log_size":192446,"up":108,"acting":90,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":191,"num_bytes_recovered":4194380,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234421,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1592320,"kb_used_data":1099000,"kb_used_omap":0,"kb_used_meta":493056,"kb_avail":92152832,"statfs":{"total":95995035648,"available":94364499968,"internally_reserved":0,"allocated":1125376000,"data_stored":1123956822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":504889344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":11.353,"5min":3.855,"15min":2.8380000000000001},"min":{"1min":0.219,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":149.25999999999999,"5min":149.25999999999999,"15min":149.25999999999999},"last":0.86099999999999999},{"interface":"front","average":{"1min":12.917,"5min":4.3019999999999996,"15min":3.0609999999999999},"min":{"1min":0.248,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":149.11799999999999,"5min":149.11799999999999,"15min":149.11799999999999},"last":2.1000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":3.2250000000000001,"15min":3.5779999999999998},"min":{"1min":0.29699999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.0599999999999996,"5min":91.335999999999999,"15min":104.16800000000001},"last":0.79900000000000004},{"interface":"front","average":{"1min":1.972,"5min":3.738,"15min":3.8370000000000002},"min":{"1min":0.29599999999999999,"5min":0.249,"15min":0.223},"max":{"1min":6.8609999999999998,"5min":91.516999999999996,"15min":104.003},"last":2.137}]},{"osd":2,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":3.3969999999999998,"15min":3.7629999999999999},"min":{"1min":0.32700000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":7.3600000000000003,"5min":67.760000000000005,"15min":105.842},"last":0.83499999999999996},{"interface":"front","average":{"1min":2.633,"5min":3.738,"15min":4.1950000000000003},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":8.6669999999999998,"5min":67.488,"15min":105.66},"last":2.1669999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":14.476000000000001,"5min":4.5030000000000001,"15min":4.6440000000000001},"min":{"1min":0.33400000000000002,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":185.17099999999999,"5min":185.17099999999999,"15min":185.17099999999999},"last":0.88200000000000001},{"interface":"front","average":{"1min":15.061999999999999,"5min":4.593,"15min":4.649},"min":{"1min":0.63,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":184.99100000000001,"5min":184.99100000000001,"15min":184.99100000000001},"last":2.246}]},{"osd":4,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":0.79200000000000004,"15min":0.71399999999999997},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.22500000000000001},"max":{"1min":6.2569999999999997,"5min":6.2569999999999997,"15min":19.215},"last":3.4079999999999999},{"interface":"front","average":{"1min":2.4009999999999998,"5min":0.98399999999999999,"15min":0.90700000000000003},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":10.882999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":1.4239999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.2450000000000001,"15min":1.105},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":14.069000000000001,"5min":14.069000000000001,"15min":35.488},"last":3.1230000000000002},{"interface":"front","average":{"1min":2.6469999999999998,"5min":1.103,"15min":0.97999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":13.43,"5min":13.43,"15min":34.896000000000001},"last":1.6519999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":1.234,"15min":0.99199999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":8.4120000000000008,"5min":8.4120000000000008,"15min":41.902999999999999},"last":2.0449999999999999},{"interface":"front","average":{"1min":1.522,"5min":0.93700000000000006,"15min":0.754},"min":{"1min":0.36699999999999999,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":7.5659999999999998,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":1.702}]}]},{"osd":6,"up_from":0,"seq":1078036791462,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762620,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926157,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1711264,"kb_used_data":947224,"kb_used_omap":8,"kb_used_meta":763959,"kb_avail":92033888,"statfs":{"total":95995035648,"available":94242701312,"internally_reserved":0,"allocated":969957376,"data_stored":967810598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":782294625},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":1.952,"15min":3.4940000000000002},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":29.893000000000001,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.23799999999999999},{"interface":"front","average":{"1min":1.97,"5min":2.4169999999999998,"15min":3.7130000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.183},"max":{"1min":23.436,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.40699999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:15:16 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.9359999999999999,"15min":5.0519999999999996},"min":{"1min":0.215,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":3.5920000000000001,"5min":160.542,"15min":270.51100000000002},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.5720000000000001,"5min":3.4990000000000001,"15min":4.9820000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":7.5220000000000002,"5min":160.75999999999999,"15min":269.69299999999998},"last":0.32300000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4279999999999999,"15min":3.8980000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":8.9469999999999992,"5min":10.082000000000001,"15min":126.95399999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.532,"15min":4.1020000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":7.375,"5min":9.5440000000000005,"15min":127.294},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":1.4790000000000001,"15min":2.0499999999999998},"min":{"1min":0.33500000000000002,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":8.7360000000000007,"5min":9.6959999999999997,"15min":148.20599999999999},"last":3.1589999999999998},{"interface":"front","average":{"1min":1.734,"5min":1.3140000000000001,"15min":1.994},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.8230000000000004,"5min":8.4429999999999996,"15min":148.11000000000001},"last":2.1499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.7609999999999999,"15min":1.8759999999999999},"min":{"1min":0.35999999999999999,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":6.8739999999999997,"5min":52.061,"15min":64.725999999999999},"last":4.9470000000000001},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.2450000000000001,"15min":1.899},"min":{"1min":0.312,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1500000000000004,"5min":51.741,"15min":51.741},"last":3.0270000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":1.5309999999999999,"15min":1.653},"min":{"1min":0.38,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":7.1950000000000003,"5min":7.1950000000000003,"15min":15.744999999999999},"last":3.133},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.3400000000000001,"15min":1.6839999999999999},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":7.2229999999999999,"5min":8.3919999999999995,"15min":13.766999999999999},"last":9.8219999999999992}]},{"osd":7,"last update":"Sat Sep 23 20:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.2969999999999999,"15min":1.7490000000000001},"min":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.7400000000000002,"5min":12.981,"15min":65.081999999999994},"last":3.8900000000000001},{"interface":"front","average":{"1min":2.5249999999999999,"5min":1.861,"15min":1.8620000000000001},"min":{"1min":0.442,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.2290000000000001,"5min":17.151,"15min":17.151},"last":2.9940000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599096,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1288276,"kb_used_data":467148,"kb_used_omap":12,"kb_used_meta":821107,"kb_avail":92456876,"statfs":{"total":95995035648,"available":94675841024,"internally_reserved":0,"allocated":478359552,"data_stored":476169200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":840814590},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":2.0670000000000002,"15min":3.0339999999999998},"min":{"1min":0.253,"5min":0.221,"15min":0.192},"max":{"1min":5.923,"5min":65.382999999999996,"15min":76.251999999999995},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.7230000000000001,"5min":1.994,"15min":3.081},"min":{"1min":0.26600000000000001,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":5.694,"5min":65.427000000000007,"15min":76.590000000000003},"last":8.8179999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.9020000000000001,"15min":3.8250000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":9.2590000000000003,"5min":70.453000000000003,"15min":94.768000000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.8149999999999999,"5min":2.7200000000000002,"15min":3.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.21099999999999999},"max":{"1min":11.981999999999999,"5min":69.825000000000003,"15min":96.808000000000007},"last":0.69399999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:15:40 2023","interfaces":[{"interface":"back","average":{"1min":7.5960000000000001,"5min":2.9870000000000001,"15min":4.1130000000000004},"min":{"1min":0.222,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":132.82499999999999,"5min":132.82499999999999,"15min":132.82499999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":8.1519999999999992,"5min":2.9900000000000002,"15min":3.411},"min":{"1min":0.23400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":130.97300000000001,"5min":130.97300000000001,"15min":130.97300000000001},"last":0.433}]},{"osd":4,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.272,"15min":1.964},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.7270000000000003,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.599,"15min":2.1779999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26700000000000002},"max":{"1min":9.1590000000000007,"5min":13.387,"15min":76.161000000000001},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":3.1360000000000001,"15min":3.0310000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.956,"5min":135.202,"15min":172.505},"last":0.66400000000000003},{"interface":"front","average":{"1min":2.0350000000000001,"5min":1.966,"15min":2.7679999999999998},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32300000000000001},"max":{"1min":12.050000000000001,"5min":29.196999999999999,"15min":172.375},"last":1.4650000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.4930000000000001,"15min":1.5920000000000001},"min":{"1min":0.435,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.0510000000000002,"5min":14.664,"15min":14.664},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.758,"15min":1.8400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.2649999999999997,"5min":12.025,"15min":14.279999999999999},"last":1.51}]},{"osd":7,"last update":"Sat Sep 23 20:15:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.7849999999999999,"15min":1.9159999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2050000000000001,"5min":13.481999999999999,"15min":23.079999999999998},"last":0.495},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.3859999999999999,"15min":1.7609999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.7789999999999999,"5min":14.698,"15min":20.614999999999998},"last":0.626}]}]},{"osd":3,"up_from":109,"seq":468151435570,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1883672,"kb_used_data":1018960,"kb_used_omap":4,"kb_used_meta":864635,"kb_avail":91861480,"statfs":{"total":95995035648,"available":94066155520,"internally_reserved":0,"allocated":1043415040,"data_stored":1040978604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":885386349},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5940000000000003,"5min":1.8560000000000001,"15min":3.0859999999999999},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":76.632000000000005,"5min":76.632000000000005,"15min":76.632000000000005},"last":0.19600000000000001},{"interface":"front","average":{"1min":5.4580000000000002,"5min":2.266,"15min":3.4159999999999999},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.19600000000000001},"max":{"1min":81.429000000000002,"5min":81.429000000000002,"15min":81.429000000000002},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":5.0510000000000002,"15min":4.6260000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":9,"5min":183.87,"15min":183.87},"last":0.23100000000000001},{"interface":"front","average":{"1min":1.2230000000000001,"5min":4.9740000000000002,"15min":4.758},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":5.5810000000000004,"5min":185.84899999999999,"15min":185.84899999999999},"last":0.251}]},{"osd":2,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":4.2610000000000001,"15min":4.109},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":9.0289999999999999,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.2679999999999998,"5min":3.6669999999999998,"15min":3.819},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":16.550999999999998,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.1120000000000001,"15min":2.786},"min":{"1min":0.32400000000000001,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":5.21,"5min":7.4960000000000004,"15min":109.66800000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6619999999999999,"15min":3.2469999999999999},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":9.6470000000000002,"5min":11.815,"15min":110.295},"last":4.9800000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":3.6349999999999998,"15min":3.9129999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6799999999999997,"5min":200.53200000000001,"15min":200.53200000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":3.411,"5min":3.7599999999999998,"15min":3.9769999999999999},"min":{"1min":0.48799999999999999,"5min":0.34799999999999998,"15min":0.26900000000000002},"max":{"1min":9.9429999999999996,"5min":200.846,"15min":200.846},"last":0.95899999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.6539999999999999,"15min":2.8660000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.32300000000000001},"max":{"1min":8.9610000000000003,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.641,"5min":1.2669999999999999,"15min":2.9020000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":5.8959999999999999,"5min":8.2629999999999999,"15min":82.233000000000004},"last":0.40400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.9470000000000001,"15min":3.3969999999999998},"min":{"1min":0.42799999999999999,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":9.0600000000000005,"5min":139.465,"15min":139.465},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.8540000000000001,"15min":3.27},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":5.3010000000000002,"5min":141.56200000000001,"15min":141.56200000000001},"last":5.4059999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019331,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1026492,"kb_used_data":581300,"kb_used_omap":0,"kb_used_meta":445184,"kb_avail":92718660,"statfs":{"total":95995035648,"available":94943907840,"internally_reserved":0,"allocated":595251200,"data_stored":593836106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":455868416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:25 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":1.2390000000000001,"15min":2.1720000000000002},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.9269999999999996,"5min":14.063000000000001,"15min":89.004999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.831,"5min":1.268,"15min":2.169},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":10.289,"5min":14.048999999999999,"15min":89.024000000000001},"last":1.7669999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:15:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":3.395,"15min":5.9109999999999996},"min":{"1min":0.27800000000000002,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":5.8259999999999996,"5min":138.47999999999999,"15min":161.613},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.6070000000000002,"5min":3.46,"15min":5.9889999999999999},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.217},"max":{"1min":10.368,"5min":138.429,"15min":162.017},"last":1.841}]},{"osd":2,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":3.7320000000000002,"15min":5.6970000000000001},"min":{"1min":0.30499999999999999,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.8570000000000002,"5min":144.916,"15min":182.96700000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.4020000000000001,"5min":3.694,"15min":5.6100000000000003},"min":{"1min":0.40300000000000002,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":11.528,"5min":144.83099999999999,"15min":183.244},"last":1.8129999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:15:31 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":3.0459999999999998,"15min":5.1020000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":15.707000000000001,"5min":100.361,"15min":184.37100000000001},"last":1.4450000000000001},{"interface":"front","average":{"1min":2.9700000000000002,"5min":3.3420000000000001,"15min":5.1900000000000004},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.28599999999999998},"max":{"1min":15.73,"5min":100.20699999999999,"15min":184.22300000000001},"last":1.337}]},{"osd":5,"last update":"Sat Sep 23 20:15:25 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":0.84099999999999997,"15min":1.177},"min":{"1min":0.26400000000000001,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":12.257999999999999,"5min":12.257999999999999,"15min":176.81899999999999},"last":6.5789999999999997},{"interface":"front","average":{"1min":2.1139999999999999,"5min":0.84199999999999997,"15min":1.2330000000000001},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":6.9770000000000003,"5min":6.9770000000000003,"15min":176.172},"last":1.3069999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":0.85899999999999999,"15min":0.627},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":8.5660000000000007,"5min":10.337999999999999,"15min":10.337999999999999},"last":1.871},{"interface":"front","average":{"1min":2.3050000000000002,"5min":1.119,"15min":0.79500000000000004},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":11.348000000000001,"5min":11.348000000000001,"15min":11.348000000000001},"last":1.407}]},{"osd":7,"last update":"Sat Sep 23 20:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":0.82999999999999996,"15min":1.429},"min":{"1min":0.442,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.71,"5min":7.71,"15min":76.507000000000005},"last":1.4730000000000001},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.88900000000000001,"15min":1.5169999999999999},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":5.8869999999999996,"5min":5.8869999999999996,"15min":76.462000000000003},"last":1.97}]}]},{"osd":5,"up_from":38,"seq":163208757695,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1910260,"kb_used_data":1337004,"kb_used_omap":0,"kb_used_meta":573184,"kb_avail":91834892,"statfs":{"total":95995035648,"available":94038929408,"internally_reserved":0,"allocated":1369092096,"data_stored":1367681443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":586940416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:23 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":1.3100000000000001,"15min":1.8100000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.304,"5min":10.304,"15min":121.94799999999999},"last":0.23000000000000001},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.2809999999999999,"15min":2.0489999999999999},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":8.1699999999999999,"5min":9.4250000000000007,"15min":121.964},"last":1.974}]},{"osd":1,"last update":"Sat Sep 23 20:15:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":5.0540000000000003,"15min":5.3209999999999997},"min":{"1min":0.22,"5min":0.22,"15min":0.21299999999999999},"max":{"1min":7.383,"5min":150.16999999999999,"15min":171.68299999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.6749999999999998,"5min":4.7960000000000003,"15min":5.4020000000000001},"min":{"1min":0.32100000000000001,"5min":0.24099999999999999,"15min":0.21199999999999999},"max":{"1min":7.3099999999999996,"5min":150.12700000000001,"15min":171.64099999999999},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":3.5649999999999999,"15min":4.2679999999999998},"min":{"1min":0.317,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2569999999999997,"5min":120.297,"15min":240.57300000000001},"last":0.443},{"interface":"front","average":{"1min":1.47,"5min":3.661,"15min":4.335},"min":{"1min":0.38,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.2859999999999996,"5min":120.325,"15min":240.464},"last":0.47899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:15:49 2023","interfaces":[{"interface":"back","average":{"1min":10.634,"5min":6.1920000000000002,"15min":5.556},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":195.84700000000001,"5min":195.84700000000001,"15min":201.50899999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":10.66,"5min":6.0990000000000002,"15min":5.4089999999999998},"min":{"1min":0.39500000000000002,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":195.87899999999999,"5min":195.87899999999999,"15min":201.58099999999999},"last":0.312}]},{"osd":4,"last update":"Sat Sep 23 20:15:23 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":0.89600000000000002,"15min":1.345},"min":{"1min":0.25,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":8,"5min":8,"15min":191.78899999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.60499999999999998,"15min":1.2210000000000001},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":2.7160000000000002,"5min":3.2189999999999999,"15min":191.821},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.1870000000000001,"15min":0.83099999999999996},"min":{"1min":0.33300000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":5.4000000000000004,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.65400000000000003,"15min":0.58199999999999996},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":3.3180000000000001,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.4079999999999999,"15min":1.478},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":11.725,"5min":11.725,"15min":191.84299999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.02,"5min":1.3,"15min":1.4990000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":11.041,"5min":11.041,"15min":191.69},"last":0.79500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":946012160,"data_stored":946012160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3092447232,"data_stored":3092447232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":471662592,"data_stored":471662592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1023672320,"data_stored":1023672320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":587862016,"data_stored":587857944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1333727232,"data_stored":1333723160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1105858560,"data_stored":1105854488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:16:19.787 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:16:19.826 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:19.881 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:19.943 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:19 smithi135 ceph-mon[221113]: pgmap v1299: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 37977/70260 objects degraded (54.052%); 1662/70260 objects misplaced (2.365%); 6.0 MiB/s, 235 objects/s recovering 2023-09-23T20:16:19.944 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:19 smithi135 ceph-mon[213871]: pgmap v1299: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 37977/70260 objects degraded (54.052%); 1662/70260 objects misplaced (2.365%); 6.0 MiB/s, 235 objects/s recovering 2023-09-23T20:16:19.979 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:19.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:19.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:20.023 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:20.085 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:20.142 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:20.249 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:20.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:20.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:20.359 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:20.425 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:20.525 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:20.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:20.591 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:20.641 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:20.688 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4032255568' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:20.688 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:20 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:20.688 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:20 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:16:20.688 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4032255568' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:20.688 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:20 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:20.689 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:20 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:16:20.689 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:20.705 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:16:20.836 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:20.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:20.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:20.946 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:21.007 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4032255568' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:21.007 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:20 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:21.007 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:20 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:16:21.009 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:21.076 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:21.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:21.196 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:21.222 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:21.264 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:21.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:21.373 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:21.450 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:21.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[213871]: failed to encode map e466 with expected crc 2023-09-23T20:16:21.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:21.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[213871]: osdmap e466: 8 total, 8 up, 6 in 2023-09-23T20:16:21.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[213871]: failed to encode map e466 with expected crc 2023-09-23T20:16:21.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[213871]: failed to encode map e466 with expected crc 2023-09-23T20:16:21.702 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[213871]: pgmap v1301: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 686 op/s; 37977/70260 objects degraded (54.052%); 1662/70260 objects misplaced (2.365%); 3.3 MiB/s, 159 objects/s recovering 2023-09-23T20:16:21.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[221113]: failed to encode map e466 with expected crc 2023-09-23T20:16:21.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:21.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[221113]: osdmap e466: 8 total, 8 up, 6 in 2023-09-23T20:16:21.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[221113]: failed to encode map e466 with expected crc 2023-09-23T20:16:21.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[221113]: failed to encode map e466 with expected crc 2023-09-23T20:16:21.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:21 smithi135 ceph-mon[221113]: pgmap v1301: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 686 op/s; 37977/70260 objects degraded (54.052%); 1662/70260 objects misplaced (2.365%); 3.3 MiB/s, 159 objects/s recovering 2023-09-23T20:16:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:21 smithi144 ceph-mon[194833]: failed to encode map e466 with expected crc 2023-09-23T20:16:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:21 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:21 smithi144 ceph-mon[194833]: osdmap e466: 8 total, 8 up, 6 in 2023-09-23T20:16:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:21 smithi144 ceph-mon[194833]: failed to encode map e466 with expected crc 2023-09-23T20:16:22.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:21 smithi144 ceph-mon[194833]: failed to encode map e466 with expected crc 2023-09-23T20:16:22.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:21 smithi144 ceph-mon[194833]: pgmap v1301: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 686 op/s; 37977/70260 objects degraded (54.052%); 1662/70260 objects misplaced (2.365%); 3.3 MiB/s, 159 objects/s recovering 2023-09-23T20:16:22.369 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:22.408 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:22.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:22.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:22.515 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:22.571 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:22.655 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:22.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:22.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:22.770 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:22.821 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:22 smithi135 ceph-mon[213871]: failed to encode map e466 with expected crc 2023-09-23T20:16:22.822 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:22.822 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:22.822 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:22 smithi135 ceph-mon[221113]: failed to encode map e466 with expected crc 2023-09-23T20:16:22.822 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:22.822 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:22.836 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:22 smithi144 ceph-mon[194833]: failed to encode map e466 with expected crc 2023-09-23T20:16:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:23.821 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:16:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:16:23] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.43.0" 2023-09-23T20:16:23.822 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 37977/70260 objects degraded (54.052%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:23.822 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:23 smithi135 ceph-mon[213871]: pgmap v1302: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 37160/63870 objects degraded (58.181%); 1241/63870 objects misplaced (1.943%); 7.0 MiB/s, 228 objects/s recovering 2023-09-23T20:16:23.823 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 37977/70260 objects degraded (54.052%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:23.823 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:23 smithi135 ceph-mon[221113]: pgmap v1302: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 37160/63870 objects degraded (58.181%); 1241/63870 objects misplaced (1.943%); 7.0 MiB/s, 228 objects/s recovering 2023-09-23T20:16:24.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 37977/70260 objects degraded (54.052%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:24.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:23 smithi144 ceph-mon[194833]: pgmap v1302: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 37160/63870 objects degraded (58.181%); 1241/63870 objects misplaced (1.943%); 7.0 MiB/s, 228 objects/s recovering 2023-09-23T20:16:24.134 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:16:24.171 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:16:24.199 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:16:24.230 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:16:24.267 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:24.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:24.273 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:16:24.308 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:16:24.356 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:16:24.371 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:24.384 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:16:24.386 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:16:24.386 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:16:24.386 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:16:24.386 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:16:24.386 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:16:24.386 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:16:24.386 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:16:24.387 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:16:24.392 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:16:24.433 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:24.527 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:24.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:24.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:24.635 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:24.680 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:24.791 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:24.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:24.902 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:24.960 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2349667041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:24.980 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:25.073 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2349667041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:25.073 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2349667041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:25.084 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:25.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:25.190 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:25.250 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:25.806 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:16:25.806 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:16:25.958 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:25.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:25.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:25.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:25 smithi135 ceph-mon[213871]: pgmap v1303: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 935 op/s; 37160/58608 objects degraded (63.404%); 1124/58608 objects misplaced (1.918%); 5.7 MiB/s, 185 objects/s recovering 2023-09-23T20:16:25.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:25 smithi135 ceph-mon[221113]: pgmap v1303: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 935 op/s; 37160/58608 objects degraded (63.404%); 1124/58608 objects misplaced (1.918%); 5.7 MiB/s, 185 objects/s recovering 2023-09-23T20:16:26.071 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:26.126 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:26.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:25 smithi144 ceph-mon[194833]: pgmap v1303: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 935 op/s; 37160/58608 objects degraded (63.404%); 1124/58608 objects misplaced (1.918%); 5.7 MiB/s, 185 objects/s recovering 2023-09-23T20:16:26.699 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:16:26.738 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:26.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:26.844 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:26.868 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:26.929 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:27.116 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:27.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:27.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:27.225 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:27.288 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:27.444 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:27.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:27.497 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:27.550 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:16:27.550 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 36757/56850 objects degraded (64.656%), 18 pgs degraded, 19 pgs undersized","count":37},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1757,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":466,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":6,"osd_in_since":1695500073,"num_remapped_pgs":21},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":108},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":16},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait+undersized+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":18950,"data_bytes":1244814765,"bytes_used":8436756480,"bytes_avail":567533457408,"bytes_total":575970213888,"degraded_objects":36757,"degraded_total":56850,"degraded_ratio":0.64656112576956903,"misplaced_objects":1124,"misplaced_total":56850,"misplaced_ratio":0.019771328056288479,"recovering_objects_per_sec":199,"recovering_bytes_per_sec":5021113,"recovering_keys_per_sec":0,"num_objects_recovered":1889,"num_bytes_recovered":47644672,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":851},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-23T20:06:56.817144+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"fb5142fa-2e5d-413d-a713-d719d0723614":{"message":"Global Recovery Event (2m)\n [=======================.....] (remaining: 28s)","progress":0.82945734262466431,"add_to_ceph_s":true}}} 2023-09-23T20:16:27.553 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:27.604 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:27.721 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:27.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:27.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:27.825 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:27.858 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:27.902 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:27.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:27.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:28.006 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:28.075 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:28.295 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:28.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:28 smithi144 ceph-mon[194833]: pgmap v1304: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 851 op/s; 36757/56850 objects degraded (64.656%); 1124/56850 objects misplaced (1.977%); 4.8 MiB/s, 199 objects/s recovering 2023-09-23T20:16:28.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1745907605' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:16:28.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 36757/56850 objects degraded (64.656%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:28.408 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:28.426 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:16:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:28 smithi135 ceph-mon[221113]: pgmap v1304: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 851 op/s; 36757/56850 objects degraded (64.656%); 1124/56850 objects misplaced (1.977%); 4.8 MiB/s, 199 objects/s recovering 2023-09-23T20:16:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1745907605' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:16:28.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 36757/56850 objects degraded (64.656%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:28.474 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:28.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:28 smithi135 ceph-mon[213871]: pgmap v1304: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 851 op/s; 36757/56850 objects degraded (64.656%); 1124/56850 objects misplaced (1.977%); 4.8 MiB/s, 199 objects/s recovering 2023-09-23T20:16:28.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1745907605' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:16:28.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 36757/56850 objects degraded (64.656%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:28.587 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:28.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:28.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:28.693 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:28.735 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:28.806 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:16:28.806 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:16:28.850 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:28.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:28.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:28.957 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:29.025 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:29.113 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:29.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:29.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:29.221 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:29.309 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:29.331 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:29.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:29.412 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:29.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:29.513 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:29.615 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:29.720 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:29.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:29.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:29.825 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:29.880 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:29.893 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:29.979 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:29.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:29.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:30.088 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:30.150 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:30.188 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:16:30.242 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:30.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:30.346 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:30.375 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:30.419 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:30.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:30 smithi135 ceph-mon[221113]: pgmap v1305: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 940 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 36023/44964 objects degraded (80.115%); 829/44964 objects misplaced (1.844%); 3.4 MiB/s, 199 objects/s recovering 2023-09-23T20:16:30.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1410502973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:16:30.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:30 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:16:30.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:30 smithi135 ceph-mon[213871]: pgmap v1305: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 940 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 36023/44964 objects degraded (80.115%); 829/44964 objects misplaced (1.844%); 3.4 MiB/s, 199 objects/s recovering 2023-09-23T20:16:30.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1410502973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:16:30.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:30 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:16:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:30.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:30.524 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:30.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:30 smithi144 ceph-mon[194833]: pgmap v1305: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 940 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 36023/44964 objects degraded (80.115%); 829/44964 objects misplaced (1.844%); 3.4 MiB/s, 199 objects/s recovering 2023-09-23T20:16:30.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1410502973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:16:30.531 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:30 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:16:30.554 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:30.631 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:30.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:30.739 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:30.800 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:30.921 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:30.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:31.031 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:31.102 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:31.186 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:16:31.187 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:16:31.206 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:31.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:31.310 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:31.336 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:31.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:31 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:31.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:31 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:16:31.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:31 smithi144 ceph-mon[194833]: osdmap e467: 8 total, 8 up, 6 in 2023-09-23T20:16:31.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:31 smithi144 ceph-mon[194833]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:31 smithi144 ceph-mon[194833]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:31 smithi144 ceph-mon[194833]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:31 smithi144 ceph-mon[194833]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:31 smithi144 ceph-mon[194833]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[213871]: osdmap e467: 8 total, 8 up, 6 in 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[213871]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[213871]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[213871]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[213871]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[213871]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:31.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:16:31.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[221113]: osdmap e467: 8 total, 8 up, 6 in 2023-09-23T20:16:31.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[221113]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[221113]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[221113]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[221113]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:31 smithi135 ceph-mon[221113]: failed to encode map e467 with expected crc 2023-09-23T20:16:31.953 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:31.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:31.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:32.063 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:32.127 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:32.434 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:32 smithi144 ceph-mon[194833]: pgmap v1307: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 940 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 36023/44964 objects degraded (80.115%); 829/44964 objects misplaced (1.844%); 3.4 MiB/s, 199 objects/s recovering 2023-09-23T20:16:32.435 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:16:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:32 smithi135 ceph-mon[213871]: pgmap v1307: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 940 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 36023/44964 objects degraded (80.115%); 829/44964 objects misplaced (1.844%); 3.4 MiB/s, 199 objects/s recovering 2023-09-23T20:16:32.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:32 smithi135 ceph-mon[221113]: pgmap v1307: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 940 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 36023/44964 objects degraded (80.115%); 829/44964 objects misplaced (1.844%); 3.4 MiB/s, 199 objects/s recovering 2023-09-23T20:16:32.727 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:16:32.728 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:16:32.728 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:16:32.737 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:16:32.771 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:16:32.821 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:16:32.860 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:16:32.890 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:16:32.896 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:16:32.896 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:16:32.896 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:16:32.896 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:16:32.896 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:16:32.896 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:16:32.896 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:16:32.896 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:16:33.130 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:33.137 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:33.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:33.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:33.177 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:33 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2989908024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:33.177 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 36023/44964 objects degraded (80.115%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:33.180 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:33 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2989908024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:33.180 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 36023/44964 objects degraded (80.115%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:33.241 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:33.317 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:33.397 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:33.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:33.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:33.501 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:33.546 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:33 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2989908024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 36023/44964 objects degraded (80.115%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:33.827 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:16:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:16:33] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.43.0" 2023-09-23T20:16:33.842 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:16:33.854 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:33.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:33.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:33.963 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:34.027 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:34.146 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:34.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:34.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:34.179 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:34 smithi135 ceph-mon[213871]: pgmap v1308: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 805 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 34848/38484 objects degraded (90.552%); 706/38484 objects misplaced (1.835%); 1.6 MiB/s, 206 objects/s recovering 2023-09-23T20:16:34.180 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:34 smithi135 ceph-mon[221113]: pgmap v1308: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 805 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 34848/38484 objects degraded (90.552%); 706/38484 objects misplaced (1.835%); 1.6 MiB/s, 206 objects/s recovering 2023-09-23T20:16:34.250 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:34.302 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:34 smithi144 ceph-mon[194833]: pgmap v1308: 129 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 805 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 34848/38484 objects degraded (90.552%); 706/38484 objects misplaced (1.835%); 1.6 MiB/s, 206 objects/s recovering 2023-09-23T20:16:34.316 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:34.430 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:34.503 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:34.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:34.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:34.560 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:16:34.566 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:16:34.609 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:34.656 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:34.709 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:34.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:34.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:34.814 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:34.840 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:34.882 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:34.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:34.985 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:35.025 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:35.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[213871]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[213871]: osdmap e468: 8 total, 8 up, 6 in 2023-09-23T20:16:35.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[213871]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[213871]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[213871]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[213871]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[213871]: from='client.46322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:35.197 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[213871]: pgmap v1310: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 689 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 995 op/s; 34848/32952 objects degraded (105.754%); 578/32952 objects misplaced (1.754%); 2.0 MiB/s, 207 objects/s recovering 2023-09-23T20:16:35.294 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:35.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:35.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:35.299 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:35 smithi144 ceph-mon[194833]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.299 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:35 smithi144 ceph-mon[194833]: osdmap e468: 8 total, 8 up, 6 in 2023-09-23T20:16:35.299 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:35 smithi144 ceph-mon[194833]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.299 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:35 smithi144 ceph-mon[194833]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.299 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:35 smithi144 ceph-mon[194833]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.299 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:35 smithi144 ceph-mon[194833]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.299 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:35 smithi144 ceph-mon[194833]: from='client.46322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:35.299 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:35 smithi144 ceph-mon[194833]: pgmap v1310: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 689 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 995 op/s; 34848/32952 objects degraded (105.754%); 578/32952 objects misplaced (1.754%); 2.0 MiB/s, 207 objects/s recovering 2023-09-23T20:16:35.399 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:35.427 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[221113]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[221113]: osdmap e468: 8 total, 8 up, 6 in 2023-09-23T20:16:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[221113]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[221113]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[221113]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[221113]: failed to encode map e468 with expected crc 2023-09-23T20:16:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[221113]: from='client.46322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:35 smithi135 ceph-mon[221113]: pgmap v1310: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 689 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 995 op/s; 34848/32952 objects degraded (105.754%); 578/32952 objects misplaced (1.754%); 2.0 MiB/s, 207 objects/s recovering 2023-09-23T20:16:35.530 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:35.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:35.654 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:35.715 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:35.825 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:35.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:35.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:35.845 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:16:35.845 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:16:35.845 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:16:35.845 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:16:35.845 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:16:35.851 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:16:35.934 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:35.993 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:36.231 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1309,"stamp":"2023-09-23T20:16:34.192047+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":722754989,"num_objects":10984,"num_object_clones":0,"num_object_copies":32952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34848,"num_objects_misplaced":578,"num_objects_unfound":0,"num_objects_dirty":10984,"num_whiteouts":0,"num_read":5996,"num_read_kb":13236,"num_write":197664,"num_write_kb":6659327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22914,"num_bytes_recovered":1190657140,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":187203,"ondisk_log_size":187203,"up":387,"acting":368,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":391,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7039616,"kb_used_data":2980816,"kb_used_omap":25,"kb_used_meta":4058726,"kb_avail":555431296,"statfs":{"total":575970213888,"available":568761647104,"internally_reserved":0,"allocated":3052355584,"data_stored":3041216669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":4156136140},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-560463872,"num_objects":-8552,"num_object_clones":0,"num_object_copies":-25656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2312,"num_objects_misplaced":-546,"num_objects_unfound":0,"num_objects_dirty":-8552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8552,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2062,"num_bytes_recovered":16384000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.108754"},"pg_stats":[{"pgid":"9.23","version":"467'6258","reported_seq":6356,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:32.854353+0000","last_change":"2023-09-23T20:14:36.092943+0000","last_active":"2023-09-23T20:16:32.854353+0000","last_peered":"2023-09-23T20:16:32.854353+0000","last_clean":"2023-09-23T20:14:33.760240+0000","last_became_active":"2023-09-23T20:14:36.076681+0000","last_became_peered":"2023-09-23T20:14:36.076681+0000","last_unstale":"2023-09-23T20:16:32.854353+0000","last_undegraded":"2023-09-23T20:14:35.911782+0000","last_fullsized":"2023-09-23T20:14:35.868390+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6258,"log_dups_size":0,"ondisk_log_size":6258,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10436608,"num_objects":159,"num_object_clones":0,"num_object_copies":477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3149,"num_write_kb":105872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":69},{"shards":"3,7","objects":-1405}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.22","version":"467'6253","reported_seq":8845,"reported_epoch":467,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:31.574583+0000","last_change":"2023-09-23T20:16:16.542393+0000","last_active":"2023-09-23T20:16:31.574583+0000","last_peered":"2023-09-23T20:16:31.574583+0000","last_clean":"2023-09-23T20:14:33.601158+0000","last_became_active":"2023-09-23T20:14:35.991960+0000","last_became_peered":"2023-09-23T20:14:35.991960+0000","last_unstale":"2023-09-23T20:16:31.574583+0000","last_undegraded":"2023-09-23T20:14:35.848470+0000","last_fullsized":"2023-09-23T20:14:35.833465+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6253,"log_dups_size":0,"ondisk_log_size":6253,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13172736,"num_objects":201,"num_object_clones":0,"num_object_copies":603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":298,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3117,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1246,"num_bytes_recovered":19005440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":-97}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.21","version":"467'6120","reported_seq":6208,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:32.930244+0000","last_change":"2023-09-23T20:14:24.799466+0000","last_active":"2023-09-23T20:16:32.930244+0000","last_peered":"2023-09-23T20:16:32.930244+0000","last_clean":"2023-09-23T20:16:32.930244+0000","last_became_active":"2023-09-23T20:14:24.796974+0000","last_became_peered":"2023-09-23T20:14:24.796974+0000","last_unstale":"2023-09-23T20:16:32.930244+0000","last_undegraded":"2023-09-23T20:16:32.930244+0000","last_fullsized":"2023-09-23T20:16:32.930244+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6120,"log_dups_size":0,"ondisk_log_size":6120,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13205504,"num_objects":202,"num_object_clones":0,"num_object_copies":606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3026,"num_write_kb":103264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.20","version":"467'6153","reported_seq":6250,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:31.070190+0000","last_change":"2023-09-23T20:14:36.109513+0000","last_active":"2023-09-23T20:16:31.070190+0000","last_peered":"2023-09-23T20:16:31.070190+0000","last_clean":"2023-09-23T20:14:33.769971+0000","last_became_active":"2023-09-23T20:14:36.106301+0000","last_became_peered":"2023-09-23T20:14:36.106301+0000","last_unstale":"2023-09-23T20:16:31.070190+0000","last_undegraded":"2023-09-23T20:14:35.845700+0000","last_fullsized":"2023-09-23T20:14:35.836337+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6153,"log_dups_size":0,"ondisk_log_size":6153,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11075584,"num_objects":169,"num_object_clones":0,"num_object_copies":507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3071,"num_write_kb":103680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":94},{"shards":"0,2","objects":-1376}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.10","version":"467'7699","reported_seq":13636,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.001562+0000","last_change":"2023-09-23T20:14:22.339078+0000","last_active":"2023-09-23T20:16:33.001562+0000","last_peered":"2023-09-23T20:16:33.001562+0000","last_clean":"2023-09-23T20:16:33.001562+0000","last_became_active":"2023-09-23T20:14:22.319507+0000","last_became_peered":"2023-09-23T20:14:22.319507+0000","last_unstale":"2023-09-23T20:16:33.001562+0000","last_undegraded":"2023-09-23T20:16:33.001562+0000","last_fullsized":"2023-09-23T20:16:33.001562+0000","mapping_epoch":434,"log_start":"417'3389","ondisk_log_start":"417'3389","created":418,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4310,"log_dups_size":0,"ondisk_log_size":4310,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24182784,"num_objects":369,"num_object_clones":0,"num_object_copies":1107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5979,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1945,"num_bytes_recovered":127467520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":37,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716593+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:16:20.716593+0000","last_peered":"2023-09-23T20:16:20.716593+0000","last_clean":"2023-09-23T20:16:20.716593+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:16:20.716593+0000","last_undegraded":"2023-09-23T20:16:20.716593+0000","last_fullsized":"2023-09-23T20:16:20.716593+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":35,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.703834+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:16:20.703834+0000","last_peered":"2023-09-23T20:16:20.703834+0000","last_clean":"2023-09-23T20:16:20.703834+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:16:20.703834+0000","last_undegraded":"2023-09-23T20:16:20.703834+0000","last_fullsized":"2023-09-23T20:16:20.703834+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.13","version":"467'7859","reported_seq":14619,"reported_epoch":467,"state":"active+recovery_wait+undersized+remapped","last_fresh":"2023-09-23T20:16:32.993406+0000","last_change":"2023-09-23T20:16:06.703336+0000","last_active":"2023-09-23T20:16:32.993406+0000","last_peered":"2023-09-23T20:16:32.993406+0000","last_clean":"2023-09-23T20:14:05.142062+0000","last_became_active":"2023-09-23T20:14:07.304693+0000","last_became_peered":"2023-09-23T20:14:07.304693+0000","last_unstale":"2023-09-23T20:16:32.993406+0000","last_undegraded":"2023-09-23T20:16:32.993406+0000","last_fullsized":"2023-09-23T20:14:07.197546+0000","mapping_epoch":429,"log_start":"419'3535","ondisk_log_start":"419'3535","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4324,"log_dups_size":0,"ondisk_log_size":4324,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19038208,"num_objects":291,"num_object_clones":0,"num_object_copies":873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6127,"num_write_kb":205344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2223,"num_bytes_recovered":145686528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,0],"avail_no_missing":["7","0","1","3","5"],"object_location_counts":[{"shards":"0,7","objects":291}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":212,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.694163+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:16:20.694163+0000","last_peered":"2023-09-23T20:16:20.694163+0000","last_clean":"2023-09-23T20:16:20.694163+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:16:20.694163+0000","last_undegraded":"2023-09-23T20:16:20.694163+0000","last_fullsized":"2023-09-23T20:16:20.694163+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":175,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.711249+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:16:20.711249+0000","last_peered":"2023-09-23T20:16:20.711249+0000","last_clean":"2023-09-23T20:16:20.711249+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:16:20.711249+0000","last_undegraded":"2023-09-23T20:16:20.711249+0000","last_fullsized":"2023-09-23T20:16:20.711249+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"467'7941","reported_seq":14411,"reported_epoch":467,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T20:16:33.516639+0000","last_change":"2023-09-23T20:14:36.234004+0000","last_active":"2023-09-23T20:16:33.516639+0000","last_peered":"2023-09-23T20:16:33.516639+0000","last_clean":"2023-09-23T20:14:33.790624+0000","last_became_active":"2023-09-23T20:14:36.220990+0000","last_became_peered":"2023-09-23T20:14:36.220990+0000","last_unstale":"2023-09-23T20:16:33.516639+0000","last_undegraded":"2023-09-23T20:16:33.516639+0000","last_fullsized":"2023-09-23T20:16:33.516639+0000","mapping_epoch":442,"log_start":"419'3512","ondisk_log_start":"419'3512","created":418,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4429,"log_dups_size":0,"ondisk_log_size":4429,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16580608,"num_objects":253,"num_object_clones":0,"num_object_copies":759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":253,"num_objects_unfound":0,"num_objects_dirty":253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6191,"num_write_kb":206208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":139067392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1","3"],"object_location_counts":[{"shards":"0,1,5","objects":253}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":36,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715246+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:16:20.715246+0000","last_peered":"2023-09-23T20:16:20.715246+0000","last_clean":"2023-09-23T20:16:20.715246+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:16:20.715246+0000","last_undegraded":"2023-09-23T20:16:20.715246+0000","last_fullsized":"2023-09-23T20:16:20.715246+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":422,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715302+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:16:20.715302+0000","last_peered":"2023-09-23T20:16:20.715302+0000","last_clean":"2023-09-23T20:16:20.715302+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:16:20.715302+0000","last_undegraded":"2023-09-23T20:16:20.715302+0000","last_fullsized":"2023-09-23T20:16:20.715302+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1d","version":"467'9062","reported_seq":9150,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:32.976026+0000","last_change":"2023-09-23T20:14:36.076800+0000","last_active":"2023-09-23T20:16:32.976026+0000","last_peered":"2023-09-23T20:16:32.976026+0000","last_clean":"2023-09-23T20:14:23.775634+0000","last_became_active":"2023-09-23T20:14:36.070774+0000","last_became_peered":"2023-09-23T20:14:36.070774+0000","last_unstale":"2023-09-23T20:16:32.976026+0000","last_undegraded":"2023-09-23T20:14:35.892079+0000","last_fullsized":"2023-09-23T20:14:35.839513+0000","mapping_epoch":442,"log_start":"421'4099","ondisk_log_start":"421'4099","created":436,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4963,"log_dups_size":0,"ondisk_log_size":4963,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26607640,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6273,"num_write_kb":213697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":142},{"shards":"5,7","objects":-2668}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":393,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715165+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:16:20.715165+0000","last_peered":"2023-09-23T20:16:20.715165+0000","last_clean":"2023-09-23T20:16:20.715165+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:16:20.715165+0000","last_undegraded":"2023-09-23T20:16:20.715165+0000","last_fullsized":"2023-09-23T20:16:20.715165+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":321,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.693987+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:16:20.693987+0000","last_peered":"2023-09-23T20:16:20.693987+0000","last_clean":"2023-09-23T20:16:20.693987+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:16:20.693987+0000","last_undegraded":"2023-09-23T20:16:20.693987+0000","last_fullsized":"2023-09-23T20:16:20.693987+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"467'8633","reported_seq":8711,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:31.507699+0000","last_change":"2023-09-23T20:14:24.810789+0000","last_active":"2023-09-23T20:16:31.507699+0000","last_peered":"2023-09-23T20:16:31.507699+0000","last_clean":"2023-09-23T20:16:31.507699+0000","last_became_active":"2023-09-23T20:14:24.807691+0000","last_became_peered":"2023-09-23T20:14:24.807691+0000","last_unstale":"2023-09-23T20:16:31.507699+0000","last_undegraded":"2023-09-23T20:16:31.507699+0000","last_fullsized":"2023-09-23T20:16:31.507699+0000","mapping_epoch":436,"log_start":"427'4028","ondisk_log_start":"427'4028","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4605,"log_dups_size":0,"ondisk_log_size":4605,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20807680,"num_objects":318,"num_object_clones":0,"num_object_copies":954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6014,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":800,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715985+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:16:20.715985+0000","last_peered":"2023-09-23T20:16:20.715985+0000","last_clean":"2023-09-23T20:16:20.715985+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:16:20.715985+0000","last_undegraded":"2023-09-23T20:16:20.715985+0000","last_fullsized":"2023-09-23T20:16:20.715985+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":476,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.706775+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:16:20.706775+0000","last_peered":"2023-09-23T20:16:20.706775+0000","last_clean":"2023-09-23T20:16:20.706775+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:16:20.706775+0000","last_undegraded":"2023-09-23T20:16:20.706775+0000","last_fullsized":"2023-09-23T20:16:20.706775+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":331,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.694123+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:16:20.694123+0000","last_peered":"2023-09-23T20:16:20.694123+0000","last_clean":"2023-09-23T20:16:20.694123+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:16:20.694123+0000","last_undegraded":"2023-09-23T20:16:20.694123+0000","last_fullsized":"2023-09-23T20:16:20.694123+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1f","version":"467'8788","reported_seq":8875,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.322024+0000","last_change":"2023-09-23T20:14:36.232228+0000","last_active":"2023-09-23T20:16:33.322024+0000","last_peered":"2023-09-23T20:16:33.322024+0000","last_clean":"2023-09-23T20:14:33.842679+0000","last_became_active":"2023-09-23T20:14:36.217802+0000","last_became_peered":"2023-09-23T20:14:36.217802+0000","last_unstale":"2023-09-23T20:16:33.322024+0000","last_undegraded":"2023-09-23T20:14:36.129688+0000","last_fullsized":"2023-09-23T20:14:36.115967+0000","mapping_epoch":442,"log_start":"419'3410","ondisk_log_start":"419'3410","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5378,"log_dups_size":0,"ondisk_log_size":5378,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22282240,"num_objects":340,"num_object_clones":0,"num_object_copies":1020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6102,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":146},{"shards":"3,5","objects":-2687}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":160,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716742+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:16:20.716742+0000","last_peered":"2023-09-23T20:16:20.716742+0000","last_clean":"2023-09-23T20:16:20.716742+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:16:20.716742+0000","last_undegraded":"2023-09-23T20:16:20.716742+0000","last_fullsized":"2023-09-23T20:16:20.716742+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":165,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716792+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:16:20.716792+0000","last_peered":"2023-09-23T20:16:20.716792+0000","last_clean":"2023-09-23T20:16:20.716792+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:16:20.716792+0000","last_undegraded":"2023-09-23T20:16:20.716792+0000","last_fullsized":"2023-09-23T20:16:20.716792+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.1e","version":"467'8811","reported_seq":14959,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.031371+0000","last_change":"2023-09-23T20:15:21.201333+0000","last_active":"2023-09-23T20:16:33.031371+0000","last_peered":"2023-09-23T20:16:33.031371+0000","last_clean":"2023-09-23T20:16:33.031371+0000","last_became_active":"2023-09-23T20:15:21.192111+0000","last_became_peered":"2023-09-23T20:15:21.192111+0000","last_unstale":"2023-09-23T20:16:33.031371+0000","last_undegraded":"2023-09-23T20:16:33.031371+0000","last_fullsized":"2023-09-23T20:16:33.031371+0000","mapping_epoch":454,"log_start":"419'3720","ondisk_log_start":"419'3720","created":436,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24346624,"num_objects":372,"num_object_clones":0,"num_object_copies":1116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6112,"num_write_kb":207456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3024,"num_bytes_recovered":198180864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":413,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.710604+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:16:20.710604+0000","last_peered":"2023-09-23T20:16:20.710604+0000","last_clean":"2023-09-23T20:16:20.710604+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:16:20.710604+0000","last_undegraded":"2023-09-23T20:16:20.710604+0000","last_fullsized":"2023-09-23T20:16:20.710604+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":344,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.706191+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:16:20.706191+0000","last_peered":"2023-09-23T20:16:20.706191+0000","last_clean":"2023-09-23T20:16:20.706191+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:16:20.706191+0000","last_undegraded":"2023-09-23T20:16:20.706191+0000","last_fullsized":"2023-09-23T20:16:20.706191+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"467'7936","reported_seq":8014,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.056224+0000","last_change":"2023-09-23T20:13:46.811423+0000","last_active":"2023-09-23T20:16:33.056224+0000","last_peered":"2023-09-23T20:16:33.056224+0000","last_clean":"2023-09-23T20:16:33.056224+0000","last_became_active":"2023-09-23T20:13:46.808266+0000","last_became_peered":"2023-09-23T20:13:46.808266+0000","last_unstale":"2023-09-23T20:16:33.056224+0000","last_undegraded":"2023-09-23T20:16:33.056224+0000","last_fullsized":"2023-09-23T20:16:33.056224+0000","mapping_epoch":418,"log_start":"419'3632","ondisk_log_start":"419'3632","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4304,"log_dups_size":0,"ondisk_log_size":4304,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22052864,"num_objects":337,"num_object_clones":0,"num_object_copies":1011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6181,"num_write_kb":208544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":826,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.703836+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:16:20.703836+0000","last_peered":"2023-09-23T20:16:20.703836+0000","last_clean":"2023-09-23T20:16:20.703836+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:16:20.703836+0000","last_undegraded":"2023-09-23T20:16:20.703836+0000","last_fullsized":"2023-09-23T20:16:20.703836+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":160,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.712050+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:16:20.712050+0000","last_peered":"2023-09-23T20:16:20.712050+0000","last_clean":"2023-09-23T20:16:20.712050+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:16:20.712050+0000","last_undegraded":"2023-09-23T20:16:20.712050+0000","last_fullsized":"2023-09-23T20:16:20.712050+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":213,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.710823+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:16:20.710823+0000","last_peered":"2023-09-23T20:16:20.710823+0000","last_clean":"2023-09-23T20:16:20.710823+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:16:20.710823+0000","last_undegraded":"2023-09-23T20:16:20.710823+0000","last_fullsized":"2023-09-23T20:16:20.710823+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"467'7941","reported_seq":8019,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.282245+0000","last_change":"2023-09-23T20:13:46.824838+0000","last_active":"2023-09-23T20:16:33.282245+0000","last_peered":"2023-09-23T20:16:33.282245+0000","last_clean":"2023-09-23T20:16:33.282245+0000","last_became_active":"2023-09-23T20:13:46.820709+0000","last_became_peered":"2023-09-23T20:13:46.820709+0000","last_unstale":"2023-09-23T20:16:33.282245+0000","last_undegraded":"2023-09-23T20:16:33.282245+0000","last_fullsized":"2023-09-23T20:16:33.282245+0000","mapping_epoch":418,"log_start":"419'3459","ondisk_log_start":"419'3459","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4482,"log_dups_size":0,"ondisk_log_size":4482,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20807680,"num_objects":318,"num_object_clones":0,"num_object_copies":954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6226,"num_write_kb":209376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":515,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.707001+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:16:20.707001+0000","last_peered":"2023-09-23T20:16:20.707001+0000","last_clean":"2023-09-23T20:16:20.707001+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:16:20.707001+0000","last_undegraded":"2023-09-23T20:16:20.707001+0000","last_fullsized":"2023-09-23T20:16:20.707001+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":175,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.713875+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:16:20.713875+0000","last_peered":"2023-09-23T20:16:20.713875+0000","last_clean":"2023-09-23T20:16:20.713875+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:16:20.713875+0000","last_undegraded":"2023-09-23T20:16:20.713875+0000","last_fullsized":"2023-09-23T20:16:20.713875+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":465,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.704617+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:16:20.704617+0000","last_peered":"2023-09-23T20:16:20.704617+0000","last_clean":"2023-09-23T20:16:20.704617+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:16:20.704617+0000","last_undegraded":"2023-09-23T20:16:20.704617+0000","last_fullsized":"2023-09-23T20:16:20.704617+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1b","version":"467'8606","reported_seq":8684,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.302529+0000","last_change":"2023-09-23T20:14:24.808547+0000","last_active":"2023-09-23T20:16:33.302529+0000","last_peered":"2023-09-23T20:16:33.302529+0000","last_clean":"2023-09-23T20:16:33.302529+0000","last_became_active":"2023-09-23T20:14:24.806384+0000","last_became_peered":"2023-09-23T20:14:24.806384+0000","last_unstale":"2023-09-23T20:16:33.302529+0000","last_undegraded":"2023-09-23T20:16:33.302529+0000","last_fullsized":"2023-09-23T20:16:33.302529+0000","mapping_epoch":436,"log_start":"415'3109","ondisk_log_start":"415'3109","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22052864,"num_objects":337,"num_object_clones":0,"num_object_copies":1011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5995,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":576,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.694951+0000","last_change":"2023-09-23T20:14:37.887863+0000","last_active":"2023-09-23T20:16:20.694951+0000","last_peered":"2023-09-23T20:16:20.694951+0000","last_clean":"2023-09-23T20:16:20.694951+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:16:20.694951+0000","last_undegraded":"2023-09-23T20:16:20.694951+0000","last_fullsized":"2023-09-23T20:16:20.694951+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.712700+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:16:20.712700+0000","last_peered":"2023-09-23T20:16:20.712700+0000","last_clean":"2023-09-23T20:16:20.712700+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:16:20.712700+0000","last_undegraded":"2023-09-23T20:16:20.712700+0000","last_fullsized":"2023-09-23T20:16:20.712700+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":455,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.694608+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:16:20.694608+0000","last_peered":"2023-09-23T20:16:20.694608+0000","last_clean":"2023-09-23T20:16:20.694608+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:16:20.694608+0000","last_undegraded":"2023-09-23T20:16:20.694608+0000","last_fullsized":"2023-09-23T20:16:20.694608+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1a","version":"467'8730","reported_seq":8818,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.270655+0000","last_change":"2023-09-23T20:14:36.218516+0000","last_active":"2023-09-23T20:16:33.270655+0000","last_peered":"2023-09-23T20:16:33.270655+0000","last_clean":"2023-09-23T20:14:33.782041+0000","last_became_active":"2023-09-23T20:14:36.175134+0000","last_became_peered":"2023-09-23T20:14:36.175134+0000","last_unstale":"2023-09-23T20:16:33.270655+0000","last_undegraded":"2023-09-23T20:14:35.943267+0000","last_fullsized":"2023-09-23T20:14:35.848493+0000","mapping_epoch":442,"log_start":"415'3159","ondisk_log_start":"415'3159","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5571,"log_dups_size":0,"ondisk_log_size":5571,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25034752,"num_objects":382,"num_object_clones":0,"num_object_copies":1146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6032,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":135},{"shards":"5,7","objects":-2578}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":604,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.714903+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:16:20.714903+0000","last_peered":"2023-09-23T20:16:20.714903+0000","last_clean":"2023-09-23T20:16:20.714903+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:16:20.714903+0000","last_undegraded":"2023-09-23T20:16:20.714903+0000","last_fullsized":"2023-09-23T20:16:20.714903+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":110,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.706430+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:16:20.706430+0000","last_peered":"2023-09-23T20:16:20.706430+0000","last_clean":"2023-09-23T20:16:20.706430+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:16:20.706430+0000","last_undegraded":"2023-09-23T20:16:20.706430+0000","last_fullsized":"2023-09-23T20:16:20.706430+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":108,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.694977+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:16:20.694977+0000","last_peered":"2023-09-23T20:16:20.694977+0000","last_clean":"2023-09-23T20:16:20.694977+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:16:20.694977+0000","last_undegraded":"2023-09-23T20:16:20.694977+0000","last_fullsized":"2023-09-23T20:16:20.694977+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"467'7946","reported_seq":8025,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.384126+0000","last_change":"2023-09-23T20:13:46.790587+0000","last_active":"2023-09-23T20:16:33.384126+0000","last_peered":"2023-09-23T20:16:33.384126+0000","last_clean":"2023-09-23T20:16:33.384126+0000","last_became_active":"2023-09-23T20:13:46.788387+0000","last_became_peered":"2023-09-23T20:13:46.788387+0000","last_unstale":"2023-09-23T20:16:33.384126+0000","last_undegraded":"2023-09-23T20:16:33.384126+0000","last_fullsized":"2023-09-23T20:16:33.384126+0000","mapping_epoch":407,"log_start":"419'3464","ondisk_log_start":"419'3464","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4482,"log_dups_size":0,"ondisk_log_size":4482,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20447232,"num_objects":312,"num_object_clones":0,"num_object_copies":936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6216,"num_write_kb":208896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":727,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716313+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:16:20.716313+0000","last_peered":"2023-09-23T20:16:20.716313+0000","last_clean":"2023-09-23T20:16:20.716313+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:16:20.716313+0000","last_undegraded":"2023-09-23T20:16:20.716313+0000","last_fullsized":"2023-09-23T20:16:20.716313+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":158,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.710505+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:16:20.710505+0000","last_peered":"2023-09-23T20:16:20.710505+0000","last_clean":"2023-09-23T20:16:20.710505+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:16:20.710505+0000","last_undegraded":"2023-09-23T20:16:20.710505+0000","last_fullsized":"2023-09-23T20:16:20.710505+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":475,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.706499+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:16:20.706499+0000","last_peered":"2023-09-23T20:16:20.706499+0000","last_clean":"2023-09-23T20:16:20.706499+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:16:20.706499+0000","last_undegraded":"2023-09-23T20:16:20.706499+0000","last_fullsized":"2023-09-23T20:16:20.706499+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"467'7808","reported_seq":13037,"reported_epoch":467,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.052082+0000","last_change":"2023-09-23T20:16:00.310118+0000","last_active":"2023-09-23T20:16:33.052082+0000","last_peered":"2023-09-23T20:16:33.052082+0000","last_clean":"2023-09-23T20:14:33.800061+0000","last_became_active":"2023-09-23T20:14:36.025052+0000","last_became_peered":"2023-09-23T20:14:36.025052+0000","last_unstale":"2023-09-23T20:16:33.052082+0000","last_undegraded":"2023-09-23T20:14:35.860952+0000","last_fullsized":"2023-09-23T20:14:35.840056+0000","mapping_epoch":442,"log_start":"412'2295","ondisk_log_start":"412'2295","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22249472,"num_objects":339,"num_object_clones":0,"num_object_copies":1017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":452,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6083,"num_write_kb":205536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2571,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"3,4","objects":-113}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":814,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.698207+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:16:20.698207+0000","last_peered":"2023-09-23T20:16:20.698207+0000","last_clean":"2023-09-23T20:16:20.698207+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:16:20.698207+0000","last_undegraded":"2023-09-23T20:16:20.698207+0000","last_fullsized":"2023-09-23T20:16:20.698207+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":173,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.711325+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:16:20.711325+0000","last_peered":"2023-09-23T20:16:20.711325+0000","last_clean":"2023-09-23T20:16:20.711325+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:16:20.711325+0000","last_undegraded":"2023-09-23T20:16:20.711325+0000","last_fullsized":"2023-09-23T20:16:20.711325+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":467,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.693936+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:16:20.693936+0000","last_peered":"2023-09-23T20:16:20.693936+0000","last_clean":"2023-09-23T20:16:20.693936+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:16:20.693936+0000","last_undegraded":"2023-09-23T20:16:20.693936+0000","last_fullsized":"2023-09-23T20:16:20.693936+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.7","version":"467'7866","reported_seq":7945,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.378028+0000","last_change":"2023-09-23T20:13:46.802046+0000","last_active":"2023-09-23T20:16:33.378028+0000","last_peered":"2023-09-23T20:16:33.378028+0000","last_clean":"2023-09-23T20:16:33.378028+0000","last_became_active":"2023-09-23T20:13:46.800187+0000","last_became_peered":"2023-09-23T20:13:46.800187+0000","last_unstale":"2023-09-23T20:16:33.378028+0000","last_undegraded":"2023-09-23T20:16:33.378028+0000","last_fullsized":"2023-09-23T20:16:33.378028+0000","mapping_epoch":407,"log_start":"417'3419","ondisk_log_start":"417'3419","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4447,"log_dups_size":0,"ondisk_log_size":4447,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24641536,"num_objects":376,"num_object_clones":0,"num_object_copies":1128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6156,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":539,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.713066+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:16:20.713066+0000","last_peered":"2023-09-23T20:16:20.713066+0000","last_clean":"2023-09-23T20:16:20.713066+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:16:20.713066+0000","last_undegraded":"2023-09-23T20:16:20.713066+0000","last_fullsized":"2023-09-23T20:16:20.713066+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":158,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716580+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:16:20.716580+0000","last_peered":"2023-09-23T20:16:20.716580+0000","last_clean":"2023-09-23T20:16:20.716580+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:16:20.716580+0000","last_undegraded":"2023-09-23T20:16:20.716580+0000","last_fullsized":"2023-09-23T20:16:20.716580+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":55,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.714086+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:16:20.714086+0000","last_peered":"2023-09-23T20:16:20.714086+0000","last_clean":"2023-09-23T20:16:20.714086+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:16:20.714086+0000","last_undegraded":"2023-09-23T20:16:20.714086+0000","last_fullsized":"2023-09-23T20:16:20.714086+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"467'8134","reported_seq":8221,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.326993+0000","last_change":"2023-09-23T20:14:36.190976+0000","last_active":"2023-09-23T20:16:33.326993+0000","last_peered":"2023-09-23T20:16:33.326993+0000","last_clean":"2023-09-23T20:14:33.830609+0000","last_became_active":"2023-09-23T20:14:36.182918+0000","last_became_peered":"2023-09-23T20:14:36.182918+0000","last_unstale":"2023-09-23T20:16:33.326993+0000","last_undegraded":"2023-09-23T20:14:35.924455+0000","last_fullsized":"2023-09-23T20:14:35.847184+0000","mapping_epoch":442,"log_start":"419'3624","ondisk_log_start":"419'3624","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4510,"log_dups_size":0,"ondisk_log_size":4510,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17825792,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6352,"num_write_kb":211968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":157},{"shards":"2,5","objects":-2925}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":670,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.711058+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:16:20.711058+0000","last_peered":"2023-09-23T20:16:20.711058+0000","last_clean":"2023-09-23T20:16:20.711058+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:16:20.711058+0000","last_undegraded":"2023-09-23T20:16:20.711058+0000","last_fullsized":"2023-09-23T20:16:20.711058+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":515,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.703532+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:16:20.703532+0000","last_peered":"2023-09-23T20:16:20.703532+0000","last_clean":"2023-09-23T20:16:20.703532+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:16:20.703532+0000","last_undegraded":"2023-09-23T20:16:20.703532+0000","last_fullsized":"2023-09-23T20:16:20.703532+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":211,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.698753+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:16:20.698753+0000","last_peered":"2023-09-23T20:16:20.698753+0000","last_clean":"2023-09-23T20:16:20.698753+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:16:20.698753+0000","last_undegraded":"2023-09-23T20:16:20.698753+0000","last_fullsized":"2023-09-23T20:16:20.698753+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"467'6168","reported_seq":6257,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:32.904472+0000","last_change":"2023-09-23T20:14:24.782498+0000","last_active":"2023-09-23T20:16:32.904472+0000","last_peered":"2023-09-23T20:16:32.904472+0000","last_clean":"2023-09-23T20:16:32.904472+0000","last_became_active":"2023-09-23T20:14:24.780750+0000","last_became_peered":"2023-09-23T20:14:24.780750+0000","last_unstale":"2023-09-23T20:16:32.904472+0000","last_undegraded":"2023-09-23T20:16:32.904472+0000","last_fullsized":"2023-09-23T20:16:32.904472+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6168,"log_dups_size":0,"ondisk_log_size":6168,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10452992,"num_objects":159,"num_object_clones":0,"num_object_copies":477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3073,"num_write_kb":103456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":849,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.712704+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:16:20.712704+0000","last_peered":"2023-09-23T20:16:20.712704+0000","last_clean":"2023-09-23T20:16:20.712704+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:16:20.712704+0000","last_undegraded":"2023-09-23T20:16:20.712704+0000","last_fullsized":"2023-09-23T20:16:20.712704+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":213,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.695101+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:16:20.695101+0000","last_peered":"2023-09-23T20:16:20.695101+0000","last_clean":"2023-09-23T20:16:20.695101+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:16:20.695101+0000","last_undegraded":"2023-09-23T20:16:20.695101+0000","last_fullsized":"2023-09-23T20:16:20.695101+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":35,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.710331+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:16:20.710331+0000","last_peered":"2023-09-23T20:16:20.710331+0000","last_clean":"2023-09-23T20:16:20.710331+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:16:20.710331+0000","last_undegraded":"2023-09-23T20:16:20.710331+0000","last_fullsized":"2023-09-23T20:16:20.710331+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"467'6163","reported_seq":6260,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:31.104812+0000","last_change":"2023-09-23T20:14:36.074365+0000","last_active":"2023-09-23T20:16:31.104812+0000","last_peered":"2023-09-23T20:16:31.104812+0000","last_clean":"2023-09-23T20:14:33.749328+0000","last_became_active":"2023-09-23T20:14:36.071685+0000","last_became_peered":"2023-09-23T20:14:36.071685+0000","last_unstale":"2023-09-23T20:16:31.104812+0000","last_undegraded":"2023-09-23T20:14:35.849009+0000","last_fullsized":"2023-09-23T20:14:35.834839+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13041664,"num_objects":199,"num_object_clones":0,"num_object_copies":597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1547,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3081,"num_write_kb":104960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":106},{"shards":"0,2","objects":-1348}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":814,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.706838+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:16:20.706838+0000","last_peered":"2023-09-23T20:16:20.706838+0000","last_clean":"2023-09-23T20:16:20.706838+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:16:20.706838+0000","last_undegraded":"2023-09-23T20:16:20.706838+0000","last_fullsized":"2023-09-23T20:16:20.706838+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":35,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.711155+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:16:20.711155+0000","last_peered":"2023-09-23T20:16:20.711155+0000","last_clean":"2023-09-23T20:16:20.711155+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:16:20.711155+0000","last_undegraded":"2023-09-23T20:16:20.711155+0000","last_fullsized":"2023-09-23T20:16:20.711155+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":368,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715020+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:16:20.715020+0000","last_peered":"2023-09-23T20:16:20.715020+0000","last_clean":"2023-09-23T20:16:20.715020+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:16:20.715020+0000","last_undegraded":"2023-09-23T20:16:20.715020+0000","last_fullsized":"2023-09-23T20:16:20.715020+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"467'6187","reported_seq":6286,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:32.953091+0000","last_change":"2023-09-23T20:14:36.109242+0000","last_active":"2023-09-23T20:16:32.953091+0000","last_peered":"2023-09-23T20:16:32.953091+0000","last_clean":"2023-09-23T20:14:33.735119+0000","last_became_active":"2023-09-23T20:14:36.106276+0000","last_became_peered":"2023-09-23T20:14:36.106276+0000","last_unstale":"2023-09-23T20:16:32.953091+0000","last_undegraded":"2023-09-23T20:14:35.867219+0000","last_fullsized":"2023-09-23T20:14:35.838359+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6187,"log_dups_size":0,"ondisk_log_size":6187,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12861440,"num_objects":196,"num_object_clones":0,"num_object_copies":588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3078,"num_write_kb":104784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":57},{"shards":"3,7","objects":-1302}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":528,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.706034+0000","last_change":"2023-09-23T20:14:37.892742+0000","last_active":"2023-09-23T20:16:20.706034+0000","last_peered":"2023-09-23T20:16:20.706034+0000","last_clean":"2023-09-23T20:16:20.706034+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:16:20.706034+0000","last_undegraded":"2023-09-23T20:16:20.706034+0000","last_fullsized":"2023-09-23T20:16:20.706034+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":359,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.714739+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:16:20.714739+0000","last_peered":"2023-09-23T20:16:20.714739+0000","last_clean":"2023-09-23T20:16:20.714739+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:16:20.714739+0000","last_undegraded":"2023-09-23T20:16:20.714739+0000","last_fullsized":"2023-09-23T20:16:20.714739+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":35,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.705058+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:16:20.705058+0000","last_peered":"2023-09-23T20:16:20.705058+0000","last_clean":"2023-09-23T20:16:20.705058+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:16:20.705058+0000","last_undegraded":"2023-09-23T20:16:20.705058+0000","last_fullsized":"2023-09-23T20:16:20.705058+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"467'6226","reported_seq":9420,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:31.520330+0000","last_change":"2023-09-23T20:16:17.619121+0000","last_active":"2023-09-23T20:16:31.520330+0000","last_peered":"2023-09-23T20:16:31.520330+0000","last_clean":"2023-09-23T20:16:31.520330+0000","last_became_active":"2023-09-23T20:16:17.609227+0000","last_became_peered":"2023-09-23T20:16:17.609227+0000","last_unstale":"2023-09-23T20:16:31.520330+0000","last_undegraded":"2023-09-23T20:16:31.520330+0000","last_fullsized":"2023-09-23T20:16:31.520330+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6226,"log_dups_size":0,"ondisk_log_size":6226,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15335424,"num_objects":234,"num_object_clones":0,"num_object_copies":702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3090,"num_write_kb":106368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1542,"num_bytes_recovered":34668544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":805,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715272+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:16:20.715272+0000","last_peered":"2023-09-23T20:16:20.715272+0000","last_clean":"2023-09-23T20:16:20.715272+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:16:20.715272+0000","last_undegraded":"2023-09-23T20:16:20.715272+0000","last_fullsized":"2023-09-23T20:16:20.715272+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":108,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.693775+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:16:20.693775+0000","last_peered":"2023-09-23T20:16:20.693775+0000","last_clean":"2023-09-23T20:16:20.693775+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:16:20.693775+0000","last_undegraded":"2023-09-23T20:16:20.693775+0000","last_fullsized":"2023-09-23T20:16:20.693775+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":211,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.695505+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:16:20.695505+0000","last_peered":"2023-09-23T20:16:20.695505+0000","last_clean":"2023-09-23T20:16:20.695505+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:16:20.695505+0000","last_undegraded":"2023-09-23T20:16:20.695505+0000","last_fullsized":"2023-09-23T20:16:20.695505+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"467'8802","reported_seq":8897,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.224398+0000","last_change":"2023-09-23T20:14:36.198248+0000","last_active":"2023-09-23T20:16:33.224398+0000","last_peered":"2023-09-23T20:16:33.224398+0000","last_clean":"2023-09-23T20:14:33.842789+0000","last_became_active":"2023-09-23T20:14:36.169565+0000","last_became_peered":"2023-09-23T20:14:36.169565+0000","last_unstale":"2023-09-23T20:16:33.224398+0000","last_undegraded":"2023-09-23T20:14:36.113647+0000","last_fullsized":"2023-09-23T20:14:35.847403+0000","mapping_epoch":442,"log_start":"417'3373","ondisk_log_start":"417'3373","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5429,"log_dups_size":0,"ondisk_log_size":5429,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21233664,"num_objects":324,"num_object_clones":0,"num_object_copies":972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6116,"num_write_kb":206080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":131},{"shards":"3,5","objects":-2703}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":749,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.695124+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:16:20.695124+0000","last_peered":"2023-09-23T20:16:20.695124+0000","last_clean":"2023-09-23T20:16:20.695124+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:16:20.695124+0000","last_undegraded":"2023-09-23T20:16:20.695124+0000","last_fullsized":"2023-09-23T20:16:20.695124+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":457,"reported_epoch":466,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:20.710468+0000","last_change":"2023-09-23T20:14:35.895655+0000","last_active":"2023-09-23T20:16:20.710468+0000","last_peered":"2023-09-23T20:16:20.710468+0000","last_clean":"2023-09-23T20:07:25.847413+0000","last_became_active":"2023-09-23T20:14:35.895204+0000","last_became_peered":"2023-09-23T20:14:35.895204+0000","last_unstale":"2023-09-23T20:16:20.710468+0000","last_undegraded":"2023-09-23T20:14:35.838988+0000","last_fullsized":"2023-09-23T20:14:35.837779+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:17.785522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":467,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.704553+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:16:20.704553+0000","last_peered":"2023-09-23T20:16:20.704553+0000","last_clean":"2023-09-23T20:16:20.704553+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:16:20.704553+0000","last_undegraded":"2023-09-23T20:16:20.704553+0000","last_fullsized":"2023-09-23T20:16:20.704553+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715198+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:16:20.715198+0000","last_peered":"2023-09-23T20:16:20.715198+0000","last_clean":"2023-09-23T20:16:20.715198+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:16:20.715198+0000","last_undegraded":"2023-09-23T20:16:20.715198+0000","last_fullsized":"2023-09-23T20:16:20.715198+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"467'8053","reported_seq":8132,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.040814+0000","last_change":"2023-09-23T20:13:46.785589+0000","last_active":"2023-09-23T20:16:33.040814+0000","last_peered":"2023-09-23T20:16:33.040814+0000","last_clean":"2023-09-23T20:16:33.040814+0000","last_became_active":"2023-09-23T20:13:46.781975+0000","last_became_peered":"2023-09-23T20:13:46.781975+0000","last_unstale":"2023-09-23T20:16:33.040814+0000","last_undegraded":"2023-09-23T20:16:33.040814+0000","last_fullsized":"2023-09-23T20:16:33.040814+0000","mapping_epoch":407,"log_start":"420'3762","ondisk_log_start":"420'3762","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4291,"log_dups_size":0,"ondisk_log_size":4291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19759104,"num_objects":301,"num_object_clones":0,"num_object_copies":903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6297,"num_write_kb":211168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":810,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.704242+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:16:20.704242+0000","last_peered":"2023-09-23T20:16:20.704242+0000","last_clean":"2023-09-23T20:16:20.704242+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:16:20.704242+0000","last_undegraded":"2023-09-23T20:16:20.704242+0000","last_fullsized":"2023-09-23T20:16:20.704242+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1117,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.714990+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:16:20.714990+0000","last_peered":"2023-09-23T20:16:20.714990+0000","last_clean":"2023-09-23T20:16:20.714990+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:16:20.714990+0000","last_undegraded":"2023-09-23T20:16:20.714990+0000","last_fullsized":"2023-09-23T20:16:20.714990+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":35,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.703326+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:16:20.703326+0000","last_peered":"2023-09-23T20:16:20.703326+0000","last_clean":"2023-09-23T20:16:20.703326+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:16:20.703326+0000","last_undegraded":"2023-09-23T20:16:20.703326+0000","last_fullsized":"2023-09-23T20:16:20.703326+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":329,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716075+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:16:20.716075+0000","last_peered":"2023-09-23T20:16:20.716075+0000","last_clean":"2023-09-23T20:16:20.716075+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:16:20.716075+0000","last_undegraded":"2023-09-23T20:16:20.716075+0000","last_fullsized":"2023-09-23T20:16:20.716075+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"467'8875","reported_seq":8967,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.326059+0000","last_change":"2023-09-23T20:14:36.228252+0000","last_active":"2023-09-23T20:16:33.326059+0000","last_peered":"2023-09-23T20:16:33.326059+0000","last_clean":"2023-09-23T20:14:33.823351+0000","last_became_active":"2023-09-23T20:14:36.216536+0000","last_became_peered":"2023-09-23T20:14:36.216536+0000","last_unstale":"2023-09-23T20:16:33.326059+0000","last_undegraded":"2023-09-23T20:14:36.114735+0000","last_fullsized":"2023-09-23T20:14:35.945404+0000","mapping_epoch":442,"log_start":"419'3584","ondisk_log_start":"419'3584","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5291,"log_dups_size":0,"ondisk_log_size":5291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20512768,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3085,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6177,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":153},{"shards":"5,7","objects":-2772}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":705,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.713952+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:16:20.713952+0000","last_peered":"2023-09-23T20:16:20.713952+0000","last_clean":"2023-09-23T20:16:20.713952+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:16:20.713952+0000","last_undegraded":"2023-09-23T20:16:20.713952+0000","last_fullsized":"2023-09-23T20:16:20.713952+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1105,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.714793+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:16:20.714793+0000","last_peered":"2023-09-23T20:16:20.714793+0000","last_clean":"2023-09-23T20:16:20.714793+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:16:20.714793+0000","last_undegraded":"2023-09-23T20:16:20.714793+0000","last_fullsized":"2023-09-23T20:16:20.714793+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":165,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716423+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:16:20.716423+0000","last_peered":"2023-09-23T20:16:20.716423+0000","last_clean":"2023-09-23T20:16:20.716423+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:16:20.716423+0000","last_undegraded":"2023-09-23T20:16:20.716423+0000","last_fullsized":"2023-09-23T20:16:20.716423+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":132,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715235+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:16:20.715235+0000","last_peered":"2023-09-23T20:16:20.715235+0000","last_clean":"2023-09-23T20:16:20.715235+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:16:20.715235+0000","last_undegraded":"2023-09-23T20:16:20.715235+0000","last_fullsized":"2023-09-23T20:16:20.715235+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"467'7815","reported_seq":7894,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.202656+0000","last_change":"2023-09-23T20:13:46.814034+0000","last_active":"2023-09-23T20:16:33.202656+0000","last_peered":"2023-09-23T20:16:33.202656+0000","last_clean":"2023-09-23T20:16:33.202656+0000","last_became_active":"2023-09-23T20:13:46.812300+0000","last_became_peered":"2023-09-23T20:13:46.812300+0000","last_unstale":"2023-09-23T20:16:33.202656+0000","last_undegraded":"2023-09-23T20:16:33.202656+0000","last_fullsized":"2023-09-23T20:16:33.202656+0000","mapping_epoch":407,"log_start":"416'3305","ondisk_log_start":"416'3305","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4510,"log_dups_size":0,"ondisk_log_size":4510,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21725184,"num_objects":331,"num_object_clones":0,"num_object_copies":993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6099,"num_write_kb":205792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1375,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.703630+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:16:20.703630+0000","last_peered":"2023-09-23T20:16:20.703630+0000","last_clean":"2023-09-23T20:16:20.703630+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:16:20.703630+0000","last_undegraded":"2023-09-23T20:16:20.703630+0000","last_fullsized":"2023-09-23T20:16:20.703630+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":662,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.709972+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:16:20.709972+0000","last_peered":"2023-09-23T20:16:20.709972+0000","last_clean":"2023-09-23T20:16:20.709972+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:16:20.709972+0000","last_undegraded":"2023-09-23T20:16:20.709972+0000","last_fullsized":"2023-09-23T20:16:20.709972+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":332,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715126+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:16:20.715126+0000","last_peered":"2023-09-23T20:16:20.715126+0000","last_clean":"2023-09-23T20:16:20.715126+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:16:20.715126+0000","last_undegraded":"2023-09-23T20:16:20.715126+0000","last_fullsized":"2023-09-23T20:16:20.715126+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":264,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716386+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:16:20.716386+0000","last_peered":"2023-09-23T20:16:20.716386+0000","last_clean":"2023-09-23T20:16:20.716386+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:16:20.716386+0000","last_undegraded":"2023-09-23T20:16:20.716386+0000","last_fullsized":"2023-09-23T20:16:20.716386+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"467'8616","reported_seq":8695,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.380087+0000","last_change":"2023-09-23T20:14:24.789921+0000","last_active":"2023-09-23T20:16:33.380087+0000","last_peered":"2023-09-23T20:16:33.380087+0000","last_clean":"2023-09-23T20:16:33.380087+0000","last_became_active":"2023-09-23T20:14:24.787385+0000","last_became_peered":"2023-09-23T20:14:24.787385+0000","last_unstale":"2023-09-23T20:16:33.380087+0000","last_undegraded":"2023-09-23T20:16:33.380087+0000","last_fullsized":"2023-09-23T20:16:33.380087+0000","mapping_epoch":407,"log_start":"416'3252","ondisk_log_start":"416'3252","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5364,"log_dups_size":0,"ondisk_log_size":5364,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22446080,"num_objects":342,"num_object_clones":0,"num_object_copies":1026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6004,"num_write_kb":203104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":528,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.703802+0000","last_change":"2023-09-23T20:14:37.894544+0000","last_active":"2023-09-23T20:16:20.703802+0000","last_peered":"2023-09-23T20:16:20.703802+0000","last_clean":"2023-09-23T20:16:20.703802+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:16:20.703802+0000","last_undegraded":"2023-09-23T20:16:20.703802+0000","last_fullsized":"2023-09-23T20:16:20.703802+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":913,"reported_epoch":466,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:20.710571+0000","last_change":"2023-09-23T20:14:35.882819+0000","last_active":"2023-09-23T20:16:20.710571+0000","last_peered":"2023-09-23T20:16:20.710571+0000","last_clean":"2023-09-23T20:14:31.826679+0000","last_became_active":"2023-09-23T20:14:35.882372+0000","last_became_peered":"2023-09-23T20:14:35.882372+0000","last_unstale":"2023-09-23T20:16:20.710571+0000","last_undegraded":"2023-09-23T20:14:35.846466+0000","last_fullsized":"2023-09-23T20:14:35.837669+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:41.083298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":413,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.693877+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:16:20.693877+0000","last_peered":"2023-09-23T20:16:20.693877+0000","last_clean":"2023-09-23T20:16:20.693877+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:16:20.693877+0000","last_undegraded":"2023-09-23T20:16:20.693877+0000","last_fullsized":"2023-09-23T20:16:20.693877+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":282,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716453+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:16:20.716453+0000","last_peered":"2023-09-23T20:16:20.716453+0000","last_clean":"2023-09-23T20:16:20.716453+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:16:20.716453+0000","last_undegraded":"2023-09-23T20:16:20.716453+0000","last_fullsized":"2023-09-23T20:16:20.716453+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"467'8855","reported_seq":8943,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.040131+0000","last_change":"2023-09-23T20:14:36.189421+0000","last_active":"2023-09-23T20:16:33.040131+0000","last_peered":"2023-09-23T20:16:33.040131+0000","last_clean":"2023-09-23T20:14:33.834923+0000","last_became_active":"2023-09-23T20:14:36.182947+0000","last_became_peered":"2023-09-23T20:14:36.182947+0000","last_unstale":"2023-09-23T20:16:33.040131+0000","last_undegraded":"2023-09-23T20:14:35.901626+0000","last_fullsized":"2023-09-23T20:14:35.838743+0000","mapping_epoch":442,"log_start":"420'3888","ondisk_log_start":"420'3888","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4967,"log_dups_size":0,"ondisk_log_size":4967,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23822336,"num_objects":363,"num_object_clones":0,"num_object_copies":1089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6155,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":165},{"shards":"4,7","objects":-2699}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":640,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.705539+0000","last_change":"2023-09-23T20:13:57.748021+0000","last_active":"2023-09-23T20:16:20.705539+0000","last_peered":"2023-09-23T20:16:20.705539+0000","last_clean":"2023-09-23T20:16:20.705539+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:16:20.705539+0000","last_undegraded":"2023-09-23T20:16:20.705539+0000","last_fullsized":"2023-09-23T20:16:20.705539+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":167,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716659+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:16:20.716659+0000","last_peered":"2023-09-23T20:16:20.716659+0000","last_clean":"2023-09-23T20:16:20.716659+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:16:20.716659+0000","last_undegraded":"2023-09-23T20:16:20.716659+0000","last_fullsized":"2023-09-23T20:16:20.716659+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":270,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716299+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:16:20.716299+0000","last_peered":"2023-09-23T20:16:20.716299+0000","last_clean":"2023-09-23T20:16:20.716299+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:16:20.716299+0000","last_undegraded":"2023-09-23T20:16:20.716299+0000","last_fullsized":"2023-09-23T20:16:20.716299+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":293,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.703478+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:16:20.703478+0000","last_peered":"2023-09-23T20:16:20.703478+0000","last_clean":"2023-09-23T20:16:20.703478+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:16:20.703478+0000","last_undegraded":"2023-09-23T20:16:20.703478+0000","last_fullsized":"2023-09-23T20:16:20.703478+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"467'9194","reported_seq":9282,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:32.974501+0000","last_change":"2023-09-23T20:14:36.072073+0000","last_active":"2023-09-23T20:16:32.974501+0000","last_peered":"2023-09-23T20:16:32.974501+0000","last_clean":"2023-09-23T20:14:33.811030+0000","last_became_active":"2023-09-23T20:14:36.065847+0000","last_became_peered":"2023-09-23T20:14:36.065847+0000","last_unstale":"2023-09-23T20:16:32.974501+0000","last_undegraded":"2023-09-23T20:14:35.915006+0000","last_fullsized":"2023-09-23T20:14:35.838163+0000","mapping_epoch":442,"log_start":"427'4373","ondisk_log_start":"427'4373","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4821,"log_dups_size":0,"ondisk_log_size":4821,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18939904,"num_objects":289,"num_object_clones":0,"num_object_copies":867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3193,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6405,"num_write_kb":214208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":135},{"shards":"5,7","objects":-2904}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":686,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715094+0000","last_change":"2023-09-23T20:14:25.790696+0000","last_active":"2023-09-23T20:16:20.715094+0000","last_peered":"2023-09-23T20:16:20.715094+0000","last_clean":"2023-09-23T20:16:20.715094+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:16:20.715094+0000","last_undegraded":"2023-09-23T20:16:20.715094+0000","last_fullsized":"2023-09-23T20:16:20.715094+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":309,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.710274+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:16:20.710274+0000","last_peered":"2023-09-23T20:16:20.710274+0000","last_clean":"2023-09-23T20:16:20.710274+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:16:20.710274+0000","last_undegraded":"2023-09-23T20:16:20.710274+0000","last_fullsized":"2023-09-23T20:16:20.710274+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":624,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.694710+0000","last_change":"2023-09-23T20:16:02.235622+0000","last_active":"2023-09-23T20:16:20.694710+0000","last_peered":"2023-09-23T20:16:20.694710+0000","last_clean":"2023-09-23T20:16:20.694710+0000","last_became_active":"2023-09-23T20:16:02.234933+0000","last_became_peered":"2023-09-23T20:16:02.234933+0000","last_unstale":"2023-09-23T20:16:20.694710+0000","last_undegraded":"2023-09-23T20:16:20.694710+0000","last_fullsized":"2023-09-23T20:16:20.694710+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:21.855632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":99,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.693690+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:16:20.693690+0000","last_peered":"2023-09-23T20:16:20.693690+0000","last_clean":"2023-09-23T20:16:20.693690+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:16:20.693690+0000","last_undegraded":"2023-09-23T20:16:20.693690+0000","last_fullsized":"2023-09-23T20:16:20.693690+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"467'8520","reported_seq":8600,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:31.554588+0000","last_change":"2023-09-23T20:14:25.775099+0000","last_active":"2023-09-23T20:16:31.554588+0000","last_peered":"2023-09-23T20:16:31.554588+0000","last_clean":"2023-09-23T20:16:31.554588+0000","last_became_active":"2023-09-23T20:14:24.777506+0000","last_became_peered":"2023-09-23T20:14:24.777506+0000","last_unstale":"2023-09-23T20:16:31.554588+0000","last_undegraded":"2023-09-23T20:16:31.554588+0000","last_fullsized":"2023-09-23T20:16:31.554588+0000","mapping_epoch":407,"log_start":"420'3773","ondisk_log_start":"420'3773","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4747,"log_dups_size":0,"ondisk_log_size":4747,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30703616,"num_objects":468,"num_object_clones":0,"num_object_copies":1404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5900,"num_write_kb":203808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":793,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:30.176847+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:16:30.176847+0000","last_peered":"2023-09-23T20:16:30.176847+0000","last_clean":"2023-09-23T20:16:30.176847+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:16:30.176847+0000","last_undegraded":"2023-09-23T20:16:30.176847+0000","last_fullsized":"2023-09-23T20:16:30.176847+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":168,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.710433+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:16:20.710433+0000","last_peered":"2023-09-23T20:16:20.710433+0000","last_clean":"2023-09-23T20:16:20.710433+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:16:20.710433+0000","last_undegraded":"2023-09-23T20:16:20.710433+0000","last_fullsized":"2023-09-23T20:16:20.710433+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":39,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.710843+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:16:20.710843+0000","last_peered":"2023-09-23T20:16:20.710843+0000","last_clean":"2023-09-23T20:16:20.710843+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:16:20.710843+0000","last_undegraded":"2023-09-23T20:16:20.710843+0000","last_fullsized":"2023-09-23T20:16:20.710843+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":250,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.704286+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:16:20.704286+0000","last_peered":"2023-09-23T20:16:20.704286+0000","last_clean":"2023-09-23T20:16:20.704286+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:16:20.704286+0000","last_undegraded":"2023-09-23T20:16:20.704286+0000","last_fullsized":"2023-09-23T20:16:20.704286+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"467'7960","reported_seq":12187,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:16:33.046158+0000","last_change":"2023-09-23T20:14:35.970620+0000","last_active":"2023-09-23T20:16:33.046158+0000","last_peered":"2023-09-23T20:16:33.046158+0000","last_clean":"2023-09-23T20:14:33.799110+0000","last_became_active":"2023-09-23T20:14:35.967502+0000","last_became_peered":"2023-09-23T20:14:35.967502+0000","last_unstale":"2023-09-23T20:16:33.046158+0000","last_undegraded":"2023-09-23T20:16:33.046158+0000","last_fullsized":"2023-09-23T20:16:33.046158+0000","mapping_epoch":442,"log_start":"419'3645","ondisk_log_start":"419'3645","created":418,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4315,"log_dups_size":0,"ondisk_log_size":4315,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21299200,"num_objects":325,"num_object_clones":0,"num_object_copies":975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":325,"num_objects_unfound":0,"num_objects_dirty":325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6215,"num_write_kb":209280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2059,"num_bytes_recovered":134938624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":325}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":244,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.717148+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:16:20.717148+0000","last_peered":"2023-09-23T20:16:20.717148+0000","last_clean":"2023-09-23T20:16:20.717148+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:16:20.717148+0000","last_undegraded":"2023-09-23T20:16:20.717148+0000","last_fullsized":"2023-09-23T20:16:20.717148+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":244,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716364+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:16:20.716364+0000","last_peered":"2023-09-23T20:16:20.716364+0000","last_clean":"2023-09-23T20:16:20.716364+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:16:20.716364+0000","last_undegraded":"2023-09-23T20:16:20.716364+0000","last_fullsized":"2023-09-23T20:16:20.716364+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"467'8024","reported_seq":8110,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.326952+0000","last_change":"2023-09-23T20:14:36.202358+0000","last_active":"2023-09-23T20:16:33.326952+0000","last_peered":"2023-09-23T20:16:33.326952+0000","last_clean":"2023-09-23T20:14:33.783493+0000","last_became_active":"2023-09-23T20:14:36.193525+0000","last_became_peered":"2023-09-23T20:14:36.193525+0000","last_unstale":"2023-09-23T20:16:33.326952+0000","last_undegraded":"2023-09-23T20:14:36.034471+0000","last_fullsized":"2023-09-23T20:14:35.969590+0000","mapping_epoch":442,"log_start":"419'3569","ondisk_log_start":"419'3569","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4455,"log_dups_size":0,"ondisk_log_size":4455,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27525120,"num_objects":420,"num_object_clones":0,"num_object_copies":1260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6242,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":133},{"shards":"2,5","objects":-2624}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":484,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.703455+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:16:20.703455+0000","last_peered":"2023-09-23T20:16:20.703455+0000","last_clean":"2023-09-23T20:16:20.703455+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:16:20.703455+0000","last_undegraded":"2023-09-23T20:16:20.703455+0000","last_fullsized":"2023-09-23T20:16:20.703455+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":487,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:30.176273+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:16:30.176273+0000","last_peered":"2023-09-23T20:16:30.176273+0000","last_clean":"2023-09-23T20:16:30.176273+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:16:30.176273+0000","last_undegraded":"2023-09-23T20:16:30.176273+0000","last_fullsized":"2023-09-23T20:16:30.176273+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.17","version":"467'7868","reported_seq":7946,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.378101+0000","last_change":"2023-09-23T20:13:46.814598+0000","last_active":"2023-09-23T20:16:33.378101+0000","last_peered":"2023-09-23T20:16:33.378101+0000","last_clean":"2023-09-23T20:16:33.378101+0000","last_became_active":"2023-09-23T20:13:46.812685+0000","last_became_peered":"2023-09-23T20:13:46.812685+0000","last_unstale":"2023-09-23T20:16:33.378101+0000","last_undegraded":"2023-09-23T20:16:33.378101+0000","last_fullsized":"2023-09-23T20:16:33.378101+0000","mapping_epoch":418,"log_start":"419'3427","ondisk_log_start":"419'3427","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4441,"log_dups_size":0,"ondisk_log_size":4441,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18612224,"num_objects":284,"num_object_clones":0,"num_object_copies":852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6158,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":474,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.704482+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:16:20.704482+0000","last_peered":"2023-09-23T20:16:20.704482+0000","last_clean":"2023-09-23T20:16:20.704482+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:16:20.704482+0000","last_undegraded":"2023-09-23T20:16:20.704482+0000","last_fullsized":"2023-09-23T20:16:20.704482+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":463,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.710312+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:16:20.710312+0000","last_peered":"2023-09-23T20:16:20.710312+0000","last_clean":"2023-09-23T20:16:20.710312+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:16:20.710312+0000","last_undegraded":"2023-09-23T20:16:20.710312+0000","last_fullsized":"2023-09-23T20:16:20.710312+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"467'7883","reported_seq":14075,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T20:16:33.325164+0000","last_change":"2023-09-23T20:16:02.232381+0000","last_active":"2023-09-23T20:16:33.325164+0000","last_peered":"2023-09-23T20:16:33.325164+0000","last_clean":"2023-09-23T20:16:33.325164+0000","last_became_active":"2023-09-23T20:16:02.185816+0000","last_became_peered":"2023-09-23T20:16:02.185816+0000","last_unstale":"2023-09-23T20:16:33.325164+0000","last_undegraded":"2023-09-23T20:16:33.325164+0000","last_fullsized":"2023-09-23T20:16:33.325164+0000","mapping_epoch":461,"log_start":"417'3432","ondisk_log_start":"417'3432","created":418,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4451,"log_dups_size":0,"ondisk_log_size":4451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22446080,"num_objects":343,"num_object_clones":0,"num_object_copies":1029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6159,"num_write_kb":208032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3047,"num_bytes_recovered":134610944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":179,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.716598+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:16:20.716598+0000","last_peered":"2023-09-23T20:16:20.716598+0000","last_clean":"2023-09-23T20:16:20.716598+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:16:20.716598+0000","last_undegraded":"2023-09-23T20:16:20.716598+0000","last_fullsized":"2023-09-23T20:16:20.716598+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":326,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.714880+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:16:20.714880+0000","last_peered":"2023-09-23T20:16:20.714880+0000","last_clean":"2023-09-23T20:16:20.714880+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:16:20.714880+0000","last_undegraded":"2023-09-23T20:16:20.714880+0000","last_fullsized":"2023-09-23T20:16:20.714880+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"467'7829","reported_seq":16599,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:33.073401+0000","last_change":"2023-09-23T20:14:38.010024+0000","last_active":"2023-09-23T20:16:33.073401+0000","last_peered":"2023-09-23T20:16:33.073401+0000","last_clean":"2023-09-23T20:14:33.867233+0000","last_became_active":"2023-09-23T20:14:38.004874+0000","last_became_peered":"2023-09-23T20:14:38.004874+0000","last_unstale":"2023-09-23T20:16:33.073401+0000","last_undegraded":"2023-09-23T20:14:37.959307+0000","last_fullsized":"2023-09-23T20:14:37.876450+0000","mapping_epoch":444,"log_start":"419'3557","ondisk_log_start":"419'3557","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4272,"log_dups_size":0,"ondisk_log_size":4272,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20381696,"num_objects":311,"num_object_clones":0,"num_object_copies":933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6099,"num_write_kb":205120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2895,"num_bytes_recovered":189726720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":127},{"shards":"3,7","objects":184}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":312,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715087+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:16:20.715087+0000","last_peered":"2023-09-23T20:16:20.715087+0000","last_clean":"2023-09-23T20:16:20.715087+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:16:20.715087+0000","last_undegraded":"2023-09-23T20:16:20.715087+0000","last_fullsized":"2023-09-23T20:16:20.715087+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":36,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T20:16:20.715175+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:16:20.715175+0000","last_peered":"2023-09-23T20:16:20.715175+0000","last_clean":"2023-09-23T20:16:20.715175+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:16:20.715175+0000","last_undegraded":"2023-09-23T20:16:20.715175+0000","last_fullsized":"2023-09-23T20:16:20.715175+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":36,"stat_sum":{"num_bytes":717946904,"num_objects":10956,"num_object_clones":0,"num_object_copies":32868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34828,"num_objects_misplaced":578,"num_objects_unfound":0,"num_objects_dirty":10956,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":196750,"num_write_kb":6646529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22680,"num_bytes_recovered":1186398208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5844316160,"data_stored":5844303944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":182788,"ondisk_log_size":182788,"up":108,"acting":91,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":191,"num_bytes_recovered":4194380,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234425,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":974320,"kb_used_data":453032,"kb_used_omap":0,"kb_used_meta":521280,"kb_avail":92770832,"statfs":{"total":95995035648,"available":94997331968,"internally_reserved":0,"allocated":463904768,"data_stored":462471198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":533790720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":11.353,"5min":3.855,"15min":2.8380000000000001},"min":{"1min":0.219,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":149.25999999999999,"5min":149.25999999999999,"15min":149.25999999999999},"last":58.109000000000002},{"interface":"front","average":{"1min":12.917,"5min":4.3019999999999996,"15min":3.0609999999999999},"min":{"1min":0.248,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":149.11799999999999,"5min":149.11799999999999,"15min":149.11799999999999},"last":58.042000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":3.2250000000000001,"15min":3.5779999999999998},"min":{"1min":0.29699999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.0599999999999996,"5min":91.335999999999999,"15min":104.16800000000001},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.972,"5min":3.738,"15min":3.8370000000000002},"min":{"1min":0.29599999999999999,"5min":0.249,"15min":0.223},"max":{"1min":6.8609999999999998,"5min":91.516999999999996,"15min":104.003},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":3.3969999999999998,"15min":3.7629999999999999},"min":{"1min":0.32700000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":7.3600000000000003,"5min":67.760000000000005,"15min":105.842},"last":0.96899999999999997},{"interface":"front","average":{"1min":2.633,"5min":3.738,"15min":4.1950000000000003},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":8.6669999999999998,"5min":67.488,"15min":105.66},"last":0.42899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":14.476000000000001,"5min":4.5030000000000001,"15min":4.6440000000000001},"min":{"1min":0.33400000000000002,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":185.17099999999999,"5min":185.17099999999999,"15min":185.17099999999999},"last":1.091},{"interface":"front","average":{"1min":15.061999999999999,"5min":4.593,"15min":4.649},"min":{"1min":0.63,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":184.99100000000001,"5min":184.99100000000001,"15min":184.99100000000001},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":0.96699999999999997,"15min":0.76300000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.22500000000000001},"max":{"1min":7.6440000000000001,"5min":7.6440000000000001,"15min":19.215},"last":1.036},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.151,"15min":0.96499999999999997},"min":{"1min":0.42699999999999999,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":3.9809999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":1.0149999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.595,"15min":1.1990000000000001},"min":{"1min":0.51100000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":8.4440000000000008,"5min":14.069000000000001,"15min":35.488},"last":1.244},{"interface":"front","average":{"1min":1.367,"5min":1.284,"15min":1.0389999999999999},"min":{"1min":0.41699999999999998,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":7.6109999999999998,"5min":13.43,"15min":34.896000000000001},"last":0.79600000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":1.234,"15min":0.99199999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":8.4120000000000008,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.499},{"interface":"front","average":{"1min":1.522,"5min":0.93700000000000006,"15min":0.754},"min":{"1min":0.36699999999999999,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":7.5659999999999998,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.82999999999999996}]}]},{"osd":6,"up_from":0,"seq":1078036791465,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762624,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926161,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1325464,"kb_used_data":535248,"kb_used_omap":8,"kb_used_meta":790199,"kb_avail":92419688,"statfs":{"total":95995035648,"available":94637760512,"internally_reserved":0,"allocated":548093952,"data_stored":545925194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":809164385},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":1.952,"15min":3.4940000000000002},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":29.893000000000001,"5min":72.534000000000006,"15min":94.055999999999997},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.97,"5min":2.4169999999999998,"15min":3.7130000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.183},"max":{"1min":23.436,"5min":72.667000000000002,"15min":91.632999999999996},"last":1.5940000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:16:17 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":2.863,"15min":4.944},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.1820000000000004,"5min":160.542,"15min":270.51100000000002},"last":0.20499999999999999},{"interface":"front","average":{"1min":2.0219999999999998,"5min":3.5009999999999999,"15min":4.8970000000000002},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":18.433,"5min":160.75999999999999,"15min":269.69299999999998},"last":1.1859999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4279999999999999,"15min":3.8980000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":8.9469999999999992,"5min":10.082000000000001,"15min":126.95399999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.532,"15min":4.1020000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":7.375,"5min":9.5440000000000005,"15min":127.294},"last":0.28799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.726,"15min":2.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":5.859,"5min":9.6959999999999997,"15min":148.20599999999999},"last":1.238},{"interface":"front","average":{"1min":5.9790000000000001,"5min":2.29,"15min":2.2589999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":105.282,"5min":105.282,"15min":148.11000000000001},"last":0.33200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.9630000000000001,"15min":1.859},"min":{"1min":0.378,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.5430000000000001,"5min":52.061,"15min":64.725999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.4649999999999999,"15min":1.8320000000000001},"min":{"1min":0.41699999999999998,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1369999999999996,"5min":51.741,"15min":51.741},"last":2.274}]},{"osd":6,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":1.5309999999999999,"15min":1.653},"min":{"1min":0.38,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":7.1950000000000003,"5min":7.1950000000000003,"15min":15.744999999999999},"last":1.3440000000000001},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.3400000000000001,"15min":1.6839999999999999},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":7.2229999999999999,"5min":8.3919999999999995,"15min":13.766999999999999},"last":0.38600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.569,"15min":1.75},"min":{"1min":0.53700000000000003,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.9980000000000002,"5min":12.981,"15min":65.081999999999994},"last":1.899},{"interface":"front","average":{"1min":2.6909999999999998,"5min":2.161,"15min":1.8440000000000001},"min":{"1min":0.49199999999999999,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.6600000000000001,"5min":17.151,"15min":17.151},"last":3.1440000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599099,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1139152,"kb_used_data":322120,"kb_used_omap":12,"kb_used_meta":817011,"kb_avail":92606000,"statfs":{"total":95995035648,"available":94828544000,"internally_reserved":0,"allocated":329850880,"data_stored":327634791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":836620286},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.1880000000000002,"15min":2.464},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":5.3879999999999999,"5min":65.382999999999996,"15min":76.251999999999995},"last":0.35099999999999998},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.282,"15min":2.718},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.265000000000001,"5min":65.427000000000007,"15min":76.590000000000003},"last":2.0219999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":3.0899999999999999,"15min":3.5649999999999999},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":7.5270000000000001,"5min":70.453000000000003,"15min":94.768000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.8969999999999998,"15min":3.2450000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.218},"max":{"1min":7.2009999999999996,"5min":69.825000000000003,"15min":96.808000000000007},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:15:40 2023","interfaces":[{"interface":"back","average":{"1min":7.5960000000000001,"5min":2.9870000000000001,"15min":4.1130000000000004},"min":{"1min":0.222,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":132.82499999999999,"5min":132.82499999999999,"15min":132.82499999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":8.1519999999999992,"5min":2.9900000000000002,"15min":3.411},"min":{"1min":0.23400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":130.97300000000001,"5min":130.97300000000001,"15min":130.97300000000001},"last":0.46100000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.4399999999999999,"15min":1.79},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":9.3399999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.432},{"interface":"front","average":{"1min":1.667,"5min":1.7649999999999999,"15min":2.105},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.641,"5min":13.387,"15min":76.161000000000001},"last":10.135}]},{"osd":5,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":3.3460000000000001,"15min":2.8519999999999999},"min":{"1min":0.34999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.3859999999999992,"5min":135.202,"15min":172.505},"last":6.9089999999999998},{"interface":"front","average":{"1min":1.96,"5min":2.0619999999999998,"15min":2.6749999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.1260000000000003,"5min":29.196999999999999,"15min":172.375},"last":2.0609999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.4930000000000001,"15min":1.5920000000000001},"min":{"1min":0.435,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.0510000000000002,"5min":14.664,"15min":14.664},"last":6.8310000000000004},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.758,"15min":1.8400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.2649999999999997,"5min":12.025,"15min":14.279999999999999},"last":2.0870000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":2.0990000000000002,"15min":1.883},"min":{"1min":0.32500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2970000000000006,"5min":13.481999999999999,"15min":13.481999999999999},"last":3.7719999999999998},{"interface":"front","average":{"1min":2.1720000000000002,"5min":1.6399999999999999,"15min":1.581},"min":{"1min":0.443,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.266,"5min":14.698,"15min":14.698},"last":6.9429999999999996}]}]},{"osd":3,"up_from":109,"seq":468151435573,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1565912,"kb_used_data":675024,"kb_used_omap":4,"kb_used_meta":890875,"kb_avail":92179240,"statfs":{"total":95995035648,"available":94391541760,"internally_reserved":0,"allocated":691224576,"data_stored":688758609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":912256109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5940000000000003,"5min":1.8560000000000001,"15min":3.0859999999999999},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":76.632000000000005,"5min":76.632000000000005,"15min":76.632000000000005},"last":1.4990000000000001},{"interface":"front","average":{"1min":5.4580000000000002,"5min":2.266,"15min":3.4159999999999999},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.19600000000000001},"max":{"1min":81.429000000000002,"5min":81.429000000000002,"15min":81.429000000000002},"last":0.32900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":5.0510000000000002,"15min":4.6260000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":9,"5min":183.87,"15min":183.87},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.2230000000000001,"5min":4.9740000000000002,"15min":4.758},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":5.5810000000000004,"5min":185.84899999999999,"15min":185.84899999999999},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":4.2610000000000001,"15min":4.109},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":9.0289999999999999,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":2.2679999999999998,"5min":3.6669999999999998,"15min":3.819},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":16.550999999999998,"5min":163.92099999999999,"15min":163.92099999999999},"last":0.34499999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.1120000000000001,"15min":2.786},"min":{"1min":0.32400000000000001,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":5.21,"5min":7.4960000000000004,"15min":109.66800000000001},"last":0.376},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6619999999999999,"15min":3.2469999999999999},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":9.6470000000000002,"5min":11.815,"15min":110.295},"last":2.0430000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":3.6349999999999998,"15min":3.9129999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6799999999999997,"5min":200.53200000000001,"15min":200.53200000000001},"last":4.6520000000000001},{"interface":"front","average":{"1min":3.411,"5min":3.7599999999999998,"15min":3.9769999999999999},"min":{"1min":0.48799999999999999,"5min":0.34799999999999998,"15min":0.26900000000000002},"max":{"1min":9.9429999999999996,"5min":200.846,"15min":200.846},"last":5.0700000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.6539999999999999,"15min":2.8660000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.32300000000000001},"max":{"1min":8.9610000000000003,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.641,"5min":1.2669999999999999,"15min":2.9020000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":5.8959999999999999,"5min":8.2629999999999999,"15min":82.233000000000004},"last":0.40200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.9470000000000001,"15min":3.3969999999999998},"min":{"1min":0.42799999999999999,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":9.0600000000000005,"5min":139.465,"15min":139.465},"last":2.8210000000000002},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.8540000000000001,"15min":3.27},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":5.3010000000000002,"5min":141.56200000000001,"15min":141.56200000000001},"last":3.996}]}]},{"osd":4,"up_from":30,"seq":128849019335,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":772588,"kb_used_data":326244,"kb_used_omap":0,"kb_used_meta":446336,"kb_avail":92972564,"statfs":{"total":95995035648,"available":95203905536,"internally_reserved":0,"allocated":334073856,"data_stored":332644370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":457048064},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.4650000000000001,"15min":1.639},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.2549999999999999,"5min":14.063000000000001,"15min":56.061999999999998},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.6519999999999999,"5min":1.4630000000000001,"15min":1.619},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":6.9740000000000002,"5min":14.048999999999999,"15min":56.075000000000003},"last":0.59899999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":3.7559999999999998,"15min":5.7779999999999996},"min":{"1min":0.28499999999999998,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":14.763,"5min":138.47999999999999,"15min":161.613},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.1789999999999998,"5min":3.7029999999999998,"15min":5.7830000000000004},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.217},"max":{"1min":13.368,"5min":138.429,"15min":162.017},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.8969999999999998,"15min":5.3760000000000003},"min":{"1min":0.28699999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.6920000000000002,"5min":144.916,"15min":182.96700000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.0299999999999998,"5min":3.8300000000000001,"15min":5.3369999999999997},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.23100000000000001},"max":{"1min":9.4550000000000001,"5min":144.83099999999999,"15min":183.244},"last":0.88900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.3159999999999998,"15min":4.6589999999999998},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":4.5819999999999999,"5min":100.361,"15min":184.37100000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.738,"5min":3.4990000000000001,"15min":4.7199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":4.7270000000000003,"5min":100.20699999999999,"15min":184.22300000000001},"last":0.68000000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.111,"15min":1.252},"min":{"1min":0.28199999999999997,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.0739999999999998,"5min":12.257999999999999,"15min":176.81899999999999},"last":3.48},{"interface":"front","average":{"1min":1.7689999999999999,"5min":1.111,"15min":1.2849999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.246},"max":{"1min":7.9470000000000001,"5min":7.9470000000000001,"15min":176.172},"last":3.2160000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":0.85899999999999999,"15min":0.627},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":8.5660000000000007,"5min":10.337999999999999,"15min":10.337999999999999},"last":0.92200000000000004},{"interface":"front","average":{"1min":2.3050000000000002,"5min":1.119,"15min":0.79500000000000004},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":11.348000000000001,"5min":11.348000000000001,"15min":11.348000000000001},"last":0.878}]},{"osd":7,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.196,"15min":1.5349999999999999},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.5090000000000003,"5min":7.71,"15min":76.507000000000005},"last":0.71099999999999997},{"interface":"front","average":{"1min":3.1040000000000001,"5min":1.4099999999999999,"15min":1.6739999999999999},"min":{"1min":0.39900000000000002,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":7.8250000000000002,"5min":7.8250000000000002,"15min":76.462000000000003},"last":0.50900000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757699,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1262180,"kb_used_data":669148,"kb_used_omap":0,"kb_used_meta":593024,"kb_avail":92482972,"statfs":{"total":95995035648,"available":94702563328,"internally_reserved":0,"allocated":685207552,"data_stored":683782507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":607256576},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":7.7800000000000002,"5min":2.794,"15min":2.214},"min":{"1min":0.23000000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":126.657,"5min":126.657,"15min":126.657},"last":7.431},{"interface":"front","average":{"1min":7.2990000000000004,"5min":2.6539999999999999,"15min":2.4289999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":126.697,"5min":126.697,"15min":126.697},"last":0.21099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:15:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":5.0540000000000003,"15min":5.3209999999999997},"min":{"1min":0.22,"5min":0.22,"15min":0.21299999999999999},"max":{"1min":7.383,"5min":150.16999999999999,"15min":171.68299999999999},"last":6.7130000000000001},{"interface":"front","average":{"1min":2.6749999999999998,"5min":4.7960000000000003,"15min":5.4020000000000001},"min":{"1min":0.32100000000000001,"5min":0.24099999999999999,"15min":0.21199999999999999},"max":{"1min":7.3099999999999996,"5min":150.12700000000001,"15min":171.64099999999999},"last":3.782}]},{"osd":2,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":3.5649999999999999,"15min":4.2679999999999998},"min":{"1min":0.317,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2569999999999997,"5min":120.297,"15min":240.57300000000001},"last":2.448},{"interface":"front","average":{"1min":1.47,"5min":3.661,"15min":4.335},"min":{"1min":0.38,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.2859999999999996,"5min":120.325,"15min":240.464},"last":2.4780000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:15:49 2023","interfaces":[{"interface":"back","average":{"1min":10.634,"5min":6.1920000000000002,"15min":5.556},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":195.84700000000001,"5min":195.84700000000001,"15min":201.50899999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":10.66,"5min":6.0990000000000002,"15min":5.4089999999999998},"min":{"1min":0.39500000000000002,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":195.87899999999999,"5min":195.87899999999999,"15min":201.58099999999999},"last":0.40899999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.0629999999999999,"15min":1.3879999999999999},"min":{"1min":0.28499999999999998,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":4.8159999999999998,"5min":8,"15min":191.78899999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.67400000000000004,"15min":1.2150000000000001},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":3.371,"5min":3.371,"15min":191.821},"last":0.38100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.1870000000000001,"15min":0.83099999999999996},"min":{"1min":0.33300000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":5.4000000000000004,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.65400000000000003,"15min":0.58199999999999996},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":3.3180000000000001,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":2.399}]},{"osd":7,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.4079999999999999,"15min":1.478},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":11.725,"5min":11.725,"15min":191.84299999999999},"last":2.173},{"interface":"front","average":{"1min":2.02,"5min":1.3,"15min":1.4990000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":11.041,"5min":11.041,"15min":191.69},"last":2.4249999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":518389760,"data_stored":518389760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2913468416,"data_stored":2913468416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":317063168,"data_stored":317063168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":669450240,"data_stored":669450240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":324210688,"data_stored":324206616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":655757312,"data_stored":655753240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":445976576,"data_stored":445972504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:16:36.251 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:36 smithi135 ceph-mon[221113]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.251 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:36 smithi135 ceph-mon[221113]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.251 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:36 smithi135 ceph-mon[221113]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.251 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:36 smithi135 ceph-mon[221113]: osdmap e469: 8 total, 8 up, 6 in 2023-09-23T20:16:36.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:36 smithi135 ceph-mon[213871]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:36 smithi135 ceph-mon[213871]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:36 smithi135 ceph-mon[213871]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.256 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:36 smithi135 ceph-mon[213871]: osdmap e469: 8 total, 8 up, 6 in 2023-09-23T20:16:36.273 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:36.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:36.351 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:36.381 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:36.409 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:36.459 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:36.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:36 smithi144 ceph-mon[194833]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.462 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:36 smithi144 ceph-mon[194833]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.463 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:36 smithi144 ceph-mon[194833]: failed to encode map e469 with expected crc 2023-09-23T20:16:36.463 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:36 smithi144 ceph-mon[194833]: osdmap e469: 8 total, 8 up, 6 in 2023-09-23T20:16:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:36.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:36.564 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:36.632 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:36.841 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:36.848 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:16:36.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:36.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:36.956 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:37.038 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:37.173 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:37.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:37.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:37.222 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:16:37.282 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:37.327 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:37.374 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:37.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:37.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:37.422 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:37.478 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:37.505 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:37.577 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:37.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:37.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:37.685 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:37.720 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:38.163 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: pgmap v1312: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 660 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 34550/31545 objects degraded (109.526%); 578/31545 objects misplaced (1.832%); 1.1 MiB/s, 195 objects/s recovering 2023-09-23T20:16:38.163 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:38.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:38.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1218550466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:38.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:38.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:38.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:38.164 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: osdmap e470: 8 total, 8 up, 6 in 2023-09-23T20:16:38.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 34550/31545 objects degraded (109.526%), 17 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:38.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: pgmap v1312: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 660 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 34550/31545 objects degraded (109.526%); 578/31545 objects misplaced (1.832%); 1.1 MiB/s, 195 objects/s recovering 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1218550466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:38.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: osdmap e470: 8 total, 8 up, 6 in 2023-09-23T20:16:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.383 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 34550/31545 objects degraded (109.526%), 17 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: pgmap v1312: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 660 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 34550/31545 objects degraded (109.526%); 578/31545 objects misplaced (1.832%); 1.1 MiB/s, 195 objects/s recovering 2023-09-23T20:16:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1218550466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: osdmap e470: 8 total, 8 up, 6 in 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: failed to encode map e470 with expected crc 2023-09-23T20:16:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 34550/31545 objects degraded (109.526%), 17 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:38.694 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:38.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:38.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:38.801 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:38.866 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:38.961 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:38.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:38.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:39.064 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:39.122 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:39.227 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:39.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:39.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:39.241 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:16:39.338 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:39.357 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:16:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:39 smithi144 ceph-mon[194833]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:39 smithi144 ceph-mon[194833]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:39 smithi144 ceph-mon[194833]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:39 smithi144 ceph-mon[194833]: osdmap e471: 8 total, 8 up, 6 in 2023-09-23T20:16:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:39 smithi144 ceph-mon[194833]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:39 smithi144 ceph-mon[194833]: pgmap v1315: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 476 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 697 op/s; 33908/22701 objects degraded (149.368%); 383/22701 objects misplaced (1.687%); 1.2 MiB/s, 172 objects/s recovering 2023-09-23T20:16:39.387 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:16:39.432 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:16:39.441 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:39.480 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:16:39.512 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:16:39.524 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:16:39.524 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:16:39.564 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:39.565 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:16:39.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[221113]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[221113]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[221113]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[221113]: osdmap e471: 8 total, 8 up, 6 in 2023-09-23T20:16:39.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[221113]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[221113]: pgmap v1315: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 476 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 697 op/s; 33908/22701 objects degraded (149.368%); 383/22701 objects misplaced (1.687%); 1.2 MiB/s, 172 objects/s recovering 2023-09-23T20:16:39.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:39.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:39.573 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[213871]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.573 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[213871]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.573 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[213871]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.574 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[213871]: osdmap e471: 8 total, 8 up, 6 in 2023-09-23T20:16:39.574 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[213871]: failed to encode map e471 with expected crc 2023-09-23T20:16:39.574 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:39 smithi135 ceph-mon[213871]: pgmap v1315: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 476 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 697 op/s; 33908/22701 objects degraded (149.368%); 383/22701 objects misplaced (1.687%); 1.2 MiB/s, 172 objects/s recovering 2023-09-23T20:16:39.605 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:16:39.640 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:16:39.643 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:16:39.643 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:16:39.643 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:16:39.643 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:16:39.643 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:16:39.643 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:16:39.643 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:16:39.643 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:16:39.674 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:39.735 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:39.824 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:39.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:39.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:39.929 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:39.977 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:40.073 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:40.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:40.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:40.178 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:40.204 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:40.249 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:40.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:40.353 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:40.378 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:40.433 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:40.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:40.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:40.541 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:40.606 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:40 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1698817524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:40.612 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:40.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:40 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1698817524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:40.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:40 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1698817524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:40.779 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:16:40.779 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:16:40.801 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:40.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:40.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:40.907 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:41.002 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:41.004 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:41.103 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:41.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:41.208 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:41.248 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:41 smithi144 ceph-mon[194833]: pgmap v1316: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 476 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 33908/22701 objects degraded (149.368%); 383/22701 objects misplaced (1.687%); 1013 KiB/s, 140 objects/s recovering 2023-09-23T20:16:41.252 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:41.336 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:41.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:41.447 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:41.484 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:41.497 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:41 smithi135 ceph-mon[213871]: pgmap v1316: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 476 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 33908/22701 objects degraded (149.368%); 383/22701 objects misplaced (1.687%); 1013 KiB/s, 140 objects/s recovering 2023-09-23T20:16:41.498 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:41 smithi135 ceph-mon[221113]: pgmap v1316: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 476 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 33908/22701 objects degraded (149.368%); 383/22701 objects misplaced (1.687%); 1013 KiB/s, 140 objects/s recovering 2023-09-23T20:16:41.507 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:41.659 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:41.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:41.774 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:41.840 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:41.932 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:41.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:41.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:42.036 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:42.062 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:42.105 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:42.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:42.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:42.208 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:42.258 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:42.510 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:42.536 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:42.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:42.650 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:42.678 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:42.859 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:42.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:42.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:42.915 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:16:42.964 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:43.020 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:43.053 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:42 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 33908/22701 objects degraded (149.368%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:43.053 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:42 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 33908/22701 objects degraded (149.368%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:43.123 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:43.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:42 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 33908/22701 objects degraded (149.368%), 16 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:43.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:43.237 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:43.287 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:43.382 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:43.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:43.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:43.487 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:43.567 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:43.669 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:16:43.671 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:16:43.733 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:16:43.733 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:16:43.856 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:16:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:16:43] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2023-09-23T20:16:43.856 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:43 smithi135 ceph-mon[213871]: pgmap v1317: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 436 MiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 33343/20778 objects degraded (160.473%); 383/20778 objects misplaced (1.843%); 1.0 MiB/s, 178 objects/s recovering 2023-09-23T20:16:43.862 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:43 smithi135 ceph-mon[221113]: pgmap v1317: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 436 MiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 33343/20778 objects degraded (160.473%); 383/20778 objects misplaced (1.843%); 1.0 MiB/s, 178 objects/s recovering 2023-09-23T20:16:44.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:43 smithi144 ceph-mon[194833]: pgmap v1317: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 436 MiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 33343/20778 objects degraded (160.473%); 383/20778 objects misplaced (1.843%); 1.0 MiB/s, 178 objects/s recovering 2023-09-23T20:16:44.272 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:44.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:44.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:44.378 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:16:44.407 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:44.694 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:16:44.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:44.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:16:44.791 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:16:44.806 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:44.825 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1317,"stamp":"2023-09-23T20:16:43.089936+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":456809901,"num_objects":6926,"num_object_clones":0,"num_object_copies":20778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33343,"num_objects_misplaced":383,"num_objects_unfound":0,"num_objects_dirty":6926,"num_whiteouts":0,"num_read":5996,"num_read_kb":13236,"num_write":201722,"num_write_kb":6659327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24324,"num_bytes_recovered":1199307892,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":185921,"ondisk_log_size":185921,"up":387,"acting":370,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":391,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6221836,"kb_used_data":2157660,"kb_used_omap":25,"kb_used_meta":4064102,"kb_avail":556249076,"statfs":{"total":575970213888,"available":569599053824,"internally_reserved":0,"allocated":2209443840,"data_stored":2198213395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":4161641164},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-265945088,"num_objects":-4058,"num_object_clones":0,"num_object_copies":-12174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1505,"num_objects_misplaced":-195,"num_objects_unfound":0,"num_objects_dirty":-4058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4058,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1410,"num_bytes_recovered":8650752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.893815"},"pg_stats":[{"pgid":"9.23","version":"470'6303","reported_seq":6404,"reported_epoch":470,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:37.768476+0000","last_change":"2023-09-23T20:14:36.092943+0000","last_active":"2023-09-23T20:16:37.768476+0000","last_peered":"2023-09-23T20:16:37.768476+0000","last_clean":"2023-09-23T20:14:33.760240+0000","last_became_active":"2023-09-23T20:14:36.076681+0000","last_became_peered":"2023-09-23T20:14:36.076681+0000","last_unstale":"2023-09-23T20:16:37.768476+0000","last_undegraded":"2023-09-23T20:14:35.911782+0000","last_fullsized":"2023-09-23T20:14:35.868390+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7487488,"num_objects":114,"num_object_clones":0,"num_object_copies":342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3194,"num_write_kb":105872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":24},{"shards":"3,7","objects":-1450}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.22","version":"471'6371","reported_seq":9573,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:41.533069+0000","last_change":"2023-09-23T20:16:35.437261+0000","last_active":"2023-09-23T20:16:41.533069+0000","last_peered":"2023-09-23T20:16:41.533069+0000","last_clean":"2023-09-23T20:16:41.533069+0000","last_became_active":"2023-09-23T20:16:35.425222+0000","last_became_peered":"2023-09-23T20:16:35.425222+0000","last_unstale":"2023-09-23T20:16:41.533069+0000","last_undegraded":"2023-09-23T20:16:41.533069+0000","last_fullsized":"2023-09-23T20:16:41.533069+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6371,"log_dups_size":0,"ondisk_log_size":6371,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5439488,"num_objects":83,"num_object_clones":0,"num_object_copies":249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3235,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1544,"num_bytes_recovered":19005440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.21","version":"470'6177","reported_seq":6268,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.992125+0000","last_change":"2023-09-23T20:14:24.799466+0000","last_active":"2023-09-23T20:16:37.992125+0000","last_peered":"2023-09-23T20:16:37.992125+0000","last_clean":"2023-09-23T20:16:37.992125+0000","last_became_active":"2023-09-23T20:14:24.796974+0000","last_became_peered":"2023-09-23T20:14:24.796974+0000","last_unstale":"2023-09-23T20:16:37.992125+0000","last_undegraded":"2023-09-23T20:16:37.992125+0000","last_fullsized":"2023-09-23T20:16:37.992125+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9469952,"num_objects":145,"num_object_clones":0,"num_object_copies":435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3083,"num_write_kb":103264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.20","version":"471'6268","reported_seq":6370,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:41.081183+0000","last_change":"2023-09-23T20:14:36.109513+0000","last_active":"2023-09-23T20:16:41.081183+0000","last_peered":"2023-09-23T20:16:41.081183+0000","last_clean":"2023-09-23T20:14:33.769971+0000","last_became_active":"2023-09-23T20:14:36.106301+0000","last_became_peered":"2023-09-23T20:14:36.106301+0000","last_unstale":"2023-09-23T20:16:41.081183+0000","last_undegraded":"2023-09-23T20:14:35.845700+0000","last_fullsized":"2023-09-23T20:14:35.836337+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6268,"log_dups_size":0,"ondisk_log_size":6268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3538944,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3186,"num_write_kb":103680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,2","objects":-1491}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.10","version":"470'7807","reported_seq":13747,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.989508+0000","last_change":"2023-09-23T20:14:22.339078+0000","last_active":"2023-09-23T20:16:37.989508+0000","last_peered":"2023-09-23T20:16:37.989508+0000","last_clean":"2023-09-23T20:16:37.989508+0000","last_became_active":"2023-09-23T20:14:22.319507+0000","last_became_peered":"2023-09-23T20:14:22.319507+0000","last_unstale":"2023-09-23T20:16:37.989508+0000","last_undegraded":"2023-09-23T20:16:37.989508+0000","last_fullsized":"2023-09-23T20:16:37.989508+0000","mapping_epoch":434,"log_start":"419'3489","ondisk_log_start":"419'3489","created":418,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4318,"log_dups_size":0,"ondisk_log_size":4318,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17104896,"num_objects":261,"num_object_clones":0,"num_object_copies":783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6087,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1945,"num_bytes_recovered":127467520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":41,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.239418+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:16:37.239418+0000","last_peered":"2023-09-23T20:16:37.239418+0000","last_clean":"2023-09-23T20:16:37.239418+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:16:37.239418+0000","last_undegraded":"2023-09-23T20:16:37.239418+0000","last_fullsized":"2023-09-23T20:16:37.239418+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":39,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.227057+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:16:37.227057+0000","last_peered":"2023-09-23T20:16:37.227057+0000","last_clean":"2023-09-23T20:16:37.227057+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:16:37.227057+0000","last_undegraded":"2023-09-23T20:16:37.227057+0000","last_fullsized":"2023-09-23T20:16:37.227057+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.13","version":"470'7971","reported_seq":14734,"reported_epoch":470,"state":"active+recovery_wait+undersized+remapped","last_fresh":"2023-09-23T20:16:37.758834+0000","last_change":"2023-09-23T20:16:06.703336+0000","last_active":"2023-09-23T20:16:37.758834+0000","last_peered":"2023-09-23T20:16:37.758834+0000","last_clean":"2023-09-23T20:14:05.142062+0000","last_became_active":"2023-09-23T20:14:07.304693+0000","last_became_peered":"2023-09-23T20:14:07.304693+0000","last_unstale":"2023-09-23T20:16:37.758834+0000","last_undegraded":"2023-09-23T20:16:37.758834+0000","last_fullsized":"2023-09-23T20:14:07.197546+0000","mapping_epoch":429,"log_start":"421'3735","ondisk_log_start":"421'3735","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4236,"log_dups_size":0,"ondisk_log_size":4236,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11698176,"num_objects":179,"num_object_clones":0,"num_object_copies":537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6239,"num_write_kb":205344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2223,"num_bytes_recovered":145686528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,0],"avail_no_missing":["7","0","1","3","5"],"object_location_counts":[{"shards":"0,7","objects":179}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":216,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.232355+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:16:37.232355+0000","last_peered":"2023-09-23T20:16:37.232355+0000","last_clean":"2023-09-23T20:16:37.232355+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:16:37.232355+0000","last_undegraded":"2023-09-23T20:16:37.232355+0000","last_fullsized":"2023-09-23T20:16:37.232355+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":178,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.208074+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:16:35.208074+0000","last_peered":"2023-09-23T20:16:35.208074+0000","last_clean":"2023-09-23T20:16:35.208074+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:16:35.208074+0000","last_undegraded":"2023-09-23T20:16:35.208074+0000","last_fullsized":"2023-09-23T20:16:35.208074+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"471'8029","reported_seq":14503,"reported_epoch":471,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T20:16:38.331381+0000","last_change":"2023-09-23T20:14:36.234004+0000","last_active":"2023-09-23T20:16:38.331381+0000","last_peered":"2023-09-23T20:16:38.331381+0000","last_clean":"2023-09-23T20:14:33.790624+0000","last_became_active":"2023-09-23T20:14:36.220990+0000","last_became_peered":"2023-09-23T20:14:36.220990+0000","last_unstale":"2023-09-23T20:16:38.331381+0000","last_undegraded":"2023-09-23T20:16:38.331381+0000","last_fullsized":"2023-09-23T20:16:38.331381+0000","mapping_epoch":442,"log_start":"419'3612","ondisk_log_start":"419'3612","created":418,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4417,"log_dups_size":0,"ondisk_log_size":4417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10813440,"num_objects":165,"num_object_clones":0,"num_object_copies":495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":165,"num_objects_unfound":0,"num_objects_dirty":165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6279,"num_write_kb":206208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":139067392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1","3"],"object_location_counts":[{"shards":"0,1,5","objects":165}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":40,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.241308+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:16:37.241308+0000","last_peered":"2023-09-23T20:16:37.241308+0000","last_clean":"2023-09-23T20:16:37.241308+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:16:37.241308+0000","last_undegraded":"2023-09-23T20:16:37.241308+0000","last_fullsized":"2023-09-23T20:16:37.241308+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":426,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.235936+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:16:37.235936+0000","last_peered":"2023-09-23T20:16:37.235936+0000","last_clean":"2023-09-23T20:16:37.235936+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:16:37.235936+0000","last_undegraded":"2023-09-23T20:16:37.235936+0000","last_fullsized":"2023-09-23T20:16:37.235936+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1d","version":"470'9174","reported_seq":9265,"reported_epoch":470,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:37.920476+0000","last_change":"2023-09-23T20:14:36.076800+0000","last_active":"2023-09-23T20:16:37.920476+0000","last_peered":"2023-09-23T20:16:37.920476+0000","last_clean":"2023-09-23T20:14:23.775634+0000","last_became_active":"2023-09-23T20:14:36.070774+0000","last_became_peered":"2023-09-23T20:14:36.070774+0000","last_unstale":"2023-09-23T20:16:37.920476+0000","last_undegraded":"2023-09-23T20:14:35.892079+0000","last_fullsized":"2023-09-23T20:14:35.839513+0000","mapping_epoch":442,"log_start":"427'4307","ondisk_log_start":"427'4307","created":436,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4867,"log_dups_size":0,"ondisk_log_size":4867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19267608,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6385,"num_write_kb":213697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":30},{"shards":"5,7","objects":-2780}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":397,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.235804+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:16:37.235804+0000","last_peered":"2023-09-23T20:16:37.235804+0000","last_clean":"2023-09-23T20:16:37.235804+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:16:37.235804+0000","last_undegraded":"2023-09-23T20:16:37.235804+0000","last_fullsized":"2023-09-23T20:16:37.235804+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":325,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.232310+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:16:37.232310+0000","last_peered":"2023-09-23T20:16:37.232310+0000","last_clean":"2023-09-23T20:16:37.232310+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:16:37.232310+0000","last_undegraded":"2023-09-23T20:16:37.232310+0000","last_fullsized":"2023-09-23T20:16:37.232310+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"471'8872","reported_seq":8954,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:41.568178+0000","last_change":"2023-09-23T20:14:24.810789+0000","last_active":"2023-09-23T20:16:41.568178+0000","last_peered":"2023-09-23T20:16:41.568178+0000","last_clean":"2023-09-23T20:16:41.568178+0000","last_became_active":"2023-09-23T20:14:24.807691+0000","last_became_peered":"2023-09-23T20:14:24.807691+0000","last_unstale":"2023-09-23T20:16:41.568178+0000","last_undegraded":"2023-09-23T20:16:41.568178+0000","last_fullsized":"2023-09-23T20:16:41.568178+0000","mapping_epoch":436,"log_start":"430'4516","ondisk_log_start":"430'4516","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4356,"log_dups_size":0,"ondisk_log_size":4356,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5144576,"num_objects":79,"num_object_clones":0,"num_object_copies":237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6253,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":804,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.239960+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:16:37.239960+0000","last_peered":"2023-09-23T20:16:37.239960+0000","last_clean":"2023-09-23T20:16:37.239960+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:16:37.239960+0000","last_undegraded":"2023-09-23T20:16:37.239960+0000","last_fullsized":"2023-09-23T20:16:37.239960+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":480,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.228630+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:16:37.228630+0000","last_peered":"2023-09-23T20:16:37.228630+0000","last_clean":"2023-09-23T20:16:37.228630+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:16:37.228630+0000","last_undegraded":"2023-09-23T20:16:37.228630+0000","last_fullsized":"2023-09-23T20:16:37.228630+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":335,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.232303+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:16:37.232303+0000","last_peered":"2023-09-23T20:16:37.232303+0000","last_clean":"2023-09-23T20:16:37.232303+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:16:37.232303+0000","last_undegraded":"2023-09-23T20:16:37.232303+0000","last_fullsized":"2023-09-23T20:16:37.232303+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1f","version":"471'8899","reported_seq":8990,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.459214+0000","last_change":"2023-09-23T20:14:36.232228+0000","last_active":"2023-09-23T20:16:38.459214+0000","last_peered":"2023-09-23T20:16:38.459214+0000","last_clean":"2023-09-23T20:14:33.842679+0000","last_became_active":"2023-09-23T20:14:36.217802+0000","last_became_peered":"2023-09-23T20:14:36.217802+0000","last_unstale":"2023-09-23T20:16:38.459214+0000","last_undegraded":"2023-09-23T20:14:36.129688+0000","last_fullsized":"2023-09-23T20:14:36.115967+0000","mapping_epoch":442,"log_start":"419'3630","ondisk_log_start":"419'3630","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5269,"log_dups_size":0,"ondisk_log_size":5269,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15007744,"num_objects":229,"num_object_clones":0,"num_object_copies":687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6213,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":35},{"shards":"3,5","objects":-2798}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":163,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.209413+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:16:35.209413+0000","last_peered":"2023-09-23T20:16:35.209413+0000","last_clean":"2023-09-23T20:16:35.209413+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:16:35.209413+0000","last_undegraded":"2023-09-23T20:16:35.209413+0000","last_fullsized":"2023-09-23T20:16:35.209413+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":169,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.239595+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:16:37.239595+0000","last_peered":"2023-09-23T20:16:37.239595+0000","last_clean":"2023-09-23T20:16:37.239595+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:16:37.239595+0000","last_undegraded":"2023-09-23T20:16:37.239595+0000","last_fullsized":"2023-09-23T20:16:37.239595+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.1e","version":"470'8920","reported_seq":15071,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.053754+0000","last_change":"2023-09-23T20:15:21.201333+0000","last_active":"2023-09-23T20:16:38.053754+0000","last_peered":"2023-09-23T20:16:38.053754+0000","last_clean":"2023-09-23T20:16:38.053754+0000","last_became_active":"2023-09-23T20:15:21.192111+0000","last_became_peered":"2023-09-23T20:15:21.192111+0000","last_unstale":"2023-09-23T20:16:38.053754+0000","last_undegraded":"2023-09-23T20:16:38.053754+0000","last_fullsized":"2023-09-23T20:16:38.053754+0000","mapping_epoch":454,"log_start":"421'3985","ondisk_log_start":"421'3985","created":436,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17203200,"num_objects":263,"num_object_clones":0,"num_object_copies":789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6221,"num_write_kb":207456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3024,"num_bytes_recovered":198180864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":416,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207245+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:16:35.207245+0000","last_peered":"2023-09-23T20:16:35.207245+0000","last_clean":"2023-09-23T20:16:35.207245+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:16:35.207245+0000","last_undegraded":"2023-09-23T20:16:35.207245+0000","last_fullsized":"2023-09-23T20:16:35.207245+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":348,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.236636+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:16:37.236636+0000","last_peered":"2023-09-23T20:16:37.236636+0000","last_clean":"2023-09-23T20:16:37.236636+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:16:37.236636+0000","last_undegraded":"2023-09-23T20:16:37.236636+0000","last_fullsized":"2023-09-23T20:16:37.236636+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"470'8040","reported_seq":8121,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.887270+0000","last_change":"2023-09-23T20:13:46.811423+0000","last_active":"2023-09-23T20:16:37.887270+0000","last_peered":"2023-09-23T20:16:37.887270+0000","last_clean":"2023-09-23T20:16:37.887270+0000","last_became_active":"2023-09-23T20:13:46.808266+0000","last_became_peered":"2023-09-23T20:13:46.808266+0000","last_unstale":"2023-09-23T20:16:37.887270+0000","last_undegraded":"2023-09-23T20:16:37.887270+0000","last_fullsized":"2023-09-23T20:16:37.887270+0000","mapping_epoch":418,"log_start":"419'3732","ondisk_log_start":"419'3732","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4308,"log_dups_size":0,"ondisk_log_size":4308,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15237120,"num_objects":233,"num_object_clones":0,"num_object_copies":699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6285,"num_write_kb":208544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":830,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.226496+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:16:37.226496+0000","last_peered":"2023-09-23T20:16:37.226496+0000","last_clean":"2023-09-23T20:16:37.226496+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:16:37.226496+0000","last_undegraded":"2023-09-23T20:16:37.226496+0000","last_fullsized":"2023-09-23T20:16:37.226496+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":163,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.208477+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:16:35.208477+0000","last_peered":"2023-09-23T20:16:35.208477+0000","last_clean":"2023-09-23T20:16:35.208477+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:16:35.208477+0000","last_undegraded":"2023-09-23T20:16:35.208477+0000","last_fullsized":"2023-09-23T20:16:35.208477+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":216,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207435+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:16:35.207435+0000","last_peered":"2023-09-23T20:16:35.207435+0000","last_clean":"2023-09-23T20:16:35.207435+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:16:35.207435+0000","last_undegraded":"2023-09-23T20:16:35.207435+0000","last_fullsized":"2023-09-23T20:16:35.207435+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"471'8056","reported_seq":8138,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.517833+0000","last_change":"2023-09-23T20:13:46.824838+0000","last_active":"2023-09-23T20:16:38.517833+0000","last_peered":"2023-09-23T20:16:38.517833+0000","last_clean":"2023-09-23T20:16:38.517833+0000","last_became_active":"2023-09-23T20:13:46.820709+0000","last_became_peered":"2023-09-23T20:13:46.820709+0000","last_unstale":"2023-09-23T20:16:38.517833+0000","last_undegraded":"2023-09-23T20:16:38.517833+0000","last_fullsized":"2023-09-23T20:16:38.517833+0000","mapping_epoch":418,"log_start":"419'3559","ondisk_log_start":"419'3559","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4497,"log_dups_size":0,"ondisk_log_size":4497,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13271040,"num_objects":203,"num_object_clones":0,"num_object_copies":609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6341,"num_write_kb":209376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":519,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.226519+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:16:37.226519+0000","last_peered":"2023-09-23T20:16:37.226519+0000","last_clean":"2023-09-23T20:16:37.226519+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:16:37.226519+0000","last_undegraded":"2023-09-23T20:16:37.226519+0000","last_fullsized":"2023-09-23T20:16:37.226519+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":178,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207051+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:16:35.207051+0000","last_peered":"2023-09-23T20:16:35.207051+0000","last_clean":"2023-09-23T20:16:35.207051+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:16:35.207051+0000","last_undegraded":"2023-09-23T20:16:35.207051+0000","last_fullsized":"2023-09-23T20:16:35.207051+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":469,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.227518+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:16:37.227518+0000","last_peered":"2023-09-23T20:16:37.227518+0000","last_clean":"2023-09-23T20:16:37.227518+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:16:37.227518+0000","last_undegraded":"2023-09-23T20:16:37.227518+0000","last_fullsized":"2023-09-23T20:16:37.227518+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1b","version":"471'8700","reported_seq":8782,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.503187+0000","last_change":"2023-09-23T20:14:24.808547+0000","last_active":"2023-09-23T20:16:38.503187+0000","last_peered":"2023-09-23T20:16:38.503187+0000","last_clean":"2023-09-23T20:16:38.503187+0000","last_became_active":"2023-09-23T20:14:24.806384+0000","last_became_peered":"2023-09-23T20:14:24.806384+0000","last_unstale":"2023-09-23T20:16:38.503187+0000","last_undegraded":"2023-09-23T20:16:38.503187+0000","last_fullsized":"2023-09-23T20:16:38.503187+0000","mapping_epoch":436,"log_start":"416'3280","ondisk_log_start":"416'3280","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5420,"log_dups_size":0,"ondisk_log_size":5420,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15892480,"num_objects":243,"num_object_clones":0,"num_object_copies":729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6089,"num_write_kb":202592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":580,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.231439+0000","last_change":"2023-09-23T20:14:37.887863+0000","last_active":"2023-09-23T20:16:37.231439+0000","last_peered":"2023-09-23T20:16:37.231439+0000","last_clean":"2023-09-23T20:16:37.231439+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:16:37.231439+0000","last_undegraded":"2023-09-23T20:16:37.231439+0000","last_fullsized":"2023-09-23T20:16:37.231439+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":157,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.210076+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:16:35.210076+0000","last_peered":"2023-09-23T20:16:35.210076+0000","last_clean":"2023-09-23T20:16:35.210076+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:16:35.210076+0000","last_undegraded":"2023-09-23T20:16:35.210076+0000","last_fullsized":"2023-09-23T20:16:35.210076+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":459,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.231113+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:16:37.231113+0000","last_peered":"2023-09-23T20:16:37.231113+0000","last_clean":"2023-09-23T20:16:37.231113+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:16:37.231113+0000","last_undegraded":"2023-09-23T20:16:37.231113+0000","last_fullsized":"2023-09-23T20:16:37.231113+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1a","version":"471'8837","reported_seq":8929,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.487361+0000","last_change":"2023-09-23T20:14:36.218516+0000","last_active":"2023-09-23T20:16:38.487361+0000","last_peered":"2023-09-23T20:16:38.487361+0000","last_clean":"2023-09-23T20:14:33.782041+0000","last_became_active":"2023-09-23T20:14:36.175134+0000","last_became_peered":"2023-09-23T20:14:36.175134+0000","last_unstale":"2023-09-23T20:16:38.487361+0000","last_undegraded":"2023-09-23T20:14:35.943267+0000","last_fullsized":"2023-09-23T20:14:35.848493+0000","mapping_epoch":442,"log_start":"417'3382","ondisk_log_start":"417'3382","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18022400,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6139,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":28},{"shards":"5,7","objects":-2685}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":608,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.234665+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:16:37.234665+0000","last_peered":"2023-09-23T20:16:37.234665+0000","last_clean":"2023-09-23T20:16:37.234665+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:16:37.234665+0000","last_undegraded":"2023-09-23T20:16:37.234665+0000","last_fullsized":"2023-09-23T20:16:37.234665+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":114,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.224973+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:16:37.224973+0000","last_peered":"2023-09-23T20:16:37.224973+0000","last_clean":"2023-09-23T20:16:37.224973+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:16:37.224973+0000","last_undegraded":"2023-09-23T20:16:37.224973+0000","last_fullsized":"2023-09-23T20:16:37.224973+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":112,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.231985+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:16:37.231985+0000","last_peered":"2023-09-23T20:16:37.231985+0000","last_clean":"2023-09-23T20:16:37.231985+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:16:37.231985+0000","last_undegraded":"2023-09-23T20:16:37.231985+0000","last_fullsized":"2023-09-23T20:16:37.231985+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"471'8048","reported_seq":8131,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.218067+0000","last_change":"2023-09-23T20:13:46.790587+0000","last_active":"2023-09-23T20:16:38.218067+0000","last_peered":"2023-09-23T20:16:38.218067+0000","last_clean":"2023-09-23T20:16:38.218067+0000","last_became_active":"2023-09-23T20:13:46.788387+0000","last_became_peered":"2023-09-23T20:13:46.788387+0000","last_unstale":"2023-09-23T20:16:38.218067+0000","last_undegraded":"2023-09-23T20:16:38.218067+0000","last_fullsized":"2023-09-23T20:16:38.218067+0000","mapping_epoch":407,"log_start":"419'3564","ondisk_log_start":"419'3564","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4484,"log_dups_size":0,"ondisk_log_size":4484,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13762560,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6318,"num_write_kb":208896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":730,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.209152+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:16:35.209152+0000","last_peered":"2023-09-23T20:16:35.209152+0000","last_clean":"2023-09-23T20:16:35.209152+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:16:35.209152+0000","last_undegraded":"2023-09-23T20:16:35.209152+0000","last_fullsized":"2023-09-23T20:16:35.209152+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":161,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207952+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:16:35.207952+0000","last_peered":"2023-09-23T20:16:35.207952+0000","last_clean":"2023-09-23T20:16:35.207952+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:16:35.207952+0000","last_undegraded":"2023-09-23T20:16:35.207952+0000","last_fullsized":"2023-09-23T20:16:35.207952+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":479,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.225050+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:16:37.225050+0000","last_peered":"2023-09-23T20:16:37.225050+0000","last_clean":"2023-09-23T20:16:37.225050+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:16:37.225050+0000","last_undegraded":"2023-09-23T20:16:37.225050+0000","last_fullsized":"2023-09-23T20:16:37.225050+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"471'8032","reported_seq":14181,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:42.888254+0000","last_change":"2023-09-23T20:16:38.292999+0000","last_active":"2023-09-23T20:16:42.888254+0000","last_peered":"2023-09-23T20:16:42.888254+0000","last_clean":"2023-09-23T20:16:42.888254+0000","last_became_active":"2023-09-23T20:16:38.276406+0000","last_became_peered":"2023-09-23T20:16:38.276406+0000","last_unstale":"2023-09-23T20:16:42.888254+0000","last_undegraded":"2023-09-23T20:16:42.888254+0000","last_fullsized":"2023-09-23T20:16:42.888254+0000","mapping_epoch":470,"log_start":"413'2715","ondisk_log_start":"413'2715","created":407,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5317,"log_dups_size":0,"ondisk_log_size":5317,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7569408,"num_objects":115,"num_object_clones":0,"num_object_copies":345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6307,"num_write_kb":205536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3023,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":819,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.274971+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:16:38.274971+0000","last_peered":"2023-09-23T20:16:38.274971+0000","last_clean":"2023-09-23T20:16:38.274971+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:16:38.274971+0000","last_undegraded":"2023-09-23T20:16:38.274971+0000","last_fullsized":"2023-09-23T20:16:38.274971+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":176,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.208145+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:16:35.208145+0000","last_peered":"2023-09-23T20:16:35.208145+0000","last_clean":"2023-09-23T20:16:35.208145+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:16:35.208145+0000","last_undegraded":"2023-09-23T20:16:35.208145+0000","last_fullsized":"2023-09-23T20:16:35.208145+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":471,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.232025+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:16:37.232025+0000","last_peered":"2023-09-23T20:16:37.232025+0000","last_clean":"2023-09-23T20:16:37.232025+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:16:37.232025+0000","last_undegraded":"2023-09-23T20:16:37.232025+0000","last_fullsized":"2023-09-23T20:16:37.232025+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.7","version":"471'7969","reported_seq":8052,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.502696+0000","last_change":"2023-09-23T20:13:46.802046+0000","last_active":"2023-09-23T20:16:38.502696+0000","last_peered":"2023-09-23T20:16:38.502696+0000","last_clean":"2023-09-23T20:16:38.502696+0000","last_became_active":"2023-09-23T20:13:46.800187+0000","last_became_peered":"2023-09-23T20:13:46.800187+0000","last_unstale":"2023-09-23T20:16:38.502696+0000","last_undegraded":"2023-09-23T20:16:38.502696+0000","last_fullsized":"2023-09-23T20:16:38.502696+0000","mapping_epoch":407,"log_start":"419'3519","ondisk_log_start":"419'3519","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4450,"log_dups_size":0,"ondisk_log_size":4450,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17891328,"num_objects":273,"num_object_clones":0,"num_object_copies":819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6259,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":543,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.234689+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:16:37.234689+0000","last_peered":"2023-09-23T20:16:37.234689+0000","last_clean":"2023-09-23T20:16:37.234689+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:16:37.234689+0000","last_undegraded":"2023-09-23T20:16:37.234689+0000","last_fullsized":"2023-09-23T20:16:37.234689+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":161,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.209324+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:16:35.209324+0000","last_peered":"2023-09-23T20:16:35.209324+0000","last_clean":"2023-09-23T20:16:35.209324+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:16:35.209324+0000","last_undegraded":"2023-09-23T20:16:35.209324+0000","last_fullsized":"2023-09-23T20:16:35.209324+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207249+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:16:35.207249+0000","last_peered":"2023-09-23T20:16:35.207249+0000","last_clean":"2023-09-23T20:16:35.207249+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:16:35.207249+0000","last_undegraded":"2023-09-23T20:16:35.207249+0000","last_fullsized":"2023-09-23T20:16:35.207249+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"471'8252","reported_seq":8343,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.511043+0000","last_change":"2023-09-23T20:14:36.190976+0000","last_active":"2023-09-23T20:16:38.511043+0000","last_peered":"2023-09-23T20:16:38.511043+0000","last_clean":"2023-09-23T20:14:33.830609+0000","last_became_active":"2023-09-23T20:14:36.182918+0000","last_became_peered":"2023-09-23T20:14:36.182918+0000","last_unstale":"2023-09-23T20:16:38.511043+0000","last_undegraded":"2023-09-23T20:14:35.924455+0000","last_fullsized":"2023-09-23T20:14:35.847184+0000","mapping_epoch":442,"log_start":"420'3824","ondisk_log_start":"420'3824","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4428,"log_dups_size":0,"ondisk_log_size":4428,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10092544,"num_objects":154,"num_object_clones":0,"num_object_copies":462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6470,"num_write_kb":211968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":39},{"shards":"2,5","objects":-3043}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":673,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207906+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:16:35.207906+0000","last_peered":"2023-09-23T20:16:35.207906+0000","last_clean":"2023-09-23T20:16:35.207906+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:16:35.207906+0000","last_undegraded":"2023-09-23T20:16:35.207906+0000","last_fullsized":"2023-09-23T20:16:35.207906+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":519,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.229252+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:16:37.229252+0000","last_peered":"2023-09-23T20:16:37.229252+0000","last_clean":"2023-09-23T20:16:37.229252+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:16:37.229252+0000","last_undegraded":"2023-09-23T20:16:37.229252+0000","last_fullsized":"2023-09-23T20:16:37.229252+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":214,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.278151+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:16:35.278151+0000","last_peered":"2023-09-23T20:16:35.278151+0000","last_clean":"2023-09-23T20:16:35.278151+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:16:35.278151+0000","last_undegraded":"2023-09-23T20:16:35.278151+0000","last_fullsized":"2023-09-23T20:16:35.278151+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"470'6218","reported_seq":6310,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.847199+0000","last_change":"2023-09-23T20:14:24.782498+0000","last_active":"2023-09-23T20:16:37.847199+0000","last_peered":"2023-09-23T20:16:37.847199+0000","last_clean":"2023-09-23T20:16:37.847199+0000","last_became_active":"2023-09-23T20:14:24.780750+0000","last_became_peered":"2023-09-23T20:14:24.780750+0000","last_unstale":"2023-09-23T20:16:37.847199+0000","last_undegraded":"2023-09-23T20:16:37.847199+0000","last_fullsized":"2023-09-23T20:16:37.847199+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6218,"log_dups_size":0,"ondisk_log_size":6218,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7176192,"num_objects":109,"num_object_clones":0,"num_object_copies":327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3123,"num_write_kb":103456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":852,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.210062+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:16:35.210062+0000","last_peered":"2023-09-23T20:16:35.210062+0000","last_clean":"2023-09-23T20:16:35.210062+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:16:35.210062+0000","last_undegraded":"2023-09-23T20:16:35.210062+0000","last_fullsized":"2023-09-23T20:16:35.210062+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":217,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.231590+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:16:37.231590+0000","last_peered":"2023-09-23T20:16:37.231590+0000","last_clean":"2023-09-23T20:16:37.231590+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:16:37.231590+0000","last_undegraded":"2023-09-23T20:16:37.231590+0000","last_fullsized":"2023-09-23T20:16:37.231590+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.206512+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:16:35.206512+0000","last_peered":"2023-09-23T20:16:35.206512+0000","last_clean":"2023-09-23T20:16:35.206512+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:16:35.206512+0000","last_undegraded":"2023-09-23T20:16:35.206512+0000","last_fullsized":"2023-09-23T20:16:35.206512+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"471'6290","reported_seq":7523,"reported_epoch":471,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:41.152312+0000","last_change":"2023-09-23T20:16:36.453758+0000","last_active":"2023-09-23T20:16:41.152312+0000","last_peered":"2023-09-23T20:16:41.152312+0000","last_clean":"2023-09-23T20:14:33.749328+0000","last_became_active":"2023-09-23T20:14:36.071685+0000","last_became_peered":"2023-09-23T20:14:36.071685+0000","last_unstale":"2023-09-23T20:16:41.152312+0000","last_undegraded":"2023-09-23T20:14:35.849009+0000","last_fullsized":"2023-09-23T20:14:35.834839+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6290,"log_dups_size":0,"ondisk_log_size":6290,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4718592,"num_objects":72,"num_object_clones":0,"num_object_copies":216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":982,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3208,"num_write_kb":104960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":566,"num_bytes_recovered":2490368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,2","objects":-910}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":818,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.226378+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:16:37.226378+0000","last_peered":"2023-09-23T20:16:37.226378+0000","last_clean":"2023-09-23T20:16:37.226378+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:16:37.226378+0000","last_undegraded":"2023-09-23T20:16:37.226378+0000","last_fullsized":"2023-09-23T20:16:37.226378+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":39,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.233095+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:16:37.233095+0000","last_peered":"2023-09-23T20:16:37.233095+0000","last_clean":"2023-09-23T20:16:37.233095+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:16:37.233095+0000","last_undegraded":"2023-09-23T20:16:37.233095+0000","last_fullsized":"2023-09-23T20:16:37.233095+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":372,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.239951+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:16:37.239951+0000","last_peered":"2023-09-23T20:16:37.239951+0000","last_clean":"2023-09-23T20:16:37.239951+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:16:37.239951+0000","last_undegraded":"2023-09-23T20:16:37.239951+0000","last_fullsized":"2023-09-23T20:16:37.239951+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"470'6230","reported_seq":6332,"reported_epoch":470,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:37.909105+0000","last_change":"2023-09-23T20:14:36.109242+0000","last_active":"2023-09-23T20:16:37.909105+0000","last_peered":"2023-09-23T20:16:37.909105+0000","last_clean":"2023-09-23T20:14:33.735119+0000","last_became_active":"2023-09-23T20:14:36.106276+0000","last_became_peered":"2023-09-23T20:14:36.106276+0000","last_unstale":"2023-09-23T20:16:37.909105+0000","last_undegraded":"2023-09-23T20:14:35.867219+0000","last_fullsized":"2023-09-23T20:14:35.838359+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6230,"log_dups_size":0,"ondisk_log_size":6230,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10043392,"num_objects":153,"num_object_clones":0,"num_object_copies":459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3121,"num_write_kb":104784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":14},{"shards":"3,7","objects":-1345}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":532,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.236784+0000","last_change":"2023-09-23T20:14:37.892742+0000","last_active":"2023-09-23T20:16:37.236784+0000","last_peered":"2023-09-23T20:16:37.236784+0000","last_clean":"2023-09-23T20:16:37.236784+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:16:37.236784+0000","last_undegraded":"2023-09-23T20:16:37.236784+0000","last_fullsized":"2023-09-23T20:16:37.236784+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":363,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.239069+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:16:37.239069+0000","last_peered":"2023-09-23T20:16:37.239069+0000","last_clean":"2023-09-23T20:16:37.239069+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:16:37.239069+0000","last_undegraded":"2023-09-23T20:16:37.239069+0000","last_fullsized":"2023-09-23T20:16:37.239069+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":39,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.227128+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:16:37.227128+0000","last_peered":"2023-09-23T20:16:37.227128+0000","last_clean":"2023-09-23T20:16:37.227128+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:16:37.227128+0000","last_undegraded":"2023-09-23T20:16:37.227128+0000","last_fullsized":"2023-09-23T20:16:37.227128+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"471'6367","reported_seq":9565,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:41.556394+0000","last_change":"2023-09-23T20:16:17.619121+0000","last_active":"2023-09-23T20:16:41.556394+0000","last_peered":"2023-09-23T20:16:41.556394+0000","last_clean":"2023-09-23T20:16:41.556394+0000","last_became_active":"2023-09-23T20:16:17.609227+0000","last_became_peered":"2023-09-23T20:16:17.609227+0000","last_unstale":"2023-09-23T20:16:41.556394+0000","last_undegraded":"2023-09-23T20:16:41.556394+0000","last_fullsized":"2023-09-23T20:16:41.556394+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":407,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":6367,"log_dups_size":0,"ondisk_log_size":6367,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6094848,"num_objects":93,"num_object_clones":0,"num_object_copies":279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3231,"num_write_kb":106368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1542,"num_bytes_recovered":34668544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":809,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.235038+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:16:37.235038+0000","last_peered":"2023-09-23T20:16:37.235038+0000","last_clean":"2023-09-23T20:16:37.235038+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:16:37.235038+0000","last_undegraded":"2023-09-23T20:16:37.235038+0000","last_fullsized":"2023-09-23T20:16:37.235038+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":112,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.231951+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:16:37.231951+0000","last_peered":"2023-09-23T20:16:37.231951+0000","last_clean":"2023-09-23T20:16:37.231951+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:16:37.231951+0000","last_undegraded":"2023-09-23T20:16:37.231951+0000","last_fullsized":"2023-09-23T20:16:37.231951+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":215,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.232431+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:16:37.232431+0000","last_peered":"2023-09-23T20:16:37.232431+0000","last_clean":"2023-09-23T20:16:37.232431+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:16:37.232431+0000","last_undegraded":"2023-09-23T20:16:37.232431+0000","last_fullsized":"2023-09-23T20:16:37.232431+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"471'8904","reported_seq":9003,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.328881+0000","last_change":"2023-09-23T20:14:36.198248+0000","last_active":"2023-09-23T20:16:38.328881+0000","last_peered":"2023-09-23T20:16:38.328881+0000","last_clean":"2023-09-23T20:14:33.842789+0000","last_became_active":"2023-09-23T20:14:36.169565+0000","last_became_peered":"2023-09-23T20:14:36.169565+0000","last_unstale":"2023-09-23T20:16:38.328881+0000","last_undegraded":"2023-09-23T20:14:36.113647+0000","last_fullsized":"2023-09-23T20:14:35.847403+0000","mapping_epoch":442,"log_start":"419'3563","ondisk_log_start":"419'3563","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5341,"log_dups_size":0,"ondisk_log_size":5341,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14548992,"num_objects":222,"num_object_clones":0,"num_object_copies":666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6218,"num_write_kb":206080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":29},{"shards":"3,5","objects":-2805}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":753,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.232138+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:16:37.232138+0000","last_peered":"2023-09-23T20:16:37.232138+0000","last_clean":"2023-09-23T20:16:37.232138+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:16:37.232138+0000","last_undegraded":"2023-09-23T20:16:37.232138+0000","last_fullsized":"2023-09-23T20:16:37.232138+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":460,"reported_epoch":469,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:35.207098+0000","last_change":"2023-09-23T20:14:35.895655+0000","last_active":"2023-09-23T20:16:35.207098+0000","last_peered":"2023-09-23T20:16:35.207098+0000","last_clean":"2023-09-23T20:07:25.847413+0000","last_became_active":"2023-09-23T20:14:35.895204+0000","last_became_peered":"2023-09-23T20:14:35.895204+0000","last_unstale":"2023-09-23T20:16:35.207098+0000","last_undegraded":"2023-09-23T20:14:35.838988+0000","last_fullsized":"2023-09-23T20:14:35.837779+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:17.785522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":471,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.227446+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:16:37.227446+0000","last_peered":"2023-09-23T20:16:37.227446+0000","last_clean":"2023-09-23T20:16:37.227446+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:16:37.227446+0000","last_undegraded":"2023-09-23T20:16:37.227446+0000","last_fullsized":"2023-09-23T20:16:37.227446+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":82,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.234974+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:16:37.234974+0000","last_peered":"2023-09-23T20:16:37.234974+0000","last_clean":"2023-09-23T20:16:37.234974+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:16:37.234974+0000","last_undegraded":"2023-09-23T20:16:37.234974+0000","last_fullsized":"2023-09-23T20:16:37.234974+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"470'8161","reported_seq":8243,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.048646+0000","last_change":"2023-09-23T20:13:46.785589+0000","last_active":"2023-09-23T20:16:38.048646+0000","last_peered":"2023-09-23T20:16:38.048646+0000","last_clean":"2023-09-23T20:16:38.048646+0000","last_became_active":"2023-09-23T20:13:46.781975+0000","last_became_peered":"2023-09-23T20:13:46.781975+0000","last_unstale":"2023-09-23T20:16:38.048646+0000","last_undegraded":"2023-09-23T20:16:38.048646+0000","last_fullsized":"2023-09-23T20:16:38.048646+0000","mapping_epoch":407,"log_start":"427'3862","ondisk_log_start":"427'3862","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4299,"log_dups_size":0,"ondisk_log_size":4299,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12681216,"num_objects":193,"num_object_clones":0,"num_object_copies":579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6405,"num_write_kb":211168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":814,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.226694+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:16:37.226694+0000","last_peered":"2023-09-23T20:16:37.226694+0000","last_clean":"2023-09-23T20:16:37.226694+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:16:37.226694+0000","last_undegraded":"2023-09-23T20:16:37.226694+0000","last_fullsized":"2023-09-23T20:16:37.226694+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1121,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.234752+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:16:37.234752+0000","last_peered":"2023-09-23T20:16:37.234752+0000","last_clean":"2023-09-23T20:16:37.234752+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:16:37.234752+0000","last_undegraded":"2023-09-23T20:16:37.234752+0000","last_fullsized":"2023-09-23T20:16:37.234752+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":39,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.236228+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:16:37.236228+0000","last_peered":"2023-09-23T20:16:37.236228+0000","last_clean":"2023-09-23T20:16:37.236228+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:16:37.236228+0000","last_undegraded":"2023-09-23T20:16:37.236228+0000","last_fullsized":"2023-09-23T20:16:37.236228+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":334,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.249142+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:16:38.249142+0000","last_peered":"2023-09-23T20:16:38.249142+0000","last_clean":"2023-09-23T20:16:38.249142+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:16:38.249142+0000","last_undegraded":"2023-09-23T20:16:38.249142+0000","last_fullsized":"2023-09-23T20:16:38.249142+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"471'8990","reported_seq":9086,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.428823+0000","last_change":"2023-09-23T20:14:36.228252+0000","last_active":"2023-09-23T20:16:38.428823+0000","last_peered":"2023-09-23T20:16:38.428823+0000","last_clean":"2023-09-23T20:14:33.823351+0000","last_became_active":"2023-09-23T20:14:36.216536+0000","last_became_peered":"2023-09-23T20:14:36.216536+0000","last_unstale":"2023-09-23T20:16:38.428823+0000","last_undegraded":"2023-09-23T20:14:36.114735+0000","last_fullsized":"2023-09-23T20:14:35.945404+0000","mapping_epoch":442,"log_start":"420'3812","ondisk_log_start":"420'3812","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5178,"log_dups_size":0,"ondisk_log_size":5178,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12976128,"num_objects":198,"num_object_clones":0,"num_object_copies":594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3085,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6292,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":38},{"shards":"5,7","objects":-2887}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":708,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207122+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:16:35.207122+0000","last_peered":"2023-09-23T20:16:35.207122+0000","last_clean":"2023-09-23T20:16:35.207122+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:16:35.207122+0000","last_undegraded":"2023-09-23T20:16:35.207122+0000","last_fullsized":"2023-09-23T20:16:35.207122+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1109,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.239103+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:16:37.239103+0000","last_peered":"2023-09-23T20:16:37.239103+0000","last_clean":"2023-09-23T20:16:37.239103+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:16:37.239103+0000","last_undegraded":"2023-09-23T20:16:37.239103+0000","last_fullsized":"2023-09-23T20:16:37.239103+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":170,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.242414+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:16:37.242414+0000","last_peered":"2023-09-23T20:16:37.242414+0000","last_clean":"2023-09-23T20:16:37.242414+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:16:37.242414+0000","last_undegraded":"2023-09-23T20:16:37.242414+0000","last_fullsized":"2023-09-23T20:16:37.242414+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":136,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.235870+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:16:37.235870+0000","last_peered":"2023-09-23T20:16:37.235870+0000","last_clean":"2023-09-23T20:16:37.235870+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:16:37.235870+0000","last_undegraded":"2023-09-23T20:16:37.235870+0000","last_fullsized":"2023-09-23T20:16:37.235870+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"471'7906","reported_seq":7989,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.216257+0000","last_change":"2023-09-23T20:13:46.814034+0000","last_active":"2023-09-23T20:16:38.216257+0000","last_peered":"2023-09-23T20:16:38.216257+0000","last_clean":"2023-09-23T20:16:38.216257+0000","last_became_active":"2023-09-23T20:13:46.812300+0000","last_became_peered":"2023-09-23T20:13:46.812300+0000","last_unstale":"2023-09-23T20:16:38.216257+0000","last_undegraded":"2023-09-23T20:16:38.216257+0000","last_fullsized":"2023-09-23T20:16:38.216257+0000","mapping_epoch":407,"log_start":"419'3482","ondisk_log_start":"419'3482","created":407,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4424,"log_dups_size":0,"ondisk_log_size":4424,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15761408,"num_objects":240,"num_object_clones":0,"num_object_copies":720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6190,"num_write_kb":205792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1379,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.236432+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:16:37.236432+0000","last_peered":"2023-09-23T20:16:37.236432+0000","last_clean":"2023-09-23T20:16:37.236432+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:16:37.236432+0000","last_undegraded":"2023-09-23T20:16:37.236432+0000","last_fullsized":"2023-09-23T20:16:37.236432+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":665,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.206430+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:16:35.206430+0000","last_peered":"2023-09-23T20:16:35.206430+0000","last_clean":"2023-09-23T20:16:35.206430+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:16:35.206430+0000","last_undegraded":"2023-09-23T20:16:35.206430+0000","last_fullsized":"2023-09-23T20:16:35.206430+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":336,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.234906+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:16:37.234906+0000","last_peered":"2023-09-23T20:16:37.234906+0000","last_clean":"2023-09-23T20:16:37.234906+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:16:37.234906+0000","last_undegraded":"2023-09-23T20:16:37.234906+0000","last_fullsized":"2023-09-23T20:16:37.234906+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":267,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.209220+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:16:35.209220+0000","last_peered":"2023-09-23T20:16:35.209220+0000","last_clean":"2023-09-23T20:16:35.209220+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:16:35.209220+0000","last_undegraded":"2023-09-23T20:16:35.209220+0000","last_fullsized":"2023-09-23T20:16:35.209220+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"471'8729","reported_seq":8812,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.453811+0000","last_change":"2023-09-23T20:14:24.789921+0000","last_active":"2023-09-23T20:16:38.453811+0000","last_peered":"2023-09-23T20:16:38.453811+0000","last_clean":"2023-09-23T20:16:38.453811+0000","last_became_active":"2023-09-23T20:14:24.787385+0000","last_became_peered":"2023-09-23T20:14:24.787385+0000","last_unstale":"2023-09-23T20:16:38.453811+0000","last_undegraded":"2023-09-23T20:16:38.453811+0000","last_fullsized":"2023-09-23T20:16:38.453811+0000","mapping_epoch":407,"log_start":"419'3474","ondisk_log_start":"419'3474","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":5255,"log_dups_size":0,"ondisk_log_size":5255,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15040512,"num_objects":229,"num_object_clones":0,"num_object_copies":687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6117,"num_write_kb":203104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":532,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.236570+0000","last_change":"2023-09-23T20:14:37.894544+0000","last_active":"2023-09-23T20:16:37.236570+0000","last_peered":"2023-09-23T20:16:37.236570+0000","last_clean":"2023-09-23T20:16:37.236570+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:16:37.236570+0000","last_undegraded":"2023-09-23T20:16:37.236570+0000","last_fullsized":"2023-09-23T20:16:37.236570+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":916,"reported_epoch":469,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:35.205595+0000","last_change":"2023-09-23T20:14:35.882819+0000","last_active":"2023-09-23T20:16:35.205595+0000","last_peered":"2023-09-23T20:16:35.205595+0000","last_clean":"2023-09-23T20:14:31.826679+0000","last_became_active":"2023-09-23T20:14:35.882372+0000","last_became_peered":"2023-09-23T20:14:35.882372+0000","last_unstale":"2023-09-23T20:16:35.205595+0000","last_undegraded":"2023-09-23T20:14:35.846466+0000","last_fullsized":"2023-09-23T20:14:35.837669+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:41.083298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":417,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.231950+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:16:37.231950+0000","last_peered":"2023-09-23T20:16:37.231950+0000","last_clean":"2023-09-23T20:16:37.231950+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:16:37.231950+0000","last_undegraded":"2023-09-23T20:16:37.231950+0000","last_fullsized":"2023-09-23T20:16:37.231950+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":286,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.240459+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:16:37.240459+0000","last_peered":"2023-09-23T20:16:37.240459+0000","last_clean":"2023-09-23T20:16:37.240459+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:16:37.240459+0000","last_undegraded":"2023-09-23T20:16:37.240459+0000","last_fullsized":"2023-09-23T20:16:37.240459+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"470'8980","reported_seq":9071,"reported_epoch":470,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.049071+0000","last_change":"2023-09-23T20:14:36.189421+0000","last_active":"2023-09-23T20:16:38.049071+0000","last_peered":"2023-09-23T20:16:38.049071+0000","last_clean":"2023-09-23T20:14:33.834923+0000","last_became_active":"2023-09-23T20:14:36.182947+0000","last_became_peered":"2023-09-23T20:14:36.182947+0000","last_unstale":"2023-09-23T20:16:38.049071+0000","last_undegraded":"2023-09-23T20:14:35.901626+0000","last_fullsized":"2023-09-23T20:14:35.838743+0000","mapping_epoch":442,"log_start":"427'4114","ondisk_log_start":"427'4114","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4866,"log_dups_size":0,"ondisk_log_size":4866,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15630336,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6280,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":40},{"shards":"4,7","objects":-2824}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":643,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.425989+0000","last_change":"2023-09-23T20:13:57.748021+0000","last_active":"2023-09-23T20:16:35.425989+0000","last_peered":"2023-09-23T20:16:35.425989+0000","last_clean":"2023-09-23T20:16:35.425989+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:16:35.425989+0000","last_undegraded":"2023-09-23T20:16:35.425989+0000","last_fullsized":"2023-09-23T20:16:35.425989+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":170,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.209368+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:16:35.209368+0000","last_peered":"2023-09-23T20:16:35.209368+0000","last_clean":"2023-09-23T20:16:35.209368+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:16:35.209368+0000","last_undegraded":"2023-09-23T20:16:35.209368+0000","last_fullsized":"2023-09-23T20:16:35.209368+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":275,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.249361+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:16:38.249361+0000","last_peered":"2023-09-23T20:16:38.249361+0000","last_clean":"2023-09-23T20:16:38.249361+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:16:38.249361+0000","last_undegraded":"2023-09-23T20:16:38.249361+0000","last_fullsized":"2023-09-23T20:16:38.249361+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":297,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.236309+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:16:37.236309+0000","last_peered":"2023-09-23T20:16:37.236309+0000","last_clean":"2023-09-23T20:16:37.236309+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:16:37.236309+0000","last_undegraded":"2023-09-23T20:16:37.236309+0000","last_fullsized":"2023-09-23T20:16:37.236309+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"470'9299","reported_seq":9390,"reported_epoch":470,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.005067+0000","last_change":"2023-09-23T20:14:36.072073+0000","last_active":"2023-09-23T20:16:38.005067+0000","last_peered":"2023-09-23T20:16:38.005067+0000","last_clean":"2023-09-23T20:14:33.811030+0000","last_became_active":"2023-09-23T20:14:36.065847+0000","last_became_peered":"2023-09-23T20:14:36.065847+0000","last_unstale":"2023-09-23T20:16:38.005067+0000","last_undegraded":"2023-09-23T20:14:35.915006+0000","last_fullsized":"2023-09-23T20:14:35.838163+0000","mapping_epoch":442,"log_start":"430'4576","ondisk_log_start":"430'4576","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4723,"log_dups_size":0,"ondisk_log_size":4723,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12058624,"num_objects":184,"num_object_clones":0,"num_object_copies":552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3193,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6510,"num_write_kb":214208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":30},{"shards":"5,7","objects":-3009}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":690,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.240019+0000","last_change":"2023-09-23T20:14:25.790696+0000","last_active":"2023-09-23T20:16:37.240019+0000","last_peered":"2023-09-23T20:16:37.240019+0000","last_clean":"2023-09-23T20:16:37.240019+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:16:37.240019+0000","last_undegraded":"2023-09-23T20:16:37.240019+0000","last_fullsized":"2023-09-23T20:16:37.240019+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":312,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207695+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:16:35.207695+0000","last_peered":"2023-09-23T20:16:35.207695+0000","last_clean":"2023-09-23T20:16:35.207695+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:16:35.207695+0000","last_undegraded":"2023-09-23T20:16:35.207695+0000","last_fullsized":"2023-09-23T20:16:35.207695+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":628,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.232283+0000","last_change":"2023-09-23T20:16:02.235622+0000","last_active":"2023-09-23T20:16:37.232283+0000","last_peered":"2023-09-23T20:16:37.232283+0000","last_clean":"2023-09-23T20:16:37.232283+0000","last_became_active":"2023-09-23T20:16:02.234933+0000","last_became_peered":"2023-09-23T20:16:02.234933+0000","last_unstale":"2023-09-23T20:16:37.232283+0000","last_undegraded":"2023-09-23T20:16:37.232283+0000","last_fullsized":"2023-09-23T20:16:37.232283+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:21.855632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":103,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.231852+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:16:37.231852+0000","last_peered":"2023-09-23T20:16:37.231852+0000","last_clean":"2023-09-23T20:16:37.231852+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:16:37.231852+0000","last_undegraded":"2023-09-23T20:16:37.231852+0000","last_fullsized":"2023-09-23T20:16:37.231852+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"471'8760","reported_seq":8844,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:41.532794+0000","last_change":"2023-09-23T20:14:25.775099+0000","last_active":"2023-09-23T20:16:41.532794+0000","last_peered":"2023-09-23T20:16:41.532794+0000","last_clean":"2023-09-23T20:16:41.532794+0000","last_became_active":"2023-09-23T20:14:24.777506+0000","last_became_peered":"2023-09-23T20:14:24.777506+0000","last_unstale":"2023-09-23T20:16:41.532794+0000","last_undegraded":"2023-09-23T20:16:41.532794+0000","last_fullsized":"2023-09-23T20:16:41.532794+0000","mapping_epoch":407,"log_start":"427'4272","ondisk_log_start":"427'4272","created":407,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4488,"log_dups_size":0,"ondisk_log_size":4488,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14974976,"num_objects":228,"num_object_clones":0,"num_object_copies":684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6140,"num_write_kb":203808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":797,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.233341+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:16:37.233341+0000","last_peered":"2023-09-23T20:16:37.233341+0000","last_clean":"2023-09-23T20:16:37.233341+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:16:37.233341+0000","last_undegraded":"2023-09-23T20:16:37.233341+0000","last_fullsized":"2023-09-23T20:16:37.233341+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":171,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207856+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:16:35.207856+0000","last_peered":"2023-09-23T20:16:35.207856+0000","last_clean":"2023-09-23T20:16:35.207856+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:16:35.207856+0000","last_undegraded":"2023-09-23T20:16:35.207856+0000","last_fullsized":"2023-09-23T20:16:35.207856+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":42,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T20:16:35.207458+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:16:35.207458+0000","last_peered":"2023-09-23T20:16:35.207458+0000","last_clean":"2023-09-23T20:16:35.207458+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:16:35.207458+0000","last_undegraded":"2023-09-23T20:16:35.207458+0000","last_fullsized":"2023-09-23T20:16:35.207458+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":254,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.226819+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:16:37.226819+0000","last_peered":"2023-09-23T20:16:37.226819+0000","last_clean":"2023-09-23T20:16:37.226819+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:16:37.226819+0000","last_undegraded":"2023-09-23T20:16:37.226819+0000","last_fullsized":"2023-09-23T20:16:37.226819+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"470'8067","reported_seq":12297,"reported_epoch":470,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:16:38.034591+0000","last_change":"2023-09-23T20:14:35.970620+0000","last_active":"2023-09-23T20:16:38.034591+0000","last_peered":"2023-09-23T20:16:38.034591+0000","last_clean":"2023-09-23T20:14:33.799110+0000","last_became_active":"2023-09-23T20:14:35.967502+0000","last_became_peered":"2023-09-23T20:14:35.967502+0000","last_unstale":"2023-09-23T20:16:38.034591+0000","last_undegraded":"2023-09-23T20:16:38.034591+0000","last_fullsized":"2023-09-23T20:16:38.034591+0000","mapping_epoch":442,"log_start":"420'3745","ondisk_log_start":"420'3745","created":418,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4322,"log_dups_size":0,"ondisk_log_size":4322,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14286848,"num_objects":218,"num_object_clones":0,"num_object_copies":654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":218,"num_objects_unfound":0,"num_objects_dirty":218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6322,"num_write_kb":209280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2059,"num_bytes_recovered":134938624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":218}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":248,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.239859+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:16:37.239859+0000","last_peered":"2023-09-23T20:16:37.239859+0000","last_clean":"2023-09-23T20:16:37.239859+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:16:37.239859+0000","last_undegraded":"2023-09-23T20:16:37.239859+0000","last_fullsized":"2023-09-23T20:16:37.239859+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":248,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.240325+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:16:37.240325+0000","last_peered":"2023-09-23T20:16:37.240325+0000","last_clean":"2023-09-23T20:16:37.240325+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:16:37.240325+0000","last_undegraded":"2023-09-23T20:16:37.240325+0000","last_fullsized":"2023-09-23T20:16:37.240325+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"471'8128","reported_seq":8218,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.448326+0000","last_change":"2023-09-23T20:14:36.202358+0000","last_active":"2023-09-23T20:16:38.448326+0000","last_peered":"2023-09-23T20:16:38.448326+0000","last_clean":"2023-09-23T20:14:33.783493+0000","last_became_active":"2023-09-23T20:14:36.193525+0000","last_became_peered":"2023-09-23T20:14:36.193525+0000","last_unstale":"2023-09-23T20:16:38.448326+0000","last_undegraded":"2023-09-23T20:14:36.034471+0000","last_fullsized":"2023-09-23T20:14:35.969590+0000","mapping_epoch":442,"log_start":"419'3669","ondisk_log_start":"419'3669","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4459,"log_dups_size":0,"ondisk_log_size":4459,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20709376,"num_objects":316,"num_object_clones":0,"num_object_copies":948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6346,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":29},{"shards":"2,5","objects":-2728}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":488,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.229173+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:16:37.229173+0000","last_peered":"2023-09-23T20:16:37.229173+0000","last_clean":"2023-09-23T20:16:37.229173+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:16:37.229173+0000","last_undegraded":"2023-09-23T20:16:37.229173+0000","last_fullsized":"2023-09-23T20:16:37.229173+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":491,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.231664+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:16:37.231664+0000","last_peered":"2023-09-23T20:16:37.231664+0000","last_clean":"2023-09-23T20:16:37.231664+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:16:37.231664+0000","last_undegraded":"2023-09-23T20:16:37.231664+0000","last_fullsized":"2023-09-23T20:16:37.231664+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.17","version":"471'7971","reported_seq":8053,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.495174+0000","last_change":"2023-09-23T20:13:46.814598+0000","last_active":"2023-09-23T20:16:38.495174+0000","last_peered":"2023-09-23T20:16:38.495174+0000","last_clean":"2023-09-23T20:16:38.495174+0000","last_became_active":"2023-09-23T20:13:46.812685+0000","last_became_peered":"2023-09-23T20:13:46.812685+0000","last_unstale":"2023-09-23T20:16:38.495174+0000","last_undegraded":"2023-09-23T20:16:38.495174+0000","last_fullsized":"2023-09-23T20:16:38.495174+0000","mapping_epoch":418,"log_start":"419'3527","ondisk_log_start":"419'3527","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4444,"log_dups_size":0,"ondisk_log_size":4444,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11862016,"num_objects":181,"num_object_clones":0,"num_object_copies":543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6261,"num_write_kb":206144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":478,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.227366+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:16:37.227366+0000","last_peered":"2023-09-23T20:16:37.227366+0000","last_clean":"2023-09-23T20:16:37.227366+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:16:37.227366+0000","last_undegraded":"2023-09-23T20:16:37.227366+0000","last_fullsized":"2023-09-23T20:16:37.227366+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":467,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.224282+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:16:37.224282+0000","last_peered":"2023-09-23T20:16:37.224282+0000","last_clean":"2023-09-23T20:16:37.224282+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:16:37.224282+0000","last_undegraded":"2023-09-23T20:16:37.224282+0000","last_fullsized":"2023-09-23T20:16:37.224282+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"471'7990","reported_seq":14186,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T20:16:38.478110+0000","last_change":"2023-09-23T20:16:02.232381+0000","last_active":"2023-09-23T20:16:38.478110+0000","last_peered":"2023-09-23T20:16:38.478110+0000","last_clean":"2023-09-23T20:16:38.478110+0000","last_became_active":"2023-09-23T20:16:02.185816+0000","last_became_peered":"2023-09-23T20:16:02.185816+0000","last_unstale":"2023-09-23T20:16:38.478110+0000","last_undegraded":"2023-09-23T20:16:38.478110+0000","last_fullsized":"2023-09-23T20:16:38.478110+0000","mapping_epoch":461,"log_start":"419'3532","ondisk_log_start":"419'3532","created":418,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4458,"log_dups_size":0,"ondisk_log_size":4458,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15433728,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6266,"num_write_kb":208032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3047,"num_bytes_recovered":134610944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":183,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.240546+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:16:37.240546+0000","last_peered":"2023-09-23T20:16:37.240546+0000","last_clean":"2023-09-23T20:16:37.240546+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:16:37.240546+0000","last_undegraded":"2023-09-23T20:16:37.240546+0000","last_fullsized":"2023-09-23T20:16:37.240546+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":330,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.239210+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:16:37.239210+0000","last_peered":"2023-09-23T20:16:37.239210+0000","last_clean":"2023-09-23T20:16:37.239210+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:16:37.239210+0000","last_undegraded":"2023-09-23T20:16:37.239210+0000","last_fullsized":"2023-09-23T20:16:37.239210+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"470'7925","reported_seq":16984,"reported_epoch":470,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:16:38.010162+0000","last_change":"2023-09-23T20:14:38.010024+0000","last_active":"2023-09-23T20:16:38.010162+0000","last_peered":"2023-09-23T20:16:38.010162+0000","last_clean":"2023-09-23T20:14:33.867233+0000","last_became_active":"2023-09-23T20:14:38.004874+0000","last_became_peered":"2023-09-23T20:14:38.004874+0000","last_unstale":"2023-09-23T20:16:38.010162+0000","last_undegraded":"2023-09-23T20:14:37.959307+0000","last_fullsized":"2023-09-23T20:14:37.876450+0000","mapping_epoch":444,"log_start":"419'3657","ondisk_log_start":"419'3657","created":418,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:13:01.190474+0000","last_clean_scrub_stamp":"2023-09-23T20:13:01.190474+0000","objects_scrubbed":0,"log_size":4268,"log_dups_size":0,"ondisk_log_size":4268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14090240,"num_objects":215,"num_object_clones":0,"num_object_copies":645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":64,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6195,"num_write_kb":205120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2990,"num_bytes_recovered":195952640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":32},{"shards":"3,7","objects":183}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":316,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.235740+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:16:37.235740+0000","last_peered":"2023-09-23T20:16:37.235740+0000","last_clean":"2023-09-23T20:16:37.235740+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:16:37.235740+0000","last_undegraded":"2023-09-23T20:16:37.235740+0000","last_fullsized":"2023-09-23T20:16:37.235740+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":40,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T20:16:37.241218+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:16:37.241218+0000","last_peered":"2023-09-23T20:16:37.241218+0000","last_clean":"2023-09-23T20:16:37.241218+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:16:37.241218+0000","last_undegraded":"2023-09-23T20:16:37.241218+0000","last_fullsized":"2023-09-23T20:16:37.241218+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":36,"stat_sum":{"num_bytes":452001816,"num_objects":6898,"num_object_clones":0,"num_object_copies":20694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33323,"num_objects_misplaced":383,"num_objects_unfound":0,"num_objects_dirty":6898,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":200808,"num_write_kb":6646529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24090,"num_bytes_recovered":1195048960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4955320320,"data_stored":4955308104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":181506,"ondisk_log_size":181506,"up":108,"acting":93,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":191,"num_bytes_recovered":4194380,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234426,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":829140,"kb_used_data":313420,"kb_used_omap":0,"kb_used_meta":515712,"kb_avail":92916012,"statfs":{"total":95995035648,"available":95145996288,"internally_reserved":0,"allocated":320942080,"data_stored":319497600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":528089088},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":11.353,"5min":3.855,"15min":2.8380000000000001},"min":{"1min":0.219,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":149.25999999999999,"5min":149.25999999999999,"15min":149.25999999999999},"last":58.109000000000002},{"interface":"front","average":{"1min":12.917,"5min":4.3019999999999996,"15min":3.0609999999999999},"min":{"1min":0.248,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":149.11799999999999,"5min":149.11799999999999,"15min":149.11799999999999},"last":58.042000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":3.2250000000000001,"15min":3.5779999999999998},"min":{"1min":0.29699999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.0599999999999996,"5min":91.335999999999999,"15min":104.16800000000001},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.972,"5min":3.738,"15min":3.8370000000000002},"min":{"1min":0.29599999999999999,"5min":0.249,"15min":0.223},"max":{"1min":6.8609999999999998,"5min":91.516999999999996,"15min":104.003},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":3.3969999999999998,"15min":3.7629999999999999},"min":{"1min":0.32700000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":7.3600000000000003,"5min":67.760000000000005,"15min":105.842},"last":0.96899999999999997},{"interface":"front","average":{"1min":2.633,"5min":3.738,"15min":4.1950000000000003},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":8.6669999999999998,"5min":67.488,"15min":105.66},"last":0.42899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":14.476000000000001,"5min":4.5030000000000001,"15min":4.6440000000000001},"min":{"1min":0.33400000000000002,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":185.17099999999999,"5min":185.17099999999999,"15min":185.17099999999999},"last":1.091},{"interface":"front","average":{"1min":15.061999999999999,"5min":4.593,"15min":4.649},"min":{"1min":0.63,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":184.99100000000001,"5min":184.99100000000001,"15min":184.99100000000001},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":0.96699999999999997,"15min":0.76300000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.22500000000000001},"max":{"1min":7.6440000000000001,"5min":7.6440000000000001,"15min":19.215},"last":1.036},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.151,"15min":0.96499999999999997},"min":{"1min":0.42699999999999999,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":3.9809999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":1.0149999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.595,"15min":1.1990000000000001},"min":{"1min":0.51100000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":8.4440000000000008,"5min":14.069000000000001,"15min":35.488},"last":1.244},{"interface":"front","average":{"1min":1.367,"5min":1.284,"15min":1.0389999999999999},"min":{"1min":0.41699999999999998,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":7.6109999999999998,"5min":13.43,"15min":34.896000000000001},"last":0.79600000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":1.234,"15min":0.99199999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":8.4120000000000008,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.499},{"interface":"front","average":{"1min":1.522,"5min":0.93700000000000006,"15min":0.754},"min":{"1min":0.36699999999999999,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":7.5659999999999998,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.82999999999999996}]}]},{"osd":6,"up_from":0,"seq":1078036791467,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762625,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926162,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1224636,"kb_used_data":442612,"kb_used_omap":8,"kb_used_meta":782007,"kb_avail":92520516,"statfs":{"total":95995035648,"available":94741008384,"internally_reserved":0,"allocated":453234688,"data_stored":451055020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":800775777},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":1.952,"15min":3.4940000000000002},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":29.893000000000001,"5min":72.534000000000006,"15min":94.055999999999997},"last":2.4390000000000001},{"interface":"front","average":{"1min":1.97,"5min":2.4169999999999998,"15min":3.7130000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.183},"max":{"1min":23.436,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.372}]},{"osd":2,"last update":"Sat Sep 23 20:16:17 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":2.863,"15min":4.944},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.1820000000000004,"5min":160.542,"15min":270.51100000000002},"last":2.5249999999999999},{"interface":"front","average":{"1min":2.0219999999999998,"5min":3.5009999999999999,"15min":4.8970000000000002},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":18.433,"5min":160.75999999999999,"15min":269.69299999999998},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4279999999999999,"15min":3.8980000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":8.9469999999999992,"5min":10.082000000000001,"15min":126.95399999999999},"last":3.7410000000000001},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.532,"15min":4.1020000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":7.375,"5min":9.5440000000000005,"15min":127.294},"last":0.41399999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.726,"15min":2.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":5.859,"5min":9.6959999999999997,"15min":148.20599999999999},"last":0.434},{"interface":"front","average":{"1min":5.9790000000000001,"5min":2.29,"15min":2.2589999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":105.282,"5min":105.282,"15min":148.11000000000001},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.9630000000000001,"15min":1.859},"min":{"1min":0.378,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.5430000000000001,"5min":52.061,"15min":64.725999999999999},"last":2.3980000000000001},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.4649999999999999,"15min":1.8320000000000001},"min":{"1min":0.41699999999999998,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1369999999999996,"5min":51.741,"15min":51.741},"last":0.45700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":1.5309999999999999,"15min":1.653},"min":{"1min":0.38,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":7.1950000000000003,"5min":7.1950000000000003,"15min":15.744999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.3400000000000001,"15min":1.6839999999999999},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":7.2229999999999999,"5min":8.3919999999999995,"15min":13.766999999999999},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.569,"15min":1.75},"min":{"1min":0.53700000000000003,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.9980000000000002,"5min":12.981,"15min":65.081999999999994},"last":3.7799999999999998},{"interface":"front","average":{"1min":2.6909999999999998,"5min":2.161,"15min":1.8440000000000001},"min":{"1min":0.49199999999999999,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.6600000000000001,"5min":17.151,"15min":17.151},"last":0.53800000000000003}]}]},{"osd":2,"up_from":104,"seq":446676599101,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1053448,"kb_used_data":228224,"kb_used_omap":12,"kb_used_meta":825203,"kb_avail":92691704,"statfs":{"total":95995035648,"available":94916304896,"internally_reserved":0,"allocated":233701376,"data_stored":231463170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":845008894},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.1880000000000002,"15min":2.464},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":5.3879999999999999,"5min":65.382999999999996,"15min":76.251999999999995},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.282,"15min":2.718},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.265000000000001,"5min":65.427000000000007,"15min":76.590000000000003},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":3.0899999999999999,"15min":3.5649999999999999},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":7.5270000000000001,"5min":70.453000000000003,"15min":94.768000000000001},"last":0.372},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.8969999999999998,"15min":3.2450000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.218},"max":{"1min":7.2009999999999996,"5min":69.825000000000003,"15min":96.808000000000007},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:15:40 2023","interfaces":[{"interface":"back","average":{"1min":7.5960000000000001,"5min":2.9870000000000001,"15min":4.1130000000000004},"min":{"1min":0.222,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":132.82499999999999,"5min":132.82499999999999,"15min":132.82499999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":8.1519999999999992,"5min":2.9900000000000002,"15min":3.411},"min":{"1min":0.23400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":130.97300000000001,"5min":130.97300000000001,"15min":130.97300000000001},"last":0.45800000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.4399999999999999,"15min":1.79},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":9.3399999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.667,"5min":1.7649999999999999,"15min":2.105},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.641,"5min":13.387,"15min":76.161000000000001},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":3.3460000000000001,"15min":2.8519999999999999},"min":{"1min":0.34999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.3859999999999992,"5min":135.202,"15min":172.505},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.96,"5min":2.0619999999999998,"15min":2.6749999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.1260000000000003,"5min":29.196999999999999,"15min":172.375},"last":0.51300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.4930000000000001,"15min":1.5920000000000001},"min":{"1min":0.435,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.0510000000000002,"5min":14.664,"15min":14.664},"last":0.504},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.758,"15min":1.8400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.2649999999999997,"5min":12.025,"15min":14.279999999999999},"last":1.387}]},{"osd":7,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":2.0990000000000002,"15min":1.883},"min":{"1min":0.32500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2970000000000006,"5min":13.481999999999999,"15min":13.481999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.1720000000000002,"5min":1.6399999999999999,"15min":1.581},"min":{"1min":0.443,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.266,"5min":14.698,"15min":14.698},"last":0.48399999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435575,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1346448,"kb_used_data":455560,"kb_used_omap":4,"kb_used_meta":890875,"kb_avail":92398704,"statfs":{"total":95995035648,"available":94616272896,"internally_reserved":0,"allocated":466493440,"data_stored":464005356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":912256109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5940000000000003,"5min":1.8560000000000001,"15min":3.0859999999999999},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":76.632000000000005,"5min":76.632000000000005,"15min":76.632000000000005},"last":1.748},{"interface":"front","average":{"1min":5.4580000000000002,"5min":2.266,"15min":3.4159999999999999},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.19600000000000001},"max":{"1min":81.429000000000002,"5min":81.429000000000002,"15min":81.429000000000002},"last":4.0199999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":5.0510000000000002,"15min":4.6260000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":9,"5min":183.87,"15min":183.87},"last":0.501},{"interface":"front","average":{"1min":1.2230000000000001,"5min":4.9740000000000002,"15min":4.758},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":5.5810000000000004,"5min":185.84899999999999,"15min":185.84899999999999},"last":3.4780000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":4.2610000000000001,"15min":4.109},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":9.0289999999999999,"5min":163.83500000000001,"15min":163.83500000000001},"last":0.69799999999999995},{"interface":"front","average":{"1min":2.2679999999999998,"5min":3.6669999999999998,"15min":3.819},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":16.550999999999998,"5min":163.92099999999999,"15min":163.92099999999999},"last":3.504}]},{"osd":4,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.1120000000000001,"15min":2.786},"min":{"1min":0.32400000000000001,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":5.21,"5min":7.4960000000000004,"15min":109.66800000000001},"last":3.54},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6619999999999999,"15min":3.2469999999999999},"min":{"1min":0.34799999999999998,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":9.6470000000000002,"5min":11.815,"15min":110.295},"last":5.6779999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":3.6349999999999998,"15min":3.9129999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6799999999999997,"5min":200.53200000000001,"15min":200.53200000000001},"last":11.742000000000001},{"interface":"front","average":{"1min":3.411,"5min":3.7599999999999998,"15min":3.9769999999999999},"min":{"1min":0.48799999999999999,"5min":0.34799999999999998,"15min":0.26900000000000002},"max":{"1min":9.9429999999999996,"5min":200.846,"15min":200.846},"last":23.094999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.6539999999999999,"15min":2.8660000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.32300000000000001},"max":{"1min":8.9610000000000003,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.641,"5min":1.2669999999999999,"15min":2.9020000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":5.8959999999999999,"5min":8.2629999999999999,"15min":82.233000000000004},"last":3.399}]},{"osd":7,"last update":"Sat Sep 23 20:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.9470000000000001,"15min":3.3969999999999998},"min":{"1min":0.42799999999999999,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":9.0600000000000005,"5min":139.465,"15min":139.465},"last":1.7949999999999999},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.8540000000000001,"15min":3.27},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":5.3010000000000002,"5min":141.56200000000001,"15min":141.56200000000001},"last":3.448}]}]},{"osd":4,"up_from":30,"seq":128849019337,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":650908,"kb_used_data":188436,"kb_used_omap":0,"kb_used_meta":462464,"kb_avail":93094244,"statfs":{"total":95995035648,"available":95328505856,"internally_reserved":0,"allocated":192958464,"data_stored":191514412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":473563136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.4650000000000001,"15min":1.639},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.2549999999999999,"5min":14.063000000000001,"15min":56.061999999999998},"last":1.135},{"interface":"front","average":{"1min":1.6519999999999999,"5min":1.4630000000000001,"15min":1.619},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":6.9740000000000002,"5min":14.048999999999999,"15min":56.075000000000003},"last":2.7679999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":3.7559999999999998,"15min":5.7779999999999996},"min":{"1min":0.28499999999999998,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":14.763,"5min":138.47999999999999,"15min":161.613},"last":4.5469999999999997},{"interface":"front","average":{"1min":2.1789999999999998,"5min":3.7029999999999998,"15min":5.7830000000000004},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.217},"max":{"1min":13.368,"5min":138.429,"15min":162.017},"last":2.8090000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.8969999999999998,"15min":5.3760000000000003},"min":{"1min":0.28699999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.6920000000000002,"5min":144.916,"15min":182.96700000000001},"last":4.5119999999999996},{"interface":"front","average":{"1min":2.0299999999999998,"5min":3.8300000000000001,"15min":5.3369999999999997},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.23100000000000001},"max":{"1min":9.4550000000000001,"5min":144.83099999999999,"15min":183.244},"last":2.8399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.3159999999999998,"15min":4.6589999999999998},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":4.5819999999999999,"5min":100.361,"15min":184.37100000000001},"last":3.1549999999999998},{"interface":"front","average":{"1min":1.738,"5min":3.4990000000000001,"15min":4.7199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":4.7270000000000003,"5min":100.20699999999999,"15min":184.22300000000001},"last":3.472}]},{"osd":5,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.111,"15min":1.252},"min":{"1min":0.28199999999999997,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.0739999999999998,"5min":12.257999999999999,"15min":176.81899999999999},"last":2.8900000000000001},{"interface":"front","average":{"1min":1.7689999999999999,"5min":1.111,"15min":1.2849999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.246},"max":{"1min":7.9470000000000001,"5min":7.9470000000000001,"15min":176.172},"last":3.1909999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":0.85899999999999999,"15min":0.627},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":8.5660000000000007,"5min":10.337999999999999,"15min":10.337999999999999},"last":2.867},{"interface":"front","average":{"1min":2.3050000000000002,"5min":1.119,"15min":0.79500000000000004},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":11.348000000000001,"5min":11.348000000000001,"15min":11.348000000000001},"last":5.2779999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.196,"15min":1.5349999999999999},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.5090000000000003,"5min":7.71,"15min":76.507000000000005},"last":3.375},{"interface":"front","average":{"1min":3.1040000000000001,"5min":1.4099999999999999,"15min":1.6739999999999999},"min":{"1min":0.39900000000000002,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":7.8250000000000002,"5min":7.8250000000000002,"15min":76.462000000000003},"last":4.5730000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757700,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1117256,"kb_used_data":529408,"kb_used_omap":0,"kb_used_meta":587840,"kb_avail":92627896,"statfs":{"total":95995035648,"available":94850965504,"internally_reserved":0,"allocated":542113792,"data_stored":540677837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":601948160},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":7.7800000000000002,"5min":2.794,"15min":2.214},"min":{"1min":0.23000000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":126.657,"5min":126.657,"15min":126.657},"last":0.27400000000000002},{"interface":"front","average":{"1min":7.2990000000000004,"5min":2.6539999999999999,"15min":2.4289999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":126.697,"5min":126.697,"15min":126.697},"last":0.58399999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:15:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":5.0540000000000003,"15min":5.3209999999999997},"min":{"1min":0.22,"5min":0.22,"15min":0.21299999999999999},"max":{"1min":7.383,"5min":150.16999999999999,"15min":171.68299999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.6749999999999998,"5min":4.7960000000000003,"15min":5.4020000000000001},"min":{"1min":0.32100000000000001,"5min":0.24099999999999999,"15min":0.21199999999999999},"max":{"1min":7.3099999999999996,"5min":150.12700000000001,"15min":171.64099999999999},"last":0.623}]},{"osd":2,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":3.5649999999999999,"15min":4.2679999999999998},"min":{"1min":0.317,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2569999999999997,"5min":120.297,"15min":240.57300000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.47,"5min":3.661,"15min":4.335},"min":{"1min":0.38,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.2859999999999996,"5min":120.325,"15min":240.464},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:15:49 2023","interfaces":[{"interface":"back","average":{"1min":10.634,"5min":6.1920000000000002,"15min":5.556},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":195.84700000000001,"5min":195.84700000000001,"15min":201.50899999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":10.66,"5min":6.0990000000000002,"15min":5.4089999999999998},"min":{"1min":0.39500000000000002,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":195.87899999999999,"5min":195.87899999999999,"15min":201.58099999999999},"last":0.42699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.0629999999999999,"15min":1.3879999999999999},"min":{"1min":0.28499999999999998,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":4.8159999999999998,"5min":8,"15min":191.78899999999999},"last":3.5459999999999998},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.67400000000000004,"15min":1.2150000000000001},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":3.371,"5min":3.371,"15min":191.821},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.1870000000000001,"15min":0.83099999999999996},"min":{"1min":0.33300000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":5.4000000000000004,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.65400000000000003,"15min":0.58199999999999996},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":3.3180000000000001,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":0.52000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.4079999999999999,"15min":1.478},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":11.725,"5min":11.725,"15min":191.84299999999999},"last":0.94499999999999995},{"interface":"front","average":{"1min":2.02,"5min":1.3,"15min":1.4990000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":11.041,"5min":11.041,"15min":191.69},"last":0.65600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":431095808,"data_stored":431095808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2861105152,"data_stored":2861105152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":217317376,"data_stored":217317376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":441843712,"data_stored":441843712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":178458624,"data_stored":178454552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":519639040,"data_stored":519634968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":305860608,"data_stored":305856536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:16:44.833 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:16:44.883 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:44.968 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:44.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:44 smithi135 ceph-mon[213871]: from='client.36447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:44.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:44 smithi135 ceph-mon[221113]: from='client.36447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:44.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:44.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:45.031 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:45.078 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:45.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:44 smithi144 ceph-mon[194833]: from='client.36447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:45.163 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:45.432 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:45.570 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:45.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:45.676 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:45.732 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:45.829 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:45.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:45.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:45.907 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:16:45.941 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:46.005 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:45 smithi144 ceph-mon[194833]: pgmap v1318: 129 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 214 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 33279/10134 objects degraded (328.390%); 144/10134 objects misplaced (1.421%); 1.3 MiB/s, 142 objects/s recovering 2023-09-23T20:16:46.005 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3678458542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:46.016 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:46.160 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:46.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:46.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:45 smithi135 ceph-mon[213871]: pgmap v1318: 129 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 214 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 33279/10134 objects degraded (328.390%); 144/10134 objects misplaced (1.421%); 1.3 MiB/s, 142 objects/s recovering 2023-09-23T20:16:46.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3678458542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:46.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:45 smithi135 ceph-mon[221113]: pgmap v1318: 129 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 214 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 33279/10134 objects degraded (328.390%); 144/10134 objects misplaced (1.421%); 1.3 MiB/s, 142 objects/s recovering 2023-09-23T20:16:46.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3678458542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:16:46.265 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:46.323 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:46.440 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:46.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:46.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:46.545 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:46.586 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:46.887 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:46.888 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[213871]: failed to encode map e472 with expected crc 2023-09-23T20:16:46.888 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[213871]: failed to encode map e472 with expected crc 2023-09-23T20:16:46.888 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[213871]: failed to encode map e472 with expected crc 2023-09-23T20:16:46.888 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3678458542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:16:46.888 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[213871]: osdmap e472: 8 total, 8 up, 6 in 2023-09-23T20:16:46.888 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:46.888 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[221113]: failed to encode map e472 with expected crc 2023-09-23T20:16:46.888 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[221113]: failed to encode map e472 with expected crc 2023-09-23T20:16:46.889 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[221113]: failed to encode map e472 with expected crc 2023-09-23T20:16:46.889 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3678458542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:16:46.889 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:46 smithi135 ceph-mon[221113]: osdmap e472: 8 total, 8 up, 6 in 2023-09-23T20:16:47.221 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Removed 103852 objects 2023-09-23T20:16:47.221 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Clean up completed and total clean up time :123.132 2023-09-23T20:16:47.263 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:47.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:47.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:47.295 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:46 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:16:47.296 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:46 smithi144 ceph-mon[194833]: failed to encode map e472 with expected crc 2023-09-23T20:16:47.296 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:46 smithi144 ceph-mon[194833]: failed to encode map e472 with expected crc 2023-09-23T20:16:47.296 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:46 smithi144 ceph-mon[194833]: failed to encode map e472 with expected crc 2023-09-23T20:16:47.296 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3678458542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:16:47.296 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:46 smithi144 ceph-mon[194833]: osdmap e472: 8 total, 8 up, 6 in 2023-09-23T20:16:47.366 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:47.394 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:47.516 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:16:47.516 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:16:47.519 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:47.532 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T20:16:47.532 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T20:16:47.578 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:47.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:47.682 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:47.715 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:47.849 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:47.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:47.940 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:16:47.954 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:47.964 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:16:47.988 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:16:47.990 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:48.018 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:16:48.040 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:48.040 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:16:48.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:48.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:48.060 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:16:48.079 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:16:48.099 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:16:48.100 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:16:48.100 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:16:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:16:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:16:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:16:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:16:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:16:48.101 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:16:48.104 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:47 smithi135 ceph-mon[221113]: 9.4 scrub starts 2023-09-23T20:16:48.104 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:47 smithi135 ceph-mon[221113]: pgmap v1320: 129 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 186 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 32802/8787 objects degraded (373.301%); 144/8787 objects misplaced (1.639%); 497 KiB/s, 120 objects/s recovering 2023-09-23T20:16:48.104 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:47 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 33279/10134 objects degraded (328.390%), 15 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:48.104 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:47 smithi135 ceph-mon[213871]: 9.4 scrub starts 2023-09-23T20:16:48.104 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:47 smithi135 ceph-mon[213871]: pgmap v1320: 129 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 186 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 32802/8787 objects degraded (373.301%); 144/8787 objects misplaced (1.639%); 497 KiB/s, 120 objects/s recovering 2023-09-23T20:16:48.104 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:47 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 33279/10134 objects degraded (328.390%), 15 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:47 smithi144 ceph-mon[194833]: 9.4 scrub starts 2023-09-23T20:16:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:47 smithi144 ceph-mon[194833]: pgmap v1320: 129 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 186 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 32802/8787 objects degraded (373.301%); 144/8787 objects misplaced (1.639%); 497 KiB/s, 120 objects/s recovering 2023-09-23T20:16:48.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:47 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 33279/10134 objects degraded (328.390%), 15 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:48.145 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:48.174 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:48.317 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:48.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:48.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:48.367 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:16:48.421 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:48.448 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:48.722 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:48.743 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:48.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:48.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:48.847 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:48.873 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:49.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:48 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1594656206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:49.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:48 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1594656206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:49.259 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:48 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1594656206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:49.401 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:49.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:49.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:49.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:49.528 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:49.558 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:49.579 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:16:49.604 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:49.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:49.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:49.708 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:49.737 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:49.790 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:49.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:49.899 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:49.905 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[213871]: failed to encode map e473 with expected crc 2023-09-23T20:16:49.905 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[213871]: failed to encode map e473 with expected crc 2023-09-23T20:16:49.905 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[213871]: failed to encode map e473 with expected crc 2023-09-23T20:16:49.906 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[213871]: osdmap e473: 8 total, 8 up, 6 in 2023-09-23T20:16:49.906 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[213871]: pgmap v1322: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 38 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 932 op/s; 32722/1701 objects degraded (1923.692%); 552 KiB/s, 144 objects/s recovering 2023-09-23T20:16:49.906 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[213871]: 9.4 scrub starts 2023-09-23T20:16:49.944 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:50.093 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:50.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:50.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:50.197 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:50.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[221113]: failed to encode map e473 with expected crc 2023-09-23T20:16:50.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[221113]: failed to encode map e473 with expected crc 2023-09-23T20:16:50.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[221113]: failed to encode map e473 with expected crc 2023-09-23T20:16:50.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[221113]: osdmap e473: 8 total, 8 up, 6 in 2023-09-23T20:16:50.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[221113]: pgmap v1322: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 38 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 932 op/s; 32722/1701 objects degraded (1923.692%); 552 KiB/s, 144 objects/s recovering 2023-09-23T20:16:50.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:49 smithi135 ceph-mon[221113]: 9.4 scrub starts 2023-09-23T20:16:50.223 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:50.270 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:50.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:50.302 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:16:50.302 INFO:teuthology.orchestra.run.smithi135.stdout:{"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 32722/1701 objects degraded (1923.692%), 15 pgs degraded, 17 pgs undersized","count":32},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1779,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":474,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499350,"num_in_osds":6,"osd_in_since":1695500073,"num_remapped_pgs":18},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":13},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+recovery_wait+undersized+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":567,"data_bytes":40115605,"bytes_used":5207982080,"bytes_avail":570762231808,"bytes_total":575970213888,"degraded_objects":32722,"degraded_total":1701,"degraded_ratio":19.236919459141681,"recovering_objects_per_sec":144,"recovering_bytes_per_sec":565027,"recovering_keys_per_sec":0,"num_objects_recovered":1154,"num_bytes_recovered":4521984,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":932},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi135.srojfl","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.135:8443/","prometheus":"http://172.21.15.135:9283/"}},"servicemap":{"epoch":26,"modified":"2023-09-23T20:16:39.088715+0000","services":{}},"progress_events":{"484bea97-5a36-40df-8508-75260abdc555":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"fb5142fa-2e5d-413d-a713-d719d0723614":{"message":"Global Recovery Event (2m)\n [=======================.....] (remaining: 28s)","progress":0.85271316766738892,"add_to_ceph_s":true}}} 2023-09-23T20:16:50.374 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:49 smithi144 ceph-mon[194833]: failed to encode map e473 with expected crc 2023-09-23T20:16:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:49 smithi144 ceph-mon[194833]: failed to encode map e473 with expected crc 2023-09-23T20:16:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:49 smithi144 ceph-mon[194833]: failed to encode map e473 with expected crc 2023-09-23T20:16:50.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:49 smithi144 ceph-mon[194833]: osdmap e473: 8 total, 8 up, 6 in 2023-09-23T20:16:50.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:49 smithi144 ceph-mon[194833]: pgmap v1322: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 38 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 932 op/s; 32722/1701 objects degraded (1923.692%); 552 KiB/s, 144 objects/s recovering 2023-09-23T20:16:50.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:49 smithi144 ceph-mon[194833]: 9.4 scrub starts 2023-09-23T20:16:50.401 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:50.444 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:50.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:50.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:50.548 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:50.574 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:50.952 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:16:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:16:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:16:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52213 8.d 8:bac999e7:::benchmark_data_smithi135_460343_object52212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.894700+0000", 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 591.05431974800001, 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1738284119999998, 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:50.953 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52213 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894700+0000", 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:50.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894700+0000", 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894699+0000", 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894728+0000", 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9312e-05 2023-09-23T20:16:50.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894737+0000", 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3720000000000005e-06 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894742+0000", 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6650000000000002e-06 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894789+0000", 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7170999999999999e-05 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.894928+0000", 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000138978 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.895072+0000", 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014409199999999999 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.897222+0000", 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021498049999999999 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.897297+0000", 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5024000000000005e-05 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.904254+0000", 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.006957759 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.904265+0000", 2023-09-23T20:16:50.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0658e-05 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.925985+0000", 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.021720427 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.925997+0000", 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1661999999999999e-05 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068429+0000", 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.142431854 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068528+0000", 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9085000000000004e-05 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52220 8.8 8:1576796a:::benchmark_data_smithi135_460343_object52219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.910095+0000", 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 591.03892399400002, 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.155063207, 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52220 2023-09-23T20:16:50.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910095+0000", 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910095+0000", 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910094+0000", 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910138+0000", 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4589999999999998e-05 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910150+0000", 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1629e-05 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.910157+0000", 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0049999999999998e-06 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.911108+0000", 2023-09-23T20:16:50.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00095094199999999998 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.911114+0000", 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.533e-06 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.930596+0000", 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0194816419999999 2023-09-23T20:16:50.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.930607+0000", 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1243999999999999e-05 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.050602+0000", 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1199942940000001 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.050759+0000", 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015762999999999999 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.050943+0000", 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018359100000000001 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.058845+0000", 2023-09-23T20:16:50.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0079017199999999992 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.062852+0000", 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004007399 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.062931+0000", 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8728999999999998e-05 2023-09-23T20:16:50.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.062985+0000", 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4354000000000002e-05 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065159+0000", 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0021735790000000001 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:50.970 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52237 8.8 8:1c4a366e:::benchmark_data_smithi135_460343_object52236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.938063+0000", 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 591.01095588700002, 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1294171959999999, 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:50.971 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52237 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938063+0000", 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938063+0000", 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938063+0000", 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938133+0000", 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9815999999999997e-05 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938144+0000", 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1532000000000001e-05 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938150+0000", 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.28e-06 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938195+0000", 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4767e-05 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.938201+0000", 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1149999999999999e-06 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931044+0000", 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99284272399999995 2023-09-23T20:16:50.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931053+0000", 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6689999999999995e-06 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.054684+0000", 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1236309849999999 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.054796+0000", 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0001121 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.054912+0000", 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115862 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.062894+0000", 2023-09-23T20:16:50.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0079826789999999995 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065219+0000", 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023241949999999998 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065297+0000", 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8922000000000001e-05 2023-09-23T20:16:50.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065347+0000", 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9633999999999999e-05 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.067481+0000", 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002133556 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:50.979 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52241 8.d 8:b3c3dddb:::benchmark_data_smithi135_460343_object52240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.941349+0000", 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 591.00766996499999, 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1495913739999999, 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:50.980 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52241 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941349+0000", 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941349+0000", 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941348+0000", 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941394+0000", 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6112000000000002e-05 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941409+0000", 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4915e-05 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941415+0000", 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1809999999999998e-06 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941468+0000", 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2883000000000002e-05 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.941476+0000", 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0490000000000004e-06 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.926017+0000", 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98454064299999999 2023-09-23T20:16:50.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.926023+0000", 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2179999999999996e-06 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068547+0000", 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1425238929999999 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068641+0000", 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3647000000000006e-05 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.068731+0000", 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0111999999999994e-05 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.075257+0000", 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0065266220000000002 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.075267+0000", 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8490000000000001e-06 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.088158+0000", 2023-09-23T20:16:50.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012890798 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.088216+0000", 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7862999999999998e-05 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.090941+0000", 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027247870000000002 2023-09-23T20:16:50.988 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52250 8.8 8:17b1c085:::benchmark_data_smithi135_460343_object52249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.949594+0000", 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 590.99942499899998, 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.129605008, 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:50.989 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52250 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949594+0000", 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.990 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949594+0000", 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949594+0000", 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949668+0000", 2023-09-23T20:16:50.991 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3799000000000003e-05 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949677+0000", 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3519999999999996e-06 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949682+0000", 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7160000000000002e-06 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949736+0000", 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4299999999999998e-05 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.949746+0000", 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0351e-05 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931077+0000", 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98133084199999998 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931085+0000", 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5859999999999999e-06 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.058877+0000", 2023-09-23T20:16:50.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1277923919999999 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.059001+0000", 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012386999999999999 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.059159+0000", 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000157589 2023-09-23T20:16:50.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.065259+0000", 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0061000919999999997 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.067619+0000", 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023606730000000002 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.076559+0000", 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0089399140000000002 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.076612+0000", 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.223e-05 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.079199+0000", 2023-09-23T20:16:50.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025878229999999999 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:52251 8.8 8:198971ba:::benchmark_data_smithi135_460343_object52250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:06:59.952872+0000", 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 590.99614782100002, 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.129071428, 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:50.998 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 52251 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952872+0000", 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:50.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952872+0000", 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952871+0000", 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952933+0000", 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2553999999999998e-05 2023-09-23T20:16:51.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952943+0000", 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.054e-06 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952948+0000", 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1710000000000001e-06 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952978+0000", 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0403e-05 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:06:59.952985+0000", 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3980000000000002e-06 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931106+0000", 2023-09-23T20:16:51.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97812022600000004 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:00.931113+0000", 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5549999999999997e-06 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.061064+0000", 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.129951087 2023-09-23T20:16:51.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.061176+0000", 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111876 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.061331+0000", 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000154353 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.067493+0000", 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0061627449999999999 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.067671+0000", 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017801899999999999 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.079252+0000", 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011580228999999999 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.079288+0000", 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6318e-05 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:02.081943+0000", 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002655059 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:65653 8.d 8:bc798c3d:::benchmark_data_smithi135_460343_object65652:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:17.919923+0000", 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 573.029095998, 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.187381153, 2023-09-23T20:16:51.007 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 65653 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919923+0000", 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919923+0000", 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919923+0000", 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919972+0000", 2023-09-23T20:16:51.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9447000000000003e-05 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919980+0000", 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8860000000000004e-06 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.919985+0000", 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1150000000000002e-06 2023-09-23T20:16:51.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920041+0000", 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5970000000000001e-05 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920193+0000", 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000151831 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.920376+0000", 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018274 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.922919+0000", 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025428830000000001 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.925658+0000", 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027392150000000001 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.939445+0000", 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013786678 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.939456+0000", 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.166e-05 2023-09-23T20:16:51.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107196+0000", 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1677396499999999 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107305+0000", 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010861699999999999 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.014 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.015 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.015 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.35643.0:65657 8.d 8:bc23b147:::benchmark_data_smithi135_460343_object65656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2023-09-23T20:16:51.015 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:07:17.956099+0000", 2023-09-23T20:16:51.015 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 572.99292072900005, 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1600642969999999, 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.35643", 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 65657 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.018 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956099+0000", 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956099+0000", 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956098+0000", 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956131+0000", 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3785000000000003e-05 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956146+0000", 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.473e-05 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956152+0000", 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5740000000000003e-06 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956190+0000", 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8167999999999998e-05 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:17.956198+0000", 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5180000000000004e-06 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107324+0000", 2023-09-23T20:16:51.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1511252599999999 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107430+0000", 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000106363 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.107554+0000", 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012434300000000001 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.111957+0000", 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0044023949999999999 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.111987+0000", 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0077999999999999e-05 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.113834+0000", 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018473949999999999 2023-09-23T20:16:51.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.113906+0000", 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1398999999999999e-05 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:07:19.116163+0000", 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022574090000000002 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.025 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:94482 9.12 9:4b5a5a92:::benchmark_data_smithi135_549536_object94481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:30.828982+0000", 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 140.12003756199999, 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.011843271, 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.026 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 94482 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.828982+0000", 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.828982+0000", 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.828980+0000", 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829042+0000", 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1940000000000007e-05 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829072+0000", 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9819e-05 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829079+0000", 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4359999999999999e-06 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.029 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829293+0000", 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00021470899999999999 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829299+0000", 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3410000000000002e-06 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.823731+0000", 2023-09-23T20:16:51.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99443200200000004 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.823837+0000", 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000106334 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.823947+0000", 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000110122 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.830752+0000", 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0068046340000000004 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.840692+0000", 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0099403549999999997 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.840740+0000", 2023-09-23T20:16:51.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7531000000000003e-05 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.840786+0000", 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6105e-05 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.840825+0000", 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9251000000000001e-05 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.033 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:94484 9.12 9:4aba21f3:::benchmark_data_smithi135_549536_object94483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:30.829094+0000", 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 140.119925693, 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0118871780000001, 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.034 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 94484 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829094+0000", 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829094+0000", 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829093+0000", 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829143+0000", 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9496000000000002e-05 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829150+0000", 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6599999999999995e-06 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829155+0000", 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9350000000000002e-06 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829317+0000", 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016145100000000001 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:30.829323+0000", 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6810000000000001e-06 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.825233+0000", 2023-09-23T20:16:51.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99590925399999997 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.825310+0000", 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7032000000000004e-05 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.825401+0000", 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1553999999999998e-05 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.830767+0000", 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0053657310000000003 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.840876+0000", 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010108676 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.840922+0000", 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5938e-05 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.840956+0000", 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4482e-05 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:31.840981+0000", 2023-09-23T20:16:51.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4766e-05 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97519 9.a 9:53c7cf51:::benchmark_data_smithi135_549536_object97518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.824055+0000", 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 137.1249641, 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4196234690000002, 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.042 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97519 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824055+0000", 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824055+0000", 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824054+0000", 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824157+0000", 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103448 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824169+0000", 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1693e-05 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824176+0000", 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5930000000000002e-06 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824217+0000", 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1545000000000003e-05 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824341+0000", 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000124237 2023-09-23T20:16:51.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824747+0000", 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000405139 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827189+0000", 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0024426080000000002 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.865491+0000", 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.038301640999999997 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.865502+0000", 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0883999999999999e-05 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.831435+0000", 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.96593320299999996 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.831444+0000", 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0529999999999996e-06 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.224105+0000", 2023-09-23T20:16:51.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3926610020000001 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.227972+0000", 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038665589999999999 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.243602+0000", 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015630145000000002 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.243679+0000", 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7040999999999996e-05 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97521 9.12 9:48b9775b:::benchmark_data_smithi135_549536_object97520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.824193+0000", 2023-09-23T20:16:51.051 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 137.12482680100001, 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.40726715, 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97521 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.052 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824193+0000", 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824193+0000", 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824192+0000", 2023-09-23T20:16:51.053 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824280+0000", 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7756000000000005e-05 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824290+0000", 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0276e-05 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.054 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824296+0000", 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1360000000000002e-06 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824510+0000", 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00021367900000000001 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.055 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824629+0000", 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011964399999999999 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.824764+0000", 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000134914 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827195+0000", 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0024308310000000001 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.849661+0000", 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.022466231 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.849672+0000", 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0749e-05 2023-09-23T20:16:51.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.831480+0000", 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98180813300000003 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.831488+0000", 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3050000000000004e-06 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.223389+0000", 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3919017390000001 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.224068+0000", 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00067835300000000003 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.231361+0000", 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0072929420000000002 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.231460+0000", 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9024000000000006e-05 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.060 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97528 9.f 9:f0a803eb:::benchmark_data_smithi135_549536_object97527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.843381+0000", 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 137.10563795100001, 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3747164490000001, 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97528 2023-09-23T20:16:51.061 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.843381+0000", 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.843381+0000", 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.062 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.843381+0000", 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.843409+0000", 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7589999999999998e-05 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.843415+0000", 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0950000000000001e-06 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.843419+0000", 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6269999999999999e-06 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.844931+0000", 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001511678 2023-09-23T20:16:51.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.845062+0000", 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013144299999999999 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.845431+0000", 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00036840400000000002 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.065 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.849989+0000", 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0045586760000000002 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852962+0000", 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0029724539999999998 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852973+0000", 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1524e-05 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.837285+0000", 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98431116900000004 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.837309+0000", 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4128999999999999e-05 2023-09-23T20:16:51.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.176512+0000", 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.339202811 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.186231+0000", 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0097191250000000003 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.218003+0000", 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.031772751000000002 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.218098+0000", 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4443000000000004e-05 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.069 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97529 9.1f 9:ff98cd7d:::benchmark_data_smithi135_549536_object97528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.852498+0000", 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 137.096521419, 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.389471854, 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97529 2023-09-23T20:16:51.070 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852498+0000", 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852498+0000", 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852496+0000", 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852543+0000", 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6201e-05 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.072 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852557+0000", 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3939e-05 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852565+0000", 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4090000000000003e-06 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.857497+0000", 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004931548 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.857507+0000", 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0311e-05 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.837314+0000", 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.979806866 2023-09-23T20:16:51.074 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.837322+0000", 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2020000000000004e-06 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.227175+0000", 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.389852868 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.227335+0000", 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016001899999999999 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.227644+0000", 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00030872300000000001 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.235302+0000", 2023-09-23T20:16:51.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0076584799999999996 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.235358+0000", 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6239999999999997e-05 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.240223+0000", 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004865159 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.240286+0000", 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2419e-05 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.241970+0000", 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001683936 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97530 9.1f 9:f8544ebf:::benchmark_data_smithi135_549536_object97529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.852573+0000", 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 137.0964462, 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3907380749999998, 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97530 2023-09-23T20:16:51.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852573+0000", 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852573+0000", 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852573+0000", 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852613+0000", 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0277000000000001e-05 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852621+0000", 2023-09-23T20:16:51.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.5830000000000007e-06 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852627+0000", 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5289999999999999e-06 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.857582+0000", 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004955391 2023-09-23T20:16:51.082 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.857589+0000", 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8979999999999999e-06 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.837451+0000", 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97986205900000001 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.837459+0000", 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7090000000000002e-06 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.230291+0000", 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3928316490000001 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.230438+0000", 2023-09-23T20:16:51.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014740099999999999 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.230823+0000", 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00038542600000000002 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.235332+0000", 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.004508626 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.240213+0000", 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0048804219999999997 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.241983+0000", 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001770059 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.242048+0000", 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5067000000000006e-05 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.243311+0000", 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001263585 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.087 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97531 9.12 9:4ac73cb4:::benchmark_data_smithi135_549536_object97530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.852633+0000", 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 137.09638677699999, 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3814939929999999, 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97531 2023-09-23T20:16:51.088 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852633+0000", 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852633+0000", 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852632+0000", 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852669+0000", 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6492999999999998e-05 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852676+0000", 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0230000000000004e-06 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.852680+0000", 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5499999999999996e-06 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.853023+0000", 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00034286199999999998 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.853033+0000", 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7669999999999998e-06 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.092 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.831509+0000", 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97847627800000003 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.831516+0000", 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7739999999999999e-06 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.224361+0000", 2023-09-23T20:16:51.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.392845366 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.224475+0000", 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113906 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,1,2", 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.224860+0000", 2023-09-23T20:16:51.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00038518299999999999 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.230687+0000", 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0058267589999999999 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.230758+0000", 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0596e-05 2023-09-23T20:16:51.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.231543+0000", 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00078515099999999999 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.233941+0000", 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023986810000000002 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.234002+0000", 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0482000000000001e-05 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.234127+0000", 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012459600000000001 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97547 9.16 9:6a9b30a8:::benchmark_data_smithi135_549536_object97546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.896744+0000", 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 137.05227530299999, 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3190595389999999, 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97547 2023-09-23T20:16:51.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.896744+0000", 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.896744+0000", 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.896739+0000", 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999952 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.896770+0000", 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0870000000000001e-05 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.100 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.896786+0000", 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5446000000000001e-05 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.896792+0000", 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9440000000000002e-06 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.896856+0000", 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4189999999999994e-05 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.896881+0000", 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4593999999999999e-05 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.833123+0000", 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.936242664 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.833129+0000", 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.558e-06 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.199993+0000", 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3668640670000001 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.200159+0000", 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016576300000000001 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.200563+0000", 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00040484199999999998 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.203924+0000", 2023-09-23T20:16:51.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0033607450000000001 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205236+0000", 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001311991 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.214002+0000", 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0087662899999999995 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.214055+0000", 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2394e-05 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.215804+0000", 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0017487729999999999 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:170786 9.14 9:2df117c3:::benchmark_data_smithi135_549536_object66931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:01.201116+0000", 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 49.747903084999997, 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0314894560000001, 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 170786 2023-09-23T20:16:51.107 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.201116+0000", 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.201116+0000", 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.108 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.201114+0000", 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.201125+0000", 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0053e-05 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.201136+0000", 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1418e-05 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.109 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.201144+0000", 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7700000000000001e-06 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.205473+0000", 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043289749999999997 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.110 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.205481+0000", 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2179999999999999e-06 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.204102+0000", 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99862099199999999 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.204195+0000", 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3142999999999999e-05 2023-09-23T20:16:51.111 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.204544+0000", 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00034918500000000001 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.216607+0000", 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012063146 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232496+0000", 2023-09-23T20:16:51.112 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.015888988999999999 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232538+0000", 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1216e-05 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232579+0000", 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1730000000000002e-05 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.113 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232606+0000", 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.6410999999999998e-05 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:170791 9.14 9:2806b165:::benchmark_data_smithi135_549536_object66936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:01.212359+0000", 2023-09-23T20:16:51.114 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 49.736660702999998, 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.02035243, 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 170791 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.115 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.212359+0000", 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.212359+0000", 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.116 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.212358+0000", 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.212362+0000", 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0910000000000003e-06 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.212371+0000", 2023-09-23T20:16:51.117 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5440000000000004e-06 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.212378+0000", 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.1519999999999997e-06 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.212441+0000", 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3906999999999996e-05 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.118 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.212448+0000", 2023-09-23T20:16:51.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0190000000000001e-06 2023-09-23T20:16:51.119 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.119 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.119 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.119 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.210543+0000", 2023-09-23T20:16:51.119 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99809407699999997 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.210625+0000", 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2038000000000003e-05 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.210799+0000", 2023-09-23T20:16:51.120 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00017406799999999999 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.216617+0000", 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0058179740000000001 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.121 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232637+0000", 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016020046 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232662+0000", 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5435e-05 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232686+0000", 2023-09-23T20:16:51.122 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4314000000000001e-05 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232711+0000", 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4678999999999999e-05 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.123 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:170803 9.14 9:2f236c0a:::benchmark_data_smithi135_549536_object66948:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:01.222221+0000", 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 49.726798444000003, 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.01088391, 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 170803 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:16:51.124 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.222221+0000", 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.222221+0000", 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.125 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.222221+0000", 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.222223+0000", 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3329999999999999e-06 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.126 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.222230+0000", 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1890000000000001e-06 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.222234+0000", 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9770000000000002e-06 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.222307+0000", 2023-09-23T20:16:51.127 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2762999999999994e-05 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:01.222315+0000", 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7570000000000004e-06 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.213542+0000", 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99122734800000001 2023-09-23T20:16:51.128 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.213655+0000", 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000113103 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.213893+0000", 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000238397 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.129 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.219671+0000", 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0057779609999999999 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.232772+0000", 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.013101026 2023-09-23T20:16:51.130 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.233005+0000", 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00023284299999999999 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.233053+0000", 2023-09-23T20:16:51.131 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8050999999999998e-05 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:02.233105+0000", 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1521999999999999e-05 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.132 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:16:51.133 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:16:51.133 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:16:51.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:50 smithi144 ceph-mon[194833]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:50 smithi144 ceph-mon[194833]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:50 smithi144 ceph-mon[194833]: osdmap e474: 8 total, 8 up, 6 in 2023-09-23T20:16:51.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:50 smithi144 ceph-mon[194833]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/549210814' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:16:51.134 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:50 smithi144 ceph-mon[194833]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.201 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[213871]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.201 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[213871]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[213871]: osdmap e474: 8 total, 8 up, 6 in 2023-09-23T20:16:51.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[213871]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/549210814' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:16:51.202 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[213871]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.202 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[221113]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.202 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[221113]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.202 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[221113]: osdmap e474: 8 total, 8 up, 6 in 2023-09-23T20:16:51.202 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[221113]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.203 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/549210814' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:16:51.203 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:50 smithi135 ceph-mon[221113]: failed to encode map e474 with expected crc 2023-09-23T20:16:51.222 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:16:51.222 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:16:51.223 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:51.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:51.315 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:51.327 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:51.357 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:51.359 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:16:51.923 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_4' removed 2023-09-23T20:16:51.966 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:16:51.966 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:51 smithi135 ceph-mon[213871]: pgmap v1324: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 38 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 545 op/s; 32722/1701 objects degraded (1923.692%); 0 B/s, 92 objects/s recovering 2023-09-23T20:16:51.966 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/133370406' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:16:51.966 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:16:51.967 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:16:51.967 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:51 smithi135 ceph-mon[221113]: pgmap v1324: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 38 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 545 op/s; 32722/1701 objects degraded (1923.692%); 0 B/s, 92 objects/s recovering 2023-09-23T20:16:51.967 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/133370406' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:16:51.967 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:16:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:16:52.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:51 smithi144 ceph-mon[194833]: pgmap v1324: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 38 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 545 op/s; 32722/1701 objects degraded (1923.692%); 0 B/s, 92 objects/s recovering 2023-09-23T20:16:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/133370406' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:16:52.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:16:52.526 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:52.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:52.638 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:52.674 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:16:52.703 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:52.754 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:52.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:52.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:52.860 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:52.890 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:53.107 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.107 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.107 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.107 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:16:53.107 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: osdmap e475: 8 total, 8 up, 6 in 2023-09-23T20:16:53.107 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 32722/1701 objects degraded (1923.692%), 15 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: failed to encode map e476 with expected crc 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[221113]: osdmap e476: 8 total, 8 up, 6 in 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:16:53.108 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: osdmap e475: 8 total, 8 up, 6 in 2023-09-23T20:16:53.109 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:53.109 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.109 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 32722/1701 objects degraded (1923.692%), 15 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:53.109 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: failed to encode map e476 with expected crc 2023-09-23T20:16:53.109 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:52 smithi135 ceph-mon[213871]: osdmap e476: 8 total, 8 up, 6 in 2023-09-23T20:16:53.110 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:16:53.110 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:16:53.110 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T20:16:53.111 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T20:16:53.300 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: osdmap e475: 8 total, 8 up, 6 in 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: failed to encode map e475 with expected crc 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 32722/1701 objects degraded (1923.692%), 15 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: failed to encode map e476 with expected crc 2023-09-23T20:16:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:52 smithi144 ceph-mon[194833]: osdmap e476: 8 total, 8 up, 6 in 2023-09-23T20:16:53.438 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:53.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:53.532 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:53.542 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:53.572 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:53.585 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:53.720 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:16:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:16:53] "GET /metrics HTTP/1.1" 200 34136 "" "Prometheus/2.43.0" 2023-09-23T20:16:53.872 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:53.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:53.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:53.960 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:53.979 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:16:54.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[221113]: 9.c scrub starts 2023-09-23T20:16:54.008 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[221113]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.008 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[221113]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.008 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[221113]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.008 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[221113]: pgmap v1327: 93 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 20/84 objects degraded (23.810%) 2023-09-23T20:16:54.008 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[221113]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.008 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[221113]: osdmap e477: 8 total, 8 up, 6 in 2023-09-23T20:16:54.008 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[213871]: 9.c scrub starts 2023-09-23T20:16:54.009 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[213871]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.009 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[213871]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.009 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[213871]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.009 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[213871]: pgmap v1327: 93 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 20/84 objects degraded (23.810%) 2023-09-23T20:16:54.009 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[213871]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.009 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:53 smithi135 ceph-mon[213871]: osdmap e477: 8 total, 8 up, 6 in 2023-09-23T20:16:54.011 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:54.058 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:16:54.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:54.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:16:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:53 smithi144 ceph-mon[194833]: 9.c scrub starts 2023-09-23T20:16:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:53 smithi144 ceph-mon[194833]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:53 smithi144 ceph-mon[194833]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:53 smithi144 ceph-mon[194833]: failed to encode map e476 with expected crc 2023-09-23T20:16:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:53 smithi144 ceph-mon[194833]: pgmap v1327: 93 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 20/84 objects degraded (23.810%) 2023-09-23T20:16:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:53 smithi144 ceph-mon[194833]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:53 smithi144 ceph-mon[194833]: osdmap e477: 8 total, 8 up, 6 in 2023-09-23T20:16:54.163 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:54.191 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:54.574 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:54.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:54.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:54.681 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:16:54.708 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:54.856 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:16:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:54.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:16:54.929 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:16:54.964 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:54.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[221113]: 1.b scrub starts 2023-09-23T20:16:54.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[221113]: 1.b scrub ok 2023-09-23T20:16:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[221113]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[221113]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[221113]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/436524783' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:54.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[221113]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[213871]: 1.b scrub starts 2023-09-23T20:16:54.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[213871]: 1.b scrub ok 2023-09-23T20:16:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[213871]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[213871]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[213871]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/436524783' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:54.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:54 smithi135 ceph-mon[213871]: failed to encode map e477 with expected crc 2023-09-23T20:16:54.995 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:55.041 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:16:55.042 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:16:55.042 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:16:55.042 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:16:55.044 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:55.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:55.067 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:54 smithi144 ceph-mon[194833]: 1.b scrub starts 2023-09-23T20:16:55.067 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:54 smithi144 ceph-mon[194833]: 1.b scrub ok 2023-09-23T20:16:55.067 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:54 smithi144 ceph-mon[194833]: failed to encode map e477 with expected crc 2023-09-23T20:16:55.067 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:54 smithi144 ceph-mon[194833]: failed to encode map e477 with expected crc 2023-09-23T20:16:55.068 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:54 smithi144 ceph-mon[194833]: failed to encode map e477 with expected crc 2023-09-23T20:16:55.068 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/436524783' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:16:55.068 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:54 smithi144 ceph-mon[194833]: failed to encode map e477 with expected crc 2023-09-23T20:16:55.151 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:55.180 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:55.745 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:55.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:55.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:55.828 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:16:55.848 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:16:55.896 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:55.945 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:16:55.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:55.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:16:56.049 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:56.100 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:55 smithi144 ceph-mon[194833]: failed to encode map e477 with expected crc 2023-09-23T20:16:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:55 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:16:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:55 smithi144 ceph-mon[194833]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:55 smithi144 ceph-mon[194833]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:55 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/436524783' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:55 smithi144 ceph-mon[194833]: osdmap e478: 8 total, 8 up, 6 in 2023-09-23T20:16:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:55 smithi144 ceph-mon[194833]: pgmap v1330: 93 pgs: 2 peering, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:16:56.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:55 smithi144 ceph-mon[194833]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.156 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:56.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:56.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[221113]: failed to encode map e477 with expected crc 2023-09-23T20:16:56.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:16:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[221113]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[221113]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/436524783' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[221113]: osdmap e478: 8 total, 8 up, 6 in 2023-09-23T20:16:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[221113]: pgmap v1330: 93 pgs: 2 peering, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:16:56.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[221113]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[213871]: failed to encode map e477 with expected crc 2023-09-23T20:16:56.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:16:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[213871]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[213871]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/436524783' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:16:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[213871]: osdmap e478: 8 total, 8 up, 6 in 2023-09-23T20:16:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[213871]: pgmap v1330: 93 pgs: 2 peering, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:16:56.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:55 smithi135 ceph-mon[213871]: failed to encode map e478 with expected crc 2023-09-23T20:16:56.262 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:56.289 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:56.353 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:56.788 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:56.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:56.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:56.890 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:16:56.920 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:57.121 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:16:57.124 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[221113]: 1.3 scrub starts 2023-09-23T20:16:57.124 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[221113]: 1.3 scrub ok 2023-09-23T20:16:57.124 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/84 objects degraded (23.810%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T20:16:57.124 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:16:57.124 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:16:57.124 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[221113]: 1.5 scrub ok 2023-09-23T20:16:57.125 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[213871]: 1.3 scrub starts 2023-09-23T20:16:57.125 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[213871]: 1.3 scrub ok 2023-09-23T20:16:57.125 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/84 objects degraded (23.810%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T20:16:57.125 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:16:57.125 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:16:57.126 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:56 smithi135 ceph-mon[213871]: 1.5 scrub ok 2023-09-23T20:16:57.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:57.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:16:57.227 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:16:57.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:56 smithi144 ceph-mon[194833]: 1.3 scrub starts 2023-09-23T20:16:57.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:56 smithi144 ceph-mon[194833]: 1.3 scrub ok 2023-09-23T20:16:57.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:56 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/84 objects degraded (23.810%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T20:16:57.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:56 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:16:57.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:56 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:16:57.253 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:56 smithi144 ceph-mon[194833]: 1.5 scrub ok 2023-09-23T20:16:57.255 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:57.402 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:16:57.409 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:16:57.411 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:16:57.422 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:16:57.434 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:16:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:57.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:16:57.442 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:16:57.465 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:16:57.488 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:16:57.511 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:16:57.534 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:16:57.537 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:57.554 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:16:57.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:16:57.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:16:57.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:16:57.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:16:57.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:16:57.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:16:57.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:16:57.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:16:57.581 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:57.943 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:16:57.943 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:16:58.092 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:16:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:57 smithi135 ceph-mon[221113]: pgmap v1331: 93 pgs: 2 peering, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 791 KiB/s, 3 objects/s recovering 2023-09-23T20:16:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2773046704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:58.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:57 smithi135 ceph-mon[213871]: pgmap v1331: 93 pgs: 2 peering, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 791 KiB/s, 3 objects/s recovering 2023-09-23T20:16:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2773046704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:58.118 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:57 smithi144 ceph-mon[194833]: pgmap v1331: 93 pgs: 2 peering, 91 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 791 KiB/s, 3 objects/s recovering 2023-09-23T20:16:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2773046704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:16:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:16:58.383 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:58.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:58.487 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_5' created 2023-09-23T20:16:58.487 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:16:58.528 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:58.632 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:16:58.977 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:16:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: from='client.46373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2054468009' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:16:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:16:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:16:59.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: osdmap e479: 8 total, 8 up, 6 in 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[221113]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: from='client.46373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2054468009' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:16:59.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:16:59.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: osdmap e479: 8 total, 8 up, 6 in 2023-09-23T20:16:59.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.223 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:58 smithi135 ceph-mon[213871]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.223 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: from='client.46373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2054468009' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: osdmap e479: 8 total, 8 up, 6 in 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.224 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:58 smithi144 ceph-mon[194833]: failed to encode map e479 with expected crc 2023-09-23T20:16:59.608 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:16:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:59.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:16:59.620 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T20:16:59.664 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1331,"stamp":"2023-09-23T20:16:57.094330+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4808085,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13235,"num_write":914,"num_write_kb":12798,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":254,"num_bytes_recovered":8453236,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4415,"ondisk_log_size":4415,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5058520,"kb_used_data":908584,"kb_used_omap":25,"kb_used_meta":4149862,"kb_avail":557412392,"statfs":{"total":575970213888,"available":570790289408,"internally_reserved":0,"allocated":930390016,"data_stored":919026303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":4249459404},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":41,"apply_latency_ms":41,"commit_latency_ns":41000000,"apply_latency_ns":41000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.181432"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":48,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.808806+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:16:53.808806+0000","last_peered":"2023-09-23T20:16:53.808806+0000","last_clean":"2023-09-23T20:16:53.808806+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:16:53.808806+0000","last_undegraded":"2023-09-23T20:16:53.808806+0000","last_fullsized":"2023-09-23T20:16:53.808806+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":44,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.920805+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:16:51.920805+0000","last_peered":"2023-09-23T20:16:51.920805+0000","last_clean":"2023-09-23T20:16:51.920805+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:16:51.920805+0000","last_undegraded":"2023-09-23T20:16:51.920805+0000","last_fullsized":"2023-09-23T20:16:51.920805+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":223,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.802613+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:16:53.802613+0000","last_peered":"2023-09-23T20:16:53.802613+0000","last_clean":"2023-09-23T20:16:53.802613+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:16:53.802613+0000","last_undegraded":"2023-09-23T20:16:53.802613+0000","last_fullsized":"2023-09-23T20:16:53.802613+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":184,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.936466+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:16:51.936466+0000","last_peered":"2023-09-23T20:16:51.936466+0000","last_clean":"2023-09-23T20:16:51.936466+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:16:51.936466+0000","last_undegraded":"2023-09-23T20:16:51.936466+0000","last_fullsized":"2023-09-23T20:16:51.936466+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":47,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.808607+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:16:53.808607+0000","last_peered":"2023-09-23T20:16:53.808607+0000","last_clean":"2023-09-23T20:16:53.808607+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:16:53.808607+0000","last_undegraded":"2023-09-23T20:16:53.808607+0000","last_fullsized":"2023-09-23T20:16:53.808607+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":431,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.933340+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:16:51.933340+0000","last_peered":"2023-09-23T20:16:51.933340+0000","last_clean":"2023-09-23T20:16:51.933340+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:16:51.933340+0000","last_undegraded":"2023-09-23T20:16:51.933340+0000","last_fullsized":"2023-09-23T20:16:51.933340+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":402,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.933214+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:16:51.933214+0000","last_peered":"2023-09-23T20:16:51.933214+0000","last_clean":"2023-09-23T20:16:51.933214+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:16:51.933214+0000","last_undegraded":"2023-09-23T20:16:51.933214+0000","last_fullsized":"2023-09-23T20:16:51.933214+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":332,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.803067+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:16:53.803067+0000","last_peered":"2023-09-23T20:16:53.803067+0000","last_clean":"2023-09-23T20:16:53.803067+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:16:53.803067+0000","last_undegraded":"2023-09-23T20:16:53.803067+0000","last_fullsized":"2023-09-23T20:16:53.803067+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":811,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.808828+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:16:53.808828+0000","last_peered":"2023-09-23T20:16:53.808828+0000","last_clean":"2023-09-23T20:16:53.808828+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:16:53.808828+0000","last_undegraded":"2023-09-23T20:16:53.808828+0000","last_fullsized":"2023-09-23T20:16:53.808828+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":485,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.928540+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:16:51.928540+0000","last_peered":"2023-09-23T20:16:51.928540+0000","last_clean":"2023-09-23T20:16:51.928540+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:16:51.928540+0000","last_undegraded":"2023-09-23T20:16:51.928540+0000","last_fullsized":"2023-09-23T20:16:51.928540+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":342,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.802535+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:16:53.802535+0000","last_peered":"2023-09-23T20:16:53.802535+0000","last_clean":"2023-09-23T20:16:53.802535+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:16:53.802535+0000","last_undegraded":"2023-09-23T20:16:53.802535+0000","last_fullsized":"2023-09-23T20:16:53.802535+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":169,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.944092+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:16:51.944092+0000","last_peered":"2023-09-23T20:16:51.944092+0000","last_clean":"2023-09-23T20:16:51.944092+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:16:51.944092+0000","last_undegraded":"2023-09-23T20:16:51.944092+0000","last_fullsized":"2023-09-23T20:16:51.944092+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":177,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-09-23T20:16:55.114198+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:16:55.114198+0000","last_peered":"2023-09-23T20:16:55.114198+0000","last_clean":"2023-09-23T20:16:55.114198+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:16:55.114198+0000","last_undegraded":"2023-09-23T20:16:55.114198+0000","last_fullsized":"2023-09-23T20:16:55.114198+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":422,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.929681+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:16:51.929681+0000","last_peered":"2023-09-23T20:16:51.929681+0000","last_clean":"2023-09-23T20:16:51.929681+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:16:51.929681+0000","last_undegraded":"2023-09-23T20:16:51.929681+0000","last_fullsized":"2023-09-23T20:16:51.929681+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":353,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.925251+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:16:51.925251+0000","last_peered":"2023-09-23T20:16:51.925251+0000","last_clean":"2023-09-23T20:16:51.925251+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:16:51.925251+0000","last_undegraded":"2023-09-23T20:16:51.925251+0000","last_fullsized":"2023-09-23T20:16:51.925251+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":835,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.919923+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:16:51.919923+0000","last_peered":"2023-09-23T20:16:51.919923+0000","last_clean":"2023-09-23T20:16:51.919923+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:16:51.919923+0000","last_undegraded":"2023-09-23T20:16:51.919923+0000","last_fullsized":"2023-09-23T20:16:51.919923+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":169,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.939868+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:16:51.939868+0000","last_peered":"2023-09-23T20:16:51.939868+0000","last_clean":"2023-09-23T20:16:51.939868+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:16:51.939868+0000","last_undegraded":"2023-09-23T20:16:51.939868+0000","last_fullsized":"2023-09-23T20:16:51.939868+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":222,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.930405+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:16:51.930405+0000","last_peered":"2023-09-23T20:16:51.930405+0000","last_clean":"2023-09-23T20:16:51.930405+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:16:51.930405+0000","last_undegraded":"2023-09-23T20:16:51.930405+0000","last_fullsized":"2023-09-23T20:16:51.930405+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":524,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.924752+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:16:51.924752+0000","last_peered":"2023-09-23T20:16:51.924752+0000","last_clean":"2023-09-23T20:16:51.924752+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:16:51.924752+0000","last_undegraded":"2023-09-23T20:16:51.924752+0000","last_fullsized":"2023-09-23T20:16:51.924752+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":184,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.932350+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:16:51.932350+0000","last_peered":"2023-09-23T20:16:51.932350+0000","last_clean":"2023-09-23T20:16:51.932350+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:16:51.932350+0000","last_undegraded":"2023-09-23T20:16:51.932350+0000","last_fullsized":"2023-09-23T20:16:51.932350+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":474,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.922277+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:16:51.922277+0000","last_peered":"2023-09-23T20:16:51.922277+0000","last_clean":"2023-09-23T20:16:51.922277+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:16:51.922277+0000","last_undegraded":"2023-09-23T20:16:51.922277+0000","last_fullsized":"2023-09-23T20:16:51.922277+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":588,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-09-23T20:16:54.921871+0000","last_change":"2023-09-23T20:14:37.887863+0000","last_active":"2023-09-23T20:16:54.921871+0000","last_peered":"2023-09-23T20:16:54.921871+0000","last_clean":"2023-09-23T20:16:54.921871+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:16:54.921871+0000","last_undegraded":"2023-09-23T20:16:54.921871+0000","last_fullsized":"2023-09-23T20:16:54.921871+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.715264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.715264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":163,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.945651+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:16:51.945651+0000","last_peered":"2023-09-23T20:16:51.945651+0000","last_clean":"2023-09-23T20:16:51.945651+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:16:51.945651+0000","last_undegraded":"2023-09-23T20:16:51.945651+0000","last_fullsized":"2023-09-23T20:16:51.945651+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":466,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.801737+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:16:53.801737+0000","last_peered":"2023-09-23T20:16:53.801737+0000","last_clean":"2023-09-23T20:16:53.801737+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:16:53.801737+0000","last_undegraded":"2023-09-23T20:16:53.801737+0000","last_fullsized":"2023-09-23T20:16:53.801737+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":613,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.930691+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:16:51.930691+0000","last_peered":"2023-09-23T20:16:51.930691+0000","last_clean":"2023-09-23T20:16:51.930691+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:16:51.930691+0000","last_undegraded":"2023-09-23T20:16:51.930691+0000","last_fullsized":"2023-09-23T20:16:51.930691+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":119,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.920425+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:16:51.920425+0000","last_peered":"2023-09-23T20:16:51.920425+0000","last_clean":"2023-09-23T20:16:51.920425+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:16:51.920425+0000","last_undegraded":"2023-09-23T20:16:51.920425+0000","last_fullsized":"2023-09-23T20:16:51.920425+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":119,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.801944+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:16:53.801944+0000","last_peered":"2023-09-23T20:16:53.801944+0000","last_clean":"2023-09-23T20:16:53.801944+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:16:53.801944+0000","last_undegraded":"2023-09-23T20:16:53.801944+0000","last_fullsized":"2023-09-23T20:16:53.801944+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":736,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.943740+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:16:51.943740+0000","last_peered":"2023-09-23T20:16:51.943740+0000","last_clean":"2023-09-23T20:16:51.943740+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:16:51.943740+0000","last_undegraded":"2023-09-23T20:16:51.943740+0000","last_fullsized":"2023-09-23T20:16:51.943740+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":167,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.928980+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:16:51.928980+0000","last_peered":"2023-09-23T20:16:51.928980+0000","last_clean":"2023-09-23T20:16:51.928980+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:16:51.928980+0000","last_undegraded":"2023-09-23T20:16:51.928980+0000","last_fullsized":"2023-09-23T20:16:51.928980+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":484,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.920509+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:16:51.920509+0000","last_peered":"2023-09-23T20:16:51.920509+0000","last_clean":"2023-09-23T20:16:51.920509+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:16:51.920509+0000","last_undegraded":"2023-09-23T20:16:51.920509+0000","last_fullsized":"2023-09-23T20:16:51.920509+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":826,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.801536+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:16:53.801536+0000","last_peered":"2023-09-23T20:16:53.801536+0000","last_clean":"2023-09-23T20:16:53.801536+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:16:53.801536+0000","last_undegraded":"2023-09-23T20:16:53.801536+0000","last_fullsized":"2023-09-23T20:16:53.801536+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":182,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.936541+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:16:51.936541+0000","last_peered":"2023-09-23T20:16:51.936541+0000","last_clean":"2023-09-23T20:16:51.936541+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:16:51.936541+0000","last_undegraded":"2023-09-23T20:16:51.936541+0000","last_fullsized":"2023-09-23T20:16:51.936541+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":479,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.802276+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:16:53.802276+0000","last_peered":"2023-09-23T20:16:53.802276+0000","last_clean":"2023-09-23T20:16:53.802276+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:16:53.802276+0000","last_undegraded":"2023-09-23T20:16:53.802276+0000","last_fullsized":"2023-09-23T20:16:53.802276+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":548,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.930845+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:16:51.930845+0000","last_peered":"2023-09-23T20:16:51.930845+0000","last_clean":"2023-09-23T20:16:51.930845+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:16:51.930845+0000","last_undegraded":"2023-09-23T20:16:51.930845+0000","last_fullsized":"2023-09-23T20:16:51.930845+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":167,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.943947+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:16:51.943947+0000","last_peered":"2023-09-23T20:16:51.943947+0000","last_clean":"2023-09-23T20:16:51.943947+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:16:51.943947+0000","last_undegraded":"2023-09-23T20:16:51.943947+0000","last_fullsized":"2023-09-23T20:16:51.943947+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":64,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.933160+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:16:51.933160+0000","last_peered":"2023-09-23T20:16:51.933160+0000","last_clean":"2023-09-23T20:16:51.933160+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:16:51.933160+0000","last_undegraded":"2023-09-23T20:16:51.933160+0000","last_fullsized":"2023-09-23T20:16:51.933160+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":679,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.936293+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:16:51.936293+0000","last_peered":"2023-09-23T20:16:51.936293+0000","last_clean":"2023-09-23T20:16:51.936293+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:16:51.936293+0000","last_undegraded":"2023-09-23T20:16:51.936293+0000","last_fullsized":"2023-09-23T20:16:51.936293+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":524,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.921400+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:16:51.921400+0000","last_peered":"2023-09-23T20:16:51.921400+0000","last_clean":"2023-09-23T20:16:51.921400+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:16:51.921400+0000","last_undegraded":"2023-09-23T20:16:51.921400+0000","last_fullsized":"2023-09-23T20:16:51.921400+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":221,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T20:16:52.802600+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:16:52.802600+0000","last_peered":"2023-09-23T20:16:52.802600+0000","last_clean":"2023-09-23T20:16:52.802600+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:16:52.802600+0000","last_undegraded":"2023-09-23T20:16:52.802600+0000","last_fullsized":"2023-09-23T20:16:52.802600+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":858,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.945563+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:16:51.945563+0000","last_peered":"2023-09-23T20:16:51.945563+0000","last_clean":"2023-09-23T20:16:51.945563+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:16:51.945563+0000","last_undegraded":"2023-09-23T20:16:51.945563+0000","last_fullsized":"2023-09-23T20:16:51.945563+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":224,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.801686+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:16:53.801686+0000","last_peered":"2023-09-23T20:16:53.801686+0000","last_clean":"2023-09-23T20:16:53.801686+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:16:53.801686+0000","last_undegraded":"2023-09-23T20:16:53.801686+0000","last_fullsized":"2023-09-23T20:16:53.801686+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":44,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.929337+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:16:51.929337+0000","last_peered":"2023-09-23T20:16:51.929337+0000","last_clean":"2023-09-23T20:16:51.929337+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:16:51.929337+0000","last_undegraded":"2023-09-23T20:16:51.929337+0000","last_fullsized":"2023-09-23T20:16:51.929337+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":823,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.921042+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:16:51.921042+0000","last_peered":"2023-09-23T20:16:51.921042+0000","last_clean":"2023-09-23T20:16:51.921042+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:16:51.921042+0000","last_undegraded":"2023-09-23T20:16:51.921042+0000","last_fullsized":"2023-09-23T20:16:51.921042+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":44,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.919970+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:16:51.919970+0000","last_peered":"2023-09-23T20:16:51.919970+0000","last_clean":"2023-09-23T20:16:51.919970+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:16:51.919970+0000","last_undegraded":"2023-09-23T20:16:51.919970+0000","last_fullsized":"2023-09-23T20:16:51.919970+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":379,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.809185+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:16:53.809185+0000","last_peered":"2023-09-23T20:16:53.809185+0000","last_clean":"2023-09-23T20:16:53.809185+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:16:53.809185+0000","last_undegraded":"2023-09-23T20:16:53.809185+0000","last_fullsized":"2023-09-23T20:16:53.809185+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":537,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.925752+0000","last_change":"2023-09-23T20:14:37.892742+0000","last_active":"2023-09-23T20:16:51.925752+0000","last_peered":"2023-09-23T20:16:51.925752+0000","last_clean":"2023-09-23T20:16:51.925752+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:16:51.925752+0000","last_undegraded":"2023-09-23T20:16:51.925752+0000","last_fullsized":"2023-09-23T20:16:51.925752+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.749964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.749964+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":370,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.808654+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:16:53.808654+0000","last_peered":"2023-09-23T20:16:53.808654+0000","last_clean":"2023-09-23T20:16:53.808654+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:16:53.808654+0000","last_undegraded":"2023-09-23T20:16:53.808654+0000","last_fullsized":"2023-09-23T20:16:53.808654+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":44,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.920999+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:16:51.920999+0000","last_peered":"2023-09-23T20:16:51.920999+0000","last_clean":"2023-09-23T20:16:51.920999+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:16:51.920999+0000","last_undegraded":"2023-09-23T20:16:51.920999+0000","last_fullsized":"2023-09-23T20:16:51.920999+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":814,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.931066+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:16:51.931066+0000","last_peered":"2023-09-23T20:16:51.931066+0000","last_clean":"2023-09-23T20:16:51.931066+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:16:51.931066+0000","last_undegraded":"2023-09-23T20:16:51.931066+0000","last_fullsized":"2023-09-23T20:16:51.931066+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":119,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.802773+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:16:53.802773+0000","last_peered":"2023-09-23T20:16:53.802773+0000","last_clean":"2023-09-23T20:16:53.802773+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:16:53.802773+0000","last_undegraded":"2023-09-23T20:16:53.802773+0000","last_fullsized":"2023-09-23T20:16:53.802773+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":223,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.802350+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:16:53.802350+0000","last_peered":"2023-09-23T20:16:53.802350+0000","last_clean":"2023-09-23T20:16:53.802350+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:16:53.802350+0000","last_undegraded":"2023-09-23T20:16:53.802350+0000","last_fullsized":"2023-09-23T20:16:53.802350+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":760,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.802033+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:16:53.802033+0000","last_peered":"2023-09-23T20:16:53.802033+0000","last_clean":"2023-09-23T20:16:53.802033+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:16:53.802033+0000","last_undegraded":"2023-09-23T20:16:53.802033+0000","last_fullsized":"2023-09-23T20:16:53.802033+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":488,"reported_epoch":476,"state":"peering","last_fresh":"2023-09-23T20:16:52.800423+0000","last_change":"2023-09-23T20:16:52.800660+0000","last_active":"2023-09-23T20:16:52.800393+0000","last_peered":"2023-09-23T20:16:52.483255+0000","last_clean":"2023-09-23T20:07:25.847413+0000","last_became_active":"2023-09-23T20:14:35.895204+0000","last_became_peered":"2023-09-23T20:14:35.895204+0000","last_unstale":"2023-09-23T20:16:52.800423+0000","last_undegraded":"2023-09-23T20:16:52.800423+0000","last_fullsized":"2023-09-23T20:16:52.800423+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.000081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1,2],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":476,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.922217+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:16:51.922217+0000","last_peered":"2023-09-23T20:16:51.922217+0000","last_clean":"2023-09-23T20:16:51.922217+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:16:51.922217+0000","last_undegraded":"2023-09-23T20:16:51.922217+0000","last_fullsized":"2023-09-23T20:16:51.922217+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":87,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.931000+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:16:51.931000+0000","last_peered":"2023-09-23T20:16:51.931000+0000","last_clean":"2023-09-23T20:16:51.931000+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:16:51.931000+0000","last_undegraded":"2023-09-23T20:16:51.931000+0000","last_fullsized":"2023-09-23T20:16:51.931000+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":819,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.920499+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:16:51.920499+0000","last_peered":"2023-09-23T20:16:51.920499+0000","last_clean":"2023-09-23T20:16:51.920499+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:16:51.920499+0000","last_undegraded":"2023-09-23T20:16:51.920499+0000","last_fullsized":"2023-09-23T20:16:51.920499+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1126,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.930783+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:16:51.930783+0000","last_peered":"2023-09-23T20:16:51.930783+0000","last_clean":"2023-09-23T20:16:51.930783+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:16:51.930783+0000","last_undegraded":"2023-09-23T20:16:51.930783+0000","last_fullsized":"2023-09-23T20:16:51.930783+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":44,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.920387+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:16:51.920387+0000","last_peered":"2023-09-23T20:16:51.920387+0000","last_clean":"2023-09-23T20:16:51.920387+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:16:51.920387+0000","last_undegraded":"2023-09-23T20:16:51.920387+0000","last_fullsized":"2023-09-23T20:16:51.920387+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":341,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.809004+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:16:53.809004+0000","last_peered":"2023-09-23T20:16:53.809004+0000","last_clean":"2023-09-23T20:16:53.809004+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:16:53.809004+0000","last_undegraded":"2023-09-23T20:16:53.809004+0000","last_fullsized":"2023-09-23T20:16:53.809004+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":714,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.932441+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:16:51.932441+0000","last_peered":"2023-09-23T20:16:51.932441+0000","last_clean":"2023-09-23T20:16:51.932441+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:16:51.932441+0000","last_undegraded":"2023-09-23T20:16:51.932441+0000","last_fullsized":"2023-09-23T20:16:51.932441+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1116,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.808655+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:16:53.808655+0000","last_peered":"2023-09-23T20:16:53.808655+0000","last_clean":"2023-09-23T20:16:53.808655+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:16:53.808655+0000","last_undegraded":"2023-09-23T20:16:53.808655+0000","last_fullsized":"2023-09-23T20:16:53.808655+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":177,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.808598+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:16:53.808598+0000","last_peered":"2023-09-23T20:16:53.808598+0000","last_clean":"2023-09-23T20:16:53.808598+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:16:53.808598+0000","last_undegraded":"2023-09-23T20:16:53.808598+0000","last_fullsized":"2023-09-23T20:16:53.808598+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":141,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.933277+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:16:51.933277+0000","last_peered":"2023-09-23T20:16:51.933277+0000","last_clean":"2023-09-23T20:16:51.933277+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:16:51.933277+0000","last_undegraded":"2023-09-23T20:16:51.933277+0000","last_fullsized":"2023-09-23T20:16:51.933277+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1384,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.922119+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:16:51.922119+0000","last_peered":"2023-09-23T20:16:51.922119+0000","last_clean":"2023-09-23T20:16:51.922119+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:16:51.922119+0000","last_undegraded":"2023-09-23T20:16:51.922119+0000","last_fullsized":"2023-09-23T20:16:51.922119+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":671,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.931269+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:16:51.931269+0000","last_peered":"2023-09-23T20:16:51.931269+0000","last_clean":"2023-09-23T20:16:51.931269+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:16:51.931269+0000","last_undegraded":"2023-09-23T20:16:51.931269+0000","last_fullsized":"2023-09-23T20:16:51.931269+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":341,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.930933+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:16:51.930933+0000","last_peered":"2023-09-23T20:16:51.930933+0000","last_clean":"2023-09-23T20:16:51.930933+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:16:51.930933+0000","last_undegraded":"2023-09-23T20:16:51.930933+0000","last_fullsized":"2023-09-23T20:16:51.930933+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":273,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.943814+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:16:51.943814+0000","last_peered":"2023-09-23T20:16:51.943814+0000","last_clean":"2023-09-23T20:16:51.943814+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:16:51.943814+0000","last_undegraded":"2023-09-23T20:16:51.943814+0000","last_fullsized":"2023-09-23T20:16:51.943814+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":537,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.925175+0000","last_change":"2023-09-23T20:14:37.894544+0000","last_active":"2023-09-23T20:16:51.925175+0000","last_peered":"2023-09-23T20:16:51.925175+0000","last_clean":"2023-09-23T20:16:51.925175+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:16:51.925175+0000","last_undegraded":"2023-09-23T20:16:51.925175+0000","last_fullsized":"2023-09-23T20:16:51.925175+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:38.758701+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:38.758701+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":948,"reported_epoch":476,"state":"peering","last_fresh":"2023-09-23T20:16:52.799463+0000","last_change":"2023-09-23T20:16:52.799696+0000","last_active":"2023-09-23T20:16:52.799439+0000","last_peered":"2023-09-23T20:16:52.341239+0000","last_clean":"2023-09-23T20:14:31.826679+0000","last_became_active":"2023-09-23T20:14:35.882372+0000","last_became_peered":"2023-09-23T20:14:35.882372+0000","last_unstale":"2023-09-23T20:16:52.799463+0000","last_undegraded":"2023-09-23T20:16:52.799463+0000","last_fullsized":"2023-09-23T20:16:52.799463+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:01.330874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1,3],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":424,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.802190+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:16:53.802190+0000","last_peered":"2023-09-23T20:16:53.802190+0000","last_clean":"2023-09-23T20:16:53.802190+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:16:53.802190+0000","last_undegraded":"2023-09-23T20:16:53.802190+0000","last_fullsized":"2023-09-23T20:16:53.802190+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":294,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.809146+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:16:53.809146+0000","last_peered":"2023-09-23T20:16:53.809146+0000","last_clean":"2023-09-23T20:16:53.809146+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:16:53.809146+0000","last_undegraded":"2023-09-23T20:16:53.809146+0000","last_fullsized":"2023-09-23T20:16:53.809146+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":649,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.939128+0000","last_change":"2023-09-23T20:13:57.748021+0000","last_active":"2023-09-23T20:16:51.939128+0000","last_peered":"2023-09-23T20:16:51.939128+0000","last_clean":"2023-09-23T20:16:51.939128+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:16:51.939128+0000","last_undegraded":"2023-09-23T20:16:51.939128+0000","last_fullsized":"2023-09-23T20:16:51.939128+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:49.763320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:49.763320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001359208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":176,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.944017+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:16:51.944017+0000","last_peered":"2023-09-23T20:16:51.944017+0000","last_clean":"2023-09-23T20:16:51.944017+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:16:51.944017+0000","last_undegraded":"2023-09-23T20:16:51.944017+0000","last_fullsized":"2023-09-23T20:16:51.944017+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":282,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.809086+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:16:53.809086+0000","last_peered":"2023-09-23T20:16:53.809086+0000","last_clean":"2023-09-23T20:16:53.809086+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:16:53.809086+0000","last_undegraded":"2023-09-23T20:16:53.809086+0000","last_fullsized":"2023-09-23T20:16:53.809086+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":302,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.920686+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:16:51.920686+0000","last_peered":"2023-09-23T20:16:51.920686+0000","last_clean":"2023-09-23T20:16:51.920686+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:16:51.920686+0000","last_undegraded":"2023-09-23T20:16:51.920686+0000","last_fullsized":"2023-09-23T20:16:51.920686+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":697,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.809269+0000","last_change":"2023-09-23T20:14:25.790696+0000","last_active":"2023-09-23T20:16:53.809269+0000","last_peered":"2023-09-23T20:16:53.809269+0000","last_clean":"2023-09-23T20:16:53.809269+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:16:53.809269+0000","last_undegraded":"2023-09-23T20:16:53.809269+0000","last_fullsized":"2023-09-23T20:16:53.809269+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:53.737276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:53.737276+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":318,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.928664+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:16:51.928664+0000","last_peered":"2023-09-23T20:16:51.928664+0000","last_clean":"2023-09-23T20:16:51.928664+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:16:51.928664+0000","last_undegraded":"2023-09-23T20:16:51.928664+0000","last_fullsized":"2023-09-23T20:16:51.928664+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":636,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.801953+0000","last_change":"2023-09-23T20:16:02.235622+0000","last_active":"2023-09-23T20:16:53.801953+0000","last_peered":"2023-09-23T20:16:53.801953+0000","last_clean":"2023-09-23T20:16:53.801953+0000","last_became_active":"2023-09-23T20:16:02.234933+0000","last_became_peered":"2023-09-23T20:16:02.234933+0000","last_unstale":"2023-09-23T20:16:53.801953+0000","last_undegraded":"2023-09-23T20:16:53.801953+0000","last_fullsized":"2023-09-23T20:16:53.801953+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:21.855632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":110,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.802691+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:16:53.802691+0000","last_peered":"2023-09-23T20:16:53.802691+0000","last_clean":"2023-09-23T20:16:53.802691+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:16:53.802691+0000","last_undegraded":"2023-09-23T20:16:53.802691+0000","last_fullsized":"2023-09-23T20:16:53.802691+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":804,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.801874+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:16:53.801874+0000","last_peered":"2023-09-23T20:16:53.801874+0000","last_clean":"2023-09-23T20:16:53.801874+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:16:53.801874+0000","last_undegraded":"2023-09-23T20:16:53.801874+0000","last_fullsized":"2023-09-23T20:16:53.801874+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":177,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.928884+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:16:51.928884+0000","last_peered":"2023-09-23T20:16:51.928884+0000","last_clean":"2023-09-23T20:16:51.928884+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:16:51.928884+0000","last_undegraded":"2023-09-23T20:16:51.928884+0000","last_fullsized":"2023-09-23T20:16:51.928884+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":48,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.932317+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:16:51.932317+0000","last_peered":"2023-09-23T20:16:51.932317+0000","last_clean":"2023-09-23T20:16:51.932317+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:16:51.932317+0000","last_undegraded":"2023-09-23T20:16:51.932317+0000","last_fullsized":"2023-09-23T20:16:51.932317+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":259,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.920653+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:16:51.920653+0000","last_peered":"2023-09-23T20:16:51.920653+0000","last_clean":"2023-09-23T20:16:51.920653+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:16:51.920653+0000","last_undegraded":"2023-09-23T20:16:51.920653+0000","last_fullsized":"2023-09-23T20:16:51.920653+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":255,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.809079+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:16:53.809079+0000","last_peered":"2023-09-23T20:16:53.809079+0000","last_clean":"2023-09-23T20:16:53.809079+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:16:53.809079+0000","last_undegraded":"2023-09-23T20:16:53.809079+0000","last_fullsized":"2023-09-23T20:16:53.809079+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":255,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.809133+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:16:53.809133+0000","last_peered":"2023-09-23T20:16:53.809133+0000","last_clean":"2023-09-23T20:16:53.809133+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:16:53.809133+0000","last_undegraded":"2023-09-23T20:16:53.809133+0000","last_fullsized":"2023-09-23T20:16:53.809133+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":493,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.921316+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:16:51.921316+0000","last_peered":"2023-09-23T20:16:51.921316+0000","last_clean":"2023-09-23T20:16:51.921316+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:16:51.921316+0000","last_undegraded":"2023-09-23T20:16:51.921316+0000","last_fullsized":"2023-09-23T20:16:51.921316+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":498,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.801772+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:16:53.801772+0000","last_peered":"2023-09-23T20:16:53.801772+0000","last_clean":"2023-09-23T20:16:53.801772+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:16:53.801772+0000","last_undegraded":"2023-09-23T20:16:53.801772+0000","last_fullsized":"2023-09-23T20:16:53.801772+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":483,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.922148+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:16:51.922148+0000","last_peered":"2023-09-23T20:16:51.922148+0000","last_clean":"2023-09-23T20:16:51.922148+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:16:51.922148+0000","last_undegraded":"2023-09-23T20:16:51.922148+0000","last_fullsized":"2023-09-23T20:16:51.922148+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":472,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:52.614189+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:16:52.614189+0000","last_peered":"2023-09-23T20:16:52.614189+0000","last_clean":"2023-09-23T20:16:52.614189+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:16:52.614189+0000","last_undegraded":"2023-09-23T20:16:52.614189+0000","last_fullsized":"2023-09-23T20:16:52.614189+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":190,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.809220+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:16:53.809220+0000","last_peered":"2023-09-23T20:16:53.809220+0000","last_clean":"2023-09-23T20:16:53.809220+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:16:53.809220+0000","last_undegraded":"2023-09-23T20:16:53.809220+0000","last_fullsized":"2023-09-23T20:16:53.809220+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":337,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.808752+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:16:53.808752+0000","last_peered":"2023-09-23T20:16:53.808752+0000","last_clean":"2023-09-23T20:16:53.808752+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:16:53.808752+0000","last_undegraded":"2023-09-23T20:16:53.808752+0000","last_fullsized":"2023-09-23T20:16:53.808752+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":321,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:16:51.933140+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:16:51.933140+0000","last_peered":"2023-09-23T20:16:51.933140+0000","last_clean":"2023-09-23T20:16:51.933140+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:16:51.933140+0000","last_undegraded":"2023-09-23T20:16:51.933140+0000","last_fullsized":"2023-09-23T20:16:51.933140+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":47,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T20:16:53.808511+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:16:53.808511+0000","last_peered":"2023-09-23T20:16:53.808511+0000","last_clean":"2023-09-23T20:16:53.808511+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:16:53.808511+0000","last_undegraded":"2023-09-23T20:16:53.808511+0000","last_fullsized":"2023-09-23T20:16:53.808511+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":211,"num_bytes_recovered":8388684,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234429,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":529868,"kb_used_data":9092,"kb_used_omap":0,"kb_used_meta":520768,"kb_avail":93215284,"statfs":{"total":95995035648,"available":95452450816,"internally_reserved":0,"allocated":9310208,"data_stored":7850961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":533266432},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":8.2040000000000006,"5min":5.3499999999999996,"15min":3.266},"min":{"1min":0.254,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":58.109000000000002,"5min":149.25999999999999,"15min":149.25999999999999},"last":0.25},{"interface":"front","average":{"1min":8.3300000000000001,"5min":5.7619999999999996,"15min":3.5179999999999998},"min":{"1min":0.37,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":58.042000000000002,"5min":149.11799999999999,"15min":149.11799999999999},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":3.3929999999999998,"15min":3.476},"min":{"1min":0.29199999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.101,"5min":91.335999999999999,"15min":104.16800000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.845,"5min":3.9380000000000002,"15min":3.79},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.223},"max":{"1min":4.7270000000000003,"5min":91.516999999999996,"15min":104.003},"last":0.373}]},{"osd":2,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":3.3969999999999998,"15min":3.7629999999999999},"min":{"1min":0.32700000000000001,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":7.3600000000000003,"5min":67.760000000000005,"15min":105.842},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.633,"5min":3.738,"15min":4.1950000000000003},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":8.6669999999999998,"5min":67.488,"15min":105.66},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":5.0609999999999999,"5min":5.266,"15min":4.8499999999999996},"min":{"1min":0.46300000000000002,"5min":0.29999999999999999,"15min":0.248},"max":{"1min":43.747,"5min":185.17099999999999,"15min":185.17099999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":5.335,"5min":5.4279999999999999,"15min":4.843},"min":{"1min":0.40600000000000003,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":43.731000000000002,"5min":184.99100000000001,"15min":184.99100000000001},"last":0.32200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":0.96699999999999997,"15min":0.76300000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.22500000000000001},"max":{"1min":7.6440000000000001,"5min":7.6440000000000001,"15min":19.215},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.151,"15min":0.96499999999999997},"min":{"1min":0.42699999999999999,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":3.9809999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":0.28799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.595,"15min":1.1990000000000001},"min":{"1min":0.51100000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":8.4440000000000008,"5min":14.069000000000001,"15min":35.488},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.367,"5min":1.284,"15min":1.0389999999999999},"min":{"1min":0.41699999999999998,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":7.6109999999999998,"5min":13.43,"15min":34.896000000000001},"last":0.30199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4830000000000001,"15min":1.0649999999999999},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":6.843,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.5580000000000001,"5min":1.143,"15min":0.81599999999999995},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.8010000000000002,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.36099999999999999}]}]},{"osd":6,"up_from":0,"seq":1078036791470,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762628,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926165,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1026816,"kb_used_data":201784,"kb_used_omap":8,"kb_used_meta":825015,"kb_avail":92718336,"statfs":{"total":95995035648,"available":94943576064,"internally_reserved":0,"allocated":206626816,"data_stored":204420758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":844815969},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":1.952,"15min":3.4940000000000002},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":29.893000000000001,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.97,"5min":2.4169999999999998,"15min":3.7130000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.183},"max":{"1min":23.436,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:16:17 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":2.863,"15min":4.944},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.1820000000000004,"5min":160.542,"15min":270.51100000000002},"last":0.313},{"interface":"front","average":{"1min":2.0219999999999998,"5min":3.5009999999999999,"15min":4.8970000000000002},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":18.433,"5min":160.75999999999999,"15min":269.69299999999998},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4279999999999999,"15min":3.8980000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":8.9469999999999992,"5min":10.082000000000001,"15min":126.95399999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.532,"15min":4.1020000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":7.375,"5min":9.5440000000000005,"15min":127.294},"last":0.432}]},{"osd":4,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.726,"15min":2.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":5.859,"5min":9.6959999999999997,"15min":148.20599999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":5.9790000000000001,"5min":2.29,"15min":2.2589999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":105.282,"5min":105.282,"15min":148.11000000000001},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.9630000000000001,"15min":1.859},"min":{"1min":0.378,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.5430000000000001,"5min":52.061,"15min":64.725999999999999},"last":0.505},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.4649999999999999,"15min":1.8320000000000001},"min":{"1min":0.41699999999999998,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1369999999999996,"5min":51.741,"15min":51.741},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":1.5309999999999999,"15min":1.653},"min":{"1min":0.38,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":7.1950000000000003,"5min":7.1950000000000003,"15min":15.744999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.3400000000000001,"15min":1.6839999999999999},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":7.2229999999999999,"5min":8.3919999999999995,"15min":13.766999999999999},"last":0.72699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.569,"15min":1.75},"min":{"1min":0.53700000000000003,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.9980000000000002,"5min":12.981,"15min":65.081999999999994},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.6909999999999998,"5min":2.161,"15min":1.8440000000000001},"min":{"1min":0.49199999999999999,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.6600000000000001,"5min":17.151,"15min":17.151},"last":0.54800000000000004}]}]},{"osd":2,"up_from":104,"seq":446676599104,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":979616,"kb_used_data":136536,"kb_used_omap":12,"kb_used_meta":843059,"kb_avail":92765536,"statfs":{"total":95995035648,"available":94991908864,"internally_reserved":0,"allocated":139812864,"data_stored":137543228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":863293438},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.1880000000000002,"15min":2.464},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":5.3879999999999999,"5min":65.382999999999996,"15min":76.251999999999995},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.282,"15min":2.718},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.265000000000001,"5min":65.427000000000007,"15min":76.590000000000003},"last":0.373}]},{"osd":1,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":3.0899999999999999,"15min":3.5649999999999999},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":7.5270000000000001,"5min":70.453000000000003,"15min":94.768000000000001},"last":0.313},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.8969999999999998,"15min":3.2450000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.218},"max":{"1min":7.2009999999999996,"5min":69.825000000000003,"15min":96.808000000000007},"last":0.40699999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:16:43 2023","interfaces":[{"interface":"back","average":{"1min":13.59,"5min":5.3810000000000002,"15min":4.7519999999999998},"min":{"1min":0.27800000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":126.345,"5min":132.82499999999999,"15min":132.82499999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":13.555,"5min":5.4829999999999997,"15min":3.9689999999999999},"min":{"1min":0.39000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":126.40300000000001,"5min":130.97300000000001,"15min":130.97300000000001},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.4399999999999999,"15min":1.79},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":9.3399999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.667,"5min":1.7649999999999999,"15min":2.105},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.641,"5min":13.387,"15min":76.161000000000001},"last":0.42899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":3.3460000000000001,"15min":2.8519999999999999},"min":{"1min":0.34999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.3859999999999992,"5min":135.202,"15min":172.505},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.96,"5min":2.0619999999999998,"15min":2.6749999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.1260000000000003,"5min":29.196999999999999,"15min":172.375},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.4930000000000001,"15min":1.5920000000000001},"min":{"1min":0.435,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.0510000000000002,"5min":14.664,"15min":14.664},"last":0.504},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.758,"15min":1.8400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.2649999999999997,"5min":12.025,"15min":14.279999999999999},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":2.0990000000000002,"15min":1.883},"min":{"1min":0.32500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2970000000000006,"5min":13.481999999999999,"15min":13.481999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.1720000000000002,"5min":1.6399999999999999,"15min":1.581},"min":{"1min":0.443,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.266,"5min":14.698,"15min":14.698},"last":0.52100000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435578,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1162544,"kb_used_data":261800,"kb_used_omap":4,"kb_used_meta":900731,"kb_avail":92582608,"statfs":{"total":95995035648,"available":94804590592,"internally_reserved":0,"allocated":268083200,"data_stored":265556156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":922348653},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":3.5459999999999998,"5min":2.355,"15min":3.1880000000000002},"min":{"1min":0.19600000000000001,"5min":0.186,"15min":0.182},"max":{"1min":24.776,"5min":76.632000000000005,"15min":76.632000000000005},"last":0.32600000000000001},{"interface":"front","average":{"1min":3.8809999999999998,"5min":2.8530000000000002,"15min":3.4969999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":24.582000000000001,"5min":81.429000000000002,"15min":81.429000000000002},"last":0.41699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":3.4340000000000002,"15min":4.3600000000000003},"min":{"1min":0.217,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":3.7050000000000001,"5min":183.87,"15min":183.87},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.3149999999999999,"5min":3.52,"15min":4.7229999999999999},"min":{"1min":0.251,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":6.3869999999999996,"5min":185.84899999999999,"15min":185.84899999999999},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.0409999999999999,"15min":3.9140000000000001},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":8.0960000000000001,"5min":14.9,"15min":163.83500000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.5389999999999999,"15min":3.7629999999999999},"min":{"1min":0.252,"5min":0.25,"15min":0.25},"max":{"1min":4.9889999999999999,"5min":16.550999999999998,"15min":163.92099999999999},"last":0.48299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.1639999999999999,"15min":2.77},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":5.016,"5min":7.4960000000000004,"15min":109.66800000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.2909999999999999,"5min":1.8879999999999999,"15min":3.2250000000000001},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":5.6779999999999999,"5min":11.815,"15min":110.295},"last":1.0820000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8889999999999998,"5min":3.9740000000000002,"15min":3.9119999999999999},"min":{"1min":0.39000000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.742000000000001,"5min":200.53200000000001,"15min":200.53200000000001},"last":1.1060000000000001},{"interface":"front","average":{"1min":2.7200000000000002,"5min":4.1429999999999998,"15min":3.903},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.26900000000000002},"max":{"1min":23.094999999999999,"5min":200.846,"15min":200.846},"last":0.626}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.6539999999999999,"15min":2.8660000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.32300000000000001},"max":{"1min":8.9610000000000003,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.641,"5min":1.2669999999999999,"15min":2.9020000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":5.8959999999999999,"5min":8.2629999999999999,"15min":82.233000000000004},"last":0.67000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":3.0510000000000002,"15min":3.234},"min":{"1min":0.376,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":3.3719999999999999,"5min":139.465,"15min":139.465},"last":0.70399999999999996},{"interface":"front","average":{"1min":2.0299999999999998,"5min":3.073,"15min":3.3069999999999999},"min":{"1min":0.44700000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":5.4459999999999997,"5min":141.56200000000001,"15min":141.56200000000001},"last":0.69399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019339,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":554492,"kb_used_data":102580,"kb_used_omap":0,"kb_used_meta":451904,"kb_avail":93190660,"statfs":{"total":95995035648,"available":95427235840,"internally_reserved":0,"allocated":105041920,"data_stored":103590914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":462749696},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.4650000000000001,"15min":1.639},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.2549999999999999,"5min":14.063000000000001,"15min":56.061999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.6519999999999999,"5min":1.4630000000000001,"15min":1.619},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":6.9740000000000002,"5min":14.048999999999999,"15min":56.075000000000003},"last":0.45400000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":3.7559999999999998,"15min":5.7779999999999996},"min":{"1min":0.28499999999999998,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":14.763,"5min":138.47999999999999,"15min":161.613},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.1789999999999998,"5min":3.7029999999999998,"15min":5.7830000000000004},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.217},"max":{"1min":13.368,"5min":138.429,"15min":162.017},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.8969999999999998,"15min":5.3760000000000003},"min":{"1min":0.28699999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.6920000000000002,"5min":144.916,"15min":182.96700000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.0299999999999998,"5min":3.8300000000000001,"15min":5.3369999999999997},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.23100000000000001},"max":{"1min":9.4550000000000001,"5min":144.83099999999999,"15min":183.244},"last":0.55200000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.3159999999999998,"15min":4.6589999999999998},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":4.5819999999999999,"5min":100.361,"15min":184.37100000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.738,"5min":3.4990000000000001,"15min":4.7199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":4.7270000000000003,"5min":100.20699999999999,"15min":184.22300000000001},"last":2.7120000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.111,"15min":1.252},"min":{"1min":0.28199999999999997,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.0739999999999998,"5min":12.257999999999999,"15min":176.81899999999999},"last":0.312},{"interface":"front","average":{"1min":1.7689999999999999,"5min":1.111,"15min":1.2849999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.246},"max":{"1min":7.9470000000000001,"5min":7.9470000000000001,"15min":176.172},"last":0.432}]},{"osd":6,"last update":"Sat Sep 23 20:15:50 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":0.85899999999999999,"15min":0.627},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":8.5660000000000007,"5min":10.337999999999999,"15min":10.337999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.3050000000000002,"5min":1.119,"15min":0.79500000000000004},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":11.348000000000001,"5min":11.348000000000001,"15min":11.348000000000001},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.196,"15min":1.5349999999999999},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.5090000000000003,"5min":7.71,"15min":76.507000000000005},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.1040000000000001,"5min":1.4099999999999999,"15min":1.6739999999999999},"min":{"1min":0.39900000000000002,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":7.8250000000000002,"5min":7.8250000000000002,"15min":76.462000000000003},"last":0.46800000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757703,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":805184,"kb_used_data":196792,"kb_used_omap":0,"kb_used_meta":608384,"kb_avail":92939968,"statfs":{"total":95995035648,"available":95170527232,"internally_reserved":0,"allocated":201515008,"data_stored":200064286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":622985216},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":7.7800000000000002,"5min":2.794,"15min":2.214},"min":{"1min":0.23000000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":126.657,"5min":126.657,"15min":126.657},"last":0.29299999999999998},{"interface":"front","average":{"1min":7.2990000000000004,"5min":2.6539999999999999,"15min":2.4289999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":126.697,"5min":126.697,"15min":126.697},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":5.3200000000000003,"15min":5.351},"min":{"1min":0.25800000000000001,"5min":0.22,"15min":0.22},"max":{"1min":6.7279999999999998,"5min":150.16999999999999,"15min":171.68299999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.91700000000000004,"5min":4.8120000000000003,"15min":5.3470000000000004},"min":{"1min":0.27200000000000002,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":3.782,"5min":150.12700000000001,"15min":171.64099999999999},"last":0.378}]},{"osd":2,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":3.5649999999999999,"15min":4.2679999999999998},"min":{"1min":0.317,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2569999999999997,"5min":120.297,"15min":240.57300000000001},"last":0.499},{"interface":"front","average":{"1min":1.47,"5min":3.661,"15min":4.335},"min":{"1min":0.38,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.2859999999999996,"5min":120.325,"15min":240.464},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":15.287000000000001,"5min":9.1349999999999998,"15min":6.4210000000000003},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":130.46899999999999,"5min":195.84700000000001,"15min":201.50899999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":15.641999999999999,"5min":9.0640000000000001,"15min":6.1699999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":130.63300000000001,"5min":195.87899999999999,"15min":201.58099999999999},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.0629999999999999,"15min":1.3879999999999999},"min":{"1min":0.28499999999999998,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":4.8159999999999998,"5min":8,"15min":191.78899999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.67400000000000004,"15min":1.2150000000000001},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":3.371,"5min":3.371,"15min":191.821},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.1870000000000001,"15min":0.83099999999999996},"min":{"1min":0.33300000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":5.4000000000000004,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.65400000000000003,"15min":0.58199999999999996},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":3.3180000000000001,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.4079999999999999,"15min":1.478},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":11.725,"5min":11.725,"15min":191.84299999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.02,"5min":1.3,"15min":1.4990000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":11.041,"5min":11.041,"15min":191.69},"last":0.47599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:16:59.697 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:59.711 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:59.740 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:16:59.784 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:16:59.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:16:59.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:16:59.888 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:16:59.914 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:00.000 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:00.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:00.045 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:59 smithi144 ceph-mon[194833]: 1.13 scrub starts 2023-09-23T20:17:00.045 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:59 smithi144 ceph-mon[194833]: 1.13 scrub ok 2023-09-23T20:17:00.046 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:59 smithi144 ceph-mon[194833]: pgmap v1333: 109 pgs: 3 creating+peering, 13 unknown, 93 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 683 KiB/s, 3 objects/s recovering 2023-09-23T20:17:00.046 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:59 smithi144 ceph-mon[194833]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.046 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:59 smithi144 ceph-mon[194833]: osdmap e480: 8 total, 8 up, 6 in 2023-09-23T20:17:00.046 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:59 smithi144 ceph-mon[194833]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.046 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:59 smithi144 ceph-mon[194833]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.046 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:16:59 smithi144 ceph-mon[194833]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.104 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:00.131 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:00.199 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:00.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:00.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:00.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[221113]: 1.13 scrub starts 2023-09-23T20:17:00.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[221113]: 1.13 scrub ok 2023-09-23T20:17:00.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[221113]: pgmap v1333: 109 pgs: 3 creating+peering, 13 unknown, 93 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 683 KiB/s, 3 objects/s recovering 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[221113]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[221113]: osdmap e480: 8 total, 8 up, 6 in 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[221113]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[221113]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[221113]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[213871]: 1.13 scrub starts 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[213871]: 1.13 scrub ok 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[213871]: pgmap v1333: 109 pgs: 3 creating+peering, 13 unknown, 93 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 683 KiB/s, 3 objects/s recovering 2023-09-23T20:17:00.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[213871]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[213871]: osdmap e480: 8 total, 8 up, 6 in 2023-09-23T20:17:00.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[213871]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[213871]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:16:59 smithi135 ceph-mon[213871]: failed to encode map e480 with expected crc 2023-09-23T20:17:00.302 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:00.332 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:00.456 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:17:00.484 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:17:00.484 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:17:00.931 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:00.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:00.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:01.035 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:01.064 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:01.139 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:17:01.216 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:01.224 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:01.262 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:01.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:01.366 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:01.394 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:01.439 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:01.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:01.490 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:17:01.545 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:01.571 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[221113]: failed to encode map e481 with expected crc 2023-09-23T20:17:01.571 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[221113]: osdmap e481: 8 total, 8 up, 6 in 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[221113]: failed to encode map e481 with expected crc 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[221113]: pgmap v1336: 109 pgs: 3 creating+peering, 13 unknown, 93 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[213871]: failed to encode map e481 with expected crc 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[213871]: osdmap e481: 8 total, 8 up, 6 in 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[213871]: failed to encode map e481 with expected crc 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[213871]: pgmap v1336: 109 pgs: 3 creating+peering, 13 unknown, 93 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:01.572 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:01.573 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:01 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:01.575 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:01.681 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:01 smithi144 ceph-mon[194833]: failed to encode map e481 with expected crc 2023-09-23T20:17:01.682 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:01 smithi144 ceph-mon[194833]: osdmap e481: 8 total, 8 up, 6 in 2023-09-23T20:17:01.682 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:01 smithi144 ceph-mon[194833]: failed to encode map e481 with expected crc 2023-09-23T20:17:01.682 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:01 smithi144 ceph-mon[194833]: pgmap v1336: 109 pgs: 3 creating+peering, 13 unknown, 93 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:01.682 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:01 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:01.682 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:01 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:01.757 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:01.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:01.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:01.861 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:01.892 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:02.672 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:17:02.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[221113]: failed to encode map e481 with expected crc 2023-09-23T20:17:02.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[221113]: failed to encode map e481 with expected crc 2023-09-23T20:17:02.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:02.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:02.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:02.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3196923866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:02.700 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:02.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[213871]: failed to encode map e481 with expected crc 2023-09-23T20:17:02.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[213871]: failed to encode map e481 with expected crc 2023-09-23T20:17:02.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:02.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:02.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:02.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3196923866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:02.701 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:02 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:02.724 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:02 smithi144 ceph-mon[194833]: failed to encode map e481 with expected crc 2023-09-23T20:17:02.724 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:02 smithi144 ceph-mon[194833]: failed to encode map e481 with expected crc 2023-09-23T20:17:02.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:02.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:02.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:02.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3196923866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:02.725 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:02 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:02.771 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:17:02.889 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:17:02.889 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:17:02.982 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:02.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:02.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:03.086 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:03.117 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:03.160 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:03.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:03.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:03.263 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:03.293 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:03.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:17:03.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[221113]: 1.6 scrub ok 2023-09-23T20:17:03.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:03.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[221113]: failed to encode map e482 with expected crc 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[221113]: osdmap e482: 8 total, 8 up, 6 in 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[221113]: pgmap v1338: 109 pgs: 3 creating+peering, 5 unknown, 101 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[213871]: 1.6 scrub ok 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[213871]: failed to encode map e482 with expected crc 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:17:03.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[213871]: osdmap e482: 8 total, 8 up, 6 in 2023-09-23T20:17:03.491 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:03 smithi135 ceph-mon[213871]: pgmap v1338: 109 pgs: 3 creating+peering, 5 unknown, 101 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:03.622 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:17:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:03 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:17:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:03 smithi144 ceph-mon[194833]: 1.6 scrub ok 2023-09-23T20:17:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:03 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:03 smithi144 ceph-mon[194833]: failed to encode map e482 with expected crc 2023-09-23T20:17:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:03 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:17:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:03 smithi144 ceph-mon[194833]: osdmap e482: 8 total, 8 up, 6 in 2023-09-23T20:17:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:03 smithi144 ceph-mon[194833]: pgmap v1338: 109 pgs: 3 creating+peering, 5 unknown, 101 active+clean; 4.6 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:03.937 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:17:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:17:03] "GET /metrics HTTP/1.1" 200 34136 "" "Prometheus/2.43.0" 2023-09-23T20:17:03.951 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:03.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:03.969 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:17:04.056 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:04.088 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:04.236 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:04.285 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:04.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:04.324 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:17:04.343 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:17:04.361 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:17:04.380 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:17:04.390 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:04.398 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:17:04.417 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:17:04.418 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:04.438 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:17:04.457 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:17:04.458 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:17:04.458 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:17:04.458 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:17:04.458 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:17:04.458 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:17:04.459 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:17:04.459 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:17:04.459 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:17:04.499 INFO:teuthology.orchestra.run.smithi135.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T20:17:04.515 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:04.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:04.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:04.618 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:04.644 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:04.686 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:04.688 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:04 smithi144 ceph-mon[194833]: failed to encode map e482 with expected crc 2023-09-23T20:17:04.688 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:04 smithi144 ceph-mon[194833]: failed to encode map e482 with expected crc 2023-09-23T20:17:04.689 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/202186928' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:17:04.689 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:04 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:17:04.689 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2947665353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:04.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:04.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:04.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[221113]: failed to encode map e482 with expected crc 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[221113]: failed to encode map e482 with expected crc 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/202186928' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2947665353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[213871]: failed to encode map e482 with expected crc 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[213871]: failed to encode map e482 with expected crc 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/202186928' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:17:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2947665353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:04.791 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:04.818 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:04.957 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:04.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:04.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:05.064 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:05.096 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:05.368 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:17:05.368 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:17:05.368 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:17:05.368 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:17:05.368 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:17:05.799 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[221113]: failed to encode map e483 with expected crc 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[221113]: failed to encode map e483 with expected crc 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[221113]: osdmap e483: 8 total, 8 up, 6 in 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[221113]: pgmap v1340: 109 pgs: 109 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[213871]: failed to encode map e483 with expected crc 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[213871]: failed to encode map e483 with expected crc 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[213871]: osdmap e483: 8 total, 8 up, 6 in 2023-09-23T20:17:05.800 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:05 smithi135 ceph-mon[213871]: pgmap v1340: 109 pgs: 109 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:05 smithi144 ceph-mon[194833]: failed to encode map e483 with expected crc 2023-09-23T20:17:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:05 smithi144 ceph-mon[194833]: failed to encode map e483 with expected crc 2023-09-23T20:17:05.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:05 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:17:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:05 smithi144 ceph-mon[194833]: osdmap e483: 8 total, 8 up, 6 in 2023-09-23T20:17:05.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:05 smithi144 ceph-mon[194833]: pgmap v1340: 109 pgs: 109 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:05.988 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:17:06.055 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:17:06.145 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:06.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:06.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:06.250 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:06.278 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:06.374 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:06.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:06.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:06.401 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:17:06.479 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:06.496 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:06.506 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:06.655 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:06.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:06.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:06.759 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:06.788 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:06.885 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:06.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:06.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:06.989 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:07.028 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:07.059 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:17:07.061 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:17:07.160 DEBUG:teuthology.orchestra.run.smithi135:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T20:17:07.165 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:17:07.166 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:17:07.210 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:hints = 1 2023-09-23T20:17:07.210 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:17:07.210 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object prefix: benchmark_data_smithi135_602286 2023-09-23T20:17:07.210 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:17:07.211 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:17:07.238 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:07.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:07.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:07.342 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:07.400 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:07.499 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:07.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:07.609 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:07.624 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:17:07.662 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:07.995 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:17:07.995 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:17:07.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:17:07.996 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [] 2023-09-23T20:17:07.996 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:17:08.136 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:08.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:08.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:08.211 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 1 16 3638 3622 226.168 226.375 0.00540577 0.00440754 2023-09-23T20:17:08.241 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:08.284 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:08 smithi144 ceph-mon[194833]: from='client.56287 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:17:08.284 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:08 smithi144 ceph-mon[194833]: pgmap v1341: 109 pgs: 109 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:08.284 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:08.284 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:08.290 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:08.374 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:08.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:08.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:08 smithi135 ceph-mon[213871]: from='client.56287 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:17:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:08 smithi135 ceph-mon[213871]: pgmap v1341: 109 pgs: 109 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:08 smithi135 ceph-mon[221113]: from='client.56287 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:17:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:08 smithi135 ceph-mon[221113]: pgmap v1341: 109 pgs: 109 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-09-23T20:17:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:08.480 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:08.532 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:08.948 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:08.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:08.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:09.058 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:09.076 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1340,"stamp":"2023-09-23T20:17:05.096550+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":254,"num_bytes_recovered":8399566,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4415,"ondisk_log_size":4415,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":326,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4979440,"kb_used_data":826368,"kb_used_omap":25,"kb_used_meta":4152998,"kb_avail":557491472,"statfs":{"total":575970213888,"available":570871267328,"internally_reserved":0,"allocated":846200832,"data_stored":834792419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":4252670668},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-15,"num_read_kb":-28,"num_write":-7,"num_write_kb":-73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.616214"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":50,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.484369+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:16:58.484369+0000","last_peered":"2023-09-23T20:16:58.484369+0000","last_clean":"2023-09-23T20:16:58.484369+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:16:58.484369+0000","last_undegraded":"2023-09-23T20:16:58.484369+0000","last_fullsized":"2023-09-23T20:16:58.484369+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":50,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078050+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:17:01.078050+0000","last_peered":"2023-09-23T20:17:01.078050+0000","last_clean":"2023-09-23T20:17:01.078050+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:17:01.078050+0000","last_undegraded":"2023-09-23T20:17:01.078050+0000","last_fullsized":"2023-09-23T20:17:01.078050+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":227,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:00.484608+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:17:00.484608+0000","last_peered":"2023-09-23T20:17:00.484608+0000","last_clean":"2023-09-23T20:17:00.484608+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:17:00.484608+0000","last_undegraded":"2023-09-23T20:17:00.484608+0000","last_fullsized":"2023-09-23T20:17:00.484608+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":189,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494455+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:16:59.494455+0000","last_peered":"2023-09-23T20:16:59.494455+0000","last_clean":"2023-09-23T20:16:59.494455+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:16:59.494455+0000","last_undegraded":"2023-09-23T20:16:59.494455+0000","last_fullsized":"2023-09-23T20:16:59.494455+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":50,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.481969+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:16:59.481969+0000","last_peered":"2023-09-23T20:16:59.481969+0000","last_clean":"2023-09-23T20:16:59.481969+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:16:59.481969+0000","last_undegraded":"2023-09-23T20:16:59.481969+0000","last_fullsized":"2023-09-23T20:16:59.481969+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":437,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078451+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:17:01.078451+0000","last_peered":"2023-09-23T20:17:01.078451+0000","last_clean":"2023-09-23T20:17:01.078451+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:17:01.078451+0000","last_undegraded":"2023-09-23T20:17:01.078451+0000","last_fullsized":"2023-09-23T20:17:01.078451+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":408,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078367+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:17:01.078367+0000","last_peered":"2023-09-23T20:17:01.078367+0000","last_clean":"2023-09-23T20:17:01.078367+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:17:01.078367+0000","last_undegraded":"2023-09-23T20:17:01.078367+0000","last_fullsized":"2023-09-23T20:17:01.078367+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":335,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.482275+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:16:59.482275+0000","last_peered":"2023-09-23T20:16:59.482275+0000","last_clean":"2023-09-23T20:16:59.482275+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:16:59.482275+0000","last_undegraded":"2023-09-23T20:16:59.482275+0000","last_fullsized":"2023-09-23T20:16:59.482275+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":813,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.484759+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:16:58.484759+0000","last_peered":"2023-09-23T20:16:58.484759+0000","last_clean":"2023-09-23T20:16:58.484759+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:16:58.484759+0000","last_undegraded":"2023-09-23T20:16:58.484759+0000","last_fullsized":"2023-09-23T20:16:58.484759+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":491,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078223+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:17:01.078223+0000","last_peered":"2023-09-23T20:17:01.078223+0000","last_clean":"2023-09-23T20:17:01.078223+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:17:01.078223+0000","last_undegraded":"2023-09-23T20:17:01.078223+0000","last_fullsized":"2023-09-23T20:17:01.078223+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":345,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.483267+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:16:59.483267+0000","last_peered":"2023-09-23T20:16:59.483267+0000","last_clean":"2023-09-23T20:16:59.483267+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:16:59.483267+0000","last_undegraded":"2023-09-23T20:16:59.483267+0000","last_fullsized":"2023-09-23T20:16:59.483267+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":174,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494377+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:16:59.494377+0000","last_peered":"2023-09-23T20:16:59.494377+0000","last_clean":"2023-09-23T20:16:59.494377+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:16:59.494377+0000","last_undegraded":"2023-09-23T20:16:59.494377+0000","last_fullsized":"2023-09-23T20:16:59.494377+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":179,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.484523+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:16:58.484523+0000","last_peered":"2023-09-23T20:16:58.484523+0000","last_clean":"2023-09-23T20:16:58.484523+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:16:58.484523+0000","last_undegraded":"2023-09-23T20:16:58.484523+0000","last_fullsized":"2023-09-23T20:16:58.484523+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":427,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494572+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:16:59.494572+0000","last_peered":"2023-09-23T20:16:59.494572+0000","last_clean":"2023-09-23T20:16:59.494572+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:16:59.494572+0000","last_undegraded":"2023-09-23T20:16:59.494572+0000","last_fullsized":"2023-09-23T20:16:59.494572+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":359,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077924+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:17:01.077924+0000","last_peered":"2023-09-23T20:17:01.077924+0000","last_clean":"2023-09-23T20:17:01.077924+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:17:01.077924+0000","last_undegraded":"2023-09-23T20:17:01.077924+0000","last_fullsized":"2023-09-23T20:17:01.077924+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":841,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077471+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:17:01.077471+0000","last_peered":"2023-09-23T20:17:01.077471+0000","last_clean":"2023-09-23T20:17:01.077471+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:17:01.077471+0000","last_undegraded":"2023-09-23T20:17:01.077471+0000","last_fullsized":"2023-09-23T20:17:01.077471+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":174,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.493316+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:16:59.493316+0000","last_peered":"2023-09-23T20:16:59.493316+0000","last_clean":"2023-09-23T20:16:59.493316+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:16:59.493316+0000","last_undegraded":"2023-09-23T20:16:59.493316+0000","last_fullsized":"2023-09-23T20:16:59.493316+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":227,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494652+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:16:59.494652+0000","last_peered":"2023-09-23T20:16:59.494652+0000","last_clean":"2023-09-23T20:16:59.494652+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:16:59.494652+0000","last_undegraded":"2023-09-23T20:16:59.494652+0000","last_fullsized":"2023-09-23T20:16:59.494652+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":530,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077512+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:17:01.077512+0000","last_peered":"2023-09-23T20:17:01.077512+0000","last_clean":"2023-09-23T20:17:01.077512+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:17:01.077512+0000","last_undegraded":"2023-09-23T20:17:01.077512+0000","last_fullsized":"2023-09-23T20:17:01.077512+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":189,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.492101+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:16:59.492101+0000","last_peered":"2023-09-23T20:16:59.492101+0000","last_clean":"2023-09-23T20:16:59.492101+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:16:59.492101+0000","last_undegraded":"2023-09-23T20:16:59.492101+0000","last_fullsized":"2023-09-23T20:16:59.492101+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":480,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078255+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:17:01.078255+0000","last_peered":"2023-09-23T20:17:01.078255+0000","last_clean":"2023-09-23T20:17:01.078255+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:17:01.078255+0000","last_undegraded":"2023-09-23T20:17:01.078255+0000","last_fullsized":"2023-09-23T20:17:01.078255+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":603,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.483592+0000","last_change":"2023-09-23T20:16:58.248265+0000","last_active":"2023-09-23T20:16:59.483592+0000","last_peered":"2023-09-23T20:16:59.483592+0000","last_clean":"2023-09-23T20:16:59.483592+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:16:59.483592+0000","last_undegraded":"2023-09-23T20:16:59.483592+0000","last_fullsized":"2023-09-23T20:16:59.483592+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:58.248167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.248167+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:54.488842+0000","scrub_duration":0.0017920810000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":168,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.495229+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:16:59.495229+0000","last_peered":"2023-09-23T20:16:59.495229+0000","last_clean":"2023-09-23T20:16:59.495229+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:16:59.495229+0000","last_undegraded":"2023-09-23T20:16:59.495229+0000","last_fullsized":"2023-09-23T20:16:59.495229+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":469,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.482540+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:16:59.482540+0000","last_peered":"2023-09-23T20:16:59.482540+0000","last_clean":"2023-09-23T20:16:59.482540+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:16:59.482540+0000","last_undegraded":"2023-09-23T20:16:59.482540+0000","last_fullsized":"2023-09-23T20:16:59.482540+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":619,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077668+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:17:01.077668+0000","last_peered":"2023-09-23T20:17:01.077668+0000","last_clean":"2023-09-23T20:17:01.077668+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:17:01.077668+0000","last_undegraded":"2023-09-23T20:17:01.077668+0000","last_fullsized":"2023-09-23T20:17:01.077668+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":125,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077389+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:17:01.077389+0000","last_peered":"2023-09-23T20:17:01.077389+0000","last_clean":"2023-09-23T20:17:01.077389+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:17:01.077389+0000","last_undegraded":"2023-09-23T20:17:01.077389+0000","last_fullsized":"2023-09-23T20:17:01.077389+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":122,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.483870+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:16:59.483870+0000","last_peered":"2023-09-23T20:16:59.483870+0000","last_clean":"2023-09-23T20:16:59.483870+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:16:59.483870+0000","last_undegraded":"2023-09-23T20:16:59.483870+0000","last_fullsized":"2023-09-23T20:16:59.483870+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":11,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078135+0000","last_change":"2023-09-23T20:16:59.490329+0000","last_active":"2023-09-23T20:17:01.078135+0000","last_peered":"2023-09-23T20:17:01.078135+0000","last_clean":"2023-09-23T20:17:01.078135+0000","last_became_active":"2023-09-23T20:16:59.490069+0000","last_became_peered":"2023-09-23T20:16:59.490069+0000","last_unstale":"2023-09-23T20:17:01.078135+0000","last_undegraded":"2023-09-23T20:17:01.078135+0000","last_fullsized":"2023-09-23T20:17:01.078135+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.487350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":741,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.493985+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:16:59.493985+0000","last_peered":"2023-09-23T20:16:59.493985+0000","last_clean":"2023-09-23T20:16:59.493985+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:16:59.493985+0000","last_undegraded":"2023-09-23T20:16:59.493985+0000","last_fullsized":"2023-09-23T20:16:59.493985+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":172,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.492752+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:16:59.492752+0000","last_peered":"2023-09-23T20:16:59.492752+0000","last_clean":"2023-09-23T20:16:59.492752+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:16:59.492752+0000","last_undegraded":"2023-09-23T20:16:59.492752+0000","last_fullsized":"2023-09-23T20:16:59.492752+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":490,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077427+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:17:01.077427+0000","last_peered":"2023-09-23T20:17:01.077427+0000","last_clean":"2023-09-23T20:17:01.077427+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:17:01.077427+0000","last_undegraded":"2023-09-23T20:17:01.077427+0000","last_fullsized":"2023-09-23T20:17:01.077427+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":11,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:00.572534+0000","last_change":"2023-09-23T20:16:59.487459+0000","last_active":"2023-09-23T20:17:00.572534+0000","last_peered":"2023-09-23T20:17:00.572534+0000","last_clean":"2023-09-23T20:17:00.572534+0000","last_became_active":"2023-09-23T20:16:59.487181+0000","last_became_peered":"2023-09-23T20:16:59.487181+0000","last_unstale":"2023-09-23T20:17:00.572534+0000","last_undegraded":"2023-09-23T20:17:00.572534+0000","last_fullsized":"2023-09-23T20:17:00.572534+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:34.406760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":829,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.483005+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:16:59.483005+0000","last_peered":"2023-09-23T20:16:59.483005+0000","last_clean":"2023-09-23T20:16:59.483005+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:16:59.483005+0000","last_undegraded":"2023-09-23T20:16:59.483005+0000","last_fullsized":"2023-09-23T20:16:59.483005+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":187,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494510+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:16:59.494510+0000","last_peered":"2023-09-23T20:16:59.494510+0000","last_clean":"2023-09-23T20:16:59.494510+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:16:59.494510+0000","last_undegraded":"2023-09-23T20:16:59.494510+0000","last_fullsized":"2023-09-23T20:16:59.494510+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":482,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.483000+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:16:59.483000+0000","last_peered":"2023-09-23T20:16:59.483000+0000","last_clean":"2023-09-23T20:16:59.483000+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:16:59.483000+0000","last_undegraded":"2023-09-23T20:16:59.483000+0000","last_fullsized":"2023-09-23T20:16:59.483000+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":12,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T20:17:03.116133+0000","last_change":"2023-09-23T20:16:59.491310+0000","last_active":"2023-09-23T20:17:03.116133+0000","last_peered":"2023-09-23T20:17:03.116133+0000","last_clean":"2023-09-23T20:17:03.116133+0000","last_became_active":"2023-09-23T20:16:59.491016+0000","last_became_peered":"2023-09-23T20:16:59.491016+0000","last_unstale":"2023-09-23T20:17:03.116133+0000","last_undegraded":"2023-09-23T20:17:03.116133+0000","last_fullsized":"2023-09-23T20:17:03.116133+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:59.436079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":554,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078274+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:17:01.078274+0000","last_peered":"2023-09-23T20:17:01.078274+0000","last_clean":"2023-09-23T20:17:01.078274+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:17:01.078274+0000","last_undegraded":"2023-09-23T20:17:01.078274+0000","last_fullsized":"2023-09-23T20:17:01.078274+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":172,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494220+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:16:59.494220+0000","last_peered":"2023-09-23T20:16:59.494220+0000","last_clean":"2023-09-23T20:16:59.494220+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:16:59.494220+0000","last_undegraded":"2023-09-23T20:16:59.494220+0000","last_fullsized":"2023-09-23T20:16:59.494220+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.492212+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:16:59.492212+0000","last_peered":"2023-09-23T20:16:59.492212+0000","last_clean":"2023-09-23T20:16:59.492212+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:16:59.492212+0000","last_undegraded":"2023-09-23T20:16:59.492212+0000","last_fullsized":"2023-09-23T20:16:59.492212+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":10,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.489321+0000","last_change":"2023-09-23T20:16:59.488990+0000","last_active":"2023-09-23T20:16:59.489321+0000","last_peered":"2023-09-23T20:16:59.489321+0000","last_clean":"2023-09-23T20:16:59.489321+0000","last_became_active":"2023-09-23T20:16:59.488614+0000","last_became_peered":"2023-09-23T20:16:59.488614+0000","last_unstale":"2023-09-23T20:16:59.489321+0000","last_undegraded":"2023-09-23T20:16:59.489321+0000","last_fullsized":"2023-09-23T20:16:59.489321+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:11.359750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":684,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494308+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:16:59.494308+0000","last_peered":"2023-09-23T20:16:59.494308+0000","last_clean":"2023-09-23T20:16:59.494308+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:16:59.494308+0000","last_undegraded":"2023-09-23T20:16:59.494308+0000","last_fullsized":"2023-09-23T20:16:59.494308+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":530,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077874+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:17:01.077874+0000","last_peered":"2023-09-23T20:17:01.077874+0000","last_clean":"2023-09-23T20:17:01.077874+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:17:01.077874+0000","last_undegraded":"2023-09-23T20:17:01.077874+0000","last_fullsized":"2023-09-23T20:17:01.077874+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":227,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:00.572501+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:17:00.572501+0000","last_peered":"2023-09-23T20:17:00.572501+0000","last_clean":"2023-09-23T20:17:00.572501+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:17:00.572501+0000","last_undegraded":"2023-09-23T20:17:00.572501+0000","last_fullsized":"2023-09-23T20:17:00.572501+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":11,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.469822+0000","last_change":"2023-09-23T20:16:59.491852+0000","last_active":"2023-09-23T20:17:01.469822+0000","last_peered":"2023-09-23T20:17:01.469822+0000","last_clean":"2023-09-23T20:17:01.469822+0000","last_became_active":"2023-09-23T20:16:59.491582+0000","last_became_peered":"2023-09-23T20:16:59.491582+0000","last_unstale":"2023-09-23T20:17:01.469822+0000","last_undegraded":"2023-09-23T20:17:01.469822+0000","last_fullsized":"2023-09-23T20:17:01.469822+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:06.925753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":863,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.495140+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:16:59.495140+0000","last_peered":"2023-09-23T20:16:59.495140+0000","last_clean":"2023-09-23T20:16:59.495140+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:16:59.495140+0000","last_undegraded":"2023-09-23T20:16:59.495140+0000","last_fullsized":"2023-09-23T20:16:59.495140+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":227,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.483717+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:16:59.483717+0000","last_peered":"2023-09-23T20:16:59.483717+0000","last_clean":"2023-09-23T20:16:59.483717+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:16:59.483717+0000","last_undegraded":"2023-09-23T20:16:59.483717+0000","last_fullsized":"2023-09-23T20:16:59.483717+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":49,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.493458+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:16:59.493458+0000","last_peered":"2023-09-23T20:16:59.493458+0000","last_clean":"2023-09-23T20:16:59.493458+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:16:59.493458+0000","last_undegraded":"2023-09-23T20:16:59.493458+0000","last_fullsized":"2023-09-23T20:16:59.493458+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":10,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.489110+0000","last_change":"2023-09-23T20:16:59.488642+0000","last_active":"2023-09-23T20:16:59.489110+0000","last_peered":"2023-09-23T20:16:59.489110+0000","last_clean":"2023-09-23T20:16:59.489110+0000","last_became_active":"2023-09-23T20:16:59.488330+0000","last_became_peered":"2023-09-23T20:16:59.488330+0000","last_unstale":"2023-09-23T20:16:59.489110+0000","last_undegraded":"2023-09-23T20:16:59.489110+0000","last_fullsized":"2023-09-23T20:16:59.489110+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:46.429096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":829,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077479+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:17:01.077479+0000","last_peered":"2023-09-23T20:17:01.077479+0000","last_clean":"2023-09-23T20:17:01.077479+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:17:01.077479+0000","last_undegraded":"2023-09-23T20:17:01.077479+0000","last_fullsized":"2023-09-23T20:17:01.077479+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":50,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077561+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:17:01.077561+0000","last_peered":"2023-09-23T20:17:01.077561+0000","last_clean":"2023-09-23T20:17:01.077561+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:17:01.077561+0000","last_undegraded":"2023-09-23T20:17:01.077561+0000","last_fullsized":"2023-09-23T20:17:01.077561+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":381,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.485487+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:16:58.485487+0000","last_peered":"2023-09-23T20:16:58.485487+0000","last_clean":"2023-09-23T20:16:58.485487+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:16:58.485487+0000","last_undegraded":"2023-09-23T20:16:58.485487+0000","last_fullsized":"2023-09-23T20:16:58.485487+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":10,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.496528+0000","last_change":"2023-09-23T20:16:59.496298+0000","last_active":"2023-09-23T20:16:59.496528+0000","last_peered":"2023-09-23T20:16:59.496528+0000","last_clean":"2023-09-23T20:16:59.496528+0000","last_became_active":"2023-09-23T20:16:59.496009+0000","last_became_peered":"2023-09-23T20:16:59.496009+0000","last_unstale":"2023-09-23T20:16:59.496528+0000","last_undegraded":"2023-09-23T20:16:59.496528+0000","last_fullsized":"2023-09-23T20:16:59.496528+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:20.859500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":553,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078583+0000","last_change":"2023-09-23T20:16:53.682976+0000","last_active":"2023-09-23T20:17:01.078583+0000","last_peered":"2023-09-23T20:17:01.078583+0000","last_clean":"2023-09-23T20:17:01.078583+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:17:01.078583+0000","last_undegraded":"2023-09-23T20:17:01.078583+0000","last_fullsized":"2023-09-23T20:17:01.078583+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:26.329677+0000","scrub_duration":0.0022394540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":372,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.483979+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:16:58.483979+0000","last_peered":"2023-09-23T20:16:58.483979+0000","last_clean":"2023-09-23T20:16:58.483979+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:16:58.483979+0000","last_undegraded":"2023-09-23T20:16:58.483979+0000","last_fullsized":"2023-09-23T20:16:58.483979+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":50,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078185+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:17:01.078185+0000","last_peered":"2023-09-23T20:17:01.078185+0000","last_clean":"2023-09-23T20:17:01.078185+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:17:01.078185+0000","last_undegraded":"2023-09-23T20:17:01.078185+0000","last_fullsized":"2023-09-23T20:17:01.078185+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":10,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.496868+0000","last_change":"2023-09-23T20:16:59.496645+0000","last_active":"2023-09-23T20:16:59.496868+0000","last_peered":"2023-09-23T20:16:59.496868+0000","last_clean":"2023-09-23T20:16:59.496868+0000","last_became_active":"2023-09-23T20:16:59.496430+0000","last_became_peered":"2023-09-23T20:16:59.496430+0000","last_unstale":"2023-09-23T20:16:59.496868+0000","last_undegraded":"2023-09-23T20:16:59.496868+0000","last_fullsized":"2023-09-23T20:16:59.496868+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:59.713976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":820,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077871+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:17:01.077871+0000","last_peered":"2023-09-23T20:17:01.077871+0000","last_clean":"2023-09-23T20:17:01.077871+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:17:01.077871+0000","last_undegraded":"2023-09-23T20:17:01.077871+0000","last_fullsized":"2023-09-23T20:17:01.077871+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":122,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.481885+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:16:59.481885+0000","last_peered":"2023-09-23T20:16:59.481885+0000","last_clean":"2023-09-23T20:16:59.481885+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:16:59.481885+0000","last_undegraded":"2023-09-23T20:16:59.481885+0000","last_fullsized":"2023-09-23T20:16:59.481885+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":226,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.484603+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:16:59.484603+0000","last_peered":"2023-09-23T20:16:59.484603+0000","last_clean":"2023-09-23T20:16:59.484603+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:16:59.484603+0000","last_undegraded":"2023-09-23T20:16:59.484603+0000","last_fullsized":"2023-09-23T20:16:59.484603+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":10,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.496524+0000","last_change":"2023-09-23T20:16:59.496291+0000","last_active":"2023-09-23T20:16:59.496524+0000","last_peered":"2023-09-23T20:16:59.496524+0000","last_clean":"2023-09-23T20:16:59.496524+0000","last_became_active":"2023-09-23T20:16:59.496000+0000","last_became_peered":"2023-09-23T20:16:59.496000+0000","last_unstale":"2023-09-23T20:16:59.496524+0000","last_undegraded":"2023-09-23T20:16:59.496524+0000","last_fullsized":"2023-09-23T20:16:59.496524+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.808639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":763,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.484527+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:16:59.484527+0000","last_peered":"2023-09-23T20:16:59.484527+0000","last_clean":"2023-09-23T20:16:59.484527+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:16:59.484527+0000","last_undegraded":"2023-09-23T20:16:59.484527+0000","last_fullsized":"2023-09-23T20:16:59.484527+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":501,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494484+0000","last_change":"2023-09-23T20:16:53.809519+0000","last_active":"2023-09-23T20:16:59.494484+0000","last_peered":"2023-09-23T20:16:59.494484+0000","last_clean":"2023-09-23T20:16:59.494484+0000","last_became_active":"2023-09-23T20:16:53.809219+0000","last_became_peered":"2023-09-23T20:16:53.809219+0000","last_unstale":"2023-09-23T20:16:59.494484+0000","last_undegraded":"2023-09-23T20:16:59.494484+0000","last_fullsized":"2023-09-23T20:16:59.494484+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:01.640683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":482,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078149+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:17:01.078149+0000","last_peered":"2023-09-23T20:17:01.078149+0000","last_clean":"2023-09-23T20:17:01.078149+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:17:01.078149+0000","last_undegraded":"2023-09-23T20:17:01.078149+0000","last_fullsized":"2023-09-23T20:17:01.078149+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":93,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077856+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:17:01.077856+0000","last_peered":"2023-09-23T20:17:01.077856+0000","last_clean":"2023-09-23T20:17:01.077856+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:17:01.077856+0000","last_undegraded":"2023-09-23T20:17:01.077856+0000","last_fullsized":"2023-09-23T20:17:01.077856+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":11,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077681+0000","last_change":"2023-09-23T20:16:59.488965+0000","last_active":"2023-09-23T20:17:01.077681+0000","last_peered":"2023-09-23T20:17:01.077681+0000","last_clean":"2023-09-23T20:17:01.077681+0000","last_became_active":"2023-09-23T20:16:59.488722+0000","last_became_peered":"2023-09-23T20:16:59.488722+0000","last_unstale":"2023-09-23T20:17:01.077681+0000","last_undegraded":"2023-09-23T20:17:01.077681+0000","last_fullsized":"2023-09-23T20:17:01.077681+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:57.375203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":825,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077485+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:17:01.077485+0000","last_peered":"2023-09-23T20:17:01.077485+0000","last_clean":"2023-09-23T20:17:01.077485+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:17:01.077485+0000","last_undegraded":"2023-09-23T20:17:01.077485+0000","last_fullsized":"2023-09-23T20:17:01.077485+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1132,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077759+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:17:01.077759+0000","last_peered":"2023-09-23T20:17:01.077759+0000","last_clean":"2023-09-23T20:17:01.077759+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:17:01.077759+0000","last_undegraded":"2023-09-23T20:17:01.077759+0000","last_fullsized":"2023-09-23T20:17:01.077759+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":50,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077780+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:17:01.077780+0000","last_peered":"2023-09-23T20:17:01.077780+0000","last_clean":"2023-09-23T20:17:01.077780+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:17:01.077780+0000","last_undegraded":"2023-09-23T20:17:01.077780+0000","last_fullsized":"2023-09-23T20:17:01.077780+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":343,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.484893+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:16:58.484893+0000","last_peered":"2023-09-23T20:16:58.484893+0000","last_clean":"2023-09-23T20:16:58.484893+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:16:58.484893+0000","last_undegraded":"2023-09-23T20:16:58.484893+0000","last_fullsized":"2023-09-23T20:16:58.484893+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":11,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077406+0000","last_change":"2023-09-23T20:16:59.496065+0000","last_active":"2023-09-23T20:17:01.077406+0000","last_peered":"2023-09-23T20:17:01.077406+0000","last_clean":"2023-09-23T20:17:01.077406+0000","last_became_active":"2023-09-23T20:16:59.495782+0000","last_became_peered":"2023-09-23T20:16:59.495782+0000","last_unstale":"2023-09-23T20:17:01.077406+0000","last_undegraded":"2023-09-23T20:17:01.077406+0000","last_fullsized":"2023-09-23T20:17:01.077406+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:08.985469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":719,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.492146+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:16:59.492146+0000","last_peered":"2023-09-23T20:16:59.492146+0000","last_clean":"2023-09-23T20:16:59.492146+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:16:59.492146+0000","last_undegraded":"2023-09-23T20:16:59.492146+0000","last_fullsized":"2023-09-23T20:16:59.492146+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1118,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.484063+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:16:58.484063+0000","last_peered":"2023-09-23T20:16:58.484063+0000","last_clean":"2023-09-23T20:16:58.484063+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:16:58.484063+0000","last_undegraded":"2023-09-23T20:16:58.484063+0000","last_fullsized":"2023-09-23T20:16:58.484063+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":179,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.484489+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:16:58.484489+0000","last_peered":"2023-09-23T20:16:58.484489+0000","last_clean":"2023-09-23T20:16:58.484489+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:16:58.484489+0000","last_undegraded":"2023-09-23T20:16:58.484489+0000","last_fullsized":"2023-09-23T20:16:58.484489+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":147,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078409+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:17:01.078409+0000","last_peered":"2023-09-23T20:17:01.078409+0000","last_clean":"2023-09-23T20:17:01.078409+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:17:01.078409+0000","last_undegraded":"2023-09-23T20:17:01.078409+0000","last_fullsized":"2023-09-23T20:17:01.078409+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"0'0","reported_seq":10,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.496642+0000","last_change":"2023-09-23T20:16:59.496379+0000","last_active":"2023-09-23T20:16:59.496642+0000","last_peered":"2023-09-23T20:16:59.496642+0000","last_clean":"2023-09-23T20:16:59.496642+0000","last_became_active":"2023-09-23T20:16:59.495954+0000","last_became_peered":"2023-09-23T20:16:59.495954+0000","last_unstale":"2023-09-23T20:16:59.496642+0000","last_undegraded":"2023-09-23T20:16:59.496642+0000","last_fullsized":"2023-09-23T20:16:59.496642+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:08.819045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1390,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077259+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:17:01.077259+0000","last_peered":"2023-09-23T20:17:01.077259+0000","last_clean":"2023-09-23T20:17:01.077259+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:17:01.077259+0000","last_undegraded":"2023-09-23T20:17:01.077259+0000","last_fullsized":"2023-09-23T20:17:01.077259+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":676,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.492021+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:16:59.492021+0000","last_peered":"2023-09-23T20:16:59.492021+0000","last_clean":"2023-09-23T20:16:59.492021+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:16:59.492021+0000","last_undegraded":"2023-09-23T20:16:59.492021+0000","last_fullsized":"2023-09-23T20:16:59.492021+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":347,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077254+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:17:01.077254+0000","last_peered":"2023-09-23T20:17:01.077254+0000","last_clean":"2023-09-23T20:17:01.077254+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:17:01.077254+0000","last_undegraded":"2023-09-23T20:17:01.077254+0000","last_fullsized":"2023-09-23T20:17:01.077254+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":278,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494070+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:16:59.494070+0000","last_peered":"2023-09-23T20:16:59.494070+0000","last_clean":"2023-09-23T20:16:59.494070+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:16:59.494070+0000","last_undegraded":"2023-09-23T20:16:59.494070+0000","last_fullsized":"2023-09-23T20:16:59.494070+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":10,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494306+0000","last_change":"2023-09-23T20:16:59.494039+0000","last_active":"2023-09-23T20:16:59.494306+0000","last_peered":"2023-09-23T20:16:59.494306+0000","last_clean":"2023-09-23T20:16:59.494306+0000","last_became_active":"2023-09-23T20:16:59.493714+0000","last_became_peered":"2023-09-23T20:16:59.493714+0000","last_unstale":"2023-09-23T20:16:59.494306+0000","last_undegraded":"2023-09-23T20:16:59.494306+0000","last_fullsized":"2023-09-23T20:16:59.494306+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:15.171264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":553,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077893+0000","last_change":"2023-09-23T20:16:55.675509+0000","last_active":"2023-09-23T20:17:01.077893+0000","last_peered":"2023-09-23T20:17:01.077893+0000","last_clean":"2023-09-23T20:17:01.077893+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:17:01.077893+0000","last_undegraded":"2023-09-23T20:17:01.077893+0000","last_fullsized":"2023-09-23T20:17:01.077893+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:55.675445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:55.675445+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:10.524240+0000","scrub_duration":0.0015595749999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":961,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.491718+0000","last_change":"2023-09-23T20:16:53.808714+0000","last_active":"2023-09-23T20:16:59.491718+0000","last_peered":"2023-09-23T20:16:59.491718+0000","last_clean":"2023-09-23T20:16:59.491718+0000","last_became_active":"2023-09-23T20:16:53.808212+0000","last_became_peered":"2023-09-23T20:16:53.808212+0000","last_unstale":"2023-09-23T20:16:59.491718+0000","last_undegraded":"2023-09-23T20:16:59.491718+0000","last_fullsized":"2023-09-23T20:16:59.491718+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:46.997392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":427,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.482914+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:16:59.482914+0000","last_peered":"2023-09-23T20:16:59.482914+0000","last_clean":"2023-09-23T20:16:59.482914+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:16:59.482914+0000","last_undegraded":"2023-09-23T20:16:59.482914+0000","last_fullsized":"2023-09-23T20:16:59.482914+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":296,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.485261+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:16:58.485261+0000","last_peered":"2023-09-23T20:16:58.485261+0000","last_clean":"2023-09-23T20:16:58.485261+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:16:58.485261+0000","last_undegraded":"2023-09-23T20:16:58.485261+0000","last_fullsized":"2023-09-23T20:16:58.485261+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":11,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:00.484630+0000","last_change":"2023-09-23T20:16:59.490420+0000","last_active":"2023-09-23T20:17:00.484630+0000","last_peered":"2023-09-23T20:17:00.484630+0000","last_clean":"2023-09-23T20:17:00.484630+0000","last_became_active":"2023-09-23T20:16:59.490025+0000","last_became_peered":"2023-09-23T20:16:59.490025+0000","last_unstale":"2023-09-23T20:17:00.484630+0000","last_undegraded":"2023-09-23T20:17:00.484630+0000","last_fullsized":"2023-09-23T20:17:00.484630+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:03.184763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":667,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:02.526275+0000","last_change":"2023-09-23T20:17:02.526275+0000","last_active":"2023-09-23T20:17:02.526275+0000","last_peered":"2023-09-23T20:17:02.526275+0000","last_clean":"2023-09-23T20:17:02.526275+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:17:02.526275+0000","last_undegraded":"2023-09-23T20:17:02.526275+0000","last_fullsized":"2023-09-23T20:17:02.526275+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:57.948831+0000","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":181,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494295+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:16:59.494295+0000","last_peered":"2023-09-23T20:16:59.494295+0000","last_clean":"2023-09-23T20:16:59.494295+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:16:59.494295+0000","last_undegraded":"2023-09-23T20:16:59.494295+0000","last_fullsized":"2023-09-23T20:16:59.494295+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":284,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.484993+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:16:58.484993+0000","last_peered":"2023-09-23T20:16:58.484993+0000","last_clean":"2023-09-23T20:16:58.484993+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:16:58.484993+0000","last_undegraded":"2023-09-23T20:16:58.484993+0000","last_fullsized":"2023-09-23T20:16:58.484993+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":308,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077836+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:17:01.077836+0000","last_peered":"2023-09-23T20:17:01.077836+0000","last_clean":"2023-09-23T20:17:01.077836+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:17:01.077836+0000","last_undegraded":"2023-09-23T20:17:01.077836+0000","last_fullsized":"2023-09-23T20:17:01.077836+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":10,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.490637+0000","last_change":"2023-09-23T20:16:59.490304+0000","last_active":"2023-09-23T20:16:59.490637+0000","last_peered":"2023-09-23T20:16:59.490637+0000","last_clean":"2023-09-23T20:16:59.490637+0000","last_became_active":"2023-09-23T20:16:59.490026+0000","last_became_peered":"2023-09-23T20:16:59.490026+0000","last_unstale":"2023-09-23T20:16:59.490637+0000","last_undegraded":"2023-09-23T20:16:59.490637+0000","last_fullsized":"2023-09-23T20:16:59.490637+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:59.789512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":711,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.485571+0000","last_change":"2023-09-23T20:16:56.408683+0000","last_active":"2023-09-23T20:16:58.485571+0000","last_peered":"2023-09-23T20:16:58.485571+0000","last_clean":"2023-09-23T20:16:58.485571+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:16:58.485571+0000","last_undegraded":"2023-09-23T20:16:58.485571+0000","last_fullsized":"2023-09-23T20:16:58.485571+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:18.311022+0000","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":323,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.492487+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:16:59.492487+0000","last_peered":"2023-09-23T20:16:59.492487+0000","last_clean":"2023-09-23T20:16:59.492487+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:16:59.492487+0000","last_undegraded":"2023-09-23T20:16:59.492487+0000","last_fullsized":"2023-09-23T20:16:59.492487+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":639,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.482739+0000","last_change":"2023-09-23T20:16:02.235622+0000","last_active":"2023-09-23T20:16:59.482739+0000","last_peered":"2023-09-23T20:16:59.482739+0000","last_clean":"2023-09-23T20:16:59.482739+0000","last_became_active":"2023-09-23T20:16:02.234933+0000","last_became_peered":"2023-09-23T20:16:02.234933+0000","last_unstale":"2023-09-23T20:16:59.482739+0000","last_undegraded":"2023-09-23T20:16:59.482739+0000","last_fullsized":"2023-09-23T20:16:59.482739+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:21.855632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":113,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.481796+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:16:59.481796+0000","last_peered":"2023-09-23T20:16:59.481796+0000","last_clean":"2023-09-23T20:16:59.481796+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:16:59.481796+0000","last_undegraded":"2023-09-23T20:16:59.481796+0000","last_fullsized":"2023-09-23T20:16:59.481796+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":11,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:00.573281+0000","last_change":"2023-09-23T20:16:59.490308+0000","last_active":"2023-09-23T20:17:00.573281+0000","last_peered":"2023-09-23T20:17:00.573281+0000","last_clean":"2023-09-23T20:17:00.573281+0000","last_became_active":"2023-09-23T20:16:59.489997+0000","last_became_peered":"2023-09-23T20:16:59.489997+0000","last_unstale":"2023-09-23T20:17:00.573281+0000","last_undegraded":"2023-09-23T20:17:00.573281+0000","last_fullsized":"2023-09-23T20:17:00.573281+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:45.118523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":807,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.483357+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:16:59.483357+0000","last_peered":"2023-09-23T20:16:59.483357+0000","last_clean":"2023-09-23T20:16:59.483357+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:16:59.483357+0000","last_undegraded":"2023-09-23T20:16:59.483357+0000","last_fullsized":"2023-09-23T20:16:59.483357+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":182,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.492682+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:16:59.492682+0000","last_peered":"2023-09-23T20:16:59.492682+0000","last_clean":"2023-09-23T20:16:59.492682+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:16:59.492682+0000","last_undegraded":"2023-09-23T20:16:59.492682+0000","last_fullsized":"2023-09-23T20:16:59.492682+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":53,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.494239+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:16:59.494239+0000","last_peered":"2023-09-23T20:16:59.494239+0000","last_clean":"2023-09-23T20:16:59.494239+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:16:59.494239+0000","last_undegraded":"2023-09-23T20:16:59.494239+0000","last_fullsized":"2023-09-23T20:16:59.494239+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":265,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077569+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:17:01.077569+0000","last_peered":"2023-09-23T20:17:01.077569+0000","last_clean":"2023-09-23T20:17:01.077569+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:17:01.077569+0000","last_undegraded":"2023-09-23T20:17:01.077569+0000","last_fullsized":"2023-09-23T20:17:01.077569+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":257,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.485116+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:16:58.485116+0000","last_peered":"2023-09-23T20:16:58.485116+0000","last_clean":"2023-09-23T20:16:58.485116+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:16:58.485116+0000","last_undegraded":"2023-09-23T20:16:58.485116+0000","last_fullsized":"2023-09-23T20:16:58.485116+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":257,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.485004+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:16:58.485004+0000","last_peered":"2023-09-23T20:16:58.485004+0000","last_clean":"2023-09-23T20:16:58.485004+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:16:58.485004+0000","last_undegraded":"2023-09-23T20:16:58.485004+0000","last_fullsized":"2023-09-23T20:16:58.485004+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":499,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077856+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:17:01.077856+0000","last_peered":"2023-09-23T20:17:01.077856+0000","last_clean":"2023-09-23T20:17:01.077856+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:17:01.077856+0000","last_undegraded":"2023-09-23T20:17:01.077856+0000","last_fullsized":"2023-09-23T20:17:01.077856+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":501,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.483867+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:16:59.483867+0000","last_peered":"2023-09-23T20:16:59.483867+0000","last_clean":"2023-09-23T20:16:59.483867+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:16:59.483867+0000","last_undegraded":"2023-09-23T20:16:59.483867+0000","last_fullsized":"2023-09-23T20:16:59.483867+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":489,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.078156+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:17:01.078156+0000","last_peered":"2023-09-23T20:17:01.078156+0000","last_clean":"2023-09-23T20:17:01.078156+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:17:01.078156+0000","last_undegraded":"2023-09-23T20:17:01.078156+0000","last_fullsized":"2023-09-23T20:17:01.078156+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":478,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.469489+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:17:01.469489+0000","last_peered":"2023-09-23T20:17:01.469489+0000","last_clean":"2023-09-23T20:17:01.469489+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:17:01.469489+0000","last_undegraded":"2023-09-23T20:17:01.469489+0000","last_fullsized":"2023-09-23T20:17:01.469489+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":192,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.485349+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:16:58.485349+0000","last_peered":"2023-09-23T20:16:58.485349+0000","last_clean":"2023-09-23T20:16:58.485349+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:16:58.485349+0000","last_undegraded":"2023-09-23T20:16:58.485349+0000","last_fullsized":"2023-09-23T20:16:58.485349+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":339,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-09-23T20:16:58.484139+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:16:58.484139+0000","last_peered":"2023-09-23T20:16:58.484139+0000","last_clean":"2023-09-23T20:16:58.484139+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:16:58.484139+0000","last_undegraded":"2023-09-23T20:16:58.484139+0000","last_fullsized":"2023-09-23T20:16:58.484139+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":327,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-09-23T20:17:01.077294+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:17:01.077294+0000","last_peered":"2023-09-23T20:17:01.077294+0000","last_clean":"2023-09-23T20:17:01.077294+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:17:01.077294+0000","last_undegraded":"2023-09-23T20:17:01.077294+0000","last_fullsized":"2023-09-23T20:17:01.077294+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":50,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-09-23T20:16:59.481872+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:16:59.481872+0000","last_peered":"2023-09-23T20:16:59.481872+0000","last_clean":"2023-09-23T20:16:59.481872+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:16:59.481872+0000","last_undegraded":"2023-09-23T20:16:59.481872+0000","last_fullsized":"2023-09-23T20:16:59.481872+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":211,"num_bytes_recovered":8388684,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234431,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":530044,"kb_used_data":9140,"kb_used_omap":0,"kb_used_meta":520896,"kb_avail":93215108,"statfs":{"total":95995035648,"available":95452270592,"internally_reserved":0,"allocated":9359360,"data_stored":7898285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":533397504},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":8.2040000000000006,"5min":5.3499999999999996,"15min":3.266},"min":{"1min":0.254,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":58.109000000000002,"5min":149.25999999999999,"15min":149.25999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":8.3300000000000001,"5min":5.7619999999999996,"15min":3.5179999999999998},"min":{"1min":0.37,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":58.042000000000002,"5min":149.11799999999999,"15min":149.11799999999999},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":3.3929999999999998,"15min":3.476},"min":{"1min":0.29199999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.101,"5min":91.335999999999999,"15min":104.16800000000001},"last":0.63},{"interface":"front","average":{"1min":1.845,"5min":3.9380000000000002,"15min":3.79},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.223},"max":{"1min":4.7270000000000003,"5min":91.516999999999996,"15min":104.003},"last":0.26400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:17:01 2023","interfaces":[{"interface":"back","average":{"1min":5.3940000000000001,"5min":3.9529999999999998,"15min":4.0259999999999998},"min":{"1min":0.33100000000000002,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":61.997,"5min":67.760000000000005,"15min":105.842},"last":0.66400000000000003},{"interface":"front","average":{"1min":5.3700000000000001,"5min":4.2329999999999997,"15min":4.4809999999999999},"min":{"1min":0.27600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":61.978999999999999,"5min":67.488,"15min":105.66},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":5.0609999999999999,"5min":5.266,"15min":4.8499999999999996},"min":{"1min":0.46300000000000002,"5min":0.29999999999999999,"15min":0.248},"max":{"1min":43.747,"5min":185.17099999999999,"15min":185.17099999999999},"last":0.68300000000000005},{"interface":"front","average":{"1min":5.335,"5min":5.4279999999999999,"15min":4.843},"min":{"1min":0.40600000000000003,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":43.731000000000002,"5min":184.99100000000001,"15min":184.99100000000001},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":0.96699999999999997,"15min":0.76300000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.22500000000000001},"max":{"1min":7.6440000000000001,"5min":7.6440000000000001,"15min":19.215},"last":0.31},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.151,"15min":0.96499999999999997},"min":{"1min":0.42699999999999999,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":3.9809999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":0.60999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.595,"15min":1.1990000000000001},"min":{"1min":0.51100000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":8.4440000000000008,"5min":14.069000000000001,"15min":35.488},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.367,"5min":1.284,"15min":1.0389999999999999},"min":{"1min":0.41699999999999998,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":7.6109999999999998,"5min":13.43,"15min":34.896000000000001},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4830000000000001,"15min":1.0649999999999999},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":6.843,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.436},{"interface":"front","average":{"1min":1.5580000000000001,"5min":1.143,"15min":0.81599999999999995},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.8010000000000002,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.47599999999999998}]}]},{"osd":6,"up_from":0,"seq":1078036791472,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762630,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926167,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1007680,"kb_used_data":182648,"kb_used_omap":8,"kb_used_meta":825015,"kb_avail":92737472,"statfs":{"total":95995035648,"available":94963171328,"internally_reserved":0,"allocated":187031552,"data_stored":184808898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":844815969},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":1.952,"15min":3.4940000000000002},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":29.893000000000001,"5min":72.534000000000006,"15min":94.055999999999997},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.97,"5min":2.4169999999999998,"15min":3.7130000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.183},"max":{"1min":23.436,"5min":72.667000000000002,"15min":91.632999999999996},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:16:17 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":2.863,"15min":4.944},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.1820000000000004,"5min":160.542,"15min":270.51100000000002},"last":0.317},{"interface":"front","average":{"1min":2.0219999999999998,"5min":3.5009999999999999,"15min":4.8970000000000002},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":18.433,"5min":160.75999999999999,"15min":269.69299999999998},"last":0.27600000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4279999999999999,"15min":3.8980000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":8.9469999999999992,"5min":10.082000000000001,"15min":126.95399999999999},"last":44.405000000000001},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.532,"15min":4.1020000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":7.375,"5min":9.5440000000000005,"15min":127.294},"last":43.909999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.726,"15min":2.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":5.859,"5min":9.6959999999999997,"15min":148.20599999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":5.9790000000000001,"5min":2.29,"15min":2.2589999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":105.282,"5min":105.282,"15min":148.11000000000001},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.9630000000000001,"15min":1.859},"min":{"1min":0.378,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.5430000000000001,"5min":52.061,"15min":64.725999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.4649999999999999,"15min":1.8320000000000001},"min":{"1min":0.41699999999999998,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1369999999999996,"5min":51.741,"15min":51.741},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":1.5309999999999999,"15min":1.653},"min":{"1min":0.38,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":7.1950000000000003,"5min":7.1950000000000003,"15min":15.744999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.3400000000000001,"15min":1.6839999999999999},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.309},"max":{"1min":7.2229999999999999,"5min":8.3919999999999995,"15min":13.766999999999999},"last":0.51200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.569,"15min":1.75},"min":{"1min":0.53700000000000003,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.9980000000000002,"5min":12.981,"15min":65.081999999999994},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.6909999999999998,"5min":2.161,"15min":1.8440000000000001},"min":{"1min":0.49199999999999999,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.6600000000000001,"5min":17.151,"15min":17.151},"last":0.48999999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599105,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":970040,"kb_used_data":126960,"kb_used_omap":12,"kb_used_meta":843059,"kb_avail":92775112,"statfs":{"total":95995035648,"available":95001714688,"internally_reserved":0,"allocated":130007040,"data_stored":127729972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":863293438},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.1880000000000002,"15min":2.464},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":5.3879999999999999,"5min":65.382999999999996,"15min":76.251999999999995},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.282,"15min":2.718},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.265000000000001,"5min":65.427000000000007,"15min":76.590000000000003},"last":0.30499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":3.0899999999999999,"15min":3.5649999999999999},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":7.5270000000000001,"5min":70.453000000000003,"15min":94.768000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.8969999999999998,"15min":3.2450000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.218},"max":{"1min":7.2009999999999996,"5min":69.825000000000003,"15min":96.808000000000007},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:16:43 2023","interfaces":[{"interface":"back","average":{"1min":13.59,"5min":5.3810000000000002,"15min":4.7519999999999998},"min":{"1min":0.27800000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":126.345,"5min":132.82499999999999,"15min":132.82499999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":13.555,"5min":5.4829999999999997,"15min":3.9689999999999999},"min":{"1min":0.39000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":126.40300000000001,"5min":130.97300000000001,"15min":130.97300000000001},"last":0.27700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.4399999999999999,"15min":1.79},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":9.3399999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.667,"5min":1.7649999999999999,"15min":2.105},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.641,"5min":13.387,"15min":76.161000000000001},"last":0.63500000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":3.3460000000000001,"15min":2.8519999999999999},"min":{"1min":0.34999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.3859999999999992,"5min":135.202,"15min":172.505},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.96,"5min":2.0619999999999998,"15min":2.6749999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.1260000000000003,"5min":29.196999999999999,"15min":172.375},"last":0.68899999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.577,"15min":1.5840000000000001},"min":{"1min":0.41399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.8310000000000004,"5min":14.664,"15min":14.664},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.298,"5min":1.627,"15min":1.7949999999999999},"min":{"1min":0.34899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.5659999999999998,"5min":12.025,"15min":14.279999999999999},"last":0.61199999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":2.0990000000000002,"15min":1.883},"min":{"1min":0.32500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2970000000000006,"5min":13.481999999999999,"15min":13.481999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":2.1720000000000002,"5min":1.6399999999999999,"15min":1.581},"min":{"1min":0.443,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.266,"5min":14.698,"15min":14.698},"last":0.64700000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435579,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1147220,"kb_used_data":246476,"kb_used_omap":4,"kb_used_meta":900731,"kb_avail":92597932,"statfs":{"total":95995035648,"available":94820282368,"internally_reserved":0,"allocated":252391424,"data_stored":249853349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":922348653},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":3.5459999999999998,"5min":2.355,"15min":3.1880000000000002},"min":{"1min":0.19600000000000001,"5min":0.186,"15min":0.182},"max":{"1min":24.776,"5min":76.632000000000005,"15min":76.632000000000005},"last":0.254},{"interface":"front","average":{"1min":3.8809999999999998,"5min":2.8530000000000002,"15min":3.4969999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":24.582000000000001,"5min":81.429000000000002,"15min":81.429000000000002},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":3.4340000000000002,"15min":4.3600000000000003},"min":{"1min":0.217,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":3.7050000000000001,"5min":183.87,"15min":183.87},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.3149999999999999,"5min":3.52,"15min":4.7229999999999999},"min":{"1min":0.251,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":6.3869999999999996,"5min":185.84899999999999,"15min":185.84899999999999},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.0409999999999999,"15min":3.9140000000000001},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":8.0960000000000001,"5min":14.9,"15min":163.83500000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.5389999999999999,"15min":3.7629999999999999},"min":{"1min":0.252,"5min":0.25,"15min":0.25},"max":{"1min":4.9889999999999999,"5min":16.550999999999998,"15min":163.92099999999999},"last":0.50800000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.1639999999999999,"15min":2.77},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":5.016,"5min":7.4960000000000004,"15min":109.66800000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.2909999999999999,"5min":1.8879999999999999,"15min":3.2250000000000001},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":5.6779999999999999,"5min":11.815,"15min":110.295},"last":0.54000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8889999999999998,"5min":3.9740000000000002,"15min":3.9119999999999999},"min":{"1min":0.39000000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.742000000000001,"5min":200.53200000000001,"15min":200.53200000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.7200000000000002,"5min":4.1429999999999998,"15min":3.903},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.26900000000000002},"max":{"1min":23.094999999999999,"5min":200.846,"15min":200.846},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.6539999999999999,"15min":2.8660000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.32300000000000001},"max":{"1min":8.9610000000000003,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.641,"5min":1.2669999999999999,"15min":2.9020000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":5.8959999999999999,"5min":8.2629999999999999,"15min":82.233000000000004},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":3.0510000000000002,"15min":3.234},"min":{"1min":0.376,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":3.3719999999999999,"5min":139.465,"15min":139.465},"last":0.59199999999999997},{"interface":"front","average":{"1min":2.0299999999999998,"5min":3.073,"15min":3.3069999999999999},"min":{"1min":0.44700000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":5.4459999999999997,"5min":141.56200000000001,"15min":141.56200000000001},"last":0.49299999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019341,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":538296,"kb_used_data":83504,"kb_used_omap":0,"kb_used_meta":454784,"kb_avail":93206856,"statfs":{"total":95995035648,"available":95443820544,"internally_reserved":0,"allocated":85508096,"data_stored":84051105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":465698816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.4650000000000001,"15min":1.639},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.2549999999999999,"5min":14.063000000000001,"15min":56.061999999999998},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.6519999999999999,"5min":1.4630000000000001,"15min":1.619},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":6.9740000000000002,"5min":14.048999999999999,"15min":56.075000000000003},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":3.7559999999999998,"15min":5.7779999999999996},"min":{"1min":0.28499999999999998,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":14.763,"5min":138.47999999999999,"15min":161.613},"last":0.372},{"interface":"front","average":{"1min":2.1789999999999998,"5min":3.7029999999999998,"15min":5.7830000000000004},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.217},"max":{"1min":13.368,"5min":138.429,"15min":162.017},"last":0.48399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.8969999999999998,"15min":5.3760000000000003},"min":{"1min":0.28699999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.6920000000000002,"5min":144.916,"15min":182.96700000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.0299999999999998,"5min":3.8300000000000001,"15min":5.3369999999999997},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.23100000000000001},"max":{"1min":9.4550000000000001,"5min":144.83099999999999,"15min":183.244},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.3159999999999998,"15min":4.6589999999999998},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":4.5819999999999999,"5min":100.361,"15min":184.37100000000001},"last":0.503},{"interface":"front","average":{"1min":1.738,"5min":3.4990000000000001,"15min":4.7199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":4.7270000000000003,"5min":100.20699999999999,"15min":184.22300000000001},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.111,"15min":1.252},"min":{"1min":0.28199999999999997,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.0739999999999998,"5min":12.257999999999999,"15min":176.81899999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.7689999999999999,"5min":1.111,"15min":1.2849999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.246},"max":{"1min":7.9470000000000001,"5min":7.9470000000000001,"15min":176.172},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:16:53 2023","interfaces":[{"interface":"back","average":{"1min":6.3659999999999997,"5min":2.0259999999999998,"15min":1.018},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":114.232,"5min":114.232,"15min":114.232},"last":0.29099999999999998},{"interface":"front","average":{"1min":6.4320000000000004,"5min":2.29,"15min":1.1919999999999999},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":114.07299999999999,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.32500000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.196,"15min":1.5349999999999999},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.5090000000000003,"5min":7.71,"15min":76.507000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.1040000000000001,"5min":1.4099999999999999,"15min":1.6739999999999999},"min":{"1min":0.39900000000000002,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":7.8250000000000002,"5min":7.8250000000000002,"15min":76.462000000000003},"last":0.49399999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757705,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":786160,"kb_used_data":177640,"kb_used_omap":0,"kb_used_meta":608512,"kb_avail":92958992,"statfs":{"total":95995035648,"available":95190007808,"internally_reserved":0,"allocated":181903360,"data_stored":180450810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":623116288},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":7.7800000000000002,"5min":2.794,"15min":2.214},"min":{"1min":0.23000000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":126.657,"5min":126.657,"15min":126.657},"last":0.317},{"interface":"front","average":{"1min":7.2990000000000004,"5min":2.6539999999999999,"15min":2.4289999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":126.697,"5min":126.697,"15min":126.697},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":5.3200000000000003,"15min":5.351},"min":{"1min":0.25800000000000001,"5min":0.22,"15min":0.22},"max":{"1min":6.7279999999999998,"5min":150.16999999999999,"15min":171.68299999999999},"last":0.44},{"interface":"front","average":{"1min":0.91700000000000004,"5min":4.8120000000000003,"15min":5.3470000000000004},"min":{"1min":0.27200000000000002,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":3.782,"5min":150.12700000000001,"15min":171.64099999999999},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":3.5649999999999999,"15min":4.2679999999999998},"min":{"1min":0.317,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2569999999999997,"5min":120.297,"15min":240.57300000000001},"last":0.373},{"interface":"front","average":{"1min":1.47,"5min":3.661,"15min":4.335},"min":{"1min":0.38,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.2859999999999996,"5min":120.325,"15min":240.464},"last":0.495}]},{"osd":3,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":15.287000000000001,"5min":9.1349999999999998,"15min":6.4210000000000003},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":130.46899999999999,"5min":195.84700000000001,"15min":201.50899999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":15.641999999999999,"5min":9.0640000000000001,"15min":6.1699999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":130.63300000000001,"5min":195.87899999999999,"15min":201.58099999999999},"last":0.57699999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.0629999999999999,"15min":1.3879999999999999},"min":{"1min":0.28499999999999998,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":4.8159999999999998,"5min":8,"15min":191.78899999999999},"last":0.505},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.67400000000000004,"15min":1.2150000000000001},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":3.371,"5min":3.371,"15min":191.821},"last":0.48099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.2030000000000001,"15min":0.84199999999999997},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":3.0209999999999999,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.72099999999999997,"15min":0.60399999999999998},"min":{"1min":0.35099999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":2.9830000000000001,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":0.53700000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.4079999999999999,"15min":1.478},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":11.725,"5min":11.725,"15min":191.84299999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.02,"5min":1.3,"15min":1.4990000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":11.041,"5min":11.041,"15min":191.69},"last":0.58699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:17:09.083 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:17:09.083 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:17:09.083 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:17:09.083 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:17:09.096 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:09.156 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:09.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:09.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:09.177 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:09.211 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 2 16 6486 6470 202.084 178 0.0040021 0.00493092 2023-09-23T20:17:09.260 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:09.344 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:09.505 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:17:09.617 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:09.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:09.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:09.730 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:09.767 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:09.826 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:09.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:09.933 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:09.977 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:10.051 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:10.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:10.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:10.185 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:10.187 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:17:10.213 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 3 16 8906 8890 185.12 151.25 0.00732578 0.00539206 2023-09-23T20:17:10.231 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:10.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:10 smithi144 ceph-mon[194833]: pgmap v1342: 109 pgs: 109 active+clean; 144 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 17 MiB/s wr, 277 op/s 2023-09-23T20:17:10.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4118793464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:10.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:10 smithi135 ceph-mon[221113]: pgmap v1342: 109 pgs: 109 active+clean; 144 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 17 MiB/s wr, 277 op/s 2023-09-23T20:17:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4118793464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:10 smithi135 ceph-mon[213871]: pgmap v1342: 109 pgs: 109 active+clean; 144 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 17 MiB/s wr, 277 op/s 2023-09-23T20:17:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4118793464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:11.212 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 4 16 11916 11900 185.867 188.125 0.00366614 0.00537305 2023-09-23T20:17:11.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[213871]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[213871]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[213871]: osdmap e484: 8 total, 8 up, 6 in 2023-09-23T20:17:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[213871]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[213871]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[213871]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:11.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[221113]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[221113]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:11.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[221113]: osdmap e484: 8 total, 8 up, 6 in 2023-09-23T20:17:11.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[221113]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[221113]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:11 smithi135 ceph-mon[221113]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:11 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:11 smithi144 ceph-mon[194833]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:11 smithi144 ceph-mon[194833]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:11 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:11 smithi144 ceph-mon[194833]: osdmap e484: 8 total, 8 up, 6 in 2023-09-23T20:17:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:11 smithi144 ceph-mon[194833]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:11 smithi144 ceph-mon[194833]: failed to encode map e484 with expected crc 2023-09-23T20:17:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:11 smithi144 ceph-mon[194833]: failed to encode map e484 with expected crc 2023-09-23T20:17:12.170 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:12.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:12.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:12.212 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 5 16 14895 14879 185.928 186.188 0.00579158 0.00537259 2023-09-23T20:17:12.277 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:12.320 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:12 smithi135 ceph-mon[213871]: pgmap v1344: 109 pgs: 109 active+clean; 144 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 17 MiB/s wr, 278 op/s 2023-09-23T20:17:12.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:12 smithi135 ceph-mon[221113]: pgmap v1344: 109 pgs: 109 active+clean; 144 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 17 MiB/s wr, 278 op/s 2023-09-23T20:17:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:12 smithi144 ceph-mon[194833]: pgmap v1344: 109 pgs: 109 active+clean; 144 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 17 MiB/s wr, 278 op/s 2023-09-23T20:17:13.006 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:13.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:13.115 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:13.182 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:13.212 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 6 16 17146 17130 178.377 140.688 0.0104995 0.00559798 2023-09-23T20:17:13.257 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:13.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:13.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:13.364 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:13.393 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:13.405 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:13.441 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:13.472 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:13.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:13.559 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:17:13.579 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:13.603 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:17:13.646 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:17:13.658 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:13.687 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:17:13.729 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:17:13.763 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:17:13.803 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:17:13.811 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:13.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:13.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:13.838 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:17:13.843 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:17:13.843 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:17:13.844 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:17:13.844 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:17:13.844 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:17:13.844 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:17:13.844 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:17:13.844 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:17:13.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:13 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:13.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:13 smithi144 ceph-mon[194833]: pgmap v1345: 109 pgs: 109 active+clean; 535 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 62 MiB/s wr, 986 op/s 2023-09-23T20:17:13.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2346766029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:13.921 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:13.929 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:17:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:17:13] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.43.0" 2023-09-23T20:17:13.930 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:13 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:13.930 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:13 smithi135 ceph-mon[213871]: pgmap v1345: 109 pgs: 109 active+clean; 535 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 62 MiB/s wr, 986 op/s 2023-09-23T20:17:13.930 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2346766029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:13.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:13 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:13.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:13 smithi135 ceph-mon[221113]: pgmap v1345: 109 pgs: 109 active+clean; 535 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 62 MiB/s wr, 986 op/s 2023-09-23T20:17:13.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2346766029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:13.957 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:14.031 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:14.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:14.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:14.140 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:14.206 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:14.212 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 7 16 19269 19253 171.85 132.688 0.0105912 0.0058128 2023-09-23T20:17:15.095 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:17:15.213 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 8 16 21791 21775 170.061 157.625 0.00570479 0.00587368 2023-09-23T20:17:15.734 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:15.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:15.843 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:15.907 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:16.213 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 9 16 24061 24045 166.928 141.875 0.00757861 0.00598457 2023-09-23T20:17:16.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:16 smithi135 ceph-mon[221113]: pgmap v1346: 109 pgs: 109 active+clean; 947 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T20:17:16.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:16 smithi135 ceph-mon[213871]: pgmap v1346: 109 pgs: 109 active+clean; 947 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T20:17:16.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:16 smithi144 ceph-mon[194833]: pgmap v1346: 109 pgs: 109 active+clean; 947 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T20:17:17.212 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:17.215 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 10 16 25340 25324 158.206 79.9375 0.0093737 0.00631528 2023-09-23T20:17:17.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:17.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:17.319 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:17.369 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:17.455 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:17.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:17.560 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:17.626 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:17.706 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:17.828 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:17.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:17.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:17.938 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:18.005 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:18.207 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:18.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:18.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:18.215 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 11 16 27076 27060 153.688 108.5 0.0108404 0.00649937 2023-09-23T20:17:18.260 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:17:18.261 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:17:18.314 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:18.371 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:18.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:18 smithi135 ceph-mon[213871]: pgmap v1347: 109 pgs: 109 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T20:17:18.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:18 smithi135 ceph-mon[221113]: pgmap v1347: 109 pgs: 109 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T20:17:18.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:18 smithi144 ceph-mon[194833]: pgmap v1347: 109 pgs: 109 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T20:17:18.667 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:18.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:18.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:18.772 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:18.817 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:18.885 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:18.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:18.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:18.919 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:17:18.932 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:17:18.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:19.058 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:19.215 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 12 16 28724 28708 149.464 103 0.0025571 0.006617 2023-09-23T20:17:19.563 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:17:20.141 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:20.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:20.216 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 13 16 29768 29752 142.979 65.25 0.00557025 0.00698892 2023-09-23T20:17:20.248 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:20.274 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:20.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:20 smithi135 ceph-mon[213871]: from='client.56308 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:17:20.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:20 smithi135 ceph-mon[213871]: pgmap v1348: 109 pgs: 109 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-09-23T20:17:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:20 smithi135 ceph-mon[221113]: from='client.56308 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:17:20.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:20 smithi135 ceph-mon[221113]: pgmap v1348: 109 pgs: 109 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-09-23T20:17:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:20 smithi144 ceph-mon[194833]: from='client.56308 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:17:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:20 smithi144 ceph-mon[194833]: pgmap v1348: 109 pgs: 109 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-09-23T20:17:20.682 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:20.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:20.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:20.791 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:20.822 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:20.863 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:20.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:20.967 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:21.021 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:21.217 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 14 16 31477 31461 140.385 106.812 0.00590734 0.00711818 2023-09-23T20:17:21.304 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:17:21.310 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1347,"stamp":"2023-09-23T20:17:17.100426+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1336088499,"num_objects":20342,"num_object_clones":0,"num_object_copies":61026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20342,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":21221,"num_write_kb":1312821,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":254,"num_bytes_recovered":8399566,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":24729,"ondisk_log_size":24729,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":333,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8413728,"kb_used_data":4126832,"kb_used_omap":25,"kb_used_meta":4286822,"kb_avail":554057184,"statfs":{"total":575970213888,"available":567354556416,"internally_reserved":0,"allocated":4225875968,"data_stored":4214443106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25908,"internal_metadata":4389706444},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1331298304,"num_objects":20314,"num_object_clones":0,"num_object_copies":60942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20314,"num_write_kb":1300096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003526"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":55,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:06.158173+0000","last_change":"2023-09-23T20:14:34.849507+0000","last_active":"2023-09-23T20:17:06.158173+0000","last_peered":"2023-09-23T20:17:06.158173+0000","last_clean":"2023-09-23T20:17:06.158173+0000","last_became_active":"2023-09-23T20:14:34.849169+0000","last_became_peered":"2023-09-23T20:14:34.849169+0000","last_unstale":"2023-09-23T20:17:06.158173+0000","last_undegraded":"2023-09-23T20:17:06.158173+0000","last_fullsized":"2023-09-23T20:17:06.158173+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.837751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":52,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497112+0000","last_change":"2023-09-23T20:14:34.846170+0000","last_active":"2023-09-23T20:17:04.497112+0000","last_peered":"2023-09-23T20:17:04.497112+0000","last_clean":"2023-09-23T20:17:04.497112+0000","last_became_active":"2023-09-23T20:14:34.845552+0000","last_became_peered":"2023-09-23T20:14:34.845552+0000","last_unstale":"2023-09-23T20:17:04.497112+0000","last_undegraded":"2023-09-23T20:17:04.497112+0000","last_fullsized":"2023-09-23T20:17:04.497112+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.721265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":230,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815107+0000","last_change":"2023-09-23T20:14:34.859703+0000","last_active":"2023-09-23T20:17:04.815107+0000","last_peered":"2023-09-23T20:17:04.815107+0000","last_clean":"2023-09-23T20:17:04.815107+0000","last_became_active":"2023-09-23T20:14:34.858511+0000","last_became_peered":"2023-09-23T20:14:34.858511+0000","last_unstale":"2023-09-23T20:17:04.815107+0000","last_undegraded":"2023-09-23T20:17:04.815107+0000","last_fullsized":"2023-09-23T20:17:04.815107+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:46.903239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":192,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816880+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:17:04.816880+0000","last_peered":"2023-09-23T20:17:04.816880+0000","last_clean":"2023-09-23T20:17:04.816880+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:17:04.816880+0000","last_undegraded":"2023-09-23T20:17:04.816880+0000","last_fullsized":"2023-09-23T20:17:04.816880+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":54,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814430+0000","last_change":"2023-09-23T20:14:34.850088+0000","last_active":"2023-09-23T20:17:04.814430+0000","last_peered":"2023-09-23T20:17:04.814430+0000","last_clean":"2023-09-23T20:17:04.814430+0000","last_became_active":"2023-09-23T20:14:34.849729+0000","last_became_peered":"2023-09-23T20:14:34.849729+0000","last_unstale":"2023-09-23T20:17:04.814430+0000","last_undegraded":"2023-09-23T20:17:04.814430+0000","last_fullsized":"2023-09-23T20:17:04.814430+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:59.401259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":439,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497480+0000","last_change":"2023-09-23T20:14:34.853468+0000","last_active":"2023-09-23T20:17:04.497480+0000","last_peered":"2023-09-23T20:17:04.497480+0000","last_clean":"2023-09-23T20:17:04.497480+0000","last_became_active":"2023-09-23T20:14:34.853080+0000","last_became_peered":"2023-09-23T20:14:34.853080+0000","last_unstale":"2023-09-23T20:17:04.497480+0000","last_undegraded":"2023-09-23T20:17:04.497480+0000","last_fullsized":"2023-09-23T20:17:04.497480+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.412270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":410,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497409+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:17:04.497409+0000","last_peered":"2023-09-23T20:17:04.497409+0000","last_clean":"2023-09-23T20:17:04.497409+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:17:04.497409+0000","last_undegraded":"2023-09-23T20:17:04.497409+0000","last_fullsized":"2023-09-23T20:17:04.497409+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":338,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815997+0000","last_change":"2023-09-23T20:14:34.848572+0000","last_active":"2023-09-23T20:17:04.815997+0000","last_peered":"2023-09-23T20:17:04.815997+0000","last_clean":"2023-09-23T20:17:04.815997+0000","last_became_active":"2023-09-23T20:14:34.848166+0000","last_became_peered":"2023-09-23T20:14:34.848166+0000","last_unstale":"2023-09-23T20:17:04.815997+0000","last_undegraded":"2023-09-23T20:17:04.815997+0000","last_fullsized":"2023-09-23T20:17:04.815997+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:17.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":817,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814729+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:17:04.814729+0000","last_peered":"2023-09-23T20:17:04.814729+0000","last_clean":"2023-09-23T20:17:04.814729+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:17:04.814729+0000","last_undegraded":"2023-09-23T20:17:04.814729+0000","last_fullsized":"2023-09-23T20:17:04.814729+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":493,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497272+0000","last_change":"2023-09-23T20:14:34.857408+0000","last_active":"2023-09-23T20:17:04.497272+0000","last_peered":"2023-09-23T20:17:04.497272+0000","last_clean":"2023-09-23T20:17:04.497272+0000","last_became_active":"2023-09-23T20:14:34.857097+0000","last_became_peered":"2023-09-23T20:14:34.857097+0000","last_unstale":"2023-09-23T20:17:04.497272+0000","last_undegraded":"2023-09-23T20:17:04.497272+0000","last_fullsized":"2023-09-23T20:17:04.497272+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:30.583895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":348,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815028+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:17:04.815028+0000","last_peered":"2023-09-23T20:17:04.815028+0000","last_clean":"2023-09-23T20:17:04.815028+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:17:04.815028+0000","last_undegraded":"2023-09-23T20:17:04.815028+0000","last_fullsized":"2023-09-23T20:17:04.815028+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":177,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.817514+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:17:04.817514+0000","last_peered":"2023-09-23T20:17:04.817514+0000","last_clean":"2023-09-23T20:17:04.817514+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:17:04.817514+0000","last_undegraded":"2023-09-23T20:17:04.817514+0000","last_fullsized":"2023-09-23T20:17:04.817514+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":183,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815042+0000","last_change":"2023-09-23T20:14:34.847709+0000","last_active":"2023-09-23T20:17:04.815042+0000","last_peered":"2023-09-23T20:17:04.815042+0000","last_clean":"2023-09-23T20:17:04.815042+0000","last_became_active":"2023-09-23T20:14:34.847240+0000","last_became_peered":"2023-09-23T20:14:34.847240+0000","last_unstale":"2023-09-23T20:17:04.815042+0000","last_undegraded":"2023-09-23T20:17:04.815042+0000","last_fullsized":"2023-09-23T20:17:04.815042+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:32.878639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":430,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.817879+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:17:04.817879+0000","last_peered":"2023-09-23T20:17:04.817879+0000","last_clean":"2023-09-23T20:17:04.817879+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:17:04.817879+0000","last_undegraded":"2023-09-23T20:17:04.817879+0000","last_fullsized":"2023-09-23T20:17:04.817879+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":361,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496978+0000","last_change":"2023-09-23T20:14:34.854691+0000","last_active":"2023-09-23T20:17:04.496978+0000","last_peered":"2023-09-23T20:17:04.496978+0000","last_clean":"2023-09-23T20:17:04.496978+0000","last_became_active":"2023-09-23T20:14:34.854368+0000","last_became_peered":"2023-09-23T20:14:34.854368+0000","last_unstale":"2023-09-23T20:17:04.496978+0000","last_undegraded":"2023-09-23T20:17:04.496978+0000","last_fullsized":"2023-09-23T20:17:04.496978+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:38.534533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":843,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496522+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:17:04.496522+0000","last_peered":"2023-09-23T20:17:04.496522+0000","last_clean":"2023-09-23T20:17:04.496522+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:17:04.496522+0000","last_undegraded":"2023-09-23T20:17:04.496522+0000","last_fullsized":"2023-09-23T20:17:04.496522+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":177,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814923+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:17:04.814923+0000","last_peered":"2023-09-23T20:17:04.814923+0000","last_clean":"2023-09-23T20:17:04.814923+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:17:04.814923+0000","last_undegraded":"2023-09-23T20:17:04.814923+0000","last_fullsized":"2023-09-23T20:17:04.814923+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":230,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.818017+0000","last_change":"2023-09-23T20:14:34.853858+0000","last_active":"2023-09-23T20:17:04.818017+0000","last_peered":"2023-09-23T20:17:04.818017+0000","last_clean":"2023-09-23T20:17:04.818017+0000","last_became_active":"2023-09-23T20:14:34.849825+0000","last_became_peered":"2023-09-23T20:14:34.849825+0000","last_unstale":"2023-09-23T20:17:04.818017+0000","last_undegraded":"2023-09-23T20:17:04.818017+0000","last_fullsized":"2023-09-23T20:17:04.818017+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.072721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":532,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496674+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:17:04.496674+0000","last_peered":"2023-09-23T20:17:04.496674+0000","last_clean":"2023-09-23T20:17:04.496674+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:17:04.496674+0000","last_undegraded":"2023-09-23T20:17:04.496674+0000","last_fullsized":"2023-09-23T20:17:04.496674+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":192,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814305+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:17:04.814305+0000","last_peered":"2023-09-23T20:17:04.814305+0000","last_clean":"2023-09-23T20:17:04.814305+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:17:04.814305+0000","last_undegraded":"2023-09-23T20:17:04.814305+0000","last_fullsized":"2023-09-23T20:17:04.814305+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":482,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497308+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:17:04.497308+0000","last_peered":"2023-09-23T20:17:04.497308+0000","last_clean":"2023-09-23T20:17:04.497308+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:17:04.497308+0000","last_undegraded":"2023-09-23T20:17:04.497308+0000","last_fullsized":"2023-09-23T20:17:04.497308+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":606,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814978+0000","last_change":"2023-09-23T20:16:58.248265+0000","last_active":"2023-09-23T20:17:04.814978+0000","last_peered":"2023-09-23T20:17:04.814978+0000","last_clean":"2023-09-23T20:17:04.814978+0000","last_became_active":"2023-09-23T20:14:37.887421+0000","last_became_peered":"2023-09-23T20:14:37.887421+0000","last_unstale":"2023-09-23T20:17:04.814978+0000","last_undegraded":"2023-09-23T20:17:04.814978+0000","last_fullsized":"2023-09-23T20:17:04.814978+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:58.248167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.248167+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:54.488842+0000","scrub_duration":0.0017920810000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":171,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814576+0000","last_change":"2023-09-23T20:07:36.970142+0000","last_active":"2023-09-23T20:17:04.814576+0000","last_peered":"2023-09-23T20:17:04.814576+0000","last_clean":"2023-09-23T20:17:04.814576+0000","last_became_active":"2023-09-23T20:07:36.969797+0000","last_became_peered":"2023-09-23T20:07:36.969797+0000","last_unstale":"2023-09-23T20:17:04.814576+0000","last_undegraded":"2023-09-23T20:17:04.814576+0000","last_fullsized":"2023-09-23T20:17:04.814576+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:02.962099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":472,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814986+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:17:04.814986+0000","last_peered":"2023-09-23T20:17:04.814986+0000","last_clean":"2023-09-23T20:17:04.814986+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:17:04.814986+0000","last_undegraded":"2023-09-23T20:17:04.814986+0000","last_fullsized":"2023-09-23T20:17:04.814986+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":621,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496692+0000","last_change":"2023-09-23T20:14:37.893125+0000","last_active":"2023-09-23T20:17:04.496692+0000","last_peered":"2023-09-23T20:17:04.496692+0000","last_clean":"2023-09-23T20:17:04.496692+0000","last_became_active":"2023-09-23T20:14:37.892794+0000","last_became_peered":"2023-09-23T20:14:37.892794+0000","last_unstale":"2023-09-23T20:17:04.496692+0000","last_undegraded":"2023-09-23T20:17:04.496692+0000","last_fullsized":"2023-09-23T20:17:04.496692+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:36.804206+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":127,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496569+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:17:04.496569+0000","last_peered":"2023-09-23T20:17:04.496569+0000","last_clean":"2023-09-23T20:17:04.496569+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:17:04.496569+0000","last_undegraded":"2023-09-23T20:17:04.496569+0000","last_fullsized":"2023-09-23T20:17:04.496569+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":125,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816208+0000","last_change":"2023-09-23T20:14:34.865939+0000","last_active":"2023-09-23T20:17:04.816208+0000","last_peered":"2023-09-23T20:17:04.816208+0000","last_clean":"2023-09-23T20:17:04.816208+0000","last_became_active":"2023-09-23T20:14:34.865542+0000","last_became_peered":"2023-09-23T20:14:34.865542+0000","last_unstale":"2023-09-23T20:17:04.816208+0000","last_undegraded":"2023-09-23T20:17:04.816208+0000","last_fullsized":"2023-09-23T20:17:04.816208+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:40.857547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"484'1130","reported_seq":1143,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.524945+0000","last_change":"2023-09-23T20:16:59.490329+0000","last_active":"2023-09-23T20:17:13.524945+0000","last_peered":"2023-09-23T20:17:13.524945+0000","last_clean":"2023-09-23T20:17:13.524945+0000","last_became_active":"2023-09-23T20:16:59.490069+0000","last_became_peered":"2023-09-23T20:16:59.490069+0000","last_unstale":"2023-09-23T20:17:13.524945+0000","last_undegraded":"2023-09-23T20:17:13.524945+0000","last_fullsized":"2023-09-23T20:17:13.524945+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1130,"log_dups_size":0,"ondisk_log_size":1130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.487350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74055680,"num_objects":1130,"num_object_clones":0,"num_object_copies":3390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1130,"num_write_kb":72320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":744,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.817118+0000","last_change":"2023-09-23T20:14:37.960856+0000","last_active":"2023-09-23T20:17:04.817118+0000","last_peered":"2023-09-23T20:17:04.817118+0000","last_clean":"2023-09-23T20:17:04.817118+0000","last_became_active":"2023-09-23T20:14:37.960470+0000","last_became_peered":"2023-09-23T20:14:37.960470+0000","last_unstale":"2023-09-23T20:17:04.817118+0000","last_undegraded":"2023-09-23T20:17:04.817118+0000","last_fullsized":"2023-09-23T20:17:04.817118+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:00.337491+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":175,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816413+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:17:04.816413+0000","last_peered":"2023-09-23T20:17:04.816413+0000","last_clean":"2023-09-23T20:17:04.816413+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:17:04.816413+0000","last_undegraded":"2023-09-23T20:17:04.816413+0000","last_fullsized":"2023-09-23T20:17:04.816413+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":492,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496580+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:17:04.496580+0000","last_peered":"2023-09-23T20:17:04.496580+0000","last_clean":"2023-09-23T20:17:04.496580+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:17:04.496580+0000","last_undegraded":"2023-09-23T20:17:04.496580+0000","last_fullsized":"2023-09-23T20:17:04.496580+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"484'1560","reported_seq":1574,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:16.590710+0000","last_change":"2023-09-23T20:16:59.487459+0000","last_active":"2023-09-23T20:17:16.590710+0000","last_peered":"2023-09-23T20:17:16.590710+0000","last_clean":"2023-09-23T20:17:16.590710+0000","last_became_active":"2023-09-23T20:16:59.487181+0000","last_became_peered":"2023-09-23T20:16:59.487181+0000","last_unstale":"2023-09-23T20:17:16.590710+0000","last_undegraded":"2023-09-23T20:17:16.590710+0000","last_fullsized":"2023-09-23T20:17:16.590710+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1560,"log_dups_size":0,"ondisk_log_size":1560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:34.406760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102236160,"num_objects":1560,"num_object_clones":0,"num_object_copies":4680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1560,"num_write_kb":99840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":832,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815682+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:17:04.815682+0000","last_peered":"2023-09-23T20:17:04.815682+0000","last_clean":"2023-09-23T20:17:04.815682+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:17:04.815682+0000","last_undegraded":"2023-09-23T20:17:04.815682+0000","last_fullsized":"2023-09-23T20:17:04.815682+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":190,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816974+0000","last_change":"2023-09-23T20:14:34.874396+0000","last_active":"2023-09-23T20:17:04.816974+0000","last_peered":"2023-09-23T20:17:04.816974+0000","last_clean":"2023-09-23T20:17:04.816974+0000","last_became_active":"2023-09-23T20:14:34.867853+0000","last_became_peered":"2023-09-23T20:14:34.867853+0000","last_unstale":"2023-09-23T20:17:04.816974+0000","last_undegraded":"2023-09-23T20:17:04.816974+0000","last_fullsized":"2023-09-23T20:17:04.816974+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:08.124003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":485,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814752+0000","last_change":"2023-09-23T20:14:34.849117+0000","last_active":"2023-09-23T20:17:04.814752+0000","last_peered":"2023-09-23T20:17:04.814752+0000","last_clean":"2023-09-23T20:17:04.814752+0000","last_became_active":"2023-09-23T20:14:34.848809+0000","last_became_peered":"2023-09-23T20:14:34.848809+0000","last_unstale":"2023-09-23T20:17:04.814752+0000","last_undegraded":"2023-09-23T20:17:04.814752+0000","last_fullsized":"2023-09-23T20:17:04.814752+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:18.863363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"484'1062","reported_seq":1076,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.167497+0000","last_change":"2023-09-23T20:16:59.491310+0000","last_active":"2023-09-23T20:17:13.167497+0000","last_peered":"2023-09-23T20:17:13.167497+0000","last_clean":"2023-09-23T20:17:13.167497+0000","last_became_active":"2023-09-23T20:16:59.491016+0000","last_became_peered":"2023-09-23T20:16:59.491016+0000","last_unstale":"2023-09-23T20:17:13.167497+0000","last_undegraded":"2023-09-23T20:17:13.167497+0000","last_fullsized":"2023-09-23T20:17:13.167497+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1062,"log_dups_size":0,"ondisk_log_size":1062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:59.436079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69599232,"num_objects":1062,"num_object_clones":0,"num_object_copies":3186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1062,"num_write_kb":67968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":556,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497324+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:17:04.497324+0000","last_peered":"2023-09-23T20:17:04.497324+0000","last_clean":"2023-09-23T20:17:04.497324+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:17:04.497324+0000","last_undegraded":"2023-09-23T20:17:04.497324+0000","last_fullsized":"2023-09-23T20:17:04.497324+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":175,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.817342+0000","last_change":"2023-09-23T20:07:37.129321+0000","last_active":"2023-09-23T20:17:04.817342+0000","last_peered":"2023-09-23T20:17:04.817342+0000","last_clean":"2023-09-23T20:17:04.817342+0000","last_became_active":"2023-09-23T20:07:37.046328+0000","last_became_peered":"2023-09-23T20:07:37.046328+0000","last_unstale":"2023-09-23T20:17:04.817342+0000","last_undegraded":"2023-09-23T20:17:04.817342+0000","last_fullsized":"2023-09-23T20:17:04.817342+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:58.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814476+0000","last_change":"2023-09-23T20:14:34.868142+0000","last_active":"2023-09-23T20:17:04.814476+0000","last_peered":"2023-09-23T20:17:04.814476+0000","last_clean":"2023-09-23T20:17:04.814476+0000","last_became_active":"2023-09-23T20:14:34.867803+0000","last_became_peered":"2023-09-23T20:14:34.867803+0000","last_unstale":"2023-09-23T20:17:04.814476+0000","last_undegraded":"2023-09-23T20:17:04.814476+0000","last_fullsized":"2023-09-23T20:17:04.814476+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:29.475640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"484'1546","reported_seq":1560,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:16.150763+0000","last_change":"2023-09-23T20:16:59.488990+0000","last_active":"2023-09-23T20:17:16.150763+0000","last_peered":"2023-09-23T20:17:16.150763+0000","last_clean":"2023-09-23T20:17:16.150763+0000","last_became_active":"2023-09-23T20:16:59.488614+0000","last_became_peered":"2023-09-23T20:16:59.488614+0000","last_unstale":"2023-09-23T20:17:16.150763+0000","last_undegraded":"2023-09-23T20:17:16.150763+0000","last_fullsized":"2023-09-23T20:17:16.150763+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1546,"log_dups_size":0,"ondisk_log_size":1546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:11.359750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101318656,"num_objects":1546,"num_object_clones":0,"num_object_copies":4638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1546,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":687,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816671+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:17:04.816671+0000","last_peered":"2023-09-23T20:17:04.816671+0000","last_clean":"2023-09-23T20:17:04.816671+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:17:04.816671+0000","last_undegraded":"2023-09-23T20:17:04.816671+0000","last_fullsized":"2023-09-23T20:17:04.816671+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":532,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496758+0000","last_change":"2023-09-23T20:14:34.852725+0000","last_active":"2023-09-23T20:17:04.496758+0000","last_peered":"2023-09-23T20:17:04.496758+0000","last_clean":"2023-09-23T20:17:04.496758+0000","last_became_active":"2023-09-23T20:14:34.852124+0000","last_became_peered":"2023-09-23T20:14:34.852124+0000","last_unstale":"2023-09-23T20:17:04.496758+0000","last_undegraded":"2023-09-23T20:17:04.496758+0000","last_fullsized":"2023-09-23T20:17:04.496758+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:21.450599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":229,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.494958+0000","last_change":"2023-09-23T20:14:34.854054+0000","last_active":"2023-09-23T20:17:04.494958+0000","last_peered":"2023-09-23T20:17:04.494958+0000","last_clean":"2023-09-23T20:17:04.494958+0000","last_became_active":"2023-09-23T20:14:34.853586+0000","last_became_peered":"2023-09-23T20:14:34.853586+0000","last_unstale":"2023-09-23T20:17:04.494958+0000","last_undegraded":"2023-09-23T20:17:04.494958+0000","last_fullsized":"2023-09-23T20:17:04.494958+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:25.255011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"484'1091","reported_seq":1104,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.031730+0000","last_change":"2023-09-23T20:16:59.491852+0000","last_active":"2023-09-23T20:17:13.031730+0000","last_peered":"2023-09-23T20:17:13.031730+0000","last_clean":"2023-09-23T20:17:13.031730+0000","last_became_active":"2023-09-23T20:16:59.491582+0000","last_became_peered":"2023-09-23T20:16:59.491582+0000","last_unstale":"2023-09-23T20:17:13.031730+0000","last_undegraded":"2023-09-23T20:17:13.031730+0000","last_fullsized":"2023-09-23T20:17:13.031730+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1091,"log_dups_size":0,"ondisk_log_size":1091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:06.925753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71499776,"num_objects":1091,"num_object_clones":0,"num_object_copies":3273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1091,"num_write_kb":69824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":866,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814531+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:17:04.814531+0000","last_peered":"2023-09-23T20:17:04.814531+0000","last_clean":"2023-09-23T20:17:04.814531+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:17:04.814531+0000","last_undegraded":"2023-09-23T20:17:04.814531+0000","last_fullsized":"2023-09-23T20:17:04.814531+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":230,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815065+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:17:04.815065+0000","last_peered":"2023-09-23T20:17:04.815065+0000","last_clean":"2023-09-23T20:17:04.815065+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:17:04.815065+0000","last_undegraded":"2023-09-23T20:17:04.815065+0000","last_fullsized":"2023-09-23T20:17:04.815065+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":52,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815533+0000","last_change":"2023-09-23T20:14:34.856656+0000","last_active":"2023-09-23T20:17:04.815533+0000","last_peered":"2023-09-23T20:17:04.815533+0000","last_clean":"2023-09-23T20:17:04.815533+0000","last_became_active":"2023-09-23T20:14:34.856378+0000","last_became_peered":"2023-09-23T20:14:34.856378+0000","last_unstale":"2023-09-23T20:17:04.815533+0000","last_undegraded":"2023-09-23T20:17:04.815533+0000","last_fullsized":"2023-09-23T20:17:04.815533+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:28.763263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.3","version":"484'1503","reported_seq":1517,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:16.154243+0000","last_change":"2023-09-23T20:16:59.488642+0000","last_active":"2023-09-23T20:17:16.154243+0000","last_peered":"2023-09-23T20:17:16.154243+0000","last_clean":"2023-09-23T20:17:16.154243+0000","last_became_active":"2023-09-23T20:16:59.488330+0000","last_became_peered":"2023-09-23T20:16:59.488330+0000","last_unstale":"2023-09-23T20:17:16.154243+0000","last_undegraded":"2023-09-23T20:17:16.154243+0000","last_fullsized":"2023-09-23T20:17:16.154243+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1503,"log_dups_size":0,"ondisk_log_size":1503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:46.429096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98500608,"num_objects":1503,"num_object_clones":0,"num_object_copies":4509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1503,"num_write_kb":96192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":831,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496742+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:17:04.496742+0000","last_peered":"2023-09-23T20:17:04.496742+0000","last_clean":"2023-09-23T20:17:04.496742+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:17:04.496742+0000","last_undegraded":"2023-09-23T20:17:04.496742+0000","last_fullsized":"2023-09-23T20:17:04.496742+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":52,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496601+0000","last_change":"2023-09-23T20:14:34.848532+0000","last_active":"2023-09-23T20:17:04.496601+0000","last_peered":"2023-09-23T20:17:04.496601+0000","last_clean":"2023-09-23T20:17:04.496601+0000","last_became_active":"2023-09-23T20:14:34.848263+0000","last_became_peered":"2023-09-23T20:14:34.848263+0000","last_unstale":"2023-09-23T20:17:04.496601+0000","last_undegraded":"2023-09-23T20:17:04.496601+0000","last_fullsized":"2023-09-23T20:17:04.496601+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:35.529140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":385,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815368+0000","last_change":"2023-09-23T20:14:34.850119+0000","last_active":"2023-09-23T20:17:04.815368+0000","last_peered":"2023-09-23T20:17:04.815368+0000","last_clean":"2023-09-23T20:17:04.815368+0000","last_became_active":"2023-09-23T20:14:34.849755+0000","last_became_peered":"2023-09-23T20:14:34.849755+0000","last_unstale":"2023-09-23T20:17:04.815368+0000","last_undegraded":"2023-09-23T20:17:04.815368+0000","last_fullsized":"2023-09-23T20:17:04.815368+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:02.019726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"484'1072","reported_seq":1085,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.064004+0000","last_change":"2023-09-23T20:16:59.496298+0000","last_active":"2023-09-23T20:17:13.064004+0000","last_peered":"2023-09-23T20:17:13.064004+0000","last_clean":"2023-09-23T20:17:13.064004+0000","last_became_active":"2023-09-23T20:16:59.496009+0000","last_became_peered":"2023-09-23T20:16:59.496009+0000","last_unstale":"2023-09-23T20:17:13.064004+0000","last_undegraded":"2023-09-23T20:17:13.064004+0000","last_fullsized":"2023-09-23T20:17:13.064004+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1072,"log_dups_size":0,"ondisk_log_size":1072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:20.859500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70254592,"num_objects":1072,"num_object_clones":0,"num_object_copies":3216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1072,"num_write_kb":68608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":555,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497016+0000","last_change":"2023-09-23T20:16:53.682976+0000","last_active":"2023-09-23T20:17:04.497016+0000","last_peered":"2023-09-23T20:17:04.497016+0000","last_clean":"2023-09-23T20:17:04.497016+0000","last_became_active":"2023-09-23T20:14:37.892234+0000","last_became_peered":"2023-09-23T20:14:37.892234+0000","last_unstale":"2023-09-23T20:17:04.497016+0000","last_undegraded":"2023-09-23T20:17:04.497016+0000","last_fullsized":"2023-09-23T20:17:04.497016+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:26.329677+0000","scrub_duration":0.0022394540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":376,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814272+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:17:04.814272+0000","last_peered":"2023-09-23T20:17:04.814272+0000","last_clean":"2023-09-23T20:17:04.814272+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:17:04.814272+0000","last_undegraded":"2023-09-23T20:17:04.814272+0000","last_fullsized":"2023-09-23T20:17:04.814272+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":52,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497261+0000","last_change":"2023-09-23T20:14:34.850762+0000","last_active":"2023-09-23T20:17:04.497261+0000","last_peered":"2023-09-23T20:17:04.497261+0000","last_clean":"2023-09-23T20:17:04.497261+0000","last_became_active":"2023-09-23T20:14:34.850204+0000","last_became_peered":"2023-09-23T20:14:34.850204+0000","last_unstale":"2023-09-23T20:17:04.497261+0000","last_undegraded":"2023-09-23T20:17:04.497261+0000","last_fullsized":"2023-09-23T20:17:04.497261+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:00.499790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.1","version":"484'1075","reported_seq":1088,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.085553+0000","last_change":"2023-09-23T20:16:59.496645+0000","last_active":"2023-09-23T20:17:13.085553+0000","last_peered":"2023-09-23T20:17:13.085553+0000","last_clean":"2023-09-23T20:17:13.085553+0000","last_became_active":"2023-09-23T20:16:59.496430+0000","last_became_peered":"2023-09-23T20:16:59.496430+0000","last_unstale":"2023-09-23T20:17:13.085553+0000","last_undegraded":"2023-09-23T20:17:13.085553+0000","last_fullsized":"2023-09-23T20:17:13.085553+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1075,"log_dups_size":0,"ondisk_log_size":1075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:59.713976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70451200,"num_objects":1075,"num_object_clones":0,"num_object_copies":3225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1075,"num_write_kb":68800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":822,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497000+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:17:04.497000+0000","last_peered":"2023-09-23T20:17:04.497000+0000","last_clean":"2023-09-23T20:17:04.497000+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:17:04.497000+0000","last_undegraded":"2023-09-23T20:17:04.497000+0000","last_fullsized":"2023-09-23T20:17:04.497000+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":125,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815718+0000","last_change":"2023-09-23T20:14:34.858759+0000","last_active":"2023-09-23T20:17:04.815718+0000","last_peered":"2023-09-23T20:17:04.815718+0000","last_clean":"2023-09-23T20:17:04.815718+0000","last_became_active":"2023-09-23T20:14:34.857692+0000","last_became_peered":"2023-09-23T20:14:34.857692+0000","last_unstale":"2023-09-23T20:17:04.815718+0000","last_undegraded":"2023-09-23T20:17:04.815718+0000","last_fullsized":"2023-09-23T20:17:04.815718+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:00.759340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":229,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816550+0000","last_change":"2023-09-23T20:14:34.866254+0000","last_active":"2023-09-23T20:17:04.816550+0000","last_peered":"2023-09-23T20:17:04.816550+0000","last_clean":"2023-09-23T20:17:04.816550+0000","last_became_active":"2023-09-23T20:14:34.865928+0000","last_became_peered":"2023-09-23T20:14:34.865928+0000","last_unstale":"2023-09-23T20:17:04.816550+0000","last_undegraded":"2023-09-23T20:17:04.816550+0000","last_fullsized":"2023-09-23T20:17:04.816550+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:48.174087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"484'1005","reported_seq":1017,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.070087+0000","last_change":"2023-09-23T20:16:59.496291+0000","last_active":"2023-09-23T20:17:13.070087+0000","last_peered":"2023-09-23T20:17:13.070087+0000","last_clean":"2023-09-23T20:17:13.070087+0000","last_became_active":"2023-09-23T20:16:59.496000+0000","last_became_peered":"2023-09-23T20:16:59.496000+0000","last_unstale":"2023-09-23T20:17:13.070087+0000","last_undegraded":"2023-09-23T20:17:13.070087+0000","last_fullsized":"2023-09-23T20:17:13.070087+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1005,"log_dups_size":0,"ondisk_log_size":1005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.808639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65863680,"num_objects":1005,"num_object_clones":0,"num_object_copies":3015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1005,"num_write_kb":64320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":766,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816322+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:17:04.816322+0000","last_peered":"2023-09-23T20:17:04.816322+0000","last_clean":"2023-09-23T20:17:04.816322+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:17:04.816322+0000","last_undegraded":"2023-09-23T20:17:04.816322+0000","last_fullsized":"2023-09-23T20:17:04.816322+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":504,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.817785+0000","last_change":"2023-09-23T20:16:53.809519+0000","last_active":"2023-09-23T20:17:04.817785+0000","last_peered":"2023-09-23T20:17:04.817785+0000","last_clean":"2023-09-23T20:17:04.817785+0000","last_became_active":"2023-09-23T20:16:53.809219+0000","last_became_peered":"2023-09-23T20:16:53.809219+0000","last_unstale":"2023-09-23T20:17:04.817785+0000","last_undegraded":"2023-09-23T20:17:04.817785+0000","last_fullsized":"2023-09-23T20:17:04.817785+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:01.640683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":484,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497218+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:17:04.497218+0000","last_peered":"2023-09-23T20:17:04.497218+0000","last_clean":"2023-09-23T20:17:04.497218+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:17:04.497218+0000","last_undegraded":"2023-09-23T20:17:04.497218+0000","last_fullsized":"2023-09-23T20:17:04.497218+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":95,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496948+0000","last_change":"2023-09-23T20:14:34.855059+0000","last_active":"2023-09-23T20:17:04.496948+0000","last_peered":"2023-09-23T20:17:04.496948+0000","last_clean":"2023-09-23T20:17:04.496948+0000","last_became_active":"2023-09-23T20:14:34.852835+0000","last_became_peered":"2023-09-23T20:14:34.852835+0000","last_unstale":"2023-09-23T20:17:04.496948+0000","last_undegraded":"2023-09-23T20:17:04.496948+0000","last_fullsized":"2023-09-23T20:17:04.496948+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:57.978736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"484'1069","reported_seq":1082,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.536422+0000","last_change":"2023-09-23T20:16:59.488965+0000","last_active":"2023-09-23T20:17:13.536422+0000","last_peered":"2023-09-23T20:17:13.536422+0000","last_clean":"2023-09-23T20:17:13.536422+0000","last_became_active":"2023-09-23T20:16:59.488722+0000","last_became_peered":"2023-09-23T20:16:59.488722+0000","last_unstale":"2023-09-23T20:17:13.536422+0000","last_undegraded":"2023-09-23T20:17:13.536422+0000","last_fullsized":"2023-09-23T20:17:13.536422+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1069,"log_dups_size":0,"ondisk_log_size":1069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:57.375203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70057984,"num_objects":1069,"num_object_clones":0,"num_object_copies":3207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1069,"num_write_kb":68416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":827,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496626+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:17:04.496626+0000","last_peered":"2023-09-23T20:17:04.496626+0000","last_clean":"2023-09-23T20:17:04.496626+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:17:04.496626+0000","last_undegraded":"2023-09-23T20:17:04.496626+0000","last_fullsized":"2023-09-23T20:17:04.496626+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1134,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496783+0000","last_change":"2023-09-23T20:14:34.887753+0000","last_active":"2023-09-23T20:17:04.496783+0000","last_peered":"2023-09-23T20:17:04.496783+0000","last_clean":"2023-09-23T20:17:04.496783+0000","last_became_active":"2023-09-23T20:14:34.852324+0000","last_became_peered":"2023-09-23T20:14:34.852324+0000","last_unstale":"2023-09-23T20:17:04.496783+0000","last_undegraded":"2023-09-23T20:17:04.496783+0000","last_fullsized":"2023-09-23T20:17:04.496783+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:53.757102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":52,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496784+0000","last_change":"2023-09-23T20:14:34.854417+0000","last_active":"2023-09-23T20:17:04.496784+0000","last_peered":"2023-09-23T20:17:04.496784+0000","last_clean":"2023-09-23T20:17:04.496784+0000","last_became_active":"2023-09-23T20:14:34.854051+0000","last_became_peered":"2023-09-23T20:14:34.854051+0000","last_unstale":"2023-09-23T20:17:04.496784+0000","last_undegraded":"2023-09-23T20:17:04.496784+0000","last_fullsized":"2023-09-23T20:17:04.496784+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:48.064237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":347,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814809+0000","last_change":"2023-09-23T20:14:34.855139+0000","last_active":"2023-09-23T20:17:04.814809+0000","last_peered":"2023-09-23T20:17:04.814809+0000","last_clean":"2023-09-23T20:17:04.814809+0000","last_became_active":"2023-09-23T20:14:34.854760+0000","last_became_peered":"2023-09-23T20:14:34.854760+0000","last_unstale":"2023-09-23T20:17:04.814809+0000","last_undegraded":"2023-09-23T20:17:04.814809+0000","last_fullsized":"2023-09-23T20:17:04.814809+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:43.001673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"484'1167","reported_seq":1180,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.539780+0000","last_change":"2023-09-23T20:16:59.496065+0000","last_active":"2023-09-23T20:17:13.539780+0000","last_peered":"2023-09-23T20:17:13.539780+0000","last_clean":"2023-09-23T20:17:13.539780+0000","last_became_active":"2023-09-23T20:16:59.495782+0000","last_became_peered":"2023-09-23T20:16:59.495782+0000","last_unstale":"2023-09-23T20:17:13.539780+0000","last_undegraded":"2023-09-23T20:17:13.539780+0000","last_fullsized":"2023-09-23T20:17:13.539780+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1167,"log_dups_size":0,"ondisk_log_size":1167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:08.985469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76480512,"num_objects":1167,"num_object_clones":0,"num_object_copies":3501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1167,"num_write_kb":74688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":722,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814392+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:17:04.814392+0000","last_peered":"2023-09-23T20:17:04.814392+0000","last_clean":"2023-09-23T20:17:04.814392+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:17:04.814392+0000","last_undegraded":"2023-09-23T20:17:04.814392+0000","last_fullsized":"2023-09-23T20:17:04.814392+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1122,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814425+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:17:04.814425+0000","last_peered":"2023-09-23T20:17:04.814425+0000","last_clean":"2023-09-23T20:17:04.814425+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:17:04.814425+0000","last_undegraded":"2023-09-23T20:17:04.814425+0000","last_fullsized":"2023-09-23T20:17:04.814425+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":183,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814543+0000","last_change":"2023-09-23T20:14:34.850285+0000","last_active":"2023-09-23T20:17:04.814543+0000","last_peered":"2023-09-23T20:17:04.814543+0000","last_clean":"2023-09-23T20:17:04.814543+0000","last_became_active":"2023-09-23T20:14:34.849746+0000","last_became_peered":"2023-09-23T20:14:34.849746+0000","last_unstale":"2023-09-23T20:17:04.814543+0000","last_undegraded":"2023-09-23T20:17:04.814543+0000","last_fullsized":"2023-09-23T20:17:04.814543+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:00.742516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":149,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497433+0000","last_change":"2023-09-23T20:14:34.857450+0000","last_active":"2023-09-23T20:17:04.497433+0000","last_peered":"2023-09-23T20:17:04.497433+0000","last_clean":"2023-09-23T20:17:04.497433+0000","last_became_active":"2023-09-23T20:14:34.857142+0000","last_became_peered":"2023-09-23T20:14:34.857142+0000","last_unstale":"2023-09-23T20:17:04.497433+0000","last_undegraded":"2023-09-23T20:17:04.497433+0000","last_fullsized":"2023-09-23T20:17:04.497433+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:17.268624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"484'1029","reported_seq":1042,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:13.095307+0000","last_change":"2023-09-23T20:16:59.496379+0000","last_active":"2023-09-23T20:17:13.095307+0000","last_peered":"2023-09-23T20:17:13.095307+0000","last_clean":"2023-09-23T20:17:13.095307+0000","last_became_active":"2023-09-23T20:16:59.495954+0000","last_became_peered":"2023-09-23T20:16:59.495954+0000","last_unstale":"2023-09-23T20:17:13.095307+0000","last_undegraded":"2023-09-23T20:17:13.095307+0000","last_fullsized":"2023-09-23T20:17:13.095307+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1029,"log_dups_size":0,"ondisk_log_size":1029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:08.819045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67436544,"num_objects":1029,"num_object_clones":0,"num_object_copies":3087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1029,"num_write_kb":65856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1392,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496895+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:17:04.496895+0000","last_peered":"2023-09-23T20:17:04.496895+0000","last_clean":"2023-09-23T20:17:04.496895+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:17:04.496895+0000","last_undegraded":"2023-09-23T20:17:04.496895+0000","last_fullsized":"2023-09-23T20:17:04.496895+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":679,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814451+0000","last_change":"2023-09-23T20:14:37.892788+0000","last_active":"2023-09-23T20:17:04.814451+0000","last_peered":"2023-09-23T20:17:04.814451+0000","last_clean":"2023-09-23T20:17:04.814451+0000","last_became_active":"2023-09-23T20:14:37.892464+0000","last_became_peered":"2023-09-23T20:14:37.892464+0000","last_unstale":"2023-09-23T20:17:04.814451+0000","last_undegraded":"2023-09-23T20:17:04.814451+0000","last_fullsized":"2023-09-23T20:17:04.814451+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:43.691684+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":349,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496907+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:17:04.496907+0000","last_peered":"2023-09-23T20:17:04.496907+0000","last_clean":"2023-09-23T20:17:04.496907+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:17:04.496907+0000","last_undegraded":"2023-09-23T20:17:04.496907+0000","last_fullsized":"2023-09-23T20:17:04.496907+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":281,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.817202+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:17:04.817202+0000","last_peered":"2023-09-23T20:17:04.817202+0000","last_clean":"2023-09-23T20:17:04.817202+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:17:04.817202+0000","last_undegraded":"2023-09-23T20:17:04.817202+0000","last_fullsized":"2023-09-23T20:17:04.817202+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"484'1479","reported_seq":1493,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:16.140803+0000","last_change":"2023-09-23T20:16:59.494039+0000","last_active":"2023-09-23T20:17:16.140803+0000","last_peered":"2023-09-23T20:17:16.140803+0000","last_clean":"2023-09-23T20:17:16.140803+0000","last_became_active":"2023-09-23T20:16:59.493714+0000","last_became_peered":"2023-09-23T20:16:59.493714+0000","last_unstale":"2023-09-23T20:17:16.140803+0000","last_undegraded":"2023-09-23T20:17:16.140803+0000","last_fullsized":"2023-09-23T20:17:16.140803+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1479,"log_dups_size":0,"ondisk_log_size":1479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:15.171264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96927744,"num_objects":1479,"num_object_clones":0,"num_object_copies":4437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1479,"num_write_kb":94656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":555,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496918+0000","last_change":"2023-09-23T20:16:55.675509+0000","last_active":"2023-09-23T20:17:04.496918+0000","last_peered":"2023-09-23T20:17:04.496918+0000","last_clean":"2023-09-23T20:17:04.496918+0000","last_became_active":"2023-09-23T20:14:37.894281+0000","last_became_peered":"2023-09-23T20:14:37.894281+0000","last_unstale":"2023-09-23T20:17:04.496918+0000","last_undegraded":"2023-09-23T20:17:04.496918+0000","last_fullsized":"2023-09-23T20:17:04.496918+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:55.675445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:55.675445+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:10.524240+0000","scrub_duration":0.0015595749999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":964,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814197+0000","last_change":"2023-09-23T20:16:53.808714+0000","last_active":"2023-09-23T20:17:04.814197+0000","last_peered":"2023-09-23T20:17:04.814197+0000","last_clean":"2023-09-23T20:17:04.814197+0000","last_became_active":"2023-09-23T20:16:53.808212+0000","last_became_peered":"2023-09-23T20:16:53.808212+0000","last_unstale":"2023-09-23T20:17:04.814197+0000","last_undegraded":"2023-09-23T20:17:04.814197+0000","last_fullsized":"2023-09-23T20:17:04.814197+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:46.997392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":430,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814678+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:17:04.814678+0000","last_peered":"2023-09-23T20:17:04.814678+0000","last_clean":"2023-09-23T20:17:04.814678+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:17:04.814678+0000","last_undegraded":"2023-09-23T20:17:04.814678+0000","last_fullsized":"2023-09-23T20:17:04.814678+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":300,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815072+0000","last_change":"2023-09-23T20:14:34.856126+0000","last_active":"2023-09-23T20:17:04.815072+0000","last_peered":"2023-09-23T20:17:04.815072+0000","last_clean":"2023-09-23T20:17:04.815072+0000","last_became_active":"2023-09-23T20:14:34.855303+0000","last_became_peered":"2023-09-23T20:14:34.855303+0000","last_unstale":"2023-09-23T20:17:04.815072+0000","last_undegraded":"2023-09-23T20:17:04.815072+0000","last_fullsized":"2023-09-23T20:17:04.815072+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:44.396592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.d","version":"484'1485","reported_seq":1499,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:16.591737+0000","last_change":"2023-09-23T20:16:59.490420+0000","last_active":"2023-09-23T20:17:16.591737+0000","last_peered":"2023-09-23T20:17:16.591737+0000","last_clean":"2023-09-23T20:17:16.591737+0000","last_became_active":"2023-09-23T20:16:59.490025+0000","last_became_peered":"2023-09-23T20:16:59.490025+0000","last_unstale":"2023-09-23T20:17:16.591737+0000","last_undegraded":"2023-09-23T20:17:16.591737+0000","last_fullsized":"2023-09-23T20:17:16.591737+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1485,"log_dups_size":0,"ondisk_log_size":1485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:03.184763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97320960,"num_objects":1485,"num_object_clones":0,"num_object_copies":4455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1485,"num_write_kb":95040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":669,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.495979+0000","last_change":"2023-09-23T20:17:02.526275+0000","last_active":"2023-09-23T20:17:04.495979+0000","last_peered":"2023-09-23T20:17:04.495979+0000","last_clean":"2023-09-23T20:17:04.495979+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:17:04.495979+0000","last_undegraded":"2023-09-23T20:17:04.495979+0000","last_fullsized":"2023-09-23T20:17:04.495979+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:57.948831+0000","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":184,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.817424+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:17:04.817424+0000","last_peered":"2023-09-23T20:17:04.817424+0000","last_clean":"2023-09-23T20:17:04.817424+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:17:04.817424+0000","last_undegraded":"2023-09-23T20:17:04.817424+0000","last_fullsized":"2023-09-23T20:17:04.817424+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":288,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815503+0000","last_change":"2023-09-23T20:14:34.847979+0000","last_active":"2023-09-23T20:17:04.815503+0000","last_peered":"2023-09-23T20:17:04.815503+0000","last_clean":"2023-09-23T20:17:04.815503+0000","last_became_active":"2023-09-23T20:14:34.847572+0000","last_became_peered":"2023-09-23T20:14:34.847572+0000","last_unstale":"2023-09-23T20:17:04.815503+0000","last_undegraded":"2023-09-23T20:17:04.815503+0000","last_fullsized":"2023-09-23T20:17:04.815503+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:35.456247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":310,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496809+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:17:04.496809+0000","last_peered":"2023-09-23T20:17:04.496809+0000","last_clean":"2023-09-23T20:17:04.496809+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:17:04.496809+0000","last_undegraded":"2023-09-23T20:17:04.496809+0000","last_fullsized":"2023-09-23T20:17:04.496809+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"484'1528","reported_seq":1542,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:16.158324+0000","last_change":"2023-09-23T20:16:59.490304+0000","last_active":"2023-09-23T20:17:16.158324+0000","last_peered":"2023-09-23T20:17:16.158324+0000","last_clean":"2023-09-23T20:17:16.158324+0000","last_became_active":"2023-09-23T20:16:59.490026+0000","last_became_peered":"2023-09-23T20:16:59.490026+0000","last_unstale":"2023-09-23T20:17:16.158324+0000","last_undegraded":"2023-09-23T20:17:16.158324+0000","last_fullsized":"2023-09-23T20:17:16.158324+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1528,"log_dups_size":0,"ondisk_log_size":1528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:59.789512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100139008,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1528,"num_write_kb":97792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":715,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815431+0000","last_change":"2023-09-23T20:16:56.408683+0000","last_active":"2023-09-23T20:17:04.815431+0000","last_peered":"2023-09-23T20:17:04.815431+0000","last_clean":"2023-09-23T20:17:04.815431+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:17:04.815431+0000","last_undegraded":"2023-09-23T20:17:04.815431+0000","last_fullsized":"2023-09-23T20:17:04.815431+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:18.311022+0000","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":326,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816057+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:17:04.816057+0000","last_peered":"2023-09-23T20:17:04.816057+0000","last_clean":"2023-09-23T20:17:04.816057+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:17:04.816057+0000","last_undegraded":"2023-09-23T20:17:04.816057+0000","last_fullsized":"2023-09-23T20:17:04.816057+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":642,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815255+0000","last_change":"2023-09-23T20:16:02.235622+0000","last_active":"2023-09-23T20:17:04.815255+0000","last_peered":"2023-09-23T20:17:04.815255+0000","last_clean":"2023-09-23T20:17:04.815255+0000","last_became_active":"2023-09-23T20:16:02.234933+0000","last_became_peered":"2023-09-23T20:16:02.234933+0000","last_unstale":"2023-09-23T20:17:04.815255+0000","last_undegraded":"2023-09-23T20:17:04.815255+0000","last_fullsized":"2023-09-23T20:17:04.815255+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:21.855632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815680+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:17:04.815680+0000","last_peered":"2023-09-23T20:17:04.815680+0000","last_clean":"2023-09-23T20:17:04.815680+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:17:04.815680+0000","last_undegraded":"2023-09-23T20:17:04.815680+0000","last_fullsized":"2023-09-23T20:17:04.815680+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"484'1513","reported_seq":1527,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T20:17:16.591572+0000","last_change":"2023-09-23T20:16:59.490308+0000","last_active":"2023-09-23T20:17:16.591572+0000","last_peered":"2023-09-23T20:17:16.591572+0000","last_clean":"2023-09-23T20:17:16.591572+0000","last_became_active":"2023-09-23T20:16:59.489997+0000","last_became_peered":"2023-09-23T20:16:59.489997+0000","last_unstale":"2023-09-23T20:17:16.591572+0000","last_undegraded":"2023-09-23T20:17:16.591572+0000","last_fullsized":"2023-09-23T20:17:16.591572+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":1513,"log_dups_size":0,"ondisk_log_size":1513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:45.118523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99155968,"num_objects":1513,"num_object_clones":0,"num_object_copies":4539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1513,"num_write_kb":96832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":810,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815922+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:17:04.815922+0000","last_peered":"2023-09-23T20:17:04.815922+0000","last_clean":"2023-09-23T20:17:04.815922+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:17:04.815922+0000","last_undegraded":"2023-09-23T20:17:04.815922+0000","last_fullsized":"2023-09-23T20:17:04.815922+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":185,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816310+0000","last_change":"2023-09-23T20:07:50.773282+0000","last_active":"2023-09-23T20:17:04.816310+0000","last_peered":"2023-09-23T20:17:04.816310+0000","last_clean":"2023-09-23T20:17:04.816310+0000","last_became_active":"2023-09-23T20:07:36.987966+0000","last_became_peered":"2023-09-23T20:07:36.987966+0000","last_unstale":"2023-09-23T20:17:04.816310+0000","last_undegraded":"2023-09-23T20:17:04.816310+0000","last_fullsized":"2023-09-23T20:17:04.816310+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:54.290741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816601+0000","last_change":"2023-09-23T20:14:34.876535+0000","last_active":"2023-09-23T20:17:04.816601+0000","last_peered":"2023-09-23T20:17:04.816601+0000","last_clean":"2023-09-23T20:17:04.816601+0000","last_became_active":"2023-09-23T20:14:34.867869+0000","last_became_peered":"2023-09-23T20:14:34.867869+0000","last_unstale":"2023-09-23T20:17:04.816601+0000","last_undegraded":"2023-09-23T20:17:04.816601+0000","last_fullsized":"2023-09-23T20:17:04.816601+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:04.434269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":267,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496685+0000","last_change":"2023-09-23T20:13:57.750756+0000","last_active":"2023-09-23T20:17:04.496685+0000","last_peered":"2023-09-23T20:17:04.496685+0000","last_clean":"2023-09-23T20:17:04.496685+0000","last_became_active":"2023-09-23T20:13:57.750460+0000","last_became_peered":"2023-09-23T20:13:57.750460+0000","last_unstale":"2023-09-23T20:17:04.496685+0000","last_undegraded":"2023-09-23T20:17:04.496685+0000","last_fullsized":"2023-09-23T20:17:04.496685+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:27.240921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":261,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.816338+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:17:04.816338+0000","last_peered":"2023-09-23T20:17:04.816338+0000","last_clean":"2023-09-23T20:17:04.816338+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:17:04.816338+0000","last_undegraded":"2023-09-23T20:17:04.816338+0000","last_fullsized":"2023-09-23T20:17:04.816338+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":261,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815257+0000","last_change":"2023-09-23T20:14:34.847699+0000","last_active":"2023-09-23T20:17:04.815257+0000","last_peered":"2023-09-23T20:17:04.815257+0000","last_clean":"2023-09-23T20:17:04.815257+0000","last_became_active":"2023-09-23T20:14:34.847381+0000","last_became_peered":"2023-09-23T20:14:34.847381+0000","last_unstale":"2023-09-23T20:17:04.815257+0000","last_undegraded":"2023-09-23T20:17:04.815257+0000","last_fullsized":"2023-09-23T20:17:04.815257+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:22.549359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":501,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.496686+0000","last_change":"2023-09-23T20:14:34.865570+0000","last_active":"2023-09-23T20:17:04.496686+0000","last_peered":"2023-09-23T20:17:04.496686+0000","last_clean":"2023-09-23T20:17:04.496686+0000","last_became_active":"2023-09-23T20:14:34.855954+0000","last_became_peered":"2023-09-23T20:14:34.855954+0000","last_unstale":"2023-09-23T20:17:04.496686+0000","last_undegraded":"2023-09-23T20:17:04.496686+0000","last_fullsized":"2023-09-23T20:17:04.496686+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:26.553487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":504,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815141+0000","last_change":"2023-09-23T20:13:57.747956+0000","last_active":"2023-09-23T20:17:04.815141+0000","last_peered":"2023-09-23T20:17:04.815141+0000","last_clean":"2023-09-23T20:17:04.815141+0000","last_became_active":"2023-09-23T20:13:57.747619+0000","last_became_peered":"2023-09-23T20:13:57.747619+0000","last_unstale":"2023-09-23T20:17:04.815141+0000","last_undegraded":"2023-09-23T20:17:04.815141+0000","last_fullsized":"2023-09-23T20:17:04.815141+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:30.178098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":491,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497183+0000","last_change":"2023-09-23T20:14:34.862135+0000","last_active":"2023-09-23T20:17:04.497183+0000","last_peered":"2023-09-23T20:17:04.497183+0000","last_clean":"2023-09-23T20:17:04.497183+0000","last_became_active":"2023-09-23T20:14:34.851454+0000","last_became_peered":"2023-09-23T20:14:34.851454+0000","last_unstale":"2023-09-23T20:17:04.497183+0000","last_undegraded":"2023-09-23T20:17:04.497183+0000","last_fullsized":"2023-09-23T20:17:04.497183+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:44.077328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":480,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814542+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:17:04.814542+0000","last_peered":"2023-09-23T20:17:04.814542+0000","last_clean":"2023-09-23T20:17:04.814542+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:17:04.814542+0000","last_undegraded":"2023-09-23T20:17:04.814542+0000","last_fullsized":"2023-09-23T20:17:04.814542+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":196,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.815191+0000","last_change":"2023-09-23T20:14:34.856335+0000","last_active":"2023-09-23T20:17:04.815191+0000","last_peered":"2023-09-23T20:17:04.815191+0000","last_clean":"2023-09-23T20:17:04.815191+0000","last_became_active":"2023-09-23T20:14:34.855750+0000","last_became_peered":"2023-09-23T20:14:34.855750+0000","last_unstale":"2023-09-23T20:17:04.815191+0000","last_undegraded":"2023-09-23T20:17:04.815191+0000","last_fullsized":"2023-09-23T20:17:04.815191+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:59.668066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":343,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814503+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:17:04.814503+0000","last_peered":"2023-09-23T20:17:04.814503+0000","last_clean":"2023-09-23T20:17:04.814503+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:17:04.814503+0000","last_undegraded":"2023-09-23T20:17:04.814503+0000","last_fullsized":"2023-09-23T20:17:04.814503+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":329,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.497382+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:17:04.497382+0000","last_peered":"2023-09-23T20:17:04.497382+0000","last_clean":"2023-09-23T20:17:04.497382+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:17:04.497382+0000","last_undegraded":"2023-09-23T20:17:04.497382+0000","last_fullsized":"2023-09-23T20:17:04.497382+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":54,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T20:17:04.814306+0000","last_change":"2023-09-23T20:14:34.850427+0000","last_active":"2023-09-23T20:17:04.814306+0000","last_peered":"2023-09-23T20:17:04.814306+0000","last_clean":"2023-09-23T20:17:04.814306+0000","last_became_active":"2023-09-23T20:14:34.850168+0000","last_became_peered":"2023-09-23T20:14:34.850168+0000","last_unstale":"2023-09-23T20:17:04.814306+0000","last_undegraded":"2023-09-23T20:17:04.814306+0000","last_fullsized":"2023-09-23T20:17:04.814306+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:55.646227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1331298304,"num_objects":20314,"num_object_clones":0,"num_object_copies":60942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20314,"num_write_kb":1300096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3802136576,"data_stored":3802136576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":20314,"ondisk_log_size":20314,"up":48,"acting":48,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":10882,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":211,"num_bytes_recovered":8388684,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234433,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917012,"kb_used_data":386188,"kb_used_omap":0,"kb_used_meta":530816,"kb_avail":92828140,"statfs":{"total":95995035648,"available":95056015360,"internally_reserved":0,"allocated":395456512,"data_stored":393995067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":543555584},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":8.2040000000000006,"5min":5.3499999999999996,"15min":3.266},"min":{"1min":0.254,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":58.109000000000002,"5min":149.25999999999999,"15min":149.25999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":8.3300000000000001,"5min":5.7619999999999996,"15min":3.5179999999999998},"min":{"1min":0.37,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":58.042000000000002,"5min":149.11799999999999,"15min":149.11799999999999},"last":0.255}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":3.3929999999999998,"15min":3.476},"min":{"1min":0.29199999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.101,"5min":91.335999999999999,"15min":104.16800000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.845,"5min":3.9380000000000002,"15min":3.79},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.223},"max":{"1min":4.7270000000000003,"5min":91.516999999999996,"15min":104.003},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:17:01 2023","interfaces":[{"interface":"back","average":{"1min":5.3940000000000001,"5min":3.9529999999999998,"15min":4.0259999999999998},"min":{"1min":0.33100000000000002,"5min":0.25800000000000001,"15min":0.251},"max":{"1min":61.997,"5min":67.760000000000005,"15min":105.842},"last":0.86199999999999999},{"interface":"front","average":{"1min":5.3700000000000001,"5min":4.2329999999999997,"15min":4.4809999999999999},"min":{"1min":0.27600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":61.978999999999999,"5min":67.488,"15min":105.66},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":5.0609999999999999,"5min":5.266,"15min":4.8499999999999996},"min":{"1min":0.46300000000000002,"5min":0.29999999999999999,"15min":0.248},"max":{"1min":43.747,"5min":185.17099999999999,"15min":185.17099999999999},"last":2.2109999999999999},{"interface":"front","average":{"1min":5.335,"5min":5.4279999999999999,"15min":4.843},"min":{"1min":0.40600000000000003,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":43.731000000000002,"5min":184.99100000000001,"15min":184.99100000000001},"last":1.679}]},{"osd":4,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":0.96699999999999997,"15min":0.76300000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.22500000000000001},"max":{"1min":7.6440000000000001,"5min":7.6440000000000001,"15min":19.215},"last":0.312},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.151,"15min":0.96499999999999997},"min":{"1min":0.42699999999999999,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":3.9809999999999999,"5min":10.882999999999999,"15min":41.822000000000003},"last":0.35999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:16:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.595,"15min":1.1990000000000001},"min":{"1min":0.51100000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":8.4440000000000008,"5min":14.069000000000001,"15min":35.488},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.367,"5min":1.284,"15min":1.0389999999999999},"min":{"1min":0.41699999999999998,"5min":0.31,"15min":0.27300000000000002},"max":{"1min":7.6109999999999998,"5min":13.43,"15min":34.896000000000001},"last":0.35199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4830000000000001,"15min":1.0649999999999999},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":6.843,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.5580000000000001,"5min":1.143,"15min":0.81599999999999995},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.8010000000000002,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.38}]}]},{"osd":6,"up_from":0,"seq":1078036791474,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":425201762632,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":94,"seq":403726926169,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1673376,"kb_used_data":834392,"kb_used_omap":8,"kb_used_meta":838967,"kb_avail":92071776,"statfs":{"total":95995035648,"available":94281498624,"internally_reserved":0,"allocated":854417408,"data_stored":852187136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":859102817},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:17:05 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.218,"15min":3.2090000000000001},"min":{"1min":0.23799999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":2.6440000000000001,"5min":29.893000000000001,"15min":94.055999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.495,"5min":1.411,"15min":3.3559999999999999},"min":{"1min":0.23499999999999999,"5min":0.20799999999999999,"15min":0.187},"max":{"1min":1.5940000000000001,"5min":23.436,"15min":91.632999999999996},"last":1.6850000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:16:17 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":2.863,"15min":4.944},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.1820000000000004,"5min":160.542,"15min":270.51100000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.0219999999999998,"5min":3.5009999999999999,"15min":4.8970000000000002},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":18.433,"5min":160.75999999999999,"15min":269.69299999999998},"last":2.1560000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:17:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1600000000000001,"5min":1.7210000000000001,"15min":3.6480000000000001},"min":{"1min":0.29399999999999998,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":44.405000000000001,"5min":44.405000000000001,"15min":126.95399999999999},"last":1.722},{"interface":"front","average":{"1min":3.2109999999999999,"5min":1.879,"15min":3.8250000000000002},"min":{"1min":0.28799999999999998,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":43.909999999999997,"5min":43.909999999999997,"15min":127.294},"last":1.8220000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.726,"15min":2.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":5.859,"5min":9.6959999999999997,"15min":148.20599999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":5.9790000000000001,"5min":2.29,"15min":2.2589999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":105.282,"5min":105.282,"15min":148.11000000000001},"last":1.788}]},{"osd":5,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.9630000000000001,"15min":1.859},"min":{"1min":0.378,"5min":0.307,"15min":0.26900000000000002},"max":{"1min":5.5430000000000001,"5min":52.061,"15min":64.725999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.4649999999999999,"15min":1.8320000000000001},"min":{"1min":0.41699999999999998,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":7.1369999999999996,"5min":51.741,"15min":51.741},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.6279999999999999,"15min":1.651},"min":{"1min":0.374,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":4.1609999999999996,"5min":7.1950000000000003,"15min":15.744999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.292,"15min":1.637},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":9.8219999999999992,"5min":9.8219999999999992,"15min":13.766999999999999},"last":2.3050000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.569,"15min":1.75},"min":{"1min":0.53700000000000003,"5min":0.38700000000000001,"15min":0.31900000000000001},"max":{"1min":4.9980000000000002,"5min":12.981,"15min":65.081999999999994},"last":1.3779999999999999},{"interface":"front","average":{"1min":2.6909999999999998,"5min":2.161,"15min":1.8440000000000001},"min":{"1min":0.49199999999999999,"5min":0.38700000000000001,"15min":0.32300000000000001},"max":{"1min":7.6600000000000001,"5min":17.151,"15min":17.151},"last":0.95599999999999996}]}]},{"osd":2,"up_from":104,"seq":446676599108,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1698848,"kb_used_data":801624,"kb_used_omap":12,"kb_used_meta":897203,"kb_avail":92046304,"statfs":{"total":95995035648,"available":94255415296,"internally_reserved":0,"allocated":820862976,"data_stored":818578145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":918736894},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.1880000000000002,"15min":2.464},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":5.3879999999999999,"5min":65.382999999999996,"15min":76.251999999999995},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.282,"15min":2.718},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.265000000000001,"5min":65.427000000000007,"15min":76.590000000000003},"last":1.45}]},{"osd":1,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":3.0899999999999999,"15min":3.5649999999999999},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":7.5270000000000001,"5min":70.453000000000003,"15min":94.768000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.8969999999999998,"15min":3.2450000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.218},"max":{"1min":7.2009999999999996,"5min":69.825000000000003,"15min":96.808000000000007},"last":0.46200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:16:43 2023","interfaces":[{"interface":"back","average":{"1min":13.59,"5min":5.3810000000000002,"15min":4.7519999999999998},"min":{"1min":0.27800000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":126.345,"5min":132.82499999999999,"15min":132.82499999999999},"last":0.86899999999999999},{"interface":"front","average":{"1min":13.555,"5min":5.4829999999999997,"15min":3.9689999999999999},"min":{"1min":0.39000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":126.40300000000001,"5min":130.97300000000001,"15min":130.97300000000001},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.4399999999999999,"15min":1.79},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":9.3399999999999999,"5min":14.757999999999999,"15min":75.608000000000004},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.667,"5min":1.7649999999999999,"15min":2.105},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.641,"5min":13.387,"15min":76.161000000000001},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":3.3460000000000001,"15min":2.8519999999999999},"min":{"1min":0.34999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.3859999999999992,"5min":135.202,"15min":172.505},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.96,"5min":2.0619999999999998,"15min":2.6749999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.1260000000000003,"5min":29.196999999999999,"15min":172.375},"last":1.5069999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.577,"15min":1.5840000000000001},"min":{"1min":0.41399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.8310000000000004,"5min":14.664,"15min":14.664},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.298,"5min":1.627,"15min":1.7949999999999999},"min":{"1min":0.34899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.5659999999999998,"5min":12.025,"15min":14.279999999999999},"last":1.4830000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:16:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":2.0990000000000002,"15min":1.883},"min":{"1min":0.32500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2970000000000006,"5min":13.481999999999999,"15min":13.481999999999999},"last":0.499},{"interface":"front","average":{"1min":2.1720000000000002,"5min":1.6399999999999999,"15min":1.581},"min":{"1min":0.443,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.266,"5min":14.698,"15min":14.698},"last":0.53800000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435582,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1804788,"kb_used_data":867948,"kb_used_omap":4,"kb_used_meta":936827,"kb_avail":91940364,"statfs":{"total":95995035648,"available":94146932736,"internally_reserved":0,"allocated":888778752,"data_stored":886232719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":959310957},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":3.5459999999999998,"5min":2.355,"15min":3.1880000000000002},"min":{"1min":0.19600000000000001,"5min":0.186,"15min":0.182},"max":{"1min":24.776,"5min":76.632000000000005,"15min":76.632000000000005},"last":1.135},{"interface":"front","average":{"1min":3.8809999999999998,"5min":2.8530000000000002,"15min":3.4969999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":24.582000000000001,"5min":81.429000000000002,"15min":81.429000000000002},"last":0.23799999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":3.4340000000000002,"15min":4.3600000000000003},"min":{"1min":0.217,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":3.7050000000000001,"5min":183.87,"15min":183.87},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.3149999999999999,"5min":3.52,"15min":4.7229999999999999},"min":{"1min":0.251,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":6.3869999999999996,"5min":185.84899999999999,"15min":185.84899999999999},"last":0.58999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.0409999999999999,"15min":3.9140000000000001},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":8.0960000000000001,"5min":14.9,"15min":163.83500000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.5389999999999999,"15min":3.7629999999999999},"min":{"1min":0.252,"5min":0.25,"15min":0.25},"max":{"1min":4.9889999999999999,"5min":16.550999999999998,"15min":163.92099999999999},"last":0.625}]},{"osd":4,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.1639999999999999,"15min":2.77},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":5.016,"5min":7.4960000000000004,"15min":109.66800000000001},"last":0.65600000000000003},{"interface":"front","average":{"1min":2.2909999999999999,"5min":1.8879999999999999,"15min":3.2250000000000001},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":5.6779999999999999,"5min":11.815,"15min":110.295},"last":0.71599999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8889999999999998,"5min":3.9740000000000002,"15min":3.9119999999999999},"min":{"1min":0.39000000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.742000000000001,"5min":200.53200000000001,"15min":200.53200000000001},"last":3.2959999999999998},{"interface":"front","average":{"1min":2.7200000000000002,"5min":4.1429999999999998,"15min":3.903},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.26900000000000002},"max":{"1min":23.094999999999999,"5min":200.846,"15min":200.846},"last":0.70099999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:17:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.7410000000000001,"15min":2.5630000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32300000000000001},"max":{"1min":3.75,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.538,"5min":1.3620000000000001,"15min":2.621},"min":{"1min":0.40200000000000002,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":5.1769999999999996,"5min":8.2629999999999999,"15min":82.233000000000004},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":3.0510000000000002,"15min":3.234},"min":{"1min":0.376,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":3.3719999999999999,"5min":139.465,"15min":139.465},"last":0.64700000000000002},{"interface":"front","average":{"1min":2.0299999999999998,"5min":3.073,"15min":3.3069999999999999},"min":{"1min":0.44700000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":5.4459999999999997,"5min":141.56200000000001,"15min":141.56200000000001},"last":0.73799999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019343,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1174096,"kb_used_data":693256,"kb_used_omap":0,"kb_used_meta":480832,"kb_avail":92571056,"statfs":{"total":95995035648,"available":94792761344,"internally_reserved":0,"allocated":709894144,"data_stored":708436783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":492371968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.4650000000000001,"15min":1.639},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":5.2549999999999999,"5min":14.063000000000001,"15min":56.061999999999998},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.6519999999999999,"5min":1.4630000000000001,"15min":1.619},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.218},"max":{"1min":6.9740000000000002,"5min":14.048999999999999,"15min":56.075000000000003},"last":1.0880000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":3.7559999999999998,"15min":5.7779999999999996},"min":{"1min":0.28499999999999998,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":14.763,"5min":138.47999999999999,"15min":161.613},"last":0.434},{"interface":"front","average":{"1min":2.1789999999999998,"5min":3.7029999999999998,"15min":5.7830000000000004},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.217},"max":{"1min":13.368,"5min":138.429,"15min":162.017},"last":0.999}]},{"osd":2,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.8969999999999998,"15min":5.3760000000000003},"min":{"1min":0.28699999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":5.6920000000000002,"5min":144.916,"15min":182.96700000000001},"last":1.1910000000000001},{"interface":"front","average":{"1min":2.0299999999999998,"5min":3.8300000000000001,"15min":5.3369999999999997},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.23100000000000001},"max":{"1min":9.4550000000000001,"5min":144.83099999999999,"15min":183.244},"last":1.0629999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:16:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.3159999999999998,"15min":4.6589999999999998},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":4.5819999999999999,"5min":100.361,"15min":184.37100000000001},"last":1.6819999999999999},{"interface":"front","average":{"1min":1.738,"5min":3.4990000000000001,"15min":4.7199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":4.7270000000000003,"5min":100.20699999999999,"15min":184.22300000000001},"last":1.9039999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.111,"15min":1.252},"min":{"1min":0.28199999999999997,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.0739999999999998,"5min":12.257999999999999,"15min":176.81899999999999},"last":1.036},{"interface":"front","average":{"1min":1.7689999999999999,"5min":1.111,"15min":1.2849999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.246},"max":{"1min":7.9470000000000001,"5min":7.9470000000000001,"15min":176.172},"last":1.714}]},{"osd":6,"last update":"Sat Sep 23 20:16:53 2023","interfaces":[{"interface":"back","average":{"1min":6.3659999999999997,"5min":2.0259999999999998,"15min":1.018},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.23499999999999999},"max":{"1min":114.232,"5min":114.232,"15min":114.232},"last":1.2609999999999999},{"interface":"front","average":{"1min":6.4320000000000004,"5min":2.29,"15min":1.1919999999999999},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.247},"max":{"1min":114.07299999999999,"5min":114.07299999999999,"15min":114.07299999999999},"last":2.2130000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.196,"15min":1.5349999999999999},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":7.5090000000000003,"5min":7.71,"15min":76.507000000000005},"last":1.837},{"interface":"front","average":{"1min":3.1040000000000001,"5min":1.4099999999999999,"15min":1.6739999999999999},"min":{"1min":0.39900000000000002,"5min":0.28999999999999998,"15min":0.222},"max":{"1min":7.8250000000000002,"5min":7.8250000000000002,"15min":76.462000000000003},"last":0.81100000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757707,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1145608,"kb_used_data":543424,"kb_used_omap":0,"kb_used_meta":602176,"kb_avail":92599544,"statfs":{"total":95995035648,"available":94821933056,"internally_reserved":0,"allocated":556466176,"data_stored":555013256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616628224},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":7.7800000000000002,"5min":2.794,"15min":2.214},"min":{"1min":0.23000000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":126.657,"5min":126.657,"15min":126.657},"last":1.2969999999999999},{"interface":"front","average":{"1min":7.2990000000000004,"5min":2.6539999999999999,"15min":2.4289999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":126.697,"5min":126.697,"15min":126.697},"last":0.63300000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":5.3200000000000003,"15min":5.351},"min":{"1min":0.25800000000000001,"5min":0.22,"15min":0.22},"max":{"1min":6.7279999999999998,"5min":150.16999999999999,"15min":171.68299999999999},"last":1.27},{"interface":"front","average":{"1min":0.91700000000000004,"5min":4.8120000000000003,"15min":5.3470000000000004},"min":{"1min":0.27200000000000002,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":3.782,"5min":150.12700000000001,"15min":171.64099999999999},"last":0.879}]},{"osd":2,"last update":"Sat Sep 23 20:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":3.726,"15min":4.3490000000000002},"min":{"1min":0.29999999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.393000000000001,"5min":120.297,"15min":240.57300000000001},"last":1.202},{"interface":"front","average":{"1min":1.8839999999999999,"5min":3.7869999999999999,"15min":4.4169999999999998},"min":{"1min":0.33100000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.372999999999999,"5min":120.325,"15min":240.464},"last":1.236}]},{"osd":3,"last update":"Sat Sep 23 20:16:51 2023","interfaces":[{"interface":"back","average":{"1min":15.287000000000001,"5min":9.1349999999999998,"15min":6.4210000000000003},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":130.46899999999999,"5min":195.84700000000001,"15min":201.50899999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":15.641999999999999,"5min":9.0640000000000001,"15min":6.1699999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":130.63300000000001,"5min":195.87899999999999,"15min":201.58099999999999},"last":0.80400000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.0629999999999999,"15min":1.3879999999999999},"min":{"1min":0.28499999999999998,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":4.8159999999999998,"5min":8,"15min":191.78899999999999},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.67400000000000004,"15min":1.2150000000000001},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":3.371,"5min":3.371,"15min":191.821},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.2030000000000001,"15min":0.84199999999999997},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":3.0209999999999999,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.72099999999999997,"15min":0.60399999999999998},"min":{"1min":0.35099999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":2.9830000000000001,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":0.77000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":1.6240000000000001,"15min":0.92800000000000005},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":6.8620000000000001,"5min":11.725,"15min":11.725},"last":1.599},{"interface":"front","average":{"1min":1.087,"5min":1.387,"15min":0.88500000000000001},"min":{"1min":0.40600000000000003,"5min":0.32800000000000001,"15min":0.26300000000000001},"max":{"1min":3.661,"5min":11.041,"15min":11.041},"last":1.784}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":767295488,"data_stored":767295488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":791543808,"data_stored":791543808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":701628416,"data_stored":701628416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":685375488,"data_stored":685375488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":441974784,"data_stored":441974784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":414318592,"data_stored":414318592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:17:21.317 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T20:17:21.413 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:21.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:21.521 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:21.678 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:21.726 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:21.812 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:21.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:21.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:21.926 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:21.960 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:22.029 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:22.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:22.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:22.134 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:22.200 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:22.218 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 15 16 32902 32886 136.953 89.0625 0.0171262 0.00729692 2023-09-23T20:17:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:22 smithi144 ceph-mon[194833]: pgmap v1349: 109 pgs: 109 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T20:17:22.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:22.400 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:22 smithi135 ceph-mon[213871]: pgmap v1349: 109 pgs: 109 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T20:17:22.400 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:22.400 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:22 smithi135 ceph-mon[221113]: pgmap v1349: 109 pgs: 109 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T20:17:22.400 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:22.408 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:22.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:22.505 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:22.517 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:22.548 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:22.617 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:22.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:22.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:22.725 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:22.798 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:23.179 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:23 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3625608371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:17:23.179 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:23 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:17:23.181 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:17:23.218 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 16 16 34142 34126 133.238 77.5 0.0020208 0.00739613 2023-09-23T20:17:23.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:23 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3625608371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:17:23.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:23 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:17:23.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:23 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3625608371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:17:23.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:23 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:17:23.869 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:23.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:23.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:23.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:17:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:17:23] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-09-23T20:17:23.977 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:24.020 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:24.101 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:24.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:24.208 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:24.218 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 17 16 35708 35692 131.158 97.875 0.00219143 0.00756426 2023-09-23T20:17:24.235 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:24.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:24 smithi144 ceph-mon[194833]: pgmap v1350: 109 pgs: 109 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T20:17:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:24 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:24 smithi144 ceph-mon[194833]: failed to encode map e485 with expected crc 2023-09-23T20:17:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:24 smithi144 ceph-mon[194833]: failed to encode map e485 with expected crc 2023-09-23T20:17:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:24 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:17:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:24 smithi144 ceph-mon[194833]: osdmap e485: 8 total, 8 up, 6 in 2023-09-23T20:17:24.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[213871]: pgmap v1350: 109 pgs: 109 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T20:17:24.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:24.475 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[213871]: failed to encode map e485 with expected crc 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[213871]: failed to encode map e485 with expected crc 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[213871]: osdmap e485: 8 total, 8 up, 6 in 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[221113]: pgmap v1350: 109 pgs: 109 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[221113]: failed to encode map e485 with expected crc 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[221113]: failed to encode map e485 with expected crc 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:17:24.476 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:24 smithi135 ceph-mon[221113]: osdmap e485: 8 total, 8 up, 6 in 2023-09-23T20:17:24.693 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:24.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:24.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:24.799 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:24.835 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:24.879 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:25.219 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 18 16 37303 37287 129.407 99.6875 0.0128019 0.00772338 2023-09-23T20:17:25.325 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:25.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:25.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:25.432 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:25.505 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:25.625 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:17:25.660 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:17:25.704 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:17:25.742 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:17:25.766 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:17:25.784 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:17:25.802 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:17:25.820 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:17:25.821 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:17:25.821 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:17:25.821 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:17:25.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:17:25.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:17:25.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:17:25.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:17:25.822 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:17:26.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:26 smithi135 ceph-mon[213871]: pgmap v1352: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:17:26.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3626161087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:26.219 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 19 16 38614 38598 126.909 81.9375 0.00712711 0.00787444 2023-09-23T20:17:26.440 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:26.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:26.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:26.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:26 smithi135 ceph-mon[221113]: pgmap v1352: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:17:26.450 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3626161087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:26.548 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:26.607 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:26 smithi144 ceph-mon[194833]: pgmap v1352: 109 pgs: 109 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:17:26.607 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3626161087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:26.610 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:26.689 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:26.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:26.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:26.797 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:26.826 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:17:26.849 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:26.901 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:26.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:26.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:27.006 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:27.032 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:27.220 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:17:27.220713+0000 min lat: 0.00112211 max lat: 0.306187 avg lat: 0.00799241 2023-09-23T20:17:27.220 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:17:27.220 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 20 16 40038 40022 125.01 89 0.01583 0.00799241 2023-09-23T20:17:27.432 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:27.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:27.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:27.540 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:27.626 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:27.765 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:27.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:27.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:27.876 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:27.900 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:27.942 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:27.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:28.047 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:28.073 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:28.114 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:28.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:28.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:28.220 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 21 16 41157 41141 122.388 69.9375 0.0100522 0.00816504 2023-09-23T20:17:28.220 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:28.281 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:28 smithi135 ceph-mon[213871]: pgmap v1353: 109 pgs: 109 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-09-23T20:17:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:28 smithi135 ceph-mon[221113]: pgmap v1353: 109 pgs: 109 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-09-23T20:17:28.505 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:28.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:28.559 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:28.618 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:28 smithi144 ceph-mon[194833]: pgmap v1353: 109 pgs: 109 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-09-23T20:17:28.712 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:29.062 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:29.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:29.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:29.167 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:29.217 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:17:29.217 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:17:29.217 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:17:29.218 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:17:29.218 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:17:29.220 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 22 16 42504 42488 120.652 84.1875 0.0155714 0.0082821 2023-09-23T20:17:29.242 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:29.589 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:17:30.220 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 23 16 43860 43844 119.091 84.75 0.00262504 0.00837769 2023-09-23T20:17:30.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:30 smithi135 ceph-mon[221113]: pgmap v1354: 109 pgs: 109 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:17:30.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:30 smithi135 ceph-mon[213871]: pgmap v1354: 109 pgs: 109 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:17:30.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:30 smithi144 ceph-mon[194833]: pgmap v1354: 109 pgs: 109 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:17:30.914 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:30.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:31.020 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:31.058 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:31.136 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:31.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:31.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:31.189 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:17:31.220 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 24 16 44949 44933 116.963 68.0625 0.00792726 0.00854523 2023-09-23T20:17:31.244 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:31.365 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:31.453 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:31.554 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:31.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:31.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:31.660 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:31.730 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:31.954 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:31.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:31.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:32.060 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:32.168 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:32.223 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 25 16 46131 46115 115.229 73.875 0.00995051 0.00867093 2023-09-23T20:17:32.244 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 1 pgp_num to 21 2023-09-23T20:17:32.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:32 smithi135 ceph-mon[221113]: pgmap v1355: 109 pgs: 109 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:17:32.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3847638138' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:17:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:32 smithi135 ceph-mon[213871]: pgmap v1355: 109 pgs: 109 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:17:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3847638138' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:17:32.495 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:32.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:32.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:32.601 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:32 smithi144 ceph-mon[194833]: pgmap v1355: 109 pgs: 109 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:17:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3847638138' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:17:32.657 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:32.803 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:32.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:32.908 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:32.957 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:33.059 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:33.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:33.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:33.169 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:33.195 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:33.233 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 26 16 47589 47573 114.299 91.125 0.0104486 0.00874388 2023-09-23T20:17:33.246 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:17:33.296 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:33.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:33.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:33.405 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:33.456 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[221113]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3847638138' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:17:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[221113]: osdmap e486: 8 total, 8 up, 6 in 2023-09-23T20:17:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[221113]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[221113]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2555773762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:33.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[221113]: pgmap v1357: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:17:33.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[213871]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3847638138' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:17:33.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[213871]: osdmap e486: 8 total, 8 up, 6 in 2023-09-23T20:17:33.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[213871]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[213871]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2555773762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:33.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:33.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:33 smithi135 ceph-mon[213871]: pgmap v1357: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:17:33.531 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:33.536 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:33 smithi144 ceph-mon[194833]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.536 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:33 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3847638138' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:17:33.536 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:33 smithi144 ceph-mon[194833]: osdmap e486: 8 total, 8 up, 6 in 2023-09-23T20:17:33.536 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:33 smithi144 ceph-mon[194833]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.536 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:33 smithi144 ceph-mon[194833]: failed to encode map e486 with expected crc 2023-09-23T20:17:33.536 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:33 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2555773762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:33.536 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:33 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:33.536 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:33 smithi144 ceph-mon[194833]: pgmap v1357: 109 pgs: 109 active+clean; 2.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:17:33.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:33.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:33.638 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:33.682 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:33.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:17:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:17:33] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-09-23T20:17:34.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 27 16 48825 48809 112.92 77.25 0.0141837 0.00885031 2023-09-23T20:17:34.346 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:34.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:34.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:34.454 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:34.479 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:17:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:17:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: osdmap e487: 8 total, 8 up, 6 in 2023-09-23T20:17:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:34 smithi144 ceph-mon[194833]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:17:34.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:17:34.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: osdmap e487: 8 total, 8 up, 6 in 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[213871]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:34.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: osdmap e487: 8 total, 8 up, 6 in 2023-09-23T20:17:34.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:34 smithi135 ceph-mon[221113]: failed to encode map e487 with expected crc 2023-09-23T20:17:34.959 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:34.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:34.998 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:35.071 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:35.106 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:35.193 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:35.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:35.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:35.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 28 16 49960 49944 111.421 70.9375 0.0111127 0.00897105 2023-09-23T20:17:35.300 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:35.347 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:35 smithi135 ceph-mon[221113]: pgmap v1359: 109 pgs: 109 active+clean; 2.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:17:35.349 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:35 smithi135 ceph-mon[213871]: pgmap v1359: 109 pgs: 109 active+clean; 2.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:17:35.380 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:35 smithi144 ceph-mon[194833]: pgmap v1359: 109 pgs: 109 active+clean; 2.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:17:35.547 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:35.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:35.652 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:35.696 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:17:35.737 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:35.743 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:17:35.802 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:17:35.845 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:17:35.848 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:35.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:35.885 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:17:35.925 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:17:35.956 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:35.968 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:17:36.010 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:17:36.036 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:17:36.036 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:17:36.037 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:17:36.037 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:17:36.037 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:17:36.037 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:17:36.037 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:17:36.037 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:17:36.039 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:36.113 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:36.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:36.216 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:36.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 29 16 51085 51069 110.004 70.3125 0.0049457 0.00904496 2023-09-23T20:17:36.243 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:36.284 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:36.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:36.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:36.391 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:36.433 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/380316274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:36.433 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/380316274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:36.440 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:36.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/380316274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:36.842 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:36.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:36.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:36.957 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:36.992 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:37.035 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:37.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:37.042 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:17:37.140 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:37.181 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:37.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 30 16 52186 52170 108.631 68.8125 0.0136418 0.0091999 2023-09-23T20:17:37.248 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:37.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:37.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:37.359 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:37.407 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:37.481 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:37.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:37.592 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:37.626 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:37.694 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:37.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:37.803 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:37.838 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:37.914 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:37.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:37.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:38.010 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:38.021 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:38.039 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:17:38.039 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:17:38.131 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:38.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 31 16 53488 53472 107.752 81.375 0.00915322 0.0092758 2023-09-23T20:17:38.317 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:38.326 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:38 smithi135 ceph-mon[221113]: pgmap v1360: 109 pgs: 109 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:17:38.326 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:38.326 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:38.327 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:38 smithi135 ceph-mon[213871]: pgmap v1360: 109 pgs: 109 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:17:38.327 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:38.327 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:38.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:38 smithi144 ceph-mon[194833]: pgmap v1360: 109 pgs: 109 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T20:17:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:38.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:38.429 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:38.478 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:38.577 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:38.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:38.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:38.685 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:38.729 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:38.810 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:38.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:38.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:38.915 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:38.941 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:38.990 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:38.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:39.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:39.101 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:39.159 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:39.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 32 16 54690 54674 106.732 75.125 0.0150844 0.00936459 2023-09-23T20:17:39.281 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:17:39.283 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:39.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:39.392 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:39.499 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:39.644 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:17:39.714 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:39.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:39.803 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7] out_osds: [6, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:17:39.803 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:17:39.803 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-09-23T20:17:39.804 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd in 1 2023-09-23T20:17:39.820 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:39.862 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:39.914 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:39.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:39.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:40.020 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:40.045 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:40.086 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:40.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:40.191 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:40.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 33 16 55624 55608 105.267 58.375 0.01489 0.0094949 2023-09-23T20:17:40.240 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:40.312 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:40.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:40.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:40.421 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:40.479 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:40 smithi135 ceph-mon[221113]: pgmap v1361: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T20:17:40.490 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:40 smithi135 ceph-mon[213871]: pgmap v1361: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T20:17:40.495 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:40.625 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:40.628 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:40 smithi144 ceph-mon[194833]: pgmap v1361: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T20:17:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:40.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:40.686 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:40.744 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:40.806 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:40.846 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:40.915 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:40.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:41.021 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:41.104 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:41.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 34 16 56746 56730 104.234 70.125 0.0103711 0.00958857 2023-09-23T20:17:41.579 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:41.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:41.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:41.685 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:41.807 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:41.951 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:41.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:41.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:42.063 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:42.135 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:42.209 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:17:42.227 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 35 16 57937 57921 103.383 74.4375 0.00792231 0.00966765 2023-09-23T20:17:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:42 smithi135 ceph-mon[221113]: pgmap v1362: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:17:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3742872331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:42 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:42 smithi135 ceph-mon[213871]: pgmap v1362: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:17:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3742872331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:42 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:42 smithi144 ceph-mon[194833]: pgmap v1362: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:17:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3742872331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:42 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:17:43.229 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 36 16 58943 58927 102.258 62.875 0.00804071 0.00976817 2023-09-23T20:17:43.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:43.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[221113]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[221113]: osdmap e488: 8 total, 8 up, 6 in 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[221113]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[221113]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[221113]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[221113]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[213871]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:17:43.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[213871]: osdmap e488: 8 total, 8 up, 6 in 2023-09-23T20:17:43.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[213871]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[213871]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[213871]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.371 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:43 smithi135 ceph-mon[213871]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:43 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:43.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:43 smithi144 ceph-mon[194833]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:43 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:17:43.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:43 smithi144 ceph-mon[194833]: osdmap e488: 8 total, 8 up, 6 in 2023-09-23T20:17:43.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:43 smithi144 ceph-mon[194833]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:43 smithi144 ceph-mon[194833]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:43 smithi144 ceph-mon[194833]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:43 smithi144 ceph-mon[194833]: failed to encode map e488 with expected crc 2023-09-23T20:17:43.657 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:43.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:43.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:43.764 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:43.791 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:43.833 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:43.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:43.942 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:43.967 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:17:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:17:43] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.43.0" 2023-09-23T20:17:43.970 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:44.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:44 smithi135 ceph-mon[213871]: pgmap v1364: 109 pgs: 109 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:17:44.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:44 smithi135 ceph-mon[221113]: pgmap v1364: 109 pgs: 109 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:17:44.226 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 37 16 59973 59957 101.234 64.375 0.0106286 0.00987368 2023-09-23T20:17:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:44 smithi144 ceph-mon[194833]: pgmap v1364: 109 pgs: 109 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:17:45.056 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:45.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:45.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:45.165 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:45.201 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:45.228 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 38 16 61002 60986 100.26 64.3125 0.0176807 0.00996768 2023-09-23T20:17:45.268 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:45.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:45.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:45.373 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:17:45.448 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:45 smithi135 ceph-mon[221113]: pgmap v1365: 109 pgs: 109 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:17:45.449 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:45 smithi135 ceph-mon[213871]: pgmap v1365: 109 pgs: 109 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:17:45.455 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:45.617 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:17:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:45 smithi144 ceph-mon[194833]: pgmap v1365: 109 pgs: 109 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:17:45.650 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:17:45.654 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:17:45.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:45.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:17:45.698 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:17:45.735 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:17:45.766 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:45.777 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:17:45.825 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:17:45.857 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:45.872 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:17:45.908 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:17:45.933 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:17:45.934 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:17:45.934 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:17:45.934 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:17:45.934 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:17:45.934 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:17:45.934 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:17:45.934 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:17:46.179 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:46 smithi135 conmon[213845]: 2023-09-23T20:17:46.171+0000 7f9708921700 -1 mon.a@0(leader).osd e488 definitely_dead 0 2023-09-23T20:17:46.182 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:17:46.183 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:17:46.183 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T20:17:46.183 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:17:46.215 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:46.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:46.228 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 39 16 62049 62033 99.3673 65.4375 0.0174163 0.0100585 2023-09-23T20:17:46.245 INFO:teuthology.orchestra.run.smithi135.stderr:marked in osd.1. 2023-09-23T20:17:46.324 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:46.365 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:46.414 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:46.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:46.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2033650798' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1272787354' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:17:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:17:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2033650798' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1272787354' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:17:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:17:46.518 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:46.541 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2033650798' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:46.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1272787354' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:17:46.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T20:17:46.544 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:46.585 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:46.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:46.693 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:46.720 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:46.761 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:46.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:46.866 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:46.892 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:46.934 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:46.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:46.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:17:47.040 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:47.066 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:47.108 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:47.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:47.217 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:47.228 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:17:47.228871+0000 min lat: 0.00112211 max lat: 0.309352 avg lat: 0.0100655 2023-09-23T20:17:47.228 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:17:47.228 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 40 16 62171 62155 97.0745 7.625 0.00278472 0.0100655 2023-09-23T20:17:47.243 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:47.301 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:47.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:47.354 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[221113]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.354 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:17:47.354 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[221113]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T20:17:47.354 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[221113]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.354 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[221113]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.354 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[221113]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.355 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[221113]: pgmap v1367: 109 pgs: 4 peering, 105 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 61 MiB/s wr, 970 op/s 2023-09-23T20:17:47.355 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[213871]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.355 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:17:47.355 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[213871]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T20:17:47.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[213871]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[213871]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[213871]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.356 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:47 smithi135 ceph-mon[213871]: pgmap v1367: 109 pgs: 4 peering, 105 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 61 MiB/s wr, 970 op/s 2023-09-23T20:17:47.410 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:47.436 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:47.478 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:47.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:47.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:47.583 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:47.609 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:47 smithi144 ceph-mon[194833]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:47 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T20:17:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:47 smithi144 ceph-mon[194833]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T20:17:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:47 smithi144 ceph-mon[194833]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:47 smithi144 ceph-mon[194833]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:47 smithi144 ceph-mon[194833]: failed to encode map e489 with expected crc 2023-09-23T20:17:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:47 smithi144 ceph-mon[194833]: pgmap v1367: 109 pgs: 4 peering, 105 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 61 MiB/s wr, 970 op/s 2023-09-23T20:17:47.923 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:17:47.970 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:47.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:48.074 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:48.103 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:48.149 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:48.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:48.228 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 41 16 62171 62155 94.7076 0 - 0.0100655 2023-09-23T20:17:48.256 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:48.303 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:48 smithi144 ceph-mon[194833]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:48 smithi144 ceph-mon[194833]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:48 smithi144 ceph-mon[194833]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:48 smithi144 ceph-mon[194833]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:48 smithi144 ceph-mon[194833]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T20:17:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:48 smithi144 ceph-mon[194833]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:48 smithi144 ceph-mon[194833]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:48 smithi144 ceph-mon[194833]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.520 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:48.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:48.532 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[221113]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.532 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[221113]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.532 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[221113]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.532 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[221113]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.532 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[221113]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T20:17:48.532 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[221113]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[221113]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[221113]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[213871]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[213871]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[213871]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[213871]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[213871]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[213871]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[213871]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.533 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:48 smithi135 ceph-mon[213871]: failed to encode map e490 with expected crc 2023-09-23T20:17:48.547 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-09-23T20:17:48.630 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:48.680 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:48.710 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:48.764 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:48.813 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:48.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:48.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:48.924 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:49.019 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:49.229 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 42 16 62779 62763 93.3559 19 0.00845602 0.0107036 2023-09-23T20:17:49.235 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:49.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:49.350 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:49.401 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[213871]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.401 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[213871]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T20:17:49.401 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[213871]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.401 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[213871]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.401 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[213871]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[213871]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[213871]: pgmap v1370: 109 pgs: 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 93 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 9.3 KiB/s rd, 71 MiB/s wr, 1.14k op/s 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[221113]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[221113]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[221113]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[221113]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[221113]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[221113]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.402 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:49 smithi135 ceph-mon[221113]: pgmap v1370: 109 pgs: 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 93 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 9.3 KiB/s rd, 71 MiB/s wr, 1.14k op/s 2023-09-23T20:17:49.444 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:49.623 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:49 smithi144 ceph-mon[194833]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:49 smithi144 ceph-mon[194833]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T20:17:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:49 smithi144 ceph-mon[194833]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:49 smithi144 ceph-mon[194833]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:49 smithi144 ceph-mon[194833]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:49 smithi144 ceph-mon[194833]: failed to encode map e491 with expected crc 2023-09-23T20:17:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:49 smithi144 ceph-mon[194833]: pgmap v1370: 109 pgs: 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 93 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 9.3 KiB/s rd, 71 MiB/s wr, 1.14k op/s 2023-09-23T20:17:49.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:49.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:49.733 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:49.811 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:49.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:17:50.229 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 43 16 63614 63598 92.3987 52.1875 0.00666362 0.0108145 2023-09-23T20:17:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:50 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T20:17:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:50 smithi144 ceph-mon[194833]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:50 smithi144 ceph-mon[194833]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:50 smithi144 ceph-mon[194833]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:50 smithi144 ceph-mon[194833]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:50 smithi144 ceph-mon[194833]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T20:17:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:50 smithi144 ceph-mon[194833]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T20:17:50.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[213871]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[213871]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[213871]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[213871]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[213871]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T20:17:50.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[213871]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T20:17:50.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[221113]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[221113]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[221113]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[221113]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[221113]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T20:17:50.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:50 smithi135 ceph-mon[221113]: failed to encode map e492 with expected crc 2023-09-23T20:17:50.788 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:50.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:50.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:50.903 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:50.930 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:50.971 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:50.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:50.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:51.078 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:51.142 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:51.232 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 44 16 64257 64241 91.2072 40.1875 0.0190699 0.0109589 2023-09-23T20:17:51.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[213871]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[213871]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[213871]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T20:17:51.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[213871]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[213871]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[213871]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[213871]: pgmap v1373: 109 pgs: 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 93 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 28 MiB/s wr, 454 op/s 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[221113]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[221113]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[221113]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[221113]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[221113]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[221113]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:51 smithi135 ceph-mon[221113]: pgmap v1373: 109 pgs: 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 93 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 28 MiB/s wr, 454 op/s 2023-09-23T20:17:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:51 smithi144 ceph-mon[194833]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:51 smithi144 ceph-mon[194833]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:51 smithi144 ceph-mon[194833]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T20:17:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:51 smithi144 ceph-mon[194833]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:51 smithi144 ceph-mon[194833]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:51 smithi144 ceph-mon[194833]: failed to encode map e493 with expected crc 2023-09-23T20:17:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:51 smithi144 ceph-mon[194833]: pgmap v1373: 109 pgs: 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 93 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 28 MiB/s wr, 454 op/s 2023-09-23T20:17:51.653 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:51.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:51.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:51.738 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:51.768 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:51.862 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:52.117 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:52.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:52.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:52.235 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 45 16 65015 64999 90.2281 47.375 0.0151102 0.0110777 2023-09-23T20:17:52.235 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:52.309 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:52.590 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:52.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:52.612 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:17:52.612 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:17:52.612 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:17:52.612 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:17:52.612 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:17:52.700 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:52.729 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:52.783 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:52.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:52.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:52.890 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:52.921 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:52.972 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:52.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:52.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:53.082 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:17:53.117 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:53.180 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:17:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:53.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:17:53.236 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 46 16 65786 65770 89.3122 48.1875 0.0155783 0.0111906 2023-09-23T20:17:53.287 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:17:53.375 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:53 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:53.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2394848739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:53.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:53.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:53 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2394848739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:53 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2394848739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:17:53.548 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:17:53.548 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:17:53.548 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T20:17:53.549 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T20:17:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:17:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:17:53] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.43.0" 2023-09-23T20:17:53.986 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:17:54.236 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 47 16 66429 66413 88.2673 40.1875 0.0632874 0.0113246 2023-09-23T20:17:54.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:54 smithi135 ceph-mon[213871]: pgmap v1374: 109 pgs: 1 active+undersized+remapped, 8 remapped+peering, 100 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.5 KiB/s rd, 34 MiB/s wr, 546 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:17:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:54 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:54 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:54 smithi135 ceph-mon[213871]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T20:17:54.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:54 smithi135 ceph-mon[221113]: pgmap v1374: 109 pgs: 1 active+undersized+remapped, 8 remapped+peering, 100 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.5 KiB/s rd, 34 MiB/s wr, 546 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:17:54.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:54 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:54.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:54 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:54.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:54 smithi135 ceph-mon[221113]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T20:17:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:54 smithi144 ceph-mon[194833]: pgmap v1374: 109 pgs: 1 active+undersized+remapped, 8 remapped+peering, 100 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.5 KiB/s rd, 34 MiB/s wr, 546 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:17:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:54 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:17:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:54 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:17:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:54 smithi144 ceph-mon[194833]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T20:17:55.013 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:17:55.034 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:17:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:55.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:17:55.142 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:55.233 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:55.237 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 48 16 67208 67192 87.443 48.6875 0.00394307 0.0114301 2023-09-23T20:17:55.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[213871]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[213871]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[213871]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[213871]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[213871]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[213871]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[221113]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[221113]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[221113]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[221113]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[221113]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:55 smithi135 ceph-mon[221113]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:55 smithi144 ceph-mon[194833]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:55 smithi144 ceph-mon[194833]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:55 smithi144 ceph-mon[194833]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:55 smithi144 ceph-mon[194833]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:55 smithi144 ceph-mon[194833]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:55 smithi144 ceph-mon[194833]: failed to encode map e494 with expected crc 2023-09-23T20:17:55.935 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:55.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:56.047 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:56.172 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:56.217 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:56 smithi135 ceph-mon[213871]: pgmap v1376: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 106 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 516 op/s; 7752/195735 objects degraded (3.960%); 3738/195735 objects misplaced (1.910%); 2.0 MiB/s, 26 objects/s recovering 2023-09-23T20:17:56.218 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:56 smithi135 ceph-mon[213871]: Health check failed: Degraded data redundancy: 7752/195735 objects degraded (3.960%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:17:56.218 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:56 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T20:17:56.218 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:56 smithi135 ceph-mon[221113]: pgmap v1376: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 106 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 516 op/s; 7752/195735 objects degraded (3.960%); 3738/195735 objects misplaced (1.910%); 2.0 MiB/s, 26 objects/s recovering 2023-09-23T20:17:56.218 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:56 smithi135 ceph-mon[221113]: Health check failed: Degraded data redundancy: 7752/195735 objects degraded (3.960%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:17:56.218 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:56 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T20:17:56.241 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 49 16 67913 67897 86.5562 44.0625 0.0257103 0.0115471 2023-09-23T20:17:56.338 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:56.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:56.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:56.442 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:17:56.475 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:56 smithi144 ceph-mon[194833]: pgmap v1376: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 106 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 516 op/s; 7752/195735 objects degraded (3.960%); 3738/195735 objects misplaced (1.910%); 2.0 MiB/s, 26 objects/s recovering 2023-09-23T20:17:56.475 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:56 smithi144 ceph-mon[194833]: Health check failed: Degraded data redundancy: 7752/195735 objects degraded (3.960%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:17:56.475 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:56 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T20:17:56.475 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:56.497 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:56.501 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:56.533 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:17:56.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:56.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:17:56.638 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:56.669 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:56.682 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:17:56.725 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:56.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:56.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:56.738 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:17:56.789 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:17:56.831 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:17:56.831 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:56.878 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:17:56.912 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:17:56.944 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:56.954 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:17:56.994 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:17:57.006 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:17:57.006 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:17:57.006 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:17:57.007 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:17:57.007 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:17:57.007 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:17:57.007 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:17:57.007 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:17:57.237 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 50 16 68635 68619 85.7279 45.125 0.0355036 0.0116578 2023-09-23T20:17:57.267 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:17:57.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:57.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:17:57.348 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:57 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4225297814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:57.348 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:57 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4225297814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:57.385 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:57.418 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:57.479 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:57.484 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:57 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4225297814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:17:57.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:57.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:57.586 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:17:57.663 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:58.100 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:17:58.237 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 51 16 69408 69392 84.9945 48.3125 0.108019 0.0117464 2023-09-23T20:17:58.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:58 smithi135 ceph-mon[221113]: pgmap v1377: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 106 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 7752/200496 objects degraded (3.866%); 3738/200496 objects misplaced (1.864%); 1.5 MiB/s, 20 objects/s recovering 2023-09-23T20:17:58.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:58.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:58 smithi135 ceph-mon[213871]: pgmap v1377: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 106 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 7752/200496 objects degraded (3.866%); 3738/200496 objects misplaced (1.864%); 1.5 MiB/s, 20 objects/s recovering 2023-09-23T20:17:58.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:58 smithi144 ceph-mon[194833]: pgmap v1377: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 106 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 7752/200496 objects degraded (3.866%); 3738/200496 objects misplaced (1.864%); 1.5 MiB/s, 20 objects/s recovering 2023-09-23T20:17:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:17:58.906 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:17:58.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:58.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:17:59.018 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:17:59.043 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:59.090 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:17:59.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:59.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:17:59.198 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:17:59.229 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:59.238 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 52 16 70066 70050 84.1491 41.125 0.0139414 0.0118783 2023-09-23T20:17:59.282 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:17:59.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:17:59.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:17:59.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:17:59 smithi144 ceph-mon[194833]: pgmap v1378: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 759 op/s; 7752/206574 objects degraded (3.753%); 3574/206574 objects misplaced (1.730%); 2.5 MiB/s, 36 objects/s recovering 2023-09-23T20:17:59.389 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:17:59.465 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:17:59.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:17:59 smithi135 ceph-mon[213871]: pgmap v1378: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 759 op/s; 7752/206574 objects degraded (3.753%); 3574/206574 objects misplaced (1.730%); 2.5 MiB/s, 36 objects/s recovering 2023-09-23T20:17:59.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:17:59 smithi135 ceph-mon[221113]: pgmap v1378: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 759 op/s; 7752/206574 objects degraded (3.753%); 3574/206574 objects misplaced (1.730%); 2.5 MiB/s, 36 objects/s recovering 2023-09-23T20:18:00.239 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 53 16 70859 70843 83.4967 49.5625 0.0191728 0.0119708 2023-09-23T20:18:00.242 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:00.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:00.354 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:00.395 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:00.444 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:00.759 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:00.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:00.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:00.866 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:00.978 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:01.120 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:01.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:01.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:01.228 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:01.239 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 54 16 71584 71568 82.7899 45.3125 0.0285657 0.0120722 2023-09-23T20:18:01.255 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:01.272 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:18:01.272 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:18:01.308 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:01.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:01.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:01.413 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:01.452 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:18:01.528 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:01.730 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:01 smithi135 conmon[213845]: 2023-09-23T20:18:01.719+0000 7f9708921700 -1 mon.a@0(leader).osd e494 definitely_dead 0 2023-09-23T20:18:02.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:02 smithi135 ceph-mon[213871]: pgmap v1379: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 7752/206574 objects degraded (3.753%); 3574/206574 objects misplaced (1.730%); 2.2 MiB/s, 32 objects/s recovering 2023-09-23T20:18:02.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/988784695' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:18:02.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:02 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:18:02.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:02 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:02.220 INFO:teuthology.orchestra.run.smithi135.stderr:marked in osd.6. 2023-09-23T20:18:02.239 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 55 16 72248 72232 82.0394 41.5 0.0421182 0.0121803 2023-09-23T20:18:02.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:02 smithi135 ceph-mon[221113]: pgmap v1379: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 7752/206574 objects degraded (3.753%); 3574/206574 objects misplaced (1.730%); 2.2 MiB/s, 32 objects/s recovering 2023-09-23T20:18:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/988784695' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:18:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:02 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:18:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:02 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:02 smithi144 ceph-mon[194833]: pgmap v1379: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 7752/206574 objects degraded (3.753%); 3574/206574 objects misplaced (1.730%); 2.2 MiB/s, 32 objects/s recovering 2023-09-23T20:18:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/988784695' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:18:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:02 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:18:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:02 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:03.239 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 56 16 72369 72353 80.71 7.5625 0.00363513 0.012184 2023-09-23T20:18:03.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[221113]: failed to encode map e495 with expected crc 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[221113]: failed to encode map e495 with expected crc 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[221113]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7752/206574 objects degraded (3.753%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[221113]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[213871]: failed to encode map e495 with expected crc 2023-09-23T20:18:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[213871]: failed to encode map e495 with expected crc 2023-09-23T20:18:03.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:18:03.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[213871]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T20:18:03.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:03.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:03.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7752/206574 objects degraded (3.753%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:03.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:03 smithi135 ceph-mon[213871]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T20:18:03.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:03 smithi144 ceph-mon[194833]: failed to encode map e495 with expected crc 2023-09-23T20:18:03.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:03 smithi144 ceph-mon[194833]: failed to encode map e495 with expected crc 2023-09-23T20:18:03.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:03 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:18:03.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:03 smithi144 ceph-mon[194833]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T20:18:03.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:03 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:03.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:03 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:03.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:03 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7752/206574 objects degraded (3.753%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:03.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:03 smithi144 ceph-mon[194833]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T20:18:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:18:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:18:03] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.43.0" 2023-09-23T20:18:04.241 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 57 16 72374 72358 79.297 0.3125 1.98314 0.0122652 2023-09-23T20:18:04.327 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:04.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:04.439 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:04.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: pgmap v1382: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 698 op/s; 7752/212523 objects degraded (3.648%); 3574/212523 objects misplaced (1.682%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:18:04.462 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:04.463 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:04.463 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[213871]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T20:18:04.463 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.463 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.463 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.463 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.463 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: pgmap v1382: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 698 op/s; 7752/212523 objects degraded (3.648%); 3574/212523 objects misplaced (1.682%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:18:04.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:04.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:04.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:04 smithi135 ceph-mon[221113]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T20:18:04.465 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T20:18:04.528 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: failed to encode map e496 with expected crc 2023-09-23T20:18:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: pgmap v1382: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 698 op/s; 7752/212523 objects degraded (3.648%); 3574/212523 objects misplaced (1.682%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:18:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:04.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:04 smithi144 ceph-mon[194833]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T20:18:04.671 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:18:04.673 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:04.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:04.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:04.778 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:18:04.818 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:04.886 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:18:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:04.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:18:04.991 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:18:05.036 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:05.151 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:18:05.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:05.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:18:05.171 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:05.212 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:05 smithi135 ceph-mon[213871]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.212 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:05 smithi135 ceph-mon[213871]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.212 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:05 smithi135 ceph-mon[213871]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.212 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:05 smithi135 ceph-mon[213871]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.241 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 58 16 72971 72955 78.5734 37.3125 0.0382051 0.0127188 2023-09-23T20:18:05.258 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:05.384 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:05.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:05 smithi135 ceph-mon[221113]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:05 smithi135 ceph-mon[221113]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.477 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:05 smithi135 ceph-mon[221113]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.478 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:05 smithi135 ceph-mon[221113]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.572 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:05.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:05.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:05 smithi144 ceph-mon[194833]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:05 smithi144 ceph-mon[194833]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:05 smithi144 ceph-mon[194833]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:05 smithi144 ceph-mon[194833]: failed to encode map e497 with expected crc 2023-09-23T20:18:05.694 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:05.803 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:05.843 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:06.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[213871]: pgmap v1384: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 537 op/s; 7758/216258 objects degraded (3.587%); 3364/216258 objects misplaced (1.556%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:18:06.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:06.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:06.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1502777797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:06.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:06.241 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 59 16 73563 73547 77.869 37 0.0161462 0.0128363 2023-09-23T20:18:06.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[221113]: pgmap v1384: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 537 op/s; 7758/216258 objects degraded (3.587%); 3364/216258 objects misplaced (1.556%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:18:06.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:06.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:06.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1502777797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:06.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:06 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:06 smithi144 ceph-mon[194833]: pgmap v1384: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 537 op/s; 7758/216258 objects degraded (3.587%); 3364/216258 objects misplaced (1.556%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:18:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:06 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:06 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1502777797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:06 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:06.711 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:18:07.241 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:18:07.242412+0000 min lat: 0.00112211 max lat: 2.19724 avg lat: 0.0129361 2023-09-23T20:18:07.242 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:18:07.242 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 60 16 74213 74197 77.2485 40.625 0.00531608 0.0129361 2023-09-23T20:18:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 494 op/s; 7758/218715 objects degraded (3.547%); 7755/218715 objects misplaced (3.546%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:18:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T20:18:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 494 op/s; 7758/218715 objects degraded (3.547%); 7755/218715 objects misplaced (3.546%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:18:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:07.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:07.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:18:07.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T20:18:07.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: failed to encode map e498 with expected crc 2023-09-23T20:18:07.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 494 op/s; 7758/218715 objects degraded (3.547%); 7755/218715 objects misplaced (3.546%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:18:07.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:07.973 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:08.243 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 61 16 74942 74926 76.7274 45.5625 0.00699506 0.0130254 2023-09-23T20:18:08.565 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:08.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:08.670 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:08.704 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:08.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:08 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7758/218715 objects degraded (3.547%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:08.972 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:08 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7758/218715 objects degraded (3.547%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:09.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:08 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7758/218715 objects degraded (3.547%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:09.248 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 62 16 75528 75512 76.0808 36.625 0.0341575 0.0131385 2023-09-23T20:18:09.465 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:18:09.466 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:18:09.466 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T20:18:09.466 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:18:09.466 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:18:09.466 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T20:18:09.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:09 smithi144 ceph-mon[194833]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 645 op/s; 12192/223281 objects degraded (5.460%); 7547/223281 objects misplaced (3.380%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:18:09.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:09 smithi135 ceph-mon[221113]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 645 op/s; 12192/223281 objects degraded (5.460%); 7547/223281 objects misplaced (3.380%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:18:09.982 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:09 smithi135 ceph-mon[213871]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 645 op/s; 12192/223281 objects degraded (5.460%); 7547/223281 objects misplaced (3.380%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:18:10.007 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:18:10.243 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 63 16 76293 76277 75.6322 47.8125 0.0376069 0.0132155 2023-09-23T20:18:10.549 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:10.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:10.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:10.670 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:10.719 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:10.776 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:10.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:10.880 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:10.907 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:10.976 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:10.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:10.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:11.079 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:11.179 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:11.243 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 64 16 76867 76851 75.0112 35.875 0.0312783 0.0133247 2023-09-23T20:18:11.913 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:11.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:12.024 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:12.130 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:12.180 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:18:12.181 INFO:teuthology.orchestra.run.smithi144.stdout: "success": "" 2023-09-23T20:18:12.181 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:18:12.245 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 65 16 77447 77431 74.413 36.25 0.0193964 0.013432 2023-09-23T20:18:12.266 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:12.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:12.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:12.373 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:12.414 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:12 smithi135 ceph-mon[221113]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 28 MiB/s wr, 447 op/s; 12192/223281 objects degraded (5.460%); 7547/223281 objects misplaced (3.380%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:18:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:12 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:12.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:12 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:12.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:12 smithi135 ceph-mon[213871]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 28 MiB/s wr, 447 op/s; 12192/223281 objects degraded (5.460%); 7547/223281 objects misplaced (3.380%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:18:12.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:12 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:12.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:12 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:12.574 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:12.576 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:12 smithi144 ceph-mon[194833]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 28 MiB/s wr, 447 op/s; 12192/223281 objects degraded (5.460%); 7547/223281 objects misplaced (3.380%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:18:12.577 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:12 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:12.577 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:12 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:12.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:12.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:12.678 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:12.712 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:12.756 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:18:12.804 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:18:12.835 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:18:12.851 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:18:12.883 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:18:12.898 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:12.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:12.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:12.929 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:18:12.974 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:18:13.003 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:18:13.031 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:13.037 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:18:13.084 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:18:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:13.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:18:13.092 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:18:13.130 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:18:13.131 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:18:13.131 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.131 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:77919 10.7 e498/479 10:ebd49c17:::benchmark_data_smithi135_602286_object77918:head v 498'4868, mlcod=498'4863)", 2023-09-23T20:18:13.131 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:13.098564+0000", 2023-09-23T20:18:13.131 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.027683313000000001, 2023-09-23T20:18:13.131 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.027714549000000002, 2023-09-23T20:18:13.131 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.098564+0000", 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.132 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.098564+0000", 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.098564+0000", 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.133 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.098588+0000", 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4366999999999999e-05 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.098594+0000", 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1469999999999998e-06 2023-09-23T20:18:13.134 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.098599+0000", 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1250000000000001e-06 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.110753+0000", 2023-09-23T20:18:13.135 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012153703 2023-09-23T20:18:13.136 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.136 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.136 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:13.136 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.110773+0000", 2023-09-23T20:18:13.136 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0177e-05 2023-09-23T20:18:13.136 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.136 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:13.136 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:77926 10.e 10:779ef718:::benchmark_data_smithi135_602286_object77925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:13.107138+0000", 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.019109088999999999, 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019217956000000001, 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:13.137 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:13.138 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 77926 2023-09-23T20:18:13.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.138 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:13.138 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.138 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:13.138 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107138+0000", 2023-09-23T20:18:13.138 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.138 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107138+0000", 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107136+0000", 2023-09-23T20:18:13.139 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107167+0000", 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1749999999999999e-05 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:13.140 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107191+0000", 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.3847000000000001e-05 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107203+0000", 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1202e-05 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:13.141 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107264+0000", 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1415999999999996e-05 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107448+0000", 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018364100000000001 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.114800+0000", 2023-09-23T20:18:13.142 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0073522559999999997 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.119547+0000", 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047472829999999997 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.119573+0000", 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.6173000000000002e-05 2023-09-23T20:18:13.143 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:77927 10.8 10:17d641eb:::benchmark_data_smithi135_602286_object77926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:13.107218+0000", 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.019028637000000001, 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019232681000000001, 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:13.144 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 77927 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107218+0000", 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.145 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107218+0000", 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107218+0000", 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.146 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107286+0000", 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8139999999999995e-05 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107295+0000", 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1540000000000008e-06 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107302+0000", 2023-09-23T20:18:13.147 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8639999999999998e-06 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107350+0000", 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7905999999999998e-05 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107496+0000", 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000146067 2023-09-23T20:18:13.148 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107658+0000", 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016234999999999999 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118274+0000", 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010615295 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:13.149 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:77928 10.8 10:1831dc1d:::benchmark_data_smithi135_602286_object77927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:13.107318+0000", 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.018929478999999999, 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019219262000000001, 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:13.150 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 77928 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107318+0000", 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:13.151 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107318+0000", 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107317+0000", 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107360+0000", 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2985999999999998e-05 2023-09-23T20:18:13.152 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107368+0000", 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0739999999999993e-06 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107375+0000", 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4370000000000004e-06 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.110542+0000", 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031674030000000001 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.110692+0000", 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015006700000000001 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.110839+0000", 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014651200000000001 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.125570+0000", 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014731105 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:13.155 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:77932 10.8 10:1039487e:::benchmark_data_smithi135_602286_object77931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:13.107381+0000", 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.018866579000000001, 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019240363999999999, 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:13.156 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 77932 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107381+0000", 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107381+0000", 2023-09-23T20:18:13.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107380+0000", 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107425+0000", 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4731999999999998e-05 2023-09-23T20:18:13.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107432+0000", 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4370000000000001e-06 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107438+0000", 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7080000000000002e-06 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.113639+0000", 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0062015289999999999 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.113827+0000", 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018791500000000001 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:18:13.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.114006+0000", 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017854899999999999 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.125608+0000", 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011602334000000001 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.161 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:77933 10.3 10:cdabab11:::benchmark_data_smithi135_602286_object77932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:13.107443+0000", 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.018804202999999999, 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.019261363, 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 77933 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.162 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107443+0000", 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107443+0000", 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:13.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107442+0000", 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107488+0000", 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5158999999999999e-05 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107495+0000", 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2189999999999998e-06 2023-09-23T20:18:13.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107501+0000", 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5549999999999999e-06 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107593+0000", 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1467999999999999e-05 2023-09-23T20:18:13.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107719+0000", 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000126415 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.107852+0000", 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000132366 2023-09-23T20:18:13.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.116481+0000", 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0086295429999999999 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:77934 10.5 10:a52fb70b:::benchmark_data_smithi135_602286_object77933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T20:18:13.167 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:13.118488+0000", 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0077594889999999996, 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0083023690000000004, 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 77934 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.168 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118488+0000", 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118488+0000", 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118486+0000", 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118537+0000", 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.062e-05 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118554+0000", 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7533999999999998e-05 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118561+0000", 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0129999999999996e-06 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:13.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118629+0000", 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7346000000000002e-05 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.118843+0000", 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000214861 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.119005+0000", 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00016118299999999999 2023-09-23T20:18:13.172 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:77937 10.7 e498/479)", 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:13.126542+0000", 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 4294967295.9997053, 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00031887799999999998, 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T20:18:13.173 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.126542+0000", 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.126542+0000", 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.126541+0000", 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.126564+0000", 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2526000000000001e-05 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.126577+0000", 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2894999999999999e-05 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:13.126584+0000", 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6990000000000003e-06 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.176 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:13.177 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:18:13.177 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 8 2023-09-23T20:18:13.177 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:18:13.177 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:18:13.177 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:18:13.177 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:18:13.177 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:18:13.177 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:18:13.178 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:18:13.178 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:18:13.178 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:18:13.189 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:13.217 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:13.246 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 66 16 78003 77987 73.8122 34.75 0.0369278 0.0135406 2023-09-23T20:18:13.268 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:13.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:13.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:13.373 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:13.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:13 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/700094547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:13.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:13 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12192/223281 objects degraded (5.460%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:13.466 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:13 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/700094547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:13.466 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:13 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12192/223281 objects degraded (5.460%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:13.472 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:13 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/700094547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:13 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12192/223281 objects degraded (5.460%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:13.722 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:18:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:18:13] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.43.0" 2023-09-23T20:18:14.245 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 67 16 78722 78706 73.3814 44.9375 0.00961066 0.0136214 2023-09-23T20:18:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:14 smithi144 ceph-mon[194833]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 29 MiB/s wr, 457 op/s; 12192/228798 objects degraded (5.329%); 7337/228798 objects misplaced (3.207%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:18:14.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:14 smithi135 ceph-mon[221113]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 29 MiB/s wr, 457 op/s; 12192/228798 objects degraded (5.329%); 7337/228798 objects misplaced (3.207%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:18:14.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:14 smithi135 ceph-mon[213871]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 29 MiB/s wr, 457 op/s; 12192/228798 objects degraded (5.329%); 7337/228798 objects misplaced (3.207%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:18:15.247 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 68 16 79482 79466 72.9993 47.5 0.00896087 0.0136916 2023-09-23T20:18:15.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:15 smithi135 ceph-mon[221113]: pgmap v1390: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 548 op/s; 12192/232737 objects degraded (5.239%); 7144/232737 objects misplaced (3.070%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:18:15.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:15 smithi135 ceph-mon[213871]: pgmap v1390: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 548 op/s; 12192/232737 objects degraded (5.239%); 7144/232737 objects misplaced (3.070%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:18:15.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:15 smithi144 ceph-mon[194833]: pgmap v1390: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 548 op/s; 12192/232737 objects degraded (5.239%); 7144/232737 objects misplaced (3.070%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:18:15.908 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:15.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:16.012 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:16.042 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:16.111 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:16.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:16.221 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:18:16.249 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 69 16 80068 80052 72.4703 36.625 0.0119087 0.0137933 2023-09-23T20:18:16.260 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:16.321 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:18:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:16.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:18:16.429 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:16.539 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:18:16.571 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:16.661 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:16.742 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:16.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:16.855 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:16.884 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:16.936 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:16.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:16.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:17.041 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:17.150 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:17.252 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 70 16 80640 80624 71.9429 35.75 0.0451329 0.0138923 2023-09-23T20:18:17.484 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:17.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:17.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:17.594 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:17.683 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:17.966 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:18:17.966 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:18:17.966 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:18:17.966 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-09-23T20:18:17.966 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-09-23T20:18:18.067 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:18.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:18.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:18.182 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:18.254 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 71 16 81298 81282 71.5068 41.125 0.00518601 0.0139784 2023-09-23T20:18:18.289 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:18.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:18 smithi135 ceph-mon[221113]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 12192/236955 objects degraded (5.145%); 6945/236955 objects misplaced (2.931%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:18:18.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:18 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12192/236955 objects degraded (5.145%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:18.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:18 smithi135 ceph-mon[213871]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 12192/236955 objects degraded (5.145%); 6945/236955 objects misplaced (2.931%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:18:18.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:18 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12192/236955 objects degraded (5.145%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:18.493 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:18:18.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:18 smithi144 ceph-mon[194833]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 12192/236955 objects degraded (5.145%); 6945/236955 objects misplaced (2.931%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:18:18.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:18 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12192/236955 objects degraded (5.145%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:19.254 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 72 16 81952 81936 71.0815 40.875 0.0249329 0.0140639 2023-09-23T20:18:19.330 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:19.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:19.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:19.428 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:19.442 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:19.471 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:19.522 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:19.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:19.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:19.628 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:19.661 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:19.719 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:19.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:19.823 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:19.825 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:19.826 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:19.859 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:19.914 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:19.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:19.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:20.020 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:20.055 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:20.105 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:20.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:20.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:20.213 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:20.258 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 73 16 82526 82510 70.5989 35.875 0.0384357 0.0141579 2023-09-23T20:18:20.324 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:20 smithi144 ceph-mon[194833]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 12192/242448 objects degraded (5.029%); 6785/242448 objects misplaced (2.799%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:18:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:20 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:20.512 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:20 smithi135 ceph-mon[213871]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 12192/242448 objects degraded (5.029%); 6785/242448 objects misplaced (2.799%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:18:20.513 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:20.513 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:20 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:20.513 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:20 smithi135 ceph-mon[221113]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 12192/242448 objects degraded (5.029%); 6785/242448 objects misplaced (2.799%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:18:20.513 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:20.513 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:20 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:20.763 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:20.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:20.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:20.878 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:20.999 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:21.255 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 74 16 83043 83027 70.0817 32.3125 0.00650533 0.0142622 2023-09-23T20:18:21.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:21.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:21.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:21.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:22.261 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 75 16 83795 83779 69.772 47 0.0186426 0.014327 2023-09-23T20:18:22.303 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:18:22.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:22 smithi144 ceph-mon[194833]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 531 op/s; 12192/242448 objects degraded (5.029%); 6785/242448 objects misplaced (2.799%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:18:22.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:22 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2543602928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:22.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:22 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:22.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:22.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:22 smithi135 ceph-mon[221113]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 531 op/s; 12192/242448 objects degraded (5.029%); 6785/242448 objects misplaced (2.799%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:18:22.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:22 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2543602928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:22.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:22 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:22.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:22.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:22 smithi135 ceph-mon[213871]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 531 op/s; 12192/242448 objects degraded (5.029%); 6785/242448 objects misplaced (2.799%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:18:22.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:22 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2543602928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:22.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:22 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:22.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:23.258 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 76 16 84534 84518 69.4618 46.1875 0.0159897 0.0143902 2023-09-23T20:18:23.293 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:23.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:23.405 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:18:23.434 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:23.477 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:18:23.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:23.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:18:23.582 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:23.610 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:23 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:18:23.610 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:23 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:18:23.610 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:23 smithi144 ceph-mon[194833]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T20:18:23.610 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:23 smithi144 ceph-mon[194833]: failed to encode map e499 with expected crc 2023-09-23T20:18:23.610 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12192/242448 objects degraded (5.029%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:23.610 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:23 smithi144 ceph-mon[194833]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 629 op/s; 12192/247707 objects degraded (4.922%); 6605/247707 objects misplaced (2.666%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:18:23.612 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:23.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:18:23.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:18:23.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[213871]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T20:18:23.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[213871]: failed to encode map e499 with expected crc 2023-09-23T20:18:23.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12192/242448 objects degraded (5.029%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:23.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[213871]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 629 op/s; 12192/247707 objects degraded (4.922%); 6605/247707 objects misplaced (2.666%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:18:23.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:18:23.643 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:18:23.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[221113]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T20:18:23.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[221113]: failed to encode map e499 with expected crc 2023-09-23T20:18:23.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12192/242448 objects degraded (5.029%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:23.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:23 smithi135 ceph-mon[221113]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 629 op/s; 12192/247707 objects degraded (4.922%); 6605/247707 objects misplaced (2.666%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:18:23.684 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:23.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:23.792 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:23.875 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:23.974 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:18:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:18:23] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-09-23T20:18:24.262 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 77 16 84978 84962 68.9171 27.75 0.0138982 0.0145053 2023-09-23T20:18:25.261 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 78 16 85420 85404 68.388 27.625 0.00995568 0.014617 2023-09-23T20:18:25.664 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:25.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:25.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:25.771 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:25.803 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:25.860 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:25.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:25.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:25.965 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:26.122 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:26.262 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 79 16 85988 85972 67.9719 35.5 0.0135589 0.0147055 2023-09-23T20:18:26.267 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:26 smithi135 ceph-mon[213871]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 40 MiB/s wr, 639 op/s; 12192/251940 objects degraded (4.839%); 6443/251940 objects misplaced (2.557%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:18:26.273 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:26 smithi135 ceph-mon[221113]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 40 MiB/s wr, 639 op/s; 12192/251940 objects degraded (4.839%); 6443/251940 objects misplaced (2.557%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:18:26.340 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:26.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:26.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:26.447 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:26.585 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:26 smithi144 ceph-mon[194833]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 40 MiB/s wr, 639 op/s; 12192/251940 objects degraded (4.839%); 6443/251940 objects misplaced (2.557%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:18:26.710 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:26.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:26.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:26.827 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:26.868 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:26.877 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:18:26.920 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:26.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:26.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:26.940 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:18:26.983 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:18:27.027 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:27.077 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:18:27.110 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:27.125 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:18:27.184 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:18:27.246 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:18:27.263 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:2023-09-23T20:18:27.262925+0000 min lat: 0.00112211 max lat: 2.19724 avg lat: 0.0148008 2023-09-23T20:18:27.263 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:18:27.263 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 80 16 86523 86507 67.54 33.4375 0.0502617 0.0148008 2023-09-23T20:18:27.275 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.4 primary-affinity to 0 (802) 2023-09-23T20:18:27.294 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:18:27.319 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:27 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/373315201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:27.319 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:27 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3467203748' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:18:27.320 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:27 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/373315201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:27.320 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:27 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3467203748' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:18:27.321 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:27.321 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:18:27.321 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:18:27.321 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:18:27.321 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:18:27.321 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:18:27.321 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:18:27.322 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:18:27.322 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:18:27.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:27.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:27.433 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:27.448 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:18:27.448 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:18:27.560 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:27 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/373315201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:27 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3467203748' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:18:27.808 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:18:27.867 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:27.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:27.980 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:28.087 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:28.255 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:28.268 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 81 16 87110 87094 67.156 36.6875 0.00937185 0.0148836 2023-09-23T20:18:28.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:28.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:28.377 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:18:28.419 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[213871]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 624 op/s; 12192/255717 objects degraded (4.768%); 6215/255717 objects misplaced (2.430%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T20:18:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[213871]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3467203748' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:18:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[213871]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T20:18:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[213871]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[213871]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12192/255717 objects degraded (4.768%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[221113]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 624 op/s; 12192/255717 objects degraded (4.768%); 6215/255717 objects misplaced (2.430%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T20:18:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[221113]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3467203748' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:18:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[221113]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T20:18:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[221113]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[221113]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12192/255717 objects degraded (4.768%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:28.482 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:18:28.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:28.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:18:28.589 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:28 smithi144 ceph-mon[194833]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 624 op/s; 12192/255717 objects degraded (4.768%); 6215/255717 objects misplaced (2.430%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T20:18:28.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:28 smithi144 ceph-mon[194833]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3467203748' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:18:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:28 smithi144 ceph-mon[194833]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T20:18:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:28 smithi144 ceph-mon[194833]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:28 smithi144 ceph-mon[194833]: failed to encode map e500 with expected crc 2023-09-23T20:18:28.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12192/255717 objects degraded (4.768%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:28.677 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:29.269 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 82 16 87784 87768 66.8482 42.125 0.0250503 0.0149533 2023-09-23T20:18:29.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:29 smithi144 ceph-mon[194833]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 12192/260064 objects degraded (4.688%); 6034/260064 objects misplaced (2.320%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:18:29.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:29 smithi135 ceph-mon[221113]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 12192/260064 objects degraded (4.688%); 6034/260064 objects misplaced (2.320%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:18:29.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:29 smithi135 ceph-mon[213871]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 12192/260064 objects degraded (4.688%); 6034/260064 objects misplaced (2.320%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T20:18:29.876 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:29.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:29.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:29.985 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:18:30.017 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:30.084 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:18:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:30.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:18:30.188 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:30.235 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:30.271 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 83 16 88430 88414 66.5294 40.375 0.0296922 0.0150248 2023-09-23T20:18:30.293 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:30.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:30.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:30.399 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:30.424 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:30.467 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:30.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:30.573 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:30.652 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:31.143 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:31.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:31.256 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:31.269 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 84 16 88930 88914 66.1096 31.25 0.0274369 0.015121 2023-09-23T20:18:31.331 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:31.363 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:18:31.449 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:31.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:31.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:31.502 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:31.570 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:31.601 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:31.656 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:31.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:31.760 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:31.872 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:32.008 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:32.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:32.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:32.118 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:32.146 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:32.203 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:32.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:32.270 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 85 16 89505 89489 65.7548 35.9375 0.0190152 0.0151999 2023-09-23T20:18:32.308 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:32.343 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:32.404 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:32.410 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:32 smithi144 ceph-mon[194833]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 12192/260064 objects degraded (4.688%); 6034/260064 objects misplaced (2.320%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:18:32.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:32.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:32.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:32 smithi135 ceph-mon[213871]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 12192/260064 objects degraded (4.688%); 6034/260064 objects misplaced (2.320%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:18:32.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:32 smithi135 ceph-mon[221113]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 12192/260064 objects degraded (4.688%); 6034/260064 objects misplaced (2.320%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:18:32.513 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:32.556 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:32.613 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:32.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:32.720 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:32.802 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:32.895 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:32.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:32.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:33.017 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:33.044 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:33.090 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:33.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:33.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:33.194 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:33.231 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:33.270 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 86 16 90022 90006 65.3661 32.3125 0.0258335 0.0152932 2023-09-23T20:18:33.277 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:18:33.319 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12192/260064 objects degraded (4.688%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:33.319 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12192/260064 objects degraded (4.688%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:33.370 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:33.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:33.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:33.491 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:33.575 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:33.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12192/260064 objects degraded (4.688%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:33.732 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:33.758 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:33.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:33.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:33.894 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:33.967 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:18:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:18:33] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-09-23T20:18:33.987 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:34.219 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:34 smithi135 ceph-mon[213871]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s; 12192/264996 objects degraded (4.601%); 5797/264996 objects misplaced (2.188%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:18:34.270 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 87 16 90599 90583 65.0295 36.0625 0.0722827 0.0153703 2023-09-23T20:18:34.271 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:34.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:34.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:34.383 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:18:34.410 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:34.472 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:18:34.476 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:34 smithi144 ceph-mon[194833]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s; 12192/264996 objects degraded (4.601%); 5797/264996 objects misplaced (2.188%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:18:34.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:34.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:18:34.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:34 smithi135 ceph-mon[221113]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s; 12192/264996 objects degraded (4.601%); 5797/264996 objects misplaced (2.188%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:18:34.578 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:34.693 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:34.960 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:34.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:34.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:35.067 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:35.096 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:35.148 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:35.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:35.256 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:35.270 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 88 16 91204 91188 64.7203 37.8125 0.0218011 0.0154444 2023-09-23T20:18:35.279 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:18:35.325 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:35.372 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:35.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:35.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:35 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/365339677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:18:35.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:35 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:18:35.475 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:35.499 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:35 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/365339677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:18:35.499 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:35 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:18:35.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:35 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/365339677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:18:35.500 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:35 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:18:35.502 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:35.547 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:35.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:35.652 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:35.744 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:36.270 INFO:tasks.radosbench.radosbench.0.smithi135.stdout: 89 16 91566 91550 64.2476 22.625 0.0121341 0.0155597 2023-09-23T20:18:36.318 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:18:36.318 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:18:36.318 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:18:36.318 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T20:18:36.319 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T20:18:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[221113]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 35 MiB/s wr, 564 op/s; 12192/268890 objects degraded (4.534%); 5624/268890 objects misplaced (2.092%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:18:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[221113]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[221113]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:18:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[221113]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T20:18:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[221113]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[221113]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[213871]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 35 MiB/s wr, 564 op/s; 12192/268890 objects degraded (4.534%); 5624/268890 objects misplaced (2.092%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:18:36.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:36.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[213871]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[213871]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:18:36.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[213871]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T20:18:36.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[213871]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:36 smithi135 ceph-mon[213871]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.542 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:36.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:18:36 smithi144 systemd[1]: Stopping Ceph osd.6 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:18:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:36 smithi144 ceph-mon[194833]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 35 MiB/s wr, 564 op/s; 12192/268890 objects degraded (4.534%); 5624/268890 objects misplaced (2.092%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:18:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:36 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:36 smithi144 ceph-mon[194833]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:36 smithi144 ceph-mon[194833]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:18:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:36 smithi144 ceph-mon[194833]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T20:18:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:36 smithi144 ceph-mon[194833]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:36 smithi144 ceph-mon[194833]: failed to encode map e501 with expected crc 2023-09-23T20:18:36.651 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:36.747 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:37.131 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:18:36 smithi144 conmon[262273]: 2023-09-23T20:18:36.720+0000 7f3c516bc700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:18:37.132 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:18:36 smithi144 conmon[262273]: 2023-09-23T20:18:36.720+0000 7f3c516bc700 -1 osd.6 501 *** Got signal Terminated *** 2023-09-23T20:18:37.132 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:18:36 smithi144 conmon[262273]: 2023-09-23T20:18:36.720+0000 7f3c516bc700 -1 osd.6 501 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:18:37.267 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:37.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:37.271 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total time run: 90.0172 2023-09-23T20:18:37.271 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Total writes made: 92223 2023-09-23T20:18:37.271 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Write size: 65536 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Object size: 65536 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Bandwidth (MB/sec): 64.0315 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Bandwidth: 42.6259 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max bandwidth (MB/sec): 226.375 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average IOPS: 1024 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev IOPS: 682.015 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max IOPS: 3622 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min IOPS: 0 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Average Latency(s): 0.0156128 2023-09-23T20:18:37.272 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Stddev Latency(s): 0.0424665 2023-09-23T20:18:37.273 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Max latency(s): 2.19724 2023-09-23T20:18:37.273 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Min latency(s): 0.00112211 2023-09-23T20:18:37.281 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:18:37.324 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:37.359 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:37 smithi135 ceph-mon[213871]: osd.6 marked itself down and dead 2023-09-23T20:18:37.359 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:37 smithi135 ceph-mon[213871]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 600 op/s; 12192/273567 objects degraded (4.457%); 5320/273567 objects misplaced (1.945%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:18:37.360 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:37.360 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:37 smithi135 ceph-mon[221113]: osd.6 marked itself down and dead 2023-09-23T20:18:37.360 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:37 smithi135 ceph-mon[221113]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 600 op/s; 12192/273567 objects degraded (4.457%); 5320/273567 objects misplaced (1.945%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:18:37.360 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:37.371 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:37.406 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:37.456 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:37.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:37.560 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:37.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:37 smithi144 ceph-mon[194833]: osd.6 marked itself down and dead 2023-09-23T20:18:37.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:37 smithi144 ceph-mon[194833]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 600 op/s; 12192/273567 objects degraded (4.457%); 5320/273567 objects misplaced (1.945%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:18:37.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:37.587 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:37.629 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:37.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:37.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:37.733 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:37.745 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:18:37.745 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:18:37.765 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:38.257 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:18:37 smithi144 bash[370253]: ae2b44eb17fb4872f26a1d0d7711c6eca9ddbb6d9031ec1eccf52c89b82e6fde 2023-09-23T20:18:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:38 smithi144 ceph-mon[194833]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:18:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:38 smithi144 ceph-mon[194833]: failed to encode map e502 with expected crc 2023-09-23T20:18:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:38 smithi144 ceph-mon[194833]: osdmap e502: 8 total, 7 up, 8 in 2023-09-23T20:18:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:38 smithi144 ceph-mon[194833]: failed to encode map e502 with expected crc 2023-09-23T20:18:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 12192/273567 objects degraded (4.457%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:38 smithi144 ceph-mon[194833]: osdmap e503: 8 total, 7 up, 8 in 2023-09-23T20:18:38.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[221113]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[221113]: failed to encode map e502 with expected crc 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[221113]: osdmap e502: 8 total, 7 up, 8 in 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[221113]: failed to encode map e502 with expected crc 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 12192/273567 objects degraded (4.457%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[221113]: osdmap e503: 8 total, 7 up, 8 in 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[213871]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[213871]: failed to encode map e502 with expected crc 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[213871]: osdmap e502: 8 total, 7 up, 8 in 2023-09-23T20:18:38.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[213871]: failed to encode map e502 with expected crc 2023-09-23T20:18:38.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 12192/273567 objects degraded (4.457%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:38.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:38 smithi135 ceph-mon[213871]: osdmap e503: 8 total, 7 up, 8 in 2023-09-23T20:18:39.072 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:39.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:39.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:39.176 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:39.277 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:39.360 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:18:39.384 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[221113]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.384 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[221113]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.384 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[221113]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.384 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[221113]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[221113]: osdmap e504: 8 total, 7 up, 8 in 2023-09-23T20:18:39.385 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[221113]: pgmap v1408: 109 pgs: 2 remapped+peering, 1 stale+active+clean, 2 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 545 op/s; 7753/275199 objects degraded (2.817%); 5320/275199 objects misplaced (1.933%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:18:39.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[213871]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[213871]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[213871]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[213871]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[213871]: osdmap e504: 8 total, 7 up, 8 in 2023-09-23T20:18:39.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:39 smithi135 ceph-mon[213871]: pgmap v1408: 109 pgs: 2 remapped+peering, 1 stale+active+clean, 2 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 545 op/s; 7753/275199 objects degraded (2.817%); 5320/275199 objects misplaced (1.933%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:18:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:39 smithi144 ceph-mon[194833]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:39 smithi144 ceph-mon[194833]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:39 smithi144 ceph-mon[194833]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:39 smithi144 ceph-mon[194833]: failed to encode map e503 with expected crc 2023-09-23T20:18:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:39 smithi144 ceph-mon[194833]: osdmap e504: 8 total, 7 up, 8 in 2023-09-23T20:18:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:39 smithi144 ceph-mon[194833]: pgmap v1408: 109 pgs: 2 remapped+peering, 1 stale+active+clean, 2 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 545 op/s; 7753/275199 objects degraded (2.817%); 5320/275199 objects misplaced (1.933%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:18:40.009 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:18:40.012 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:40.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:40.138 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:40.179 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:40.241 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:40.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:40.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:40.347 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:40.390 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:40.497 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:40.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:40.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:40.503 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:40 smithi144 ceph-mon[194833]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.503 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:40 smithi144 ceph-mon[194833]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.503 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:40 smithi144 ceph-mon[194833]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.503 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:40 smithi144 ceph-mon[194833]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.560 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:40 smithi135 ceph-mon[221113]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.561 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:40 smithi135 ceph-mon[221113]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.561 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:40 smithi135 ceph-mon[221113]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.561 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:40 smithi135 ceph-mon[221113]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.571 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:40 smithi135 ceph-mon[213871]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.571 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:40 smithi135 ceph-mon[213871]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.571 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:40 smithi135 ceph-mon[213871]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.571 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:40 smithi135 ceph-mon[213871]: failed to encode map e504 with expected crc 2023-09-23T20:18:40.604 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:40.642 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:40.714 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:40.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:40.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:40.759 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:40.818 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:40.895 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:40.923 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:18:40.926 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:40.971 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:18:41.037 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:18:41.073 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:41.081 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:18:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:41.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:41.115 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:18:41.152 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:18:41.190 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:18:41.190 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:18:41.190 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:18:41.190 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:18:41.191 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:18:41.191 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:18:41.191 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:18:41.191 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:18:41.191 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:18:41.192 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:41.230 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:41.297 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:41.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:41.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:41.400 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:41.464 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:41 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2564872196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:41.465 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:41 smithi135 ceph-mon[221113]: pgmap v1409: 109 pgs: 1 active+recovering+remapped, 2 remapped+peering, 3 active+undersized, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 MiB/s wr, 451 op/s; 7753/275205 objects degraded (2.817%); 4963/275205 objects misplaced (1.803%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:18:41.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:41 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2564872196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:41.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:41 smithi135 ceph-mon[213871]: pgmap v1409: 109 pgs: 1 active+recovering+remapped, 2 remapped+peering, 3 active+undersized, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 MiB/s wr, 451 op/s; 7753/275205 objects degraded (2.817%); 4963/275205 objects misplaced (1.803%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:18:41.475 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:41 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2564872196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:41 smithi144 ceph-mon[194833]: pgmap v1409: 109 pgs: 1 active+recovering+remapped, 2 remapped+peering, 3 active+undersized, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 MiB/s wr, 451 op/s; 7753/275205 objects degraded (2.817%); 4963/275205 objects misplaced (1.803%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:18:41.812 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T20:18:41.874 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:18:41.875 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T20:18:42.554 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:42.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:42.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:42.665 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:42.693 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:43.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:43 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 7753/275205 objects degraded (2.817%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:43.419 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:43 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 7753/275205 objects degraded (2.817%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:43.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:43 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 7753/275205 objects degraded (2.817%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:43.686 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:18:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:18:43] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T20:18:44.147 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:44.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:44.227 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:18:44.227 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:18:44.260 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:44.344 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:44 smithi135 ceph-mon[213871]: pgmap v1410: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 remapped+peering, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 394 op/s; 9013/272847 objects degraded (3.303%); 6190/272847 objects misplaced (2.269%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:18:44.344 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:44 smithi135 ceph-mon[221113]: pgmap v1410: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 remapped+peering, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 394 op/s; 9013/272847 objects degraded (3.303%); 6190/272847 objects misplaced (2.269%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:18:44.353 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:44.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:44 smithi144 ceph-mon[194833]: pgmap v1410: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 remapped+peering, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 394 op/s; 9013/272847 objects degraded (3.303%); 6190/272847 objects misplaced (2.269%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:18:44.583 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:44.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:44.700 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:44.735 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:44.791 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:44.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:44.900 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:45.001 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:45.144 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:45.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:45.228 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:96230 10.7 10:eb78d70e:::benchmark_data_smithi135_602286_object4004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:45.185629+0000", 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.031623502999999997, 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.031648322999999999, 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:45.229 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 96230 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185629+0000", 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:45.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185629+0000", 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185626+0000", 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185636+0000", 2023-09-23T20:18:45.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0035e-05 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185659+0000", 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2798000000000001e-05 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185669+0000", 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0064999999999999e-05 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185851+0000", 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00018154799999999999 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185979+0000", 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012802700000000001 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:18:45.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.186096+0000", 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000117578 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.189339+0000", 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003242506 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.204659+0000", 2023-09-23T20:18:45.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.01531972 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:96232 10.7 10:ec3a15d5:::benchmark_data_smithi135_602286_object4006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:45.185699+0000", 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.031552873000000002, 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.031638301000000001, 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:45.235 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 96232 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185699+0000", 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185699+0000", 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185699+0000", 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185703+0000", 2023-09-23T20:18:45.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.861e-06 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185710+0000", 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8299999999999996e-06 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185717+0000", 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3849999999999999e-06 2023-09-23T20:18:45.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.189365+0000", 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036484249999999998 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.189498+0000", 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013306900000000001 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:18:45.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.189618+0000", 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0001198 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.192736+0000", 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031182139999999998 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.204667+0000", 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011930550999999999 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.240 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:96233 10.7 10:e2f60f93:::benchmark_data_smithi135_602286_object4007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:45.185722+0000", 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.031529752000000001, 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.031666209000000001, 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:45.241 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 96233 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185722+0000", 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185722+0000", 2023-09-23T20:18:45.242 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185722+0000", 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185725+0000", 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.52e-06 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185731+0000", 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.117e-06 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185735+0000", 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5539999999999999e-06 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:45.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.192761+0000", 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0070261719999999998 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.192882+0000", 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012082399999999999 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.193011+0000", 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012884600000000001 2023-09-23T20:18:45.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.196464+0000", 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0034530379999999999 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.204674+0000", 2023-09-23T20:18:45.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0082095459999999999 2023-09-23T20:18:45.247 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.247 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:45.247 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.247 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.247 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.247 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:96236 10.7 10:e1aa0d4e:::benchmark_data_smithi135_602286_object4010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:18:45.247 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:45.185740+0000", 2023-09-23T20:18:45.248 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.031512512999999999, 2023-09-23T20:18:45.248 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.031699138000000002, 2023-09-23T20:18:45.248 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:45.248 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 96236 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185740+0000", 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185740+0000", 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185739+0000", 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185742+0000", 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.48e-06 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185747+0000", 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6520000000000003e-06 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:45.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.185752+0000", 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2869999999999998e-06 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.196488+0000", 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010736704999999999 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.196611+0000", 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000123106 2023-09-23T20:18:45.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.196732+0000", 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000120271 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.204679+0000", 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0079471030000000005 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205739+0000", 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010606190000000001 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:96237 10.f 10:f074e0e0:::benchmark_data_smithi135_602286_object4011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:45.205779+0000", 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.011472777, 2023-09-23T20:18:45.254 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011713038, 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 96237 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205779+0000", 2023-09-23T20:18:45.255 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205779+0000", 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205778+0000", 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205783+0000", 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3680000000000001e-06 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205797+0000", 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3504e-05 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:45.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205804+0000", 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2690000000000002e-06 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205918+0000", 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00011446999999999999 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.206089+0000", 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00017076099999999999 2023-09-23T20:18:45.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.206236+0000", 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014655000000000001 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.210025+0000", 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0037890070000000001 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.212621+0000", 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025960089999999998 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:96242 10.7 10:ed7f4e55:::benchmark_data_smithi135_602286_object4016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:45.205820+0000", 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.01143186, 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011735921, 2023-09-23T20:18:45.260 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 96242 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205820+0000", 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205820+0000", 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205820+0000", 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205823+0000", 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4680000000000001e-06 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205833+0000", 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8199999999999992e-06 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:45.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.205839+0000", 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0120000000000002e-06 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.208990+0000", 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031508650000000001 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.209138+0000", 2023-09-23T20:18:45.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000148228 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.209262+0000", 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012356499999999999 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.212921+0000", 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0036594420000000002 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.265 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:96249 10.7 10:e412c869:::benchmark_data_smithi135_602286_object4023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:18:45.216997+0000", 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.00025496699999999999, 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00060464099999999995, 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:18:45.266 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 96249 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.216997+0000", 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.216997+0000", 2023-09-23T20:18:45.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.216996+0000", 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.217003+0000", 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.9759999999999998e-06 2023-09-23T20:18:45.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.217020+0000", 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7708000000000001e-05 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.217030+0000", 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.9990000000000003e-06 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.217180+0000", 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014975700000000001 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.217322+0000", 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00014171799999999999 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:18:45.217445+0000", 2023-09-23T20:18:45.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00012355800000000001 2023-09-23T20:18:45.271 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.271 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:18:45.271 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.271 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:18:45.271 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:18:45.271 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 7 2023-09-23T20:18:45.271 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:18:45.271 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:45.301 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:45.366 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:45.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:45.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:45.479 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:45.515 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:45.581 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:45.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:45.691 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:45.768 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:45.891 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:45.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:45.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:45.997 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:46.024 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:46.065 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:46.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:46.168 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:46.199 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:46.230 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:18:46.273 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:46.277 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:18:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:46.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:46.316 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:46 smithi135 ceph-mon[213871]: pgmap v1411: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 129 B/s rd, 8.4 MiB/s wr, 367 op/s; 14535/271239 objects degraded (5.359%); 6190/271239 objects misplaced (2.282%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:18:46.316 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1656541322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:46.316 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:46 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:46.317 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:46 smithi135 ceph-mon[221113]: pgmap v1411: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 129 B/s rd, 8.4 MiB/s wr, 367 op/s; 14535/271239 objects degraded (5.359%); 6190/271239 objects misplaced (2.282%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:18:46.317 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1656541322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:46.317 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:46 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:46.381 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:46.493 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:46 smithi144 ceph-mon[194833]: pgmap v1411: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 129 B/s rd, 8.4 MiB/s wr, 367 op/s; 14535/271239 objects degraded (5.359%); 6190/271239 objects misplaced (2.282%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:18:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1656541322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:46 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:18:46.875 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:18:46.875 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:18:46.875 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:18:46.876 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:18:47.005 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:47.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:47.117 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:47.221 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:47.233 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:18:47.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:47.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:18:47.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: osdmap e505: 8 total, 7 up, 8 in 2023-09-23T20:18:47.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.479 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[213871]: pgmap v1413: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 3.7 MiB/s wr, 543 op/s; 14535/264768 objects degraded (5.490%); 5822/264768 objects misplaced (2.199%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: osdmap e505: 8 total, 7 up, 8 in 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.480 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.481 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:47 smithi135 ceph-mon[221113]: pgmap v1413: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 3.7 MiB/s wr, 543 op/s; 14535/264768 objects degraded (5.490%); 5822/264768 objects misplaced (2.199%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:18:47.553 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:47.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:47.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:47.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:47.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:18:47.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: osdmap e505: 8 total, 7 up, 8 in 2023-09-23T20:18:47.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: failed to encode map e505 with expected crc 2023-09-23T20:18:47.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:47 smithi144 ceph-mon[194833]: pgmap v1413: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 3.7 MiB/s wr, 543 op/s; 14535/264768 objects degraded (5.490%); 5822/264768 objects misplaced (2.199%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:18:47.686 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:47.719 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:47.764 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:47.926 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:47.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:47.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:47.934 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:48.033 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:48.113 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:48.326 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:48.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:48.390 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:48.436 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:48.528 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:48 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 14535/264768 objects degraded (5.490%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:48.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:48 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 14535/264768 objects degraded (5.490%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:48.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:48 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 14535/264768 objects degraded (5.490%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:18:49.591 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:18:49.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:49 smithi144 ceph-mon[194833]: pgmap v1414: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 3.0 MiB/s wr, 541 op/s; 14347/261816 objects degraded (5.480%); 5822/261816 objects misplaced (2.224%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T20:18:49.651 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:49 smithi135 ceph-mon[213871]: pgmap v1414: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 3.0 MiB/s wr, 541 op/s; 14347/261816 objects degraded (5.480%); 5822/261816 objects misplaced (2.224%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T20:18:49.653 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:49 smithi135 ceph-mon[221113]: pgmap v1414: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 3.0 MiB/s wr, 541 op/s; 14347/261816 objects degraded (5.480%); 5822/261816 objects misplaced (2.224%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T20:18:50.011 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:50.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:50.122 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:50.203 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:50.887 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:50.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:51.004 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:51.108 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:51.381 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:51.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:51.495 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:51.524 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:51.585 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:51.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:51.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:51.691 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:18:51.721 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:18:51.721 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:51.775 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:18:51.777 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:18:51.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:51.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:18:51.817 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:18:51.862 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:18:51.882 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:51.899 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:18:51.959 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:18:51.982 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:52.024 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:18:52.024 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:18:52.024 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:18:52.025 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:18:52.025 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:18:52.025 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:18:52.025 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:18:52.025 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:18:52.025 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:18:52.070 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:52.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:52.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:52.099 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:18:52.101 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:18:52.175 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:52.201 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:52.244 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:52.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:52.348 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:52.450 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:52 smithi144 ceph-mon[194833]: pgmap v1415: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 1.5 MiB/s wr, 571 op/s; 14347/259476 objects degraded (5.529%); 5634/259476 objects misplaced (2.171%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:18:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3156775368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:52 smithi135 ceph-mon[213871]: pgmap v1415: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 1.5 MiB/s wr, 571 op/s; 14347/259476 objects degraded (5.529%); 5634/259476 objects misplaced (2.171%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:18:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3156775368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:52 smithi135 ceph-mon[221113]: pgmap v1415: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 1.5 MiB/s wr, 571 op/s; 14347/259476 objects degraded (5.529%); 5634/259476 objects misplaced (2.171%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:18:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3156775368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:18:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:18:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:52.822 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:52.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:52.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:52.936 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:53.008 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:53.492 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:53 smithi135 ceph-mon[221113]: from='client.36615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:18:53.493 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 14347/259476 objects degraded (5.529%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:18:53.493 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:53 smithi135 ceph-mon[213871]: from='client.36615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:18:53.493 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 14347/259476 objects degraded (5.529%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:18:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:53 smithi144 ceph-mon[194833]: from='client.36615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:18:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 14347/259476 objects degraded (5.529%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:18:53.846 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:53.855 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:18:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:18:53] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.43.0" 2023-09-23T20:18:53.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:53.956 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:54.067 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:54.304 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:54.334 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:18:54.335 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:18:54.370 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:54.377 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1415,"stamp":"2023-09-23T20:18:51.164207+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5671265167,"num_objects":86492,"num_object_clones":0,"num_object_copies":259476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14347,"num_objects_misplaced":5634,"num_objects_unfound":0,"num_objects_dirty":86492,"num_whiteouts":0,"num_read":6011,"num_read_kb":13264,"num_write":98906,"num_write_kb":5915145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4667,"num_bytes_recovered":296078963,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98899,"ondisk_log_size":98899,"up":320,"acting":321,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":339,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22560732,"kb_used_data":17068380,"kb_used_omap":38,"kb_used_meta":5492121,"kb_avail":727400484,"statfs":{"total":767960285184,"available":744858095616,"internally_reserved":0,"allocated":17478021120,"data_stored":17462272145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39803,"internal_metadata":5623932037},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,2,5,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-343670760,"num_objects":-5243,"num_object_clones":0,"num_object_copies":-15729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6594,"num_objects_misplaced":671,"num_objects_unfound":0,"num_objects_dirty":-5243,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5715,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1028,"num_bytes_recovered":67371008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008211"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.019140+0000","last_change":"2023-09-23T20:17:47.279267+0000","last_active":"2023-09-23T20:18:39.019140+0000","last_peered":"2023-09-23T20:18:39.019140+0000","last_clean":"2023-09-23T20:18:39.019140+0000","last_became_active":"2023-09-23T20:17:47.278920+0000","last_became_peered":"2023-09-23T20:17:47.278920+0000","last_unstale":"2023-09-23T20:18:39.019140+0000","last_undegraded":"2023-09-23T20:18:39.019140+0000","last_fullsized":"2023-09-23T20:18:39.019140+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:16.699236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012866+0000","last_change":"2023-09-23T20:17:47.270409+0000","last_active":"2023-09-23T20:18:39.012866+0000","last_peered":"2023-09-23T20:18:39.012866+0000","last_clean":"2023-09-23T20:18:39.012866+0000","last_became_active":"2023-09-23T20:17:47.270086+0000","last_became_peered":"2023-09-23T20:17:47.270086+0000","last_unstale":"2023-09-23T20:18:39.012866+0000","last_undegraded":"2023-09-23T20:18:39.012866+0000","last_fullsized":"2023-09-23T20:18:39.012866+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.885372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012588+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:18:39.012588+0000","last_peered":"2023-09-23T20:18:39.012588+0000","last_clean":"2023-09-23T20:18:39.012588+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:18:39.012588+0000","last_undegraded":"2023-09-23T20:18:39.012588+0000","last_fullsized":"2023-09-23T20:18:39.012588+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":213,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007519+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:18:39.007519+0000","last_peered":"2023-09-23T20:18:39.007519+0000","last_clean":"2023-09-23T20:18:39.007519+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:18:39.007519+0000","last_undegraded":"2023-09-23T20:18:39.007519+0000","last_fullsized":"2023-09-23T20:18:39.007519+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012790+0000","last_change":"2023-09-23T20:17:47.279385+0000","last_active":"2023-09-23T20:18:39.012790+0000","last_peered":"2023-09-23T20:18:39.012790+0000","last_clean":"2023-09-23T20:18:39.012790+0000","last_became_active":"2023-09-23T20:17:47.279065+0000","last_became_peered":"2023-09-23T20:17:47.279065+0000","last_unstale":"2023-09-23T20:18:39.012790+0000","last_undegraded":"2023-09-23T20:18:39.012790+0000","last_fullsized":"2023-09-23T20:18:39.012790+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:19.638157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":470,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004745+0000","last_change":"2023-09-23T20:17:47.270621+0000","last_active":"2023-09-23T20:18:39.004745+0000","last_peered":"2023-09-23T20:18:39.004745+0000","last_clean":"2023-09-23T20:18:39.004745+0000","last_became_active":"2023-09-23T20:17:47.270375+0000","last_became_peered":"2023-09-23T20:17:47.270375+0000","last_unstale":"2023-09-23T20:18:39.004745+0000","last_undegraded":"2023-09-23T20:18:39.004745+0000","last_fullsized":"2023-09-23T20:18:39.004745+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:16.961271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":431,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004695+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:18:39.004695+0000","last_peered":"2023-09-23T20:18:39.004695+0000","last_clean":"2023-09-23T20:18:39.004695+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:18:39.004695+0000","last_undegraded":"2023-09-23T20:18:39.004695+0000","last_fullsized":"2023-09-23T20:18:39.004695+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":369,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999395+0000","last_change":"2023-09-23T20:17:47.265452+0000","last_active":"2023-09-23T20:18:38.999395+0000","last_peered":"2023-09-23T20:18:38.999395+0000","last_clean":"2023-09-23T20:18:38.999395+0000","last_became_active":"2023-09-23T20:17:47.265115+0000","last_became_peered":"2023-09-23T20:17:47.265115+0000","last_unstale":"2023-09-23T20:18:38.999395+0000","last_undegraded":"2023-09-23T20:18:38.999395+0000","last_fullsized":"2023-09-23T20:18:38.999395+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:04.898742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":839,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.008754+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:18:39.008754+0000","last_peered":"2023-09-23T20:18:39.008754+0000","last_clean":"2023-09-23T20:18:39.008754+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:18:39.008754+0000","last_undegraded":"2023-09-23T20:18:39.008754+0000","last_fullsized":"2023-09-23T20:18:39.008754+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":524,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004421+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:18:39.004421+0000","last_peered":"2023-09-23T20:18:39.004421+0000","last_clean":"2023-09-23T20:18:39.004421+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:18:39.004421+0000","last_undegraded":"2023-09-23T20:18:39.004421+0000","last_fullsized":"2023-09-23T20:18:39.004421+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":370,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999977+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:18:38.999977+0000","last_peered":"2023-09-23T20:18:38.999977+0000","last_clean":"2023-09-23T20:18:38.999977+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:18:38.999977+0000","last_undegraded":"2023-09-23T20:18:38.999977+0000","last_fullsized":"2023-09-23T20:18:38.999977+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":198,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007572+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:18:39.007572+0000","last_peered":"2023-09-23T20:18:39.007572+0000","last_clean":"2023-09-23T20:18:39.007572+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:18:39.007572+0000","last_undegraded":"2023-09-23T20:18:39.007572+0000","last_fullsized":"2023-09-23T20:18:39.007572+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.019086+0000","last_change":"2023-09-23T20:17:47.278773+0000","last_active":"2023-09-23T20:18:39.019086+0000","last_peered":"2023-09-23T20:18:39.019086+0000","last_clean":"2023-09-23T20:18:39.019086+0000","last_became_active":"2023-09-23T20:17:47.278467+0000","last_became_peered":"2023-09-23T20:17:47.278467+0000","last_unstale":"2023-09-23T20:18:39.019086+0000","last_undegraded":"2023-09-23T20:18:39.019086+0000","last_fullsized":"2023-09-23T20:18:39.019086+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:41.368483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":451,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007100+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:18:39.007100+0000","last_peered":"2023-09-23T20:18:39.007100+0000","last_clean":"2023-09-23T20:18:39.007100+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:18:39.007100+0000","last_undegraded":"2023-09-23T20:18:39.007100+0000","last_fullsized":"2023-09-23T20:18:39.007100+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":24,"reported_epoch":504,"state":"active+undersized","last_fresh":"2023-09-23T20:18:39.012938+0000","last_change":"2023-09-23T20:18:38.012263+0000","last_active":"2023-09-23T20:18:39.012938+0000","last_peered":"2023-09-23T20:18:39.012938+0000","last_clean":"2023-09-23T20:17:54.133085+0000","last_became_active":"2023-09-23T20:18:38.012263+0000","last_became_peered":"2023-09-23T20:18:38.012263+0000","last_unstale":"2023-09-23T20:18:39.012938+0000","last_undegraded":"2023-09-23T20:18:39.012938+0000","last_fullsized":"2023-09-23T20:18:38.003813+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:47.449647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":864,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003763+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:18:39.003763+0000","last_peered":"2023-09-23T20:18:39.003763+0000","last_clean":"2023-09-23T20:18:39.003763+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:18:39.003763+0000","last_undegraded":"2023-09-23T20:18:39.003763+0000","last_fullsized":"2023-09-23T20:18:39.003763+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":198,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.006811+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:18:39.006811+0000","last_peered":"2023-09-23T20:18:39.006811+0000","last_clean":"2023-09-23T20:18:39.006811+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:18:39.006811+0000","last_undegraded":"2023-09-23T20:18:39.006811+0000","last_fullsized":"2023-09-23T20:18:39.006811+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018797+0000","last_change":"2023-09-23T20:17:47.266963+0000","last_active":"2023-09-23T20:18:39.018797+0000","last_peered":"2023-09-23T20:18:39.018797+0000","last_clean":"2023-09-23T20:18:39.018797+0000","last_became_active":"2023-09-23T20:17:47.266645+0000","last_became_peered":"2023-09-23T20:17:47.266645+0000","last_unstale":"2023-09-23T20:18:39.018797+0000","last_undegraded":"2023-09-23T20:18:39.018797+0000","last_fullsized":"2023-09-23T20:18:39.018797+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:05.052559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":553,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003771+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:18:39.003771+0000","last_peered":"2023-09-23T20:18:39.003771+0000","last_clean":"2023-09-23T20:18:39.003771+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:18:39.003771+0000","last_undegraded":"2023-09-23T20:18:39.003771+0000","last_fullsized":"2023-09-23T20:18:39.003771+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":213,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.006691+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:18:39.006691+0000","last_peered":"2023-09-23T20:18:39.006691+0000","last_clean":"2023-09-23T20:18:39.006691+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:18:39.006691+0000","last_undegraded":"2023-09-23T20:18:39.006691+0000","last_fullsized":"2023-09-23T20:18:39.006691+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":503,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004397+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:18:39.004397+0000","last_peered":"2023-09-23T20:18:39.004397+0000","last_clean":"2023-09-23T20:18:39.004397+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:18:39.004397+0000","last_undegraded":"2023-09-23T20:18:39.004397+0000","last_fullsized":"2023-09-23T20:18:39.004397+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":563,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.013035+0000","last_change":"2023-09-23T20:17:50.330492+0000","last_active":"2023-09-23T20:18:39.013035+0000","last_peered":"2023-09-23T20:18:39.013035+0000","last_clean":"2023-09-23T20:18:39.013035+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:18:39.013035+0000","last_undegraded":"2023-09-23T20:18:39.013035+0000","last_fullsized":"2023-09-23T20:18:39.013035+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:58.248167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.248167+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":209,"reported_epoch":504,"state":"active+undersized","last_fresh":"2023-09-23T20:18:39.006205+0000","last_change":"2023-09-23T20:18:38.011856+0000","last_active":"2023-09-23T20:18:39.006205+0000","last_peered":"2023-09-23T20:18:39.006205+0000","last_clean":"2023-09-23T20:18:35.260354+0000","last_became_active":"2023-09-23T20:18:38.011856+0000","last_became_peered":"2023-09-23T20:18:38.011856+0000","last_unstale":"2023-09-23T20:18:39.006205+0000","last_undegraded":"2023-09-23T20:18:39.006205+0000","last_fullsized":"2023-09-23T20:18:37.999747+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:58.093947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":494,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999942+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:18:38.999942+0000","last_peered":"2023-09-23T20:18:38.999942+0000","last_clean":"2023-09-23T20:18:38.999942+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:18:38.999942+0000","last_undegraded":"2023-09-23T20:18:38.999942+0000","last_fullsized":"2023-09-23T20:18:38.999942+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":662,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003826+0000","last_change":"2023-09-23T20:17:50.332129+0000","last_active":"2023-09-23T20:18:39.003826+0000","last_peered":"2023-09-23T20:18:39.003826+0000","last_clean":"2023-09-23T20:18:39.003826+0000","last_became_active":"2023-09-23T20:17:50.331669+0000","last_became_peered":"2023-09-23T20:17:50.331669+0000","last_unstale":"2023-09-23T20:18:39.003826+0000","last_undegraded":"2023-09-23T20:18:39.003826+0000","last_fullsized":"2023-09-23T20:18:39.003826+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:23.893417+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":148,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003652+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:18:39.003652+0000","last_peered":"2023-09-23T20:18:39.003652+0000","last_clean":"2023-09-23T20:18:39.003652+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:18:39.003652+0000","last_undegraded":"2023-09-23T20:18:39.003652+0000","last_fullsized":"2023-09-23T20:18:39.003652+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":156,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.000126+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:18:39.000126+0000","last_peered":"2023-09-23T20:18:39.000126+0000","last_clean":"2023-09-23T20:18:39.000126+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:18:39.000126+0000","last_undegraded":"2023-09-23T20:18:39.000126+0000","last_fullsized":"2023-09-23T20:18:39.000126+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"505'6165","reported_seq":6199,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:48.489051+0000","last_change":"2023-09-23T20:16:59.490329+0000","last_active":"2023-09-23T20:18:48.489051+0000","last_peered":"2023-09-23T20:18:48.489051+0000","last_clean":"2023-09-23T20:18:48.489051+0000","last_became_active":"2023-09-23T20:16:59.490069+0000","last_became_peered":"2023-09-23T20:16:59.490069+0000","last_unstale":"2023-09-23T20:18:48.489051+0000","last_undegraded":"2023-09-23T20:18:48.489051+0000","last_fullsized":"2023-09-23T20:18:48.489051+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6165,"log_dups_size":0,"ondisk_log_size":6165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.487350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353304576,"num_objects":5391,"num_object_clones":0,"num_object_copies":16173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6165,"num_write_kb":369792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":785,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.006954+0000","last_change":"2023-09-23T20:17:50.309119+0000","last_active":"2023-09-23T20:18:39.006954+0000","last_peered":"2023-09-23T20:18:39.006954+0000","last_clean":"2023-09-23T20:18:39.006954+0000","last_became_active":"2023-09-23T20:17:50.308732+0000","last_became_peered":"2023-09-23T20:17:50.308732+0000","last_unstale":"2023-09-23T20:18:39.006954+0000","last_undegraded":"2023-09-23T20:18:39.006954+0000","last_fullsized":"2023-09-23T20:18:39.006954+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:47.327378+0000","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":196,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007731+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:18:39.007731+0000","last_peered":"2023-09-23T20:18:39.007731+0000","last_clean":"2023-09-23T20:18:39.007731+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:18:39.007731+0000","last_undegraded":"2023-09-23T20:18:39.007731+0000","last_fullsized":"2023-09-23T20:18:39.007731+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":513,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003635+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:18:39.003635+0000","last_peered":"2023-09-23T20:18:39.003635+0000","last_clean":"2023-09-23T20:18:39.003635+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:18:39.003635+0000","last_undegraded":"2023-09-23T20:18:39.003635+0000","last_fullsized":"2023-09-23T20:18:39.003635+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"505'6021","reported_seq":6056,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:46.459522+0000","last_change":"2023-09-23T20:16:59.487459+0000","last_active":"2023-09-23T20:18:46.459522+0000","last_peered":"2023-09-23T20:18:46.459522+0000","last_clean":"2023-09-23T20:18:46.459522+0000","last_became_active":"2023-09-23T20:16:59.487181+0000","last_became_peered":"2023-09-23T20:16:59.487181+0000","last_unstale":"2023-09-23T20:18:46.459522+0000","last_undegraded":"2023-09-23T20:18:46.459522+0000","last_fullsized":"2023-09-23T20:18:46.459522+0000","mapping_epoch":479,"log_start":"483'300","ondisk_log_start":"483'300","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5721,"log_dups_size":0,"ondisk_log_size":5721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:34.406760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358940672,"num_objects":5477,"num_object_clones":0,"num_object_copies":16431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6021,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":853,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999790+0000","last_change":"2023-09-23T20:13:44.420508+0000","last_active":"2023-09-23T20:18:38.999790+0000","last_peered":"2023-09-23T20:18:38.999790+0000","last_clean":"2023-09-23T20:18:38.999790+0000","last_became_active":"2023-09-23T20:12:57.739531+0000","last_became_peered":"2023-09-23T20:12:57.739531+0000","last_unstale":"2023-09-23T20:18:38.999790+0000","last_undegraded":"2023-09-23T20:18:38.999790+0000","last_fullsized":"2023-09-23T20:18:38.999790+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:25.401354+0000","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":225,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007590+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:18:39.007590+0000","last_peered":"2023-09-23T20:18:39.007590+0000","last_clean":"2023-09-23T20:18:39.007590+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:18:39.007590+0000","last_undegraded":"2023-09-23T20:18:39.007590+0000","last_fullsized":"2023-09-23T20:18:39.007590+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012582+0000","last_change":"2023-09-23T20:17:47.264776+0000","last_active":"2023-09-23T20:18:39.012582+0000","last_peered":"2023-09-23T20:18:39.012582+0000","last_clean":"2023-09-23T20:18:39.012582+0000","last_became_active":"2023-09-23T20:17:47.264473+0000","last_became_peered":"2023-09-23T20:17:47.264473+0000","last_unstale":"2023-09-23T20:18:39.012582+0000","last_undegraded":"2023-09-23T20:18:39.012582+0000","last_fullsized":"2023-09-23T20:18:39.012582+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:40.906303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"505'6270","reported_seq":6305,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:50.562855+0000","last_change":"2023-09-23T20:16:59.491310+0000","last_active":"2023-09-23T20:18:50.562855+0000","last_peered":"2023-09-23T20:18:50.562855+0000","last_clean":"2023-09-23T20:18:50.562855+0000","last_became_active":"2023-09-23T20:16:59.491016+0000","last_became_peered":"2023-09-23T20:16:59.491016+0000","last_unstale":"2023-09-23T20:18:50.562855+0000","last_undegraded":"2023-09-23T20:18:50.562855+0000","last_fullsized":"2023-09-23T20:18:50.562855+0000","mapping_epoch":479,"log_start":"484'700","ondisk_log_start":"484'700","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5570,"log_dups_size":0,"ondisk_log_size":5570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:59.436079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352714752,"num_objects":5382,"num_object_clones":0,"num_object_copies":16146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6270,"num_write_kb":372864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":577,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004577+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:18:39.004577+0000","last_peered":"2023-09-23T20:18:39.004577+0000","last_clean":"2023-09-23T20:18:39.004577+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:18:39.004577+0000","last_undegraded":"2023-09-23T20:18:39.004577+0000","last_fullsized":"2023-09-23T20:18:39.004577+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":217,"reported_epoch":504,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:18:39.007325+0000","last_change":"2023-09-23T20:18:38.001952+0000","last_active":"2023-09-23T20:18:39.007325+0000","last_peered":"2023-09-23T20:18:39.007325+0000","last_clean":"2023-09-23T20:18:35.257306+0000","last_became_active":"2023-09-23T20:18:38.001952+0000","last_became_peered":"2023-09-23T20:18:38.001952+0000","last_unstale":"2023-09-23T20:18:39.007325+0000","last_undegraded":"2023-09-23T20:18:38.000196+0000","last_fullsized":"2023-09-23T20:18:38.000044+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:17.507792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.019150+0000","last_change":"2023-09-23T20:17:47.266977+0000","last_active":"2023-09-23T20:18:39.019150+0000","last_peered":"2023-09-23T20:18:39.019150+0000","last_clean":"2023-09-23T20:18:39.019150+0000","last_became_active":"2023-09-23T20:17:47.266657+0000","last_became_peered":"2023-09-23T20:17:47.266657+0000","last_unstale":"2023-09-23T20:18:39.019150+0000","last_undegraded":"2023-09-23T20:18:39.019150+0000","last_fullsized":"2023-09-23T20:18:39.019150+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:33.774981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"505'6177","reported_seq":6212,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:46.299020+0000","last_change":"2023-09-23T20:16:59.488990+0000","last_active":"2023-09-23T20:18:46.299020+0000","last_peered":"2023-09-23T20:18:46.299020+0000","last_clean":"2023-09-23T20:18:46.299020+0000","last_became_active":"2023-09-23T20:16:59.488614+0000","last_became_peered":"2023-09-23T20:16:59.488614+0000","last_unstale":"2023-09-23T20:18:46.299020+0000","last_undegraded":"2023-09-23T20:18:46.299020+0000","last_fullsized":"2023-09-23T20:18:46.299020+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:11.359750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363266048,"num_objects":5543,"num_object_clones":0,"num_object_copies":16629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6177,"num_write_kb":375040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":708,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007487+0000","last_change":"2023-09-23T20:13:57.751784+0000","last_active":"2023-09-23T20:18:39.007487+0000","last_peered":"2023-09-23T20:18:39.007487+0000","last_clean":"2023-09-23T20:18:39.007487+0000","last_became_active":"2023-09-23T20:13:57.751485+0000","last_became_peered":"2023-09-23T20:13:57.751485+0000","last_unstale":"2023-09-23T20:18:39.007487+0000","last_undegraded":"2023-09-23T20:18:39.007487+0000","last_fullsized":"2023-09-23T20:18:39.007487+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.766779+0000","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":563,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003974+0000","last_change":"2023-09-23T20:17:47.267657+0000","last_active":"2023-09-23T20:18:39.003974+0000","last_peered":"2023-09-23T20:18:39.003974+0000","last_clean":"2023-09-23T20:18:39.003974+0000","last_became_active":"2023-09-23T20:17:47.267288+0000","last_became_peered":"2023-09-23T20:17:47.267288+0000","last_unstale":"2023-09-23T20:18:39.003974+0000","last_undegraded":"2023-09-23T20:18:39.003974+0000","last_fullsized":"2023-09-23T20:18:39.003974+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.185253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012756+0000","last_change":"2023-09-23T20:17:47.280675+0000","last_active":"2023-09-23T20:18:39.012756+0000","last_peered":"2023-09-23T20:18:39.012756+0000","last_clean":"2023-09-23T20:18:39.012756+0000","last_became_active":"2023-09-23T20:17:47.280175+0000","last_became_peered":"2023-09-23T20:17:47.280175+0000","last_unstale":"2023-09-23T20:18:39.012756+0000","last_undegraded":"2023-09-23T20:18:39.012756+0000","last_fullsized":"2023-09-23T20:18:39.012756+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:08.499949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"505'6256","reported_seq":6294,"reported_epoch":505,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:18:50.590679+0000","last_change":"2023-09-23T20:17:48.443980+0000","last_active":"2023-09-23T20:18:50.590679+0000","last_peered":"2023-09-23T20:18:50.590679+0000","last_clean":"2023-09-23T20:17:46.225624+0000","last_became_active":"2023-09-23T20:17:48.428406+0000","last_became_peered":"2023-09-23T20:17:48.428406+0000","last_unstale":"2023-09-23T20:18:50.590679+0000","last_undegraded":"2023-09-23T20:17:48.291655+0000","last_fullsized":"2023-09-23T20:17:48.256072+0000","mapping_epoch":490,"log_start":"484'700","ondisk_log_start":"484'700","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5556,"log_dups_size":0,"ondisk_log_size":5556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:14.276994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346685440,"num_objects":5290,"num_object_clones":0,"num_object_copies":15870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3930,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6256,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[0,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":3447},{"shards":"0,4","objects":1360}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":887,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.006568+0000","last_change":"2023-09-23T20:07:40.030913+0000","last_active":"2023-09-23T20:18:39.006568+0000","last_peered":"2023-09-23T20:18:39.006568+0000","last_clean":"2023-09-23T20:18:39.006568+0000","last_became_active":"2023-09-23T20:07:40.030523+0000","last_became_peered":"2023-09-23T20:07:40.030523+0000","last_unstale":"2023-09-23T20:18:39.006568+0000","last_undegraded":"2023-09-23T20:18:39.006568+0000","last_fullsized":"2023-09-23T20:18:39.006568+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:12.253187+0000","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":252,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999788+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:18:38.999788+0000","last_peered":"2023-09-23T20:18:38.999788+0000","last_clean":"2023-09-23T20:18:38.999788+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:18:38.999788+0000","last_undegraded":"2023-09-23T20:18:38.999788+0000","last_fullsized":"2023-09-23T20:18:38.999788+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018787+0000","last_change":"2023-09-23T20:17:47.270450+0000","last_active":"2023-09-23T20:18:39.018787+0000","last_peered":"2023-09-23T20:18:39.018787+0000","last_clean":"2023-09-23T20:18:39.018787+0000","last_became_active":"2023-09-23T20:17:47.270040+0000","last_became_peered":"2023-09-23T20:17:47.270040+0000","last_unstale":"2023-09-23T20:18:39.018787+0000","last_undegraded":"2023-09-23T20:18:39.018787+0000","last_fullsized":"2023-09-23T20:18:39.018787+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:51.572016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.3","version":"505'6061","reported_seq":6096,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:46.252030+0000","last_change":"2023-09-23T20:16:59.488642+0000","last_active":"2023-09-23T20:18:46.252030+0000","last_peered":"2023-09-23T20:18:46.252030+0000","last_clean":"2023-09-23T20:18:46.252030+0000","last_became_active":"2023-09-23T20:16:59.488330+0000","last_became_peered":"2023-09-23T20:16:59.488330+0000","last_unstale":"2023-09-23T20:18:46.252030+0000","last_undegraded":"2023-09-23T20:18:46.252030+0000","last_fullsized":"2023-09-23T20:18:46.252030+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:46.429096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357367808,"num_objects":5453,"num_object_clones":0,"num_object_copies":16359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6061,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":852,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003955+0000","last_change":"2023-09-23T20:12:55.017982+0000","last_active":"2023-09-23T20:18:39.003955+0000","last_peered":"2023-09-23T20:18:39.003955+0000","last_clean":"2023-09-23T20:18:39.003955+0000","last_became_active":"2023-09-23T20:11:11.228870+0000","last_became_peered":"2023-09-23T20:11:11.228870+0000","last_unstale":"2023-09-23T20:18:39.003955+0000","last_undegraded":"2023-09-23T20:18:39.003955+0000","last_fullsized":"2023-09-23T20:18:39.003955+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:23.895016+0000","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.019011+0000","last_change":"2023-09-23T20:17:47.266513+0000","last_active":"2023-09-23T20:18:39.019011+0000","last_peered":"2023-09-23T20:18:39.019011+0000","last_clean":"2023-09-23T20:18:39.019011+0000","last_became_active":"2023-09-23T20:17:47.266163+0000","last_became_peered":"2023-09-23T20:17:47.266163+0000","last_unstale":"2023-09-23T20:18:39.019011+0000","last_undegraded":"2023-09-23T20:18:39.019011+0000","last_fullsized":"2023-09-23T20:18:39.019011+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.645381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":416,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.008456+0000","last_change":"2023-09-23T20:17:47.270131+0000","last_active":"2023-09-23T20:18:39.008456+0000","last_peered":"2023-09-23T20:18:39.008456+0000","last_clean":"2023-09-23T20:18:39.008456+0000","last_became_active":"2023-09-23T20:17:47.269818+0000","last_became_peered":"2023-09-23T20:17:47.269818+0000","last_unstale":"2023-09-23T20:18:39.008456+0000","last_undegraded":"2023-09-23T20:18:39.008456+0000","last_fullsized":"2023-09-23T20:18:39.008456+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:40.738070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"505'6202","reported_seq":11035,"reported_epoch":505,"state":"active+recovering+remapped","last_fresh":"2023-09-23T20:18:50.594363+0000","last_change":"2023-09-23T20:17:48.625479+0000","last_active":"2023-09-23T20:18:50.594363+0000","last_peered":"2023-09-23T20:18:50.594363+0000","last_clean":"2023-09-23T20:17:46.197657+0000","last_became_active":"2023-09-23T20:17:48.417981+0000","last_became_peered":"2023-09-23T20:17:48.417981+0000","last_unstale":"2023-09-23T20:18:50.594363+0000","last_undegraded":"2023-09-23T20:18:50.594363+0000","last_fullsized":"2023-09-23T20:18:50.594363+0000","mapping_epoch":490,"log_start":"484'600","ondisk_log_start":"484'600","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5602,"log_dups_size":0,"ondisk_log_size":5602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:55.829023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346161152,"num_objects":5282,"num_object_clones":0,"num_object_copies":15846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1469,"num_objects_unfound":0,"num_objects_dirty":5282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6202,"num_write_kb":367488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2396,"num_bytes_recovered":157024256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":3813}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":559,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012838+0000","last_change":"2023-09-23T20:17:50.330873+0000","last_active":"2023-09-23T20:18:39.012838+0000","last_peered":"2023-09-23T20:18:39.012838+0000","last_clean":"2023-09-23T20:18:39.012838+0000","last_became_active":"2023-09-23T20:17:50.330235+0000","last_became_peered":"2023-09-23T20:17:50.330235+0000","last_unstale":"2023-09-23T20:18:39.012838+0000","last_undegraded":"2023-09-23T20:18:39.012838+0000","last_fullsized":"2023-09-23T20:18:39.012838+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":398,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.008849+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:18:39.008849+0000","last_peered":"2023-09-23T20:18:39.008849+0000","last_clean":"2023-09-23T20:18:39.008849+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:18:39.008849+0000","last_undegraded":"2023-09-23T20:18:39.008849+0000","last_fullsized":"2023-09-23T20:18:39.008849+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018998+0000","last_change":"2023-09-23T20:17:47.268811+0000","last_active":"2023-09-23T20:18:39.018998+0000","last_peered":"2023-09-23T20:18:39.018998+0000","last_clean":"2023-09-23T20:18:39.018998+0000","last_became_active":"2023-09-23T20:17:47.268391+0000","last_became_peered":"2023-09-23T20:17:47.268391+0000","last_unstale":"2023-09-23T20:18:39.018998+0000","last_undegraded":"2023-09-23T20:18:39.018998+0000","last_fullsized":"2023-09-23T20:18:39.018998+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:32.388705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"505'6152","reported_seq":6186,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:48.038555+0000","last_change":"2023-09-23T20:16:59.496645+0000","last_active":"2023-09-23T20:18:48.038555+0000","last_peered":"2023-09-23T20:18:48.038555+0000","last_clean":"2023-09-23T20:18:48.038555+0000","last_became_active":"2023-09-23T20:16:59.496430+0000","last_became_peered":"2023-09-23T20:16:59.496430+0000","last_unstale":"2023-09-23T20:18:48.038555+0000","last_undegraded":"2023-09-23T20:18:48.038555+0000","last_fullsized":"2023-09-23T20:18:48.038555+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6152,"log_dups_size":0,"ondisk_log_size":6152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:59.713976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351535104,"num_objects":5364,"num_object_clones":0,"num_object_copies":16092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6152,"num_write_kb":368512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":843,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004441+0000","last_change":"2023-09-23T19:56:42.337030+0000","last_active":"2023-09-23T20:18:39.004441+0000","last_peered":"2023-09-23T20:18:39.004441+0000","last_clean":"2023-09-23T20:18:39.004441+0000","last_became_active":"2023-09-23T19:56:42.324641+0000","last_became_peered":"2023-09-23T19:56:42.324641+0000","last_unstale":"2023-09-23T20:18:39.004441+0000","last_undegraded":"2023-09-23T20:18:39.004441+0000","last_fullsized":"2023-09-23T20:18:39.004441+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.018522+0000","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":156,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999186+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:18:38.999186+0000","last_peered":"2023-09-23T20:18:38.999186+0000","last_clean":"2023-09-23T20:18:38.999186+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:18:38.999186+0000","last_undegraded":"2023-09-23T20:18:38.999186+0000","last_fullsized":"2023-09-23T20:18:38.999186+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012986+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:18:39.012986+0000","last_peered":"2023-09-23T20:18:39.012986+0000","last_clean":"2023-09-23T20:18:39.012986+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:18:39.012986+0000","last_undegraded":"2023-09-23T20:18:39.012986+0000","last_fullsized":"2023-09-23T20:18:39.012986+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"505'6119","reported_seq":6153,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:48.068878+0000","last_change":"2023-09-23T20:16:59.496291+0000","last_active":"2023-09-23T20:18:48.068878+0000","last_peered":"2023-09-23T20:18:48.068878+0000","last_clean":"2023-09-23T20:18:48.068878+0000","last_became_active":"2023-09-23T20:16:59.496000+0000","last_became_peered":"2023-09-23T20:16:59.496000+0000","last_unstale":"2023-09-23T20:18:48.068878+0000","last_undegraded":"2023-09-23T20:18:48.068878+0000","last_fullsized":"2023-09-23T20:18:48.068878+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6119,"log_dups_size":0,"ondisk_log_size":6119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.808639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354091008,"num_objects":5403,"num_object_clones":0,"num_object_copies":16209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6119,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":788,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.000205+0000","last_change":"2023-09-23T20:13:41.333706+0000","last_active":"2023-09-23T20:18:39.000205+0000","last_peered":"2023-09-23T20:18:39.000205+0000","last_clean":"2023-09-23T20:18:39.000205+0000","last_became_active":"2023-09-23T20:11:20.856645+0000","last_became_peered":"2023-09-23T20:11:20.856645+0000","last_unstale":"2023-09-23T20:18:39.000205+0000","last_undegraded":"2023-09-23T20:18:39.000205+0000","last_fullsized":"2023-09-23T20:18:39.000205+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:50.454163+0000","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":455,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018669+0000","last_change":"2023-09-23T20:17:50.311646+0000","last_active":"2023-09-23T20:18:39.018669+0000","last_peered":"2023-09-23T20:18:39.018669+0000","last_clean":"2023-09-23T20:18:39.018669+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:18:39.018669+0000","last_undegraded":"2023-09-23T20:18:39.018669+0000","last_fullsized":"2023-09-23T20:18:39.018669+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":505,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004280+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:18:39.004280+0000","last_peered":"2023-09-23T20:18:39.004280+0000","last_clean":"2023-09-23T20:18:39.004280+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:18:39.004280+0000","last_undegraded":"2023-09-23T20:18:39.004280+0000","last_fullsized":"2023-09-23T20:18:39.004280+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.019059+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:18:39.019059+0000","last_peered":"2023-09-23T20:18:39.019059+0000","last_clean":"2023-09-23T20:18:39.019059+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:18:39.019059+0000","last_undegraded":"2023-09-23T20:18:39.019059+0000","last_fullsized":"2023-09-23T20:18:39.019059+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"505'6064","reported_seq":7355,"reported_epoch":505,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T20:18:48.544137+0000","last_change":"2023-09-23T20:18:39.083016+0000","last_active":"2023-09-23T20:18:48.544137+0000","last_peered":"2023-09-23T20:18:48.544137+0000","last_clean":"2023-09-23T20:18:02.142478+0000","last_became_active":"2023-09-23T20:18:39.050965+0000","last_became_peered":"2023-09-23T20:18:39.050965+0000","last_unstale":"2023-09-23T20:18:48.544137+0000","last_undegraded":"2023-09-23T20:18:39.012780+0000","last_fullsized":"2023-09-23T20:18:48.544137+0000","mapping_epoch":503,"log_start":"483'100","ondisk_log_start":"483'100","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:43.005386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349569024,"num_objects":5334,"num_object_clones":0,"num_object_copies":16002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5334,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6064,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":622,"num_bytes_recovered":40763392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,2,5","objects":4691},{"shards":"0,5","objects":643}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":848,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003880+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:18:39.003880+0000","last_peered":"2023-09-23T20:18:39.003880+0000","last_clean":"2023-09-23T20:18:39.003880+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:18:39.003880+0000","last_undegraded":"2023-09-23T20:18:39.003880+0000","last_fullsized":"2023-09-23T20:18:39.003880+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1185,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.003963+0000","last_change":"2023-09-23T20:17:47.305979+0000","last_active":"2023-09-23T20:18:39.003963+0000","last_peered":"2023-09-23T20:18:39.003963+0000","last_clean":"2023-09-23T20:18:39.003963+0000","last_became_active":"2023-09-23T20:17:47.274623+0000","last_became_peered":"2023-09-23T20:17:47.274623+0000","last_unstale":"2023-09-23T20:18:39.003963+0000","last_undegraded":"2023-09-23T20:18:39.003963+0000","last_fullsized":"2023-09-23T20:18:39.003963+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:27.293931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012676+0000","last_change":"2023-09-23T20:17:47.280498+0000","last_active":"2023-09-23T20:18:39.012676+0000","last_peered":"2023-09-23T20:18:39.012676+0000","last_clean":"2023-09-23T20:18:39.012676+0000","last_became_active":"2023-09-23T20:17:47.279716+0000","last_became_peered":"2023-09-23T20:17:47.279716+0000","last_unstale":"2023-09-23T20:18:39.012676+0000","last_undegraded":"2023-09-23T20:18:39.012676+0000","last_fullsized":"2023-09-23T20:18:39.012676+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:18.625303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018712+0000","last_change":"2023-09-23T20:17:47.276964+0000","last_active":"2023-09-23T20:18:39.018712+0000","last_peered":"2023-09-23T20:18:39.018712+0000","last_clean":"2023-09-23T20:18:39.018712+0000","last_became_active":"2023-09-23T20:17:47.276540+0000","last_became_peered":"2023-09-23T20:17:47.276540+0000","last_unstale":"2023-09-23T20:18:39.018712+0000","last_undegraded":"2023-09-23T20:18:39.018712+0000","last_fullsized":"2023-09-23T20:18:39.018712+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:31.547044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"505'6226","reported_seq":6260,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:48.507245+0000","last_change":"2023-09-23T20:16:59.496065+0000","last_active":"2023-09-23T20:18:48.507245+0000","last_peered":"2023-09-23T20:18:48.507245+0000","last_clean":"2023-09-23T20:18:48.507245+0000","last_became_active":"2023-09-23T20:16:59.495782+0000","last_became_peered":"2023-09-23T20:16:59.495782+0000","last_unstale":"2023-09-23T20:18:48.507245+0000","last_undegraded":"2023-09-23T20:18:48.507245+0000","last_fullsized":"2023-09-23T20:18:48.507245+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6226,"log_dups_size":0,"ondisk_log_size":6226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:08.985469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354549760,"num_objects":5410,"num_object_clones":0,"num_object_copies":16230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6226,"num_write_kb":372352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":743,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.006855+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:18:39.006855+0000","last_peered":"2023-09-23T20:18:39.006855+0000","last_clean":"2023-09-23T20:18:39.006855+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:18:39.006855+0000","last_undegraded":"2023-09-23T20:18:39.006855+0000","last_fullsized":"2023-09-23T20:18:39.006855+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1144,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.009000+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:18:39.009000+0000","last_peered":"2023-09-23T20:18:39.009000+0000","last_clean":"2023-09-23T20:18:39.009000+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:18:39.009000+0000","last_undegraded":"2023-09-23T20:18:39.009000+0000","last_fullsized":"2023-09-23T20:18:39.009000+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.012711+0000","last_change":"2023-09-23T20:17:47.278822+0000","last_active":"2023-09-23T20:18:39.012711+0000","last_peered":"2023-09-23T20:18:39.012711+0000","last_clean":"2023-09-23T20:18:39.012711+0000","last_became_active":"2023-09-23T20:17:47.278407+0000","last_became_peered":"2023-09-23T20:17:47.278407+0000","last_unstale":"2023-09-23T20:18:39.012711+0000","last_undegraded":"2023-09-23T20:18:39.012711+0000","last_fullsized":"2023-09-23T20:18:39.012711+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:01.125231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.013058+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:18:39.013058+0000","last_peered":"2023-09-23T20:18:39.013058+0000","last_clean":"2023-09-23T20:18:39.013058+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:18:39.013058+0000","last_undegraded":"2023-09-23T20:18:39.013058+0000","last_fullsized":"2023-09-23T20:18:39.013058+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"505'6119","reported_seq":6158,"reported_epoch":505,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:18:50.546993+0000","last_change":"2023-09-23T20:17:48.438517+0000","last_active":"2023-09-23T20:18:50.546993+0000","last_peered":"2023-09-23T20:18:50.546993+0000","last_clean":"2023-09-23T20:17:46.219689+0000","last_became_active":"2023-09-23T20:17:48.427732+0000","last_became_peered":"2023-09-23T20:17:48.427732+0000","last_unstale":"2023-09-23T20:18:50.546993+0000","last_undegraded":"2023-09-23T20:17:48.289936+0000","last_fullsized":"2023-09-23T20:17:48.256145+0000","mapping_epoch":490,"log_start":"483'500","ondisk_log_start":"483'500","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5619,"log_dups_size":0,"ondisk_log_size":5619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:05.919808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341245952,"num_objects":5207,"num_object_clones":0,"num_object_copies":15621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3822,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6119,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":1385},{"shards":"0,4,7","objects":3366}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1413,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004329+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:18:39.004329+0000","last_peered":"2023-09-23T20:18:39.004329+0000","last_clean":"2023-09-23T20:18:39.004329+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:18:39.004329+0000","last_undegraded":"2023-09-23T20:18:39.004329+0000","last_fullsized":"2023-09-23T20:18:39.004329+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":720,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.006905+0000","last_change":"2023-09-23T20:17:50.313197+0000","last_active":"2023-09-23T20:18:39.006905+0000","last_peered":"2023-09-23T20:18:39.006905+0000","last_clean":"2023-09-23T20:18:39.006905+0000","last_became_active":"2023-09-23T20:17:50.312838+0000","last_became_peered":"2023-09-23T20:17:50.312838+0000","last_unstale":"2023-09-23T20:18:39.006905+0000","last_undegraded":"2023-09-23T20:18:39.006905+0000","last_fullsized":"2023-09-23T20:18:39.006905+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:31.530167+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":370,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004169+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:18:39.004169+0000","last_peered":"2023-09-23T20:18:39.004169+0000","last_clean":"2023-09-23T20:18:39.004169+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:18:39.004169+0000","last_undegraded":"2023-09-23T20:18:39.004169+0000","last_fullsized":"2023-09-23T20:18:39.004169+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":302,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007104+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:18:39.007104+0000","last_peered":"2023-09-23T20:18:39.007104+0000","last_clean":"2023-09-23T20:18:39.007104+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:18:39.007104+0000","last_undegraded":"2023-09-23T20:18:39.007104+0000","last_fullsized":"2023-09-23T20:18:39.007104+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"504'6011","reported_seq":6046,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:46.219881+0000","last_change":"2023-09-23T20:16:59.494039+0000","last_active":"2023-09-23T20:18:46.219881+0000","last_peered":"2023-09-23T20:18:46.219881+0000","last_clean":"2023-09-23T20:18:46.219881+0000","last_became_active":"2023-09-23T20:16:59.493714+0000","last_became_peered":"2023-09-23T20:16:59.493714+0000","last_unstale":"2023-09-23T20:18:46.219881+0000","last_undegraded":"2023-09-23T20:18:46.219881+0000","last_fullsized":"2023-09-23T20:18:46.219881+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6011,"log_dups_size":0,"ondisk_log_size":6011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:15.171264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358547456,"num_objects":5471,"num_object_clones":0,"num_object_copies":16413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6011,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":515,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018550+0000","last_change":"2023-09-23T20:17:50.330694+0000","last_active":"2023-09-23T20:18:39.018550+0000","last_peered":"2023-09-23T20:18:39.018550+0000","last_clean":"2023-09-23T20:18:39.018550+0000","last_became_active":"2023-09-23T20:17:50.329940+0000","last_became_peered":"2023-09-23T20:17:50.329940+0000","last_unstale":"2023-09-23T20:18:39.018550+0000","last_undegraded":"2023-09-23T20:18:39.018550+0000","last_fullsized":"2023-09-23T20:18:39.018550+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:55.675445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:55.675445+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1029,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.006423+0000","last_change":"2023-09-23T20:17:50.313157+0000","last_active":"2023-09-23T20:18:39.006423+0000","last_peered":"2023-09-23T20:18:39.006423+0000","last_clean":"2023-09-23T20:18:39.006423+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:18:39.006423+0000","last_undegraded":"2023-09-23T20:18:39.006423+0000","last_fullsized":"2023-09-23T20:18:39.006423+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":452,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999644+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:18:38.999644+0000","last_peered":"2023-09-23T20:18:38.999644+0000","last_clean":"2023-09-23T20:18:38.999644+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:18:38.999644+0000","last_undegraded":"2023-09-23T20:18:38.999644+0000","last_fullsized":"2023-09-23T20:18:38.999644+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.019000+0000","last_change":"2023-09-23T20:17:47.278112+0000","last_active":"2023-09-23T20:18:39.019000+0000","last_peered":"2023-09-23T20:18:39.019000+0000","last_clean":"2023-09-23T20:18:39.019000+0000","last_became_active":"2023-09-23T20:17:47.277728+0000","last_became_peered":"2023-09-23T20:17:47.277728+0000","last_unstale":"2023-09-23T20:18:39.019000+0000","last_undegraded":"2023-09-23T20:18:39.019000+0000","last_fullsized":"2023-09-23T20:18:39.019000+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:21.371380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"505'6026","reported_seq":6062,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:46.677952+0000","last_change":"2023-09-23T20:16:59.490420+0000","last_active":"2023-09-23T20:18:46.677952+0000","last_peered":"2023-09-23T20:18:46.677952+0000","last_clean":"2023-09-23T20:18:46.677952+0000","last_became_active":"2023-09-23T20:16:59.490025+0000","last_became_peered":"2023-09-23T20:16:59.490025+0000","last_unstale":"2023-09-23T20:18:46.677952+0000","last_undegraded":"2023-09-23T20:18:46.677952+0000","last_fullsized":"2023-09-23T20:18:46.677952+0000","mapping_epoch":479,"log_start":"483'300","ondisk_log_start":"483'300","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5726,"log_dups_size":0,"ondisk_log_size":5726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:03.184763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358023192,"num_objects":5464,"num_object_clones":0,"num_object_copies":16392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5464,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6026,"num_write_kb":367617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":691,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.005033+0000","last_change":"2023-09-23T20:17:02.526275+0000","last_active":"2023-09-23T20:18:39.005033+0000","last_peered":"2023-09-23T20:18:39.005033+0000","last_clean":"2023-09-23T20:18:39.005033+0000","last_became_active":"2023-09-23T20:13:57.747679+0000","last_became_peered":"2023-09-23T20:13:57.747679+0000","last_unstale":"2023-09-23T20:18:39.005033+0000","last_undegraded":"2023-09-23T20:18:39.005033+0000","last_fullsized":"2023-09-23T20:18:39.005033+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:57.948831+0000","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":205,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007479+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:18:39.007479+0000","last_peered":"2023-09-23T20:18:39.007479+0000","last_clean":"2023-09-23T20:18:39.007479+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:18:39.007479+0000","last_undegraded":"2023-09-23T20:18:39.007479+0000","last_fullsized":"2023-09-23T20:18:39.007479+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018622+0000","last_change":"2023-09-23T20:17:47.276790+0000","last_active":"2023-09-23T20:18:39.018622+0000","last_peered":"2023-09-23T20:18:39.018622+0000","last_clean":"2023-09-23T20:18:39.018622+0000","last_became_active":"2023-09-23T20:17:47.276198+0000","last_became_peered":"2023-09-23T20:17:47.276198+0000","last_unstale":"2023-09-23T20:18:39.018622+0000","last_undegraded":"2023-09-23T20:18:39.018622+0000","last_fullsized":"2023-09-23T20:18:39.018622+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:06.124215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":331,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004021+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:18:39.004021+0000","last_peered":"2023-09-23T20:18:39.004021+0000","last_clean":"2023-09-23T20:18:39.004021+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:18:39.004021+0000","last_undegraded":"2023-09-23T20:18:39.004021+0000","last_fullsized":"2023-09-23T20:18:39.004021+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"505'6104","reported_seq":6139,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:46.283867+0000","last_change":"2023-09-23T20:16:59.490304+0000","last_active":"2023-09-23T20:18:46.283867+0000","last_peered":"2023-09-23T20:18:46.283867+0000","last_clean":"2023-09-23T20:18:46.283867+0000","last_became_active":"2023-09-23T20:16:59.490026+0000","last_became_peered":"2023-09-23T20:16:59.490026+0000","last_unstale":"2023-09-23T20:18:46.283867+0000","last_undegraded":"2023-09-23T20:18:46.283867+0000","last_fullsized":"2023-09-23T20:18:46.283867+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6104,"log_dups_size":0,"ondisk_log_size":6104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:59.789512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364904448,"num_objects":5568,"num_object_clones":0,"num_object_copies":16704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6104,"num_write_kb":373504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":737,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.008543+0000","last_change":"2023-09-23T20:16:56.408683+0000","last_active":"2023-09-23T20:18:39.008543+0000","last_peered":"2023-09-23T20:18:39.008543+0000","last_clean":"2023-09-23T20:18:39.008543+0000","last_became_active":"2023-09-23T20:13:57.747737+0000","last_became_peered":"2023-09-23T20:13:57.747737+0000","last_unstale":"2023-09-23T20:18:39.008543+0000","last_undegraded":"2023-09-23T20:18:39.008543+0000","last_fullsized":"2023-09-23T20:18:39.008543+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:18.311022+0000","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":347,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.007254+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:18:39.007254+0000","last_peered":"2023-09-23T20:18:39.007254+0000","last_clean":"2023-09-23T20:18:39.007254+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:18:39.007254+0000","last_undegraded":"2023-09-23T20:18:39.007254+0000","last_fullsized":"2023-09-23T20:18:39.007254+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":583,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018890+0000","last_change":"2023-09-23T20:17:50.311535+0000","last_active":"2023-09-23T20:18:39.018890+0000","last_peered":"2023-09-23T20:18:39.018890+0000","last_clean":"2023-09-23T20:18:39.018890+0000","last_became_active":"2023-09-23T20:17:50.310876+0000","last_became_peered":"2023-09-23T20:17:50.310876+0000","last_unstale":"2023-09-23T20:18:39.018890+0000","last_undegraded":"2023-09-23T20:18:39.018890+0000","last_fullsized":"2023-09-23T20:18:39.018890+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.794641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":138,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999173+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:18:38.999173+0000","last_peered":"2023-09-23T20:18:38.999173+0000","last_clean":"2023-09-23T20:18:38.999173+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:18:38.999173+0000","last_undegraded":"2023-09-23T20:18:38.999173+0000","last_fullsized":"2023-09-23T20:18:38.999173+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"505'6011","reported_seq":8796,"reported_epoch":505,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:18:46.626785+0000","last_change":"2023-09-23T20:18:39.126760+0000","last_active":"2023-09-23T20:18:46.626785+0000","last_peered":"2023-09-23T20:18:46.626785+0000","last_clean":"2023-09-23T20:18:02.160044+0000","last_became_active":"2023-09-23T20:18:39.117958+0000","last_became_peered":"2023-09-23T20:18:39.117958+0000","last_unstale":"2023-09-23T20:18:46.626785+0000","last_undegraded":"2023-09-23T20:18:39.008966+0000","last_fullsized":"2023-09-23T20:18:46.626785+0000","mapping_epoch":503,"log_start":"483'300","ondisk_log_start":"483'300","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5711,"log_dups_size":0,"ondisk_log_size":5711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:45.818039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355532800,"num_objects":5425,"num_object_clones":0,"num_object_copies":16275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1260,"num_objects_misplaced":4165,"num_objects_unfound":0,"num_objects_dirty":5425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6011,"num_write_kb":365952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1370,"num_bytes_recovered":89784320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,4],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1260},{"shards":"2,3,4","objects":4165}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":832,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.000048+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:18:39.000048+0000","last_peered":"2023-09-23T20:18:39.000048+0000","last_clean":"2023-09-23T20:18:39.000048+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:18:39.000048+0000","last_undegraded":"2023-09-23T20:18:39.000048+0000","last_fullsized":"2023-09-23T20:18:39.000048+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":211,"reported_epoch":504,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:18:39.010722+0000","last_change":"2023-09-23T20:18:39.010444+0000","last_active":"2023-09-23T20:18:39.010722+0000","last_peered":"2023-09-23T20:18:39.010722+0000","last_clean":"2023-09-23T20:18:39.010722+0000","last_became_active":"2023-09-23T20:18:39.009999+0000","last_became_peered":"2023-09-23T20:18:39.009999+0000","last_unstale":"2023-09-23T20:18:39.010722+0000","last_undegraded":"2023-09-23T20:18:39.010722+0000","last_fullsized":"2023-09-23T20:18:39.010722+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:53.838719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,3],"avail_no_missing":["7","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":30,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.018742+0000","last_change":"2023-09-23T20:17:47.284676+0000","last_active":"2023-09-23T20:18:39.018742+0000","last_peered":"2023-09-23T20:18:39.018742+0000","last_clean":"2023-09-23T20:18:39.018742+0000","last_became_active":"2023-09-23T20:17:47.277321+0000","last_became_peered":"2023-09-23T20:17:47.277321+0000","last_unstale":"2023-09-23T20:18:39.018742+0000","last_undegraded":"2023-09-23T20:18:39.018742+0000","last_fullsized":"2023-09-23T20:18:39.018742+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.082445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":305,"reported_epoch":504,"state":"active+undersized","last_fresh":"2023-09-23T20:18:39.003992+0000","last_change":"2023-09-23T20:18:38.011785+0000","last_active":"2023-09-23T20:18:39.003992+0000","last_peered":"2023-09-23T20:18:39.003992+0000","last_clean":"2023-09-23T20:18:35.261768+0000","last_became_active":"2023-09-23T20:18:38.011785+0000","last_became_peered":"2023-09-23T20:18:38.011785+0000","last_unstale":"2023-09-23T20:18:39.003992+0000","last_undegraded":"2023-09-23T20:18:39.003992+0000","last_fullsized":"2023-09-23T20:18:38.001274+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:00.398059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":283,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T20:18:46.252240+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:18:46.252240+0000","last_peered":"2023-09-23T20:18:46.252240+0000","last_clean":"2023-09-23T20:18:46.252240+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:18:46.252240+0000","last_undegraded":"2023-09-23T20:18:46.252240+0000","last_fullsized":"2023-09-23T20:18:46.252240+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":292,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.008683+0000","last_change":"2023-09-23T20:17:47.281804+0000","last_active":"2023-09-23T20:18:39.008683+0000","last_peered":"2023-09-23T20:18:39.008683+0000","last_clean":"2023-09-23T20:18:39.008683+0000","last_became_active":"2023-09-23T20:17:47.281384+0000","last_became_peered":"2023-09-23T20:17:47.281384+0000","last_unstale":"2023-09-23T20:18:39.008683+0000","last_undegraded":"2023-09-23T20:18:39.008683+0000","last_fullsized":"2023-09-23T20:18:39.008683+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:03.796273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":536,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004234+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:18:39.004234+0000","last_peered":"2023-09-23T20:18:39.004234+0000","last_clean":"2023-09-23T20:18:39.004234+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:18:39.004234+0000","last_undegraded":"2023-09-23T20:18:39.004234+0000","last_fullsized":"2023-09-23T20:18:39.004234+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":534,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:38.999866+0000","last_change":"2023-09-23T20:17:47.266215+0000","last_active":"2023-09-23T20:18:38.999866+0000","last_peered":"2023-09-23T20:18:38.999866+0000","last_clean":"2023-09-23T20:18:38.999866+0000","last_became_active":"2023-09-23T20:17:47.265890+0000","last_became_peered":"2023-09-23T20:17:47.265890+0000","last_unstale":"2023-09-23T20:18:38.999866+0000","last_undegraded":"2023-09-23T20:18:38.999866+0000","last_fullsized":"2023-09-23T20:18:38.999866+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:59.854975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":526,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004275+0000","last_change":"2023-09-23T20:17:47.288753+0000","last_active":"2023-09-23T20:18:39.004275+0000","last_peered":"2023-09-23T20:18:39.004275+0000","last_clean":"2023-09-23T20:18:39.004275+0000","last_became_active":"2023-09-23T20:17:47.270488+0000","last_became_peered":"2023-09-23T20:17:47.270488+0000","last_unstale":"2023-09-23T20:18:39.004275+0000","last_undegraded":"2023-09-23T20:18:39.004275+0000","last_fullsized":"2023-09-23T20:18:39.004275+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:35.837184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":501,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.077677+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:18:39.077677+0000","last_peered":"2023-09-23T20:18:39.077677+0000","last_clean":"2023-09-23T20:18:39.077677+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:18:39.077677+0000","last_undegraded":"2023-09-23T20:18:39.077677+0000","last_fullsized":"2023-09-23T20:18:39.077677+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":227,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.008543+0000","last_change":"2023-09-23T20:17:47.281777+0000","last_active":"2023-09-23T20:18:39.008543+0000","last_peered":"2023-09-23T20:18:39.008543+0000","last_clean":"2023-09-23T20:18:39.008543+0000","last_became_active":"2023-09-23T20:17:47.281381+0000","last_became_peered":"2023-09-23T20:17:47.281381+0000","last_unstale":"2023-09-23T20:18:39.008543+0000","last_undegraded":"2023-09-23T20:18:39.008543+0000","last_fullsized":"2023-09-23T20:18:39.008543+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.083242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":365,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.009069+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:18:39.009069+0000","last_peered":"2023-09-23T20:18:39.009069+0000","last_clean":"2023-09-23T20:18:39.009069+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:18:39.009069+0000","last_undegraded":"2023-09-23T20:18:39.009069+0000","last_fullsized":"2023-09-23T20:18:39.009069+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":350,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.004635+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:18:39.004635+0000","last_peered":"2023-09-23T20:18:39.004635+0000","last_clean":"2023-09-23T20:18:39.004635+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:18:39.004635+0000","last_undegraded":"2023-09-23T20:18:39.004635+0000","last_fullsized":"2023-09-23T20:18:39.004635+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":26,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T20:18:39.020495+0000","last_change":"2023-09-23T20:17:47.267673+0000","last_active":"2023-09-23T20:18:39.020495+0000","last_peered":"2023-09-23T20:18:39.020495+0000","last_clean":"2023-09-23T20:18:39.020495+0000","last_became_active":"2023-09-23T20:17:47.267216+0000","last_became_peered":"2023-09-23T20:17:47.267216+0000","last_unstale":"2023-09-23T20:18:39.020495+0000","last_undegraded":"2023-09-23T20:18:39.020495+0000","last_fullsized":"2023-09-23T20:18:39.020495+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:37.855890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5666439192,"num_objects":86464,"num_object_clones":0,"num_object_copies":259392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14346,"num_objects_misplaced":5634,"num_objects_unfound":0,"num_objects_dirty":86464,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":97984,"num_write_kb":5902273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4388,"num_bytes_recovered":287571968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17376686080,"data_stored":17376673864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94484,"ondisk_log_size":94484,"up":46,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":10952,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":94,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":626148,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":30,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":107340,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3384,"ondisk_log_size":3384,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":8388703,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234452,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2550764,"kb_used_data":1986852,"kb_used_omap":0,"kb_used_meta":563904,"kb_avail":91194388,"statfs":{"total":95995035648,"available":93383053312,"internally_reserved":0,"allocated":2034536448,"data_stored":2033026057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":577437696},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":5.3470000000000004,"15min":3.3220000000000001},"min":{"1min":0.223,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":5.2880000000000003,"5min":149.25999999999999,"15min":149.25999999999999},"last":9.952},{"interface":"front","average":{"1min":1.02,"5min":5.6669999999999998,"15min":3.5459999999999998},"min":{"1min":0.246,"5min":0.222,"15min":0.20399999999999999},"max":{"1min":6.2880000000000003,"5min":149.11799999999999,"15min":149.11799999999999},"last":1.3899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.8240000000000001,"15min":3.4159999999999999},"min":{"1min":0.29199999999999998,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":3.2229999999999999,"5min":47.197000000000003,"15min":104.16800000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.8420000000000001,"15min":3.6520000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":5.3559999999999999,"5min":47.436999999999998,"15min":104.003},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":4.9720000000000004,"5min":3.0529999999999999,"15min":4.298},"min":{"1min":0.35799999999999998,"5min":0.32700000000000001,"15min":0.251},"max":{"1min":70.915000000000006,"5min":70.915000000000006,"15min":105.842},"last":0.29799999999999999},{"interface":"front","average":{"1min":5.7779999999999996,"5min":3.3759999999999999,"15min":4.6529999999999996},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.255},"max":{"1min":70.796999999999997,"5min":70.796999999999997,"15min":105.66},"last":3.0870000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":7.032,"5min":6.3120000000000003,"15min":5.1630000000000003},"min":{"1min":0.33700000000000002,"5min":0.33400000000000002,"15min":0.29299999999999998},"max":{"1min":112.291,"5min":185.17099999999999,"15min":185.17099999999999},"last":1.6000000000000001},{"interface":"front","average":{"1min":7.3869999999999996,"5min":6.6180000000000003,"15min":5.165},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28999999999999998},"max":{"1min":112.887,"5min":184.99100000000001,"15min":184.99100000000001},"last":1.6739999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":1.222,"15min":0.78200000000000003},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.23499999999999999},"max":{"1min":2.093,"5min":7.9199999999999999,"15min":7.9199999999999999},"last":0.377},{"interface":"front","average":{"1min":0.65700000000000003,"5min":1.379,"15min":0.97499999999999998},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":2.117,"5min":10.882999999999999,"15min":41.822000000000003},"last":0.34999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:18:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.8620000000000001,"15min":1.1819999999999999},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":4.6959999999999997,"5min":16.327000000000002,"15min":24.876000000000001},"last":0.439},{"interface":"front","average":{"1min":1.042,"5min":1.7310000000000001,"15min":1.073},"min":{"1min":0.33700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":3.5289999999999999,"5min":16.716000000000001,"15min":21.623000000000001},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.3779999999999999,"15min":1.0569999999999999},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.3420000000000001,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.163,"15min":0.82199999999999995},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1400000000000001,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.436}]}]},{"osd":6,"up_from":251,"seq":1078036791490,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273876,"kb_used_data":233868,"kb_used_omap":6,"kb_used_meta":39993,"kb_avail":93471276,"statfs":{"total":95995035648,"available":95714586624,"internally_reserved":0,"allocated":239480832,"data_stored":238014233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":40953471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":8.9100000000000001,"5min":3.8820000000000001,"15min":2.6890000000000001},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":53.780000000000001,"5min":74.787000000000006,"15min":126.64},"last":0.27000000000000002},{"interface":"front","average":{"1min":9.0090000000000003,"5min":4.0289999999999999,"15min":2.7570000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":53.768000000000001,"5min":74.751999999999995,"15min":126.658},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":4.3120000000000003,"15min":4.0830000000000002},"min":{"1min":0.26500000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":87.864999999999995,"5min":103.545,"15min":156.70699999999999},"last":5.4710000000000001},{"interface":"front","average":{"1min":5.1260000000000003,"5min":4.0279999999999996,"15min":4.0590000000000002},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.23899999999999999},"max":{"1min":87.772000000000006,"5min":100.018,"15min":156.68299999999999},"last":2.8690000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":6.891,"5min":2.4329999999999998,"15min":3.1179999999999999},"min":{"1min":0.315,"5min":0.254,"15min":0.23100000000000001},"max":{"1min":53.911999999999999,"5min":53.911999999999999,"15min":122.149},"last":0.47499999999999998},{"interface":"front","average":{"1min":7.3259999999999996,"5min":2.4129999999999998,"15min":3.0600000000000001},"min":{"1min":0.32400000000000001,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":54.125999999999998,"5min":54.125999999999998,"15min":122.23999999999999},"last":1.7789999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":5.181,"15min":3.8580000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":51.512999999999998,"5min":128.279,"15min":160.27699999999999},"last":4.8179999999999996},{"interface":"front","average":{"1min":3.4249999999999998,"5min":4.4969999999999999,"15min":3.8180000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":51.601999999999997,"5min":127.911,"15min":160.42699999999999},"last":2.9830000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.298,"15min":1.5780000000000001},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.248},"max":{"1min":2.641,"5min":34.534999999999997,"15min":145.29300000000001},"last":2.9140000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.077,"15min":1.538},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.3999999999999999,"5min":34.426000000000002,"15min":145.256},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":4.5880000000000001,"15min":2.972},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.25800000000000001},"max":{"1min":2.5329999999999999,"5min":218.58199999999999,"15min":218.58199999999999},"last":2.9500000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.343,"15min":2.8969999999999998},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.28499999999999998},"max":{"1min":2.4380000000000002,"5min":217.95599999999999,"15min":217.95599999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.1269999999999998,"15min":1.73},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.315},"max":{"1min":4.7380000000000004,"5min":137.63900000000001,"15min":137.63900000000001},"last":1},{"interface":"front","average":{"1min":0.751,"5min":2.8199999999999998,"15min":1.629},"min":{"1min":0.39700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":2.3620000000000001,"5min":136.738,"15min":136.738},"last":0.48799999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762651,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1295716,"kb_used_data":490524,"kb_used_omap":7,"kb_used_meta":805176,"kb_avail":92449436,"statfs":{"total":95995035648,"available":94668222464,"internally_reserved":0,"allocated":502296576,"data_stored":499918706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":824501050},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":4.5659999999999998,"15min":3.7650000000000001},"min":{"1min":0.19600000000000001,"5min":0.185,"15min":0.17100000000000001},"max":{"1min":8.9879999999999995,"5min":145.81299999999999,"15min":145.81299999999999},"last":2.6080000000000001},{"interface":"front","average":{"1min":3.7850000000000001,"5min":4.7999999999999998,"15min":4.1980000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19400000000000001},"max":{"1min":13.798999999999999,"5min":146.83799999999999,"15min":146.83799999999999},"last":8.3680000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:18:26 2023","interfaces":[{"interface":"back","average":{"1min":10.983000000000001,"5min":4.4349999999999996,"15min":4.359},"min":{"1min":0.23999999999999999,"5min":0.214,"15min":0.20000000000000001},"max":{"1min":152.62,"5min":152.62,"15min":152.62},"last":3.7250000000000001},{"interface":"front","average":{"1min":10.912000000000001,"5min":3.79,"15min":4.0750000000000002},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.20699999999999999},"max":{"1min":154.84299999999999,"5min":154.84299999999999,"15min":154.84299999999999},"last":6.6699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":4.2199999999999998,"5min":3.194,"15min":3.161},"min":{"1min":0.24399999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":21.056999999999999,"5min":66.527000000000001,"15min":66.527000000000001},"last":7.3869999999999996},{"interface":"front","average":{"1min":1.5389999999999999,"5min":2.0499999999999998,"15min":2.7469999999999999},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":5.5599999999999996,"5min":61.854999999999997,"15min":61.854999999999997},"last":2.5270000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.5189999999999999,"15min":1.681},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.23599999999999999},"max":{"1min":6.5359999999999996,"5min":7.7999999999999998,"15min":18.087},"last":2.633},{"interface":"front","average":{"1min":1.3169999999999999,"5min":1.403,"15min":1.7470000000000001},"min":{"1min":0.34999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.5910000000000002,"5min":9.8320000000000007,"15min":20.847000000000001},"last":2.5680000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.776,"15min":2.1509999999999998},"min":{"1min":0.313,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.8059999999999992,"5min":9.782,"15min":83.510999999999996},"last":6.6100000000000003},{"interface":"front","average":{"1min":2.1379999999999999,"5min":1.8009999999999999,"15min":2.2770000000000001},"min":{"1min":0.33300000000000002,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":5.5759999999999996,"5min":13.449,"15min":83.353999999999999},"last":1.3109999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.2390000000000001,"15min":1.665},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":4.5830000000000002,"5min":9.2279999999999998,"15min":18.117999999999999},"last":5.6760000000000002},{"interface":"front","average":{"1min":0.75,"5min":1.325,"15min":1.843},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.0310000000000001,"5min":9.8659999999999997,"15min":20.734999999999999},"last":3.7330000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":3.4820000000000002,"15min":2.589},"min":{"1min":0.41699999999999998,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":5.6070000000000002,"5min":127.03100000000001,"15min":127.03100000000001},"last":3.7719999999999998},{"interface":"front","average":{"1min":2.181,"5min":3.0579999999999998,"15min":2.266},"min":{"1min":0.40200000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.5819999999999999,"5min":125.095,"15min":125.095},"last":3.2770000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926189,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4731136,"kb_used_data":3739576,"kb_used_omap":8,"kb_used_meta":991543,"kb_avail":89014016,"statfs":{"total":95995035648,"available":91150352384,"internally_reserved":0,"allocated":3829325824,"data_stored":3826980590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1015340641},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":1.4330000000000001,"15min":2.673},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":7.7290000000000001,"5min":29.893000000000001,"15min":72.534000000000006},"last":0.96399999999999997},{"interface":"front","average":{"1min":2.294,"5min":1.379,"15min":2.8319999999999999},"min":{"1min":0.24099999999999999,"5min":0.20799999999999999,"15min":0.187},"max":{"1min":9.8439999999999994,"5min":23.436,"15min":72.667000000000002},"last":6.0750000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:18:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9409999999999998,"5min":2.911,"15min":3.7320000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":17.475000000000001,"5min":160.542,"15min":162.62899999999999},"last":5.4089999999999998},{"interface":"front","average":{"1min":2.992,"5min":3.4079999999999999,"15min":3.9140000000000001},"min":{"1min":0.23599999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":8.2189999999999994,"5min":160.75999999999999,"15min":162.52500000000001},"last":0.63900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":1.8009999999999999,"15min":3.5419999999999998},"min":{"1min":0.315,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":22.460000000000001,"5min":44.405000000000001,"15min":126.95399999999999},"last":13.462999999999999},{"interface":"front","average":{"1min":2.9489999999999998,"5min":2.0950000000000002,"15min":3.702},"min":{"1min":0.27800000000000002,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":22.539999999999999,"5min":43.909999999999997,"15min":127.294},"last":6.0439999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.7609999999999999,"15min":2.0600000000000001},"min":{"1min":0.39000000000000001,"5min":0.316,"15min":0.27600000000000002},"max":{"1min":7.5819999999999999,"5min":8.7360000000000007,"15min":148.20599999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":2.3199999999999998,"5min":2.3319999999999999,"15min":2.3290000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0150000000000006,"5min":105.282,"15min":148.11000000000001},"last":5.8170000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":2.1699999999999999,"15min":1.698},"min":{"1min":0.36499999999999999,"5min":0.35999999999999999,"15min":0.26900000000000002},"max":{"1min":7.6180000000000003,"5min":52.061,"15min":52.061},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.456,"5min":2.351,"15min":1.8660000000000001},"min":{"1min":0.37,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":5.3799999999999999,"5min":51.741,"15min":51.741},"last":0.33800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6339999999999999,"15min":1.6419999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":7.6500000000000004,"5min":7.6500000000000004,"15min":15.744999999999999},"last":4.1239999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.234,"15min":1.6020000000000001},"min":{"1min":0.44,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":9.8219999999999992,"15min":13.766999999999999},"last":0.82199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.9119999999999999,"15min":1.9910000000000001},"min":{"1min":0.39800000000000002,"5min":0.39200000000000002,"15min":0.377},"max":{"1min":8.0269999999999992,"5min":99.486000000000004,"15min":99.486000000000004},"last":0.93899999999999995},{"interface":"front","average":{"1min":2.069,"5min":3.3740000000000001,"15min":2.3210000000000002},"min":{"1min":0.42599999999999999,"5min":0.40200000000000002,"15min":0.32300000000000001},"max":{"1min":8.048,"5min":99.668999999999997,"15min":99.668999999999997},"last":0.308}]}]},{"osd":2,"up_from":104,"seq":446676599126,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3692456,"kb_used_data":2763232,"kb_used_omap":12,"kb_used_meta":929203,"kb_avail":90052696,"statfs":{"total":95995035648,"available":92213960704,"internally_reserved":0,"allocated":2829549568,"data_stored":2827165412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":951504894},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":16.698,"5min":5.444,"15min":3.5150000000000001},"min":{"1min":0.40200000000000002,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":104.93600000000001,"5min":104.93600000000001,"15min":104.93600000000001},"last":274.185},{"interface":"front","average":{"1min":18.001999999999999,"5min":5.9480000000000004,"15min":3.851},"min":{"1min":0.23200000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":106.477,"5min":106.477,"15min":106.477},"last":5.0599999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":12.108000000000001,"5min":4.0960000000000001,"15min":3.593},"min":{"1min":0.38700000000000001,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":106.357,"5min":106.357,"15min":106.357},"last":5.258},{"interface":"front","average":{"1min":11.811999999999999,"5min":4.2199999999999998,"15min":3.2959999999999998},"min":{"1min":0.371,"5min":0.222,"15min":0.218},"max":{"1min":104.905,"5min":104.905,"15min":104.905},"last":5.6079999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:18:45 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":5.5549999999999997,"15min":4.6159999999999997},"min":{"1min":0.247,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":14.129,"5min":132.82499999999999,"15min":132.82499999999999},"last":6.9699999999999998},{"interface":"front","average":{"1min":3.153,"5min":5.6369999999999996,"15min":3.9079999999999999},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":22.847999999999999,"5min":130.97300000000001,"15min":130.97300000000001},"last":5.6609999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.3009999999999999,"15min":1.8320000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.8220000000000001,"5min":10.079000000000001,"15min":75.608000000000004},"last":5.6900000000000004},{"interface":"front","average":{"1min":2.444,"5min":1.5740000000000001,"15min":2.1970000000000001},"min":{"1min":0.34599999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":12.726000000000001,"5min":12.726000000000001,"15min":76.161000000000001},"last":3.7770000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":2.9039999999999999,"15min":2.9180000000000001},"min":{"1min":0.316,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.8719999999999999,"5min":135.202,"15min":172.505},"last":5.7240000000000002},{"interface":"front","average":{"1min":2.0939999999999999,"5min":1.738,"15min":2.7839999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.778,"5min":12.778,"15min":172.375},"last":5.0289999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.329,"15min":1.542},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5510000000000002,"5min":11.843,"15min":14.664},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.425,"15min":1.744},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.077,"5min":12.025,"15min":14.279999999999999},"last":4.9660000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4849999999999999,"5min":1.984,"15min":2},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.789999999999999,"5min":12.789999999999999,"15min":13.481999999999999},"last":3.8090000000000002},{"interface":"front","average":{"1min":1.333,"5min":1.635,"15min":1.6399999999999999},"min":{"1min":0.40500000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.8460000000000001,"5min":8.9730000000000008,"15min":14.698},"last":5.7519999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435600,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3371396,"kb_used_data":2372092,"kb_used_omap":4,"kb_used_meta":999163,"kb_avail":90373756,"statfs":{"total":95995035648,"available":92542726144,"internally_reserved":0,"allocated":2429022208,"data_stored":2426369728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1023143021},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:52 2023","interfaces":[{"interface":"back","average":{"1min":9.3889999999999993,"5min":3.927,"15min":3.7789999999999999},"min":{"1min":0.191,"5min":0.189,"15min":0.186},"max":{"1min":132.47800000000001,"5min":132.47800000000001,"15min":132.47800000000001},"last":2.8820000000000001},{"interface":"front","average":{"1min":10.243,"5min":4.508,"15min":4.109},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":133.22499999999999,"5min":133.22499999999999,"15min":133.22499999999999},"last":15.013}]},{"osd":1,"last update":"Sat Sep 23 20:17:52 2023","interfaces":[{"interface":"back","average":{"1min":4.8300000000000001,"5min":3.9689999999999999,"15min":4.5609999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":92.036000000000001,"5min":183.87,"15min":183.87},"last":9.5899999999999999},{"interface":"front","average":{"1min":4.9109999999999996,"5min":3.9660000000000002,"15min":4.9199999999999999},"min":{"1min":0.248,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":92.210999999999999,"5min":185.84899999999999,"15min":185.84899999999999},"last":3.0659999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:17:52 2023","interfaces":[{"interface":"back","average":{"1min":14.343999999999999,"5min":4.2779999999999996,"15min":4.8159999999999998},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":128.369,"5min":128.369,"15min":163.83500000000001},"last":3.2370000000000001},{"interface":"front","average":{"1min":13.863,"5min":4.0609999999999999,"15min":4.6369999999999996},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":126.497,"5min":126.497,"15min":163.92099999999999},"last":3.198}]},{"osd":4,"last update":"Sat Sep 23 20:17:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.1140000000000001,"15min":2.524},"min":{"1min":0.34799999999999998,"5min":0.29299999999999998,"15min":0.27200000000000002},"max":{"1min":1.986,"5min":7.4960000000000004,"15min":109.66800000000001},"last":2.9569999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.7649999999999999,"15min":2.9860000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":5.7279999999999998,"5min":11.815,"15min":110.295},"last":11.782}]},{"osd":5,"last update":"Sat Sep 23 20:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":3.871,"15min":3.4649999999999999},"min":{"1min":0.39000000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.774,"5min":200.53200000000001,"15min":200.53200000000001},"last":11.832000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":3.9950000000000001,"15min":3.4300000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.26900000000000002},"max":{"1min":1.423,"5min":200.846,"15min":200.846},"last":2.8500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.593,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":1.5329999999999999,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.3280000000000001,"15min":2.286},"min":{"1min":0.42699999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.2040000000000002,"5min":8.2629999999999999,"15min":82.233000000000004},"last":4.1509999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:17:52 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":2.8929999999999998,"15min":3.2309999999999999},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.29599999999999999},"max":{"1min":1.5569999999999999,"5min":139.465,"15min":139.465},"last":2.8140000000000001},{"interface":"front","average":{"1min":0.69299999999999995,"5min":2.9079999999999999,"15min":3.3050000000000002},"min":{"1min":0.38700000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.766,"5min":141.56200000000001,"15min":141.56200000000001},"last":2.919}]}]},{"osd":4,"up_from":30,"seq":128849019362,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3925100,"kb_used_data":3395300,"kb_used_omap":0,"kb_used_meta":529792,"kb_avail":89820052,"statfs":{"total":95995035648,"available":91975733248,"internally_reserved":0,"allocated":3476787200,"data_stored":3475276821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":542507008},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":14.343,"5min":5.6890000000000001,"15min":3.0819999999999999},"min":{"1min":0.25,"5min":0.224,"15min":0.22},"max":{"1min":232.89500000000001,"5min":232.89500000000001,"15min":232.89500000000001},"last":1.2450000000000001},{"interface":"front","average":{"1min":14.416,"5min":5.71,"15min":3.0609999999999999},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":232.851,"5min":232.851,"15min":232.851},"last":1.2569999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":3.1480000000000001,"15min":5.226},"min":{"1min":0.26700000000000002,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":3.1579999999999999,"5min":114.05200000000001,"15min":161.613},"last":0.251},{"interface":"front","average":{"1min":1.1519999999999999,"5min":3.1499999999999999,"15min":5.2380000000000004},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.217},"max":{"1min":3.109,"5min":114.02800000000001,"15min":162.017},"last":0.32000000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:18:21 2023","interfaces":[{"interface":"back","average":{"1min":4.9160000000000004,"5min":4.9770000000000003,"15min":5.9820000000000002},"min":{"1min":0.29899999999999999,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":53.057000000000002,"5min":114.161,"15min":182.96700000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":4.883,"5min":4.9370000000000003,"15min":5.9470000000000001},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":53.796999999999997,"5min":114.26600000000001,"15min":183.244},"last":2.1299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":3.9790000000000001,"15min":5.0960000000000001},"min":{"1min":0.34699999999999998,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":13.426,"5min":115.937,"15min":184.37100000000001},"last":2.4060000000000001},{"interface":"front","average":{"1min":1.9039999999999999,"5min":3.9350000000000001,"15min":5.1559999999999997},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":10.486000000000001,"5min":115.902,"15min":184.22300000000001},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":2.2509999999999999,"15min":1.6379999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":2.9670000000000001,"5min":114.788,"15min":176.81899999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.753,"5min":2.363,"15min":1.698},"min":{"1min":0.29699999999999999,"5min":0.29499999999999998,"15min":0.26600000000000001},"max":{"1min":4.0460000000000003,"5min":113.822,"15min":176.172},"last":0.29799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:17:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.0390000000000001,"15min":1.026},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":2.5110000000000001,"5min":114.232,"15min":114.232},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.3540000000000001,"15min":1.2130000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.247},"max":{"1min":4.085,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":2.4620000000000002,"15min":1.9450000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":1.8500000000000001,"5min":113.861,"15min":113.861},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.73099999999999998,"5min":2.548,"15min":2.0550000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.222},"max":{"1min":2.8839999999999999,"5min":114.32299999999999,"15min":114.32299999999999},"last":0.372}]}]},{"osd":5,"up_from":38,"seq":163208757726,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2720288,"kb_used_data":2086936,"kb_used_omap":0,"kb_used_meta":633344,"kb_avail":91024864,"statfs":{"total":95995035648,"available":93209460736,"internally_reserved":0,"allocated":2137022464,"data_stored":2135520598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":648544256},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:35 2023","interfaces":[{"interface":"back","average":{"1min":11.675000000000001,"5min":6.9509999999999996,"15min":3.649},"min":{"1min":0.29699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":166.57900000000001,"5min":166.57900000000001,"15min":166.57900000000001},"last":2.5529999999999999},{"interface":"front","average":{"1min":10.929,"5min":6.3719999999999999,"15min":3.6840000000000002},"min":{"1min":0.30599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":165.67500000000001,"5min":165.67500000000001,"15min":165.67500000000001},"last":2.508}]},{"osd":1,"last update":"Sat Sep 23 20:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":5.1619999999999999,"15min":4.8520000000000003},"min":{"1min":0.248,"5min":0.22,"15min":0.22},"max":{"1min":17.010999999999999,"5min":150.16999999999999,"15min":171.47300000000001},"last":2.1619999999999999},{"interface":"front","average":{"1min":1.454,"5min":4.6630000000000003,"15min":4.8689999999999998},"min":{"1min":0.313,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":18.879000000000001,"5min":150.12700000000001,"15min":171.399},"last":2.024}]},{"osd":2,"last update":"Sat Sep 23 20:18:12 2023","interfaces":[{"interface":"back","average":{"1min":9.3829999999999991,"5min":3.3450000000000002,"15min":4.7640000000000002},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":161.852,"5min":161.852,"15min":240.57300000000001},"last":2.3180000000000001},{"interface":"front","average":{"1min":9.5299999999999994,"5min":3.4279999999999999,"15min":4.8399999999999999},"min":{"1min":0.41799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":161.95599999999999,"5min":161.95599999999999,"15min":240.464},"last":2.3359999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:17:52 2023","interfaces":[{"interface":"back","average":{"1min":10.457000000000001,"5min":10.824,"15min":7.0780000000000003},"min":{"1min":0.33300000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":196.97800000000001,"5min":196.97800000000001,"15min":201.50899999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":10.571999999999999,"5min":11.041,"15min":6.8330000000000002},"min":{"1min":0.35999999999999999,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":196.875,"5min":196.875,"15min":201.58099999999999},"last":3.1419999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:18:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1579999999999999,"15min":0.77800000000000002},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.24099999999999999},"max":{"1min":4.2350000000000003,"5min":8,"15min":9.7240000000000002},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.72899999999999998,"15min":0.60199999999999998},"min":{"1min":0.33700000000000002,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":1.5029999999999999,"5min":3.371,"15min":8.7360000000000007},"last":0.32300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2490000000000001,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":4.04,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.73299999999999998,"15min":0.60999999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":1.609,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":1.492}]},{"osd":7,"last update":"Sat Sep 23 20:18:12 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":1.6439999999999999,"15min":0.93700000000000006},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":1.877,"5min":11.725,"15min":11.725},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.02,"5min":1.502,"15min":0.92300000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.8499999999999996,"5min":11.041,"15min":11.041},"last":0.313}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3816165376,"data_stored":3816161304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":491716608,"data_stored":491716608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2821521408,"data_stored":2821521408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2411139072,"data_stored":2411135000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3458994176,"data_stored":3458990104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2120876032,"data_stored":2120876032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":237764608,"data_stored":237764608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2018508800,"data_stored":2018508800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:18:54.385 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:18:54.385 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-09-23T20:18:54.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:54.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:54.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:54 smithi135 ceph-mon[213871]: pgmap v1416: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 14347/253992 objects degraded (5.649%); 5430/253992 objects misplaced (2.138%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:18:54.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:54 smithi135 ceph-mon[221113]: pgmap v1416: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 14347/253992 objects degraded (5.649%); 5430/253992 objects misplaced (2.138%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:18:54.487 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:54.564 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:54 smithi144 ceph-mon[194833]: pgmap v1416: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 14347/253992 objects degraded (5.649%); 5430/253992 objects misplaced (2.138%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:18:54.730 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:54.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:54.851 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:54.943 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:55.033 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:18:55.033 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:18:55.109 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:55.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:55.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:55.223 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:55.316 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:55.626 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:18:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:55.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:18:55.634 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:55 smithi135 ceph-mon[213871]: pgmap v1417: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 14154/252273 objects degraded (5.611%); 5430/252273 objects misplaced (2.152%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:18:55.634 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:55 smithi135 ceph-mon[221113]: pgmap v1417: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 14154/252273 objects degraded (5.611%); 5430/252273 objects misplaced (2.152%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:18:55.635 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:55 smithi144 ceph-mon[194833]: pgmap v1417: 109 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 14154/252273 objects degraded (5.611%); 5430/252273 objects misplaced (2.152%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:18:55.735 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:55.762 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:55.806 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:55.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:55.909 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:55.937 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:55.999 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:56.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:56.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:56.108 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:56.134 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:56.336 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:18:56.720 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:18:56.755 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:56.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:56.870 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:56.875 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:56.910 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:56.959 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:56.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:57.067 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:18:57.197 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:57.448 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:18:57.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:57.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:18:57.556 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:57.636 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:57.686 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:57.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:57.792 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:18:57.884 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:57.952 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:18:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:57.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:18:58.055 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:18:58.087 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:58.147 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:18:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:58.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:18:58.245 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:18:58.256 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:18:58.342 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:58.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:58 smithi144 ceph-mon[194833]: pgmap v1418: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 14154/246141 objects degraded (5.750%); 5051/246141 objects misplaced (2.052%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:18:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/573931968' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:58.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:58 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 14154/246141 objects degraded (5.750%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:18:58.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:58 smithi135 ceph-mon[213871]: pgmap v1418: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 14154/246141 objects degraded (5.750%); 5051/246141 objects misplaced (2.052%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:18:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/573931968' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:58 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 14154/246141 objects degraded (5.750%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:18:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:58 smithi135 ceph-mon[221113]: pgmap v1418: 109 pgs: 1 active+recovering+remapped, 1 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 99 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 14154/246141 objects degraded (5.750%); 5051/246141 objects misplaced (2.052%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:18:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/573931968' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:18:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:58 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 14154/246141 objects degraded (5.750%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:18:59.394 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:18:59.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:59.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:18:59.467 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:59.482 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:59.482 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/573931968' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:18:59.482 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[213871]: osdmap e506: 8 total, 7 up, 8 in 2023-09-23T20:18:59.482 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[213871]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[213871]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[213871]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[213871]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/573931968' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[221113]: osdmap e506: 8 total, 7 up, 8 in 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[221113]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[221113]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.483 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[221113]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.484 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:18:59 smithi135 ceph-mon[221113]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.514 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:18:59.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:59 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:18:59.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:59 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/573931968' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:18:59.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:59 smithi144 ceph-mon[194833]: osdmap e506: 8 total, 7 up, 8 in 2023-09-23T20:18:59.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:59 smithi144 ceph-mon[194833]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:59 smithi144 ceph-mon[194833]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:59 smithi144 ceph-mon[194833]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.542 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:18:59 smithi144 ceph-mon[194833]: failed to encode map e506 with expected crc 2023-09-23T20:18:59.547 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:18:59.581 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:59.603 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:18:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:18:59.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:18:59.713 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:18:59.819 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:00.247 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:00.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:00.358 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:00.408 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:00 smithi135 ceph-mon[213871]: pgmap v1420: 109 pgs: 2 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 9013/243498 objects degraded (3.701%); 10034/243498 objects misplaced (4.121%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:19:00.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:00 smithi135 ceph-mon[221113]: pgmap v1420: 109 pgs: 2 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 9013/243498 objects degraded (3.701%); 10034/243498 objects misplaced (4.121%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:19:00.462 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:00 smithi144 ceph-mon[194833]: pgmap v1420: 109 pgs: 2 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 9013/243498 objects degraded (3.701%); 10034/243498 objects misplaced (4.121%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:19:01.301 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 1 pg_num to 31 2023-09-23T20:19:01.502 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:19:01.572 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:01 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1873794446' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:19:01.572 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:01 smithi135 ceph-mon[213871]: pgmap v1421: 109 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 9013/241140 objects degraded (3.738%); 9834/241140 objects misplaced (4.078%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:19:01.573 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:01 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1873794446' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:19:01.573 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:01 smithi135 ceph-mon[221113]: pgmap v1421: 109 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 9013/241140 objects degraded (3.738%); 9834/241140 objects misplaced (4.078%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:19:01.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:01 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1873794446' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:19:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:01 smithi144 ceph-mon[194833]: pgmap v1421: 109 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 9013/241140 objects degraded (3.738%); 9834/241140 objects misplaced (4.078%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:19:02.184 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:02.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:02.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:02.292 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:02.318 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:02 smithi135 ceph-mon[221113]: failed to encode map e507 with expected crc 2023-09-23T20:19:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1873794446' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T20:19:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:02 smithi135 ceph-mon[221113]: osdmap e507: 8 total, 7 up, 8 in 2023-09-23T20:19:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:02 smithi135 ceph-mon[213871]: failed to encode map e507 with expected crc 2023-09-23T20:19:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1873794446' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T20:19:02.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:02 smithi135 ceph-mon[213871]: osdmap e507: 8 total, 7 up, 8 in 2023-09-23T20:19:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:02 smithi144 ceph-mon[194833]: failed to encode map e507 with expected crc 2023-09-23T20:19:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1873794446' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T20:19:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:02 smithi144 ceph-mon[194833]: osdmap e507: 8 total, 7 up, 8 in 2023-09-23T20:19:02.860 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:02.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:02.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:02.973 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:03.005 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:03.060 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:03.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:03.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:03.171 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:03.201 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:03.259 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:03.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:03.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:03.364 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:03.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:03 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9013/241140 objects degraded (3.738%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:03.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:03 smithi135 ceph-mon[221113]: pgmap v1423: 109 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 9013/235548 objects degraded (3.826%); 9605/235548 objects misplaced (4.078%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:19:03.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:03 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T20:19:03.393 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:03 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9013/241140 objects degraded (3.738%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:03.393 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:03 smithi135 ceph-mon[213871]: pgmap v1423: 109 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 9013/235548 objects degraded (3.826%); 9605/235548 objects misplaced (4.078%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:19:03.393 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:03 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T20:19:03.434 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:03.606 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:03 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9013/241140 objects degraded (3.738%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:03 smithi144 ceph-mon[194833]: pgmap v1423: 109 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 9013/235548 objects degraded (3.826%); 9605/235548 objects misplaced (4.078%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:19:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:03 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T20:19:03.686 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:03.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:03.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:03.744 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:19:03.744 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:19:03.791 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:03.905 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:03.976 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:19:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:19:03] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.43.0" 2023-09-23T20:19:04.024 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:19:04.029 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:04.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:04.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:04.082 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:19:04.133 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:19:04.137 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:04.165 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:19:04.168 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:04.204 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:19:04.226 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:04.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:04.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:04.239 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:19:04.276 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:19:04.276 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:19:04.277 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:19:04.277 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:19:04.277 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:19:04.277 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:19:04.277 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:19:04.277 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:19:04.277 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:19:04.334 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:04.403 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1193040475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:04 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T20:19:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:04 smithi144 ceph-mon[194833]: osdmap e508: 8 total, 7 up, 8 in 2023-09-23T20:19:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1193040475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:04 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T20:19:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:04 smithi135 ceph-mon[213871]: osdmap e508: 8 total, 7 up, 8 in 2023-09-23T20:19:04.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1193040475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:04.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:04 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T20:19:04.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:04 smithi135 ceph-mon[221113]: osdmap e508: 8 total, 7 up, 8 in 2023-09-23T20:19:05.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:05 smithi144 ceph-mon[194833]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:05 smithi144 ceph-mon[194833]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:05 smithi144 ceph-mon[194833]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:05 smithi144 ceph-mon[194833]: osdmap e509: 8 total, 7 up, 8 in 2023-09-23T20:19:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:05 smithi144 ceph-mon[194833]: pgmap v1426: 119 pgs: 1 peering, 10 unknown, 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 9013/233730 objects degraded (3.856%); 9402/233730 objects misplaced (4.023%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T20:19:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[213871]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[213871]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[213871]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[213871]: osdmap e509: 8 total, 7 up, 8 in 2023-09-23T20:19:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[213871]: pgmap v1426: 119 pgs: 1 peering, 10 unknown, 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 9013/233730 objects degraded (3.856%); 9402/233730 objects misplaced (4.023%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T20:19:05.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[221113]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[221113]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[221113]: failed to encode map e508 with expected crc 2023-09-23T20:19:05.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[221113]: osdmap e509: 8 total, 7 up, 8 in 2023-09-23T20:19:05.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:05 smithi135 ceph-mon[221113]: pgmap v1426: 119 pgs: 1 peering, 10 unknown, 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 9013/233730 objects degraded (3.856%); 9402/233730 objects misplaced (4.023%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T20:19:05.778 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:19:05.996 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:06.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:06.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:19:06.086 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:19:06.106 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:06.199 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:06.253 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:06.370 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:06.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:06.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:06.485 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:06.547 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[221113]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.547 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[221113]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.547 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[221113]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.547 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[221113]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.547 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[221113]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[213871]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[213871]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[213871]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[213871]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:06 smithi135 ceph-mon[213871]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.549 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:06.592 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:06.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:06 smithi144 ceph-mon[194833]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:06 smithi144 ceph-mon[194833]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:06 smithi144 ceph-mon[194833]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:06 smithi144 ceph-mon[194833]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:06 smithi144 ceph-mon[194833]: failed to encode map e509 with expected crc 2023-09-23T20:19:06.696 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:06.724 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:06.881 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:06.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:06.984 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:07.027 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:07.105 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:07.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:07.211 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:07.247 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:07.297 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:07.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:07.403 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:07.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:07 smithi135 ceph-mon[221113]: pgmap v1427: 119 pgs: 1 active+recovering+remapped, 1 peering, 6 unknown, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 9013/227295 objects degraded (3.965%); 8983/227295 objects misplaced (3.952%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:07.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:07 smithi135 ceph-mon[213871]: pgmap v1427: 119 pgs: 1 active+recovering+remapped, 1 peering, 6 unknown, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 9013/227295 objects degraded (3.965%); 8983/227295 objects misplaced (3.952%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:07.510 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:07.614 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:07.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:07.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:07 smithi144 ceph-mon[194833]: pgmap v1427: 119 pgs: 1 active+recovering+remapped, 1 peering, 6 unknown, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 102 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 9013/227295 objects degraded (3.965%); 8983/227295 objects misplaced (3.952%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:07.722 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:07.758 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:07.827 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:07.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:07.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:07.931 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:08.005 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:08.087 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:19:08.133 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:19:08.133 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:19:08.133 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-09-23T20:19:08.134 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:19:08.135 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:08.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:08.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:08.241 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:08.350 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:08 smithi135 ceph-mon[213871]: 1.a scrub starts 2023-09-23T20:19:08.351 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:08 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9013/227295 objects degraded (3.965%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:08.404 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:08.535 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:08.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:08.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:08 smithi144 ceph-mon[194833]: 1.a scrub starts 2023-09-23T20:19:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:08 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9013/227295 objects degraded (3.965%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:08.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:08.696 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:08 smithi135 ceph-mon[221113]: 1.a scrub starts 2023-09-23T20:19:08.696 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:08 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9013/227295 objects degraded (3.965%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:08.705 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:08.845 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:08.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:08.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:08.913 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:08.954 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:08.975 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:09.052 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:19:09.072 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:09.224 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:19:09.224 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:19:09.224 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.224 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:111294 10.6 e510/479 10:66e153d7:::benchmark_data_smithi135_602286_object19068:head v 510'6978, mlcod=510'6976)", 2023-09-23T20:19:09.224 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:09.217536+0000", 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 0.002240082, 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022697809999999998, 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:09.217536+0000", 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:09.225 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:09.217536+0000", 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:09.217535+0000", 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.226 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:09.217541+0000", 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2400000000000004e-06 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:09.217555+0000", 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3974999999999999e-05 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:09.217562+0000", 2023-09-23T20:19:09.227 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4280000000000001e-06 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:09.217620+0000", 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8644000000000003e-05 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:09.217633+0000", 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2860999999999999e-05 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:09.228 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:09.229 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:09.229 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:09.229 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2023-09-23T20:19:09.229 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 1 2023-09-23T20:19:09.229 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:19:09.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1237155367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:09.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:19:09.368 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1237155367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:09.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[213871]: osdmap e510: 8 total, 7 up, 8 in 2023-09-23T20:19:09.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[213871]: pgmap v1429: 119 pgs: 1 active+recovering+remapped, 1 peering, 1 unknown, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 9013/224229 objects degraded (4.020%); 8791/224229 objects misplaced (3.921%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:09.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1237155367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:09.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:19:09.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1237155367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:09.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[221113]: osdmap e510: 8 total, 7 up, 8 in 2023-09-23T20:19:09.369 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:09 smithi135 ceph-mon[221113]: pgmap v1429: 119 pgs: 1 active+recovering+remapped, 1 peering, 1 unknown, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 9013/224229 objects degraded (4.020%); 8791/224229 objects misplaced (3.921%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:09.383 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:09.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:09.492 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:09.590 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1237155367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:09 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:19:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1237155367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:09 smithi144 ceph-mon[194833]: osdmap e510: 8 total, 7 up, 8 in 2023-09-23T20:19:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:09 smithi144 ceph-mon[194833]: pgmap v1429: 119 pgs: 1 active+recovering+remapped, 1 peering, 1 unknown, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 9013/224229 objects degraded (4.020%); 8791/224229 objects misplaced (3.921%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:09.952 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:19:10.352 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:10.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:10 smithi144 ceph-mon[194833]: 1.8 scrub starts 2023-09-23T20:19:10.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:10 smithi144 ceph-mon[194833]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:10 smithi144 ceph-mon[194833]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:10 smithi144 ceph-mon[194833]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.618 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:10 smithi144 ceph-mon[194833]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[213871]: 1.8 scrub starts 2023-09-23T20:19:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[213871]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[213871]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[213871]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[213871]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[221113]: 1.8 scrub starts 2023-09-23T20:19:10.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[221113]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[221113]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[221113]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:10 smithi135 ceph-mon[221113]: failed to encode map e510 with expected crc 2023-09-23T20:19:10.844 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:10.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:10.974 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:11.007 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:11.061 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:11.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:11.165 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:11.238 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:11 smithi135 ceph-mon[213871]: pgmap v1430: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 9013/221892 objects degraded (4.062%); 8507/221892 objects misplaced (3.834%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:19:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:11 smithi135 ceph-mon[221113]: pgmap v1430: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 9013/221892 objects degraded (4.062%); 8507/221892 objects misplaced (3.834%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:19:11.472 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:11.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:11.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:11.583 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:11.609 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:11.751 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:11 smithi144 ceph-mon[194833]: pgmap v1430: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 9013/221892 objects degraded (4.062%); 8507/221892 objects misplaced (3.834%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:19:12.002 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:19:12.002 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:19:12.002 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:19:12.002 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:19:12.002 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:19:12.230 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:12.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:12.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:12.341 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:12.369 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:12.640 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:12 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:19:12.776 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:19:12.910 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:12.918 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:12 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:19:12.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:12.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:12.952 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:19:13.020 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:13.131 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:12 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:19:13.142 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:13.226 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:13.375 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:13.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:13.492 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:13.529 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:13.645 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:13 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9013/221892 objects degraded (4.062%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:13.645 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:13 smithi135 ceph-mon[213871]: pgmap v1431: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 9013/216447 objects degraded (4.164%); 8295/216447 objects misplaced (3.832%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:19:13.691 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:13.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:13.785 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:19:13.796 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:13.817 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:13 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9013/221892 objects degraded (4.062%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:13.817 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:13 smithi144 ceph-mon[194833]: pgmap v1431: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 9013/216447 objects degraded (4.164%); 8295/216447 objects misplaced (3.832%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:19:13.821 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:19:13.838 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:13.854 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:19:13.891 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:19:13.897 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:13.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:13.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:13.928 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:19:13.967 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:19:13.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:19:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:19:13] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-09-23T20:19:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:13 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9013/221892 objects degraded (4.062%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:13.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:13 smithi135 ceph-mon[221113]: pgmap v1431: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 9013/216447 objects degraded (4.164%); 8295/216447 objects misplaced (3.832%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:19:14.001 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:14.027 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:14.050 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:14.068 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:14.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:14.148 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:19:14.149 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:19:14.149 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:19:14.149 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:19:14.149 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:19:14.149 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:19:14.149 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:19:14.149 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:19:14.149 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:19:14.172 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:14.203 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:14.414 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:19:14.423 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:14.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:14.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:14.441 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:19:14.528 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:14.688 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/875355736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:14.688 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:14 smithi144 ceph-mon[194833]: 1.18 scrub starts 2023-09-23T20:19:14.712 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:14.924 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:19:14.924 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:19:14.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/875355736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:14.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:14 smithi135 ceph-mon[221113]: 1.18 scrub starts 2023-09-23T20:19:14.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/875355736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:14.975 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:14 smithi135 ceph-mon[213871]: 1.18 scrub starts 2023-09-23T20:19:15.609 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:19:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:15 smithi144 ceph-mon[194833]: from='client.46535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:19:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:15 smithi144 ceph-mon[194833]: pgmap v1432: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9013/214680 objects degraded (4.198%); 8113/214680 objects misplaced (3.779%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:19:15.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:15 smithi135 ceph-mon[221113]: from='client.46535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:19:15.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:15 smithi135 ceph-mon[221113]: pgmap v1432: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9013/214680 objects degraded (4.198%); 8113/214680 objects misplaced (3.779%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:19:15.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:15 smithi135 ceph-mon[213871]: from='client.46535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:19:15.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:15 smithi135 ceph-mon[213871]: pgmap v1432: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9013/214680 objects degraded (4.198%); 8113/214680 objects misplaced (3.779%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:19:16.055 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:16.295 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:16.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:16.401 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:19:16.402 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:19:16.403 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:16.446 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:16.516 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:16.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:16.581 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1431,"stamp":"2023-09-23T20:19:13.173964+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4731282319,"num_objects":72149,"num_object_clones":0,"num_object_copies":216447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9013,"num_objects_misplaced":8295,"num_objects_unfound":0,"num_objects_dirty":72149,"num_whiteouts":0,"num_read":6011,"num_read_kb":13264,"num_write":113249,"num_write_kb":5915145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6243,"num_bytes_recovered":399363699,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103137,"ondisk_log_size":103137,"up":350,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19945156,"kb_used_data":14451972,"kb_used_omap":38,"kb_used_meta":5493017,"kb_avail":730016060,"statfs":{"total":767960285184,"available":747536445440,"internally_reserved":0,"allocated":14798819328,"data_stored":14782866708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39803,"internal_metadata":5624849541},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,3,1,2,6,1,6],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-377552896,"num_objects":-5761,"num_object_clones":0,"num_object_copies":-17283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1107,"num_objects_unfound":0,"num_objects_dirty":-5761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5761,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":535,"num_bytes_recovered":35061760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.117465"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":866,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.090157+0000","last_change":"2023-09-23T20:19:05.090157+0000","last_active":"2023-09-23T20:19:05.090157+0000","last_peered":"2023-09-23T20:19:05.090157+0000","last_clean":"2023-09-23T20:19:05.090157+0000","last_became_active":"2023-09-23T20:19:05.089810+0000","last_became_peered":"2023-09-23T20:19:05.089810+0000","last_unstale":"2023-09-23T20:19:05.090157+0000","last_undegraded":"2023-09-23T20:19:05.090157+0000","last_fullsized":"2023-09-23T20:19:05.090157+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057748+0000","last_change":"2023-09-23T20:17:47.279267+0000","last_active":"2023-09-23T20:19:05.057748+0000","last_peered":"2023-09-23T20:19:05.057748+0000","last_clean":"2023-09-23T20:19:05.057748+0000","last_became_active":"2023-09-23T20:17:47.278920+0000","last_became_peered":"2023-09-23T20:17:47.278920+0000","last_unstale":"2023-09-23T20:19:05.057748+0000","last_undegraded":"2023-09-23T20:19:05.057748+0000","last_fullsized":"2023-09-23T20:19:05.057748+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:16.699236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057080+0000","last_change":"2023-09-23T20:17:47.270409+0000","last_active":"2023-09-23T20:19:05.057080+0000","last_peered":"2023-09-23T20:19:05.057080+0000","last_clean":"2023-09-23T20:19:05.057080+0000","last_became_active":"2023-09-23T20:17:47.270086+0000","last_became_peered":"2023-09-23T20:17:47.270086+0000","last_unstale":"2023-09-23T20:19:05.057080+0000","last_undegraded":"2023-09-23T20:19:05.057080+0000","last_fullsized":"2023-09-23T20:19:05.057080+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.885372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":573,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.091704+0000","last_change":"2023-09-23T20:19:05.090444+0000","last_active":"2023-09-23T20:19:05.091704+0000","last_peered":"2023-09-23T20:19:05.091704+0000","last_clean":"2023-09-23T20:19:05.091704+0000","last_became_active":"2023-09-23T20:19:05.090104+0000","last_became_peered":"2023-09-23T20:19:05.090104+0000","last_unstale":"2023-09-23T20:19:05.091704+0000","last_undegraded":"2023-09-23T20:19:05.091704+0000","last_fullsized":"2023-09-23T20:19:05.091704+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056392+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:19:05.056392+0000","last_peered":"2023-09-23T20:19:05.056392+0000","last_clean":"2023-09-23T20:19:05.056392+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:19:05.056392+0000","last_undegraded":"2023-09-23T20:19:05.056392+0000","last_fullsized":"2023-09-23T20:19:05.056392+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":218,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.059225+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:19:05.059225+0000","last_peered":"2023-09-23T20:19:05.059225+0000","last_clean":"2023-09-23T20:19:05.059225+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:19:05.059225+0000","last_undegraded":"2023-09-23T20:19:05.059225+0000","last_fullsized":"2023-09-23T20:19:05.059225+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":856,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.067892+0000","last_change":"2023-09-23T20:19:05.067892+0000","last_active":"2023-09-23T20:19:05.067892+0000","last_peered":"2023-09-23T20:19:05.067892+0000","last_clean":"2023-09-23T20:19:05.067892+0000","last_became_active":"2023-09-23T20:19:05.067520+0000","last_became_peered":"2023-09-23T20:19:05.067520+0000","last_unstale":"2023-09-23T20:19:05.067892+0000","last_undegraded":"2023-09-23T20:19:05.067892+0000","last_fullsized":"2023-09-23T20:19:05.067892+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056273+0000","last_change":"2023-09-23T20:17:47.279385+0000","last_active":"2023-09-23T20:19:05.056273+0000","last_peered":"2023-09-23T20:19:05.056273+0000","last_clean":"2023-09-23T20:19:05.056273+0000","last_became_active":"2023-09-23T20:17:47.279065+0000","last_became_peered":"2023-09-23T20:17:47.279065+0000","last_unstale":"2023-09-23T20:19:05.056273+0000","last_undegraded":"2023-09-23T20:19:05.056273+0000","last_fullsized":"2023-09-23T20:19:05.056273+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:19.638157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":474,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.045273+0000","last_change":"2023-09-23T20:17:47.270621+0000","last_active":"2023-09-23T20:19:04.045273+0000","last_peered":"2023-09-23T20:19:04.045273+0000","last_clean":"2023-09-23T20:19:04.045273+0000","last_became_active":"2023-09-23T20:17:47.270375+0000","last_became_peered":"2023-09-23T20:17:47.270375+0000","last_unstale":"2023-09-23T20:19:04.045273+0000","last_undegraded":"2023-09-23T20:19:04.045273+0000","last_fullsized":"2023-09-23T20:19:04.045273+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:16.961271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":751,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:06.387187+0000","last_change":"2023-09-23T20:19:05.069925+0000","last_active":"2023-09-23T20:19:06.387187+0000","last_peered":"2023-09-23T20:19:06.387187+0000","last_clean":"2023-09-23T20:19:06.387187+0000","last_became_active":"2023-09-23T20:19:05.069486+0000","last_became_peered":"2023-09-23T20:19:05.069486+0000","last_unstale":"2023-09-23T20:19:06.387187+0000","last_undegraded":"2023-09-23T20:19:06.387187+0000","last_fullsized":"2023-09-23T20:19:06.387187+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":435,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.045142+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:19:04.045142+0000","last_peered":"2023-09-23T20:19:04.045142+0000","last_clean":"2023-09-23T20:19:04.045142+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:19:04.045142+0000","last_undegraded":"2023-09-23T20:19:04.045142+0000","last_fullsized":"2023-09-23T20:19:04.045142+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":374,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.051423+0000","last_change":"2023-09-23T20:17:47.265452+0000","last_active":"2023-09-23T20:19:05.051423+0000","last_peered":"2023-09-23T20:19:05.051423+0000","last_clean":"2023-09-23T20:19:05.051423+0000","last_became_active":"2023-09-23T20:17:47.265115+0000","last_became_peered":"2023-09-23T20:17:47.265115+0000","last_unstale":"2023-09-23T20:19:05.051423+0000","last_undegraded":"2023-09-23T20:19:05.051423+0000","last_fullsized":"2023-09-23T20:19:05.051423+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:04.898742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":844,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.064265+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:19:05.064265+0000","last_peered":"2023-09-23T20:19:05.064265+0000","last_clean":"2023-09-23T20:19:05.064265+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:19:05.064265+0000","last_undegraded":"2023-09-23T20:19:05.064265+0000","last_fullsized":"2023-09-23T20:19:05.064265+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":528,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.045977+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:19:04.045977+0000","last_peered":"2023-09-23T20:19:04.045977+0000","last_clean":"2023-09-23T20:19:04.045977+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:19:04.045977+0000","last_undegraded":"2023-09-23T20:19:04.045977+0000","last_fullsized":"2023-09-23T20:19:04.045977+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":375,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.052108+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:19:05.052108+0000","last_peered":"2023-09-23T20:19:05.052108+0000","last_clean":"2023-09-23T20:19:05.052108+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:19:05.052108+0000","last_undegraded":"2023-09-23T20:19:05.052108+0000","last_fullsized":"2023-09-23T20:19:05.052108+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":802,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:06.760002+0000","last_change":"2023-09-23T20:19:05.073028+0000","last_active":"2023-09-23T20:19:06.760002+0000","last_peered":"2023-09-23T20:19:06.760002+0000","last_clean":"2023-09-23T20:19:06.760002+0000","last_became_active":"2023-09-23T20:19:05.072575+0000","last_became_peered":"2023-09-23T20:19:05.072575+0000","last_unstale":"2023-09-23T20:19:06.760002+0000","last_undegraded":"2023-09-23T20:19:06.760002+0000","last_fullsized":"2023-09-23T20:19:06.760002+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":203,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.059803+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:19:05.059803+0000","last_peered":"2023-09-23T20:19:05.059803+0000","last_clean":"2023-09-23T20:19:05.059803+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:19:05.059803+0000","last_undegraded":"2023-09-23T20:19:05.059803+0000","last_fullsized":"2023-09-23T20:19:05.059803+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057566+0000","last_change":"2023-09-23T20:17:47.278773+0000","last_active":"2023-09-23T20:19:05.057566+0000","last_peered":"2023-09-23T20:19:05.057566+0000","last_clean":"2023-09-23T20:19:05.057566+0000","last_became_active":"2023-09-23T20:17:47.278467+0000","last_became_peered":"2023-09-23T20:17:47.278467+0000","last_unstale":"2023-09-23T20:19:05.057566+0000","last_undegraded":"2023-09-23T20:19:05.057566+0000","last_fullsized":"2023-09-23T20:19:05.057566+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:41.368483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":705,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.658374+0000","last_change":"2023-09-23T20:19:05.092744+0000","last_active":"2023-09-23T20:19:05.658374+0000","last_peered":"2023-09-23T20:19:05.658374+0000","last_clean":"2023-09-23T20:19:05.658374+0000","last_became_active":"2023-09-23T20:19:05.092425+0000","last_became_peered":"2023-09-23T20:19:05.092425+0000","last_unstale":"2023-09-23T20:19:05.658374+0000","last_undegraded":"2023-09-23T20:19:05.658374+0000","last_fullsized":"2023-09-23T20:19:05.658374+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":456,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.059024+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:19:05.059024+0000","last_peered":"2023-09-23T20:19:05.059024+0000","last_clean":"2023-09-23T20:19:05.059024+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:19:05.059024+0000","last_undegraded":"2023-09-23T20:19:05.059024+0000","last_fullsized":"2023-09-23T20:19:05.059024+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":29,"reported_epoch":509,"state":"active+undersized","last_fresh":"2023-09-23T20:19:05.058777+0000","last_change":"2023-09-23T20:18:38.012263+0000","last_active":"2023-09-23T20:19:05.058777+0000","last_peered":"2023-09-23T20:19:05.058777+0000","last_clean":"2023-09-23T20:17:54.133085+0000","last_became_active":"2023-09-23T20:18:38.012263+0000","last_became_peered":"2023-09-23T20:18:38.012263+0000","last_unstale":"2023-09-23T20:19:05.058777+0000","last_undegraded":"2023-09-23T20:19:05.058777+0000","last_fullsized":"2023-09-23T20:18:38.003813+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:47.449647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":868,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044423+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:19:04.044423+0000","last_peered":"2023-09-23T20:19:04.044423+0000","last_clean":"2023-09-23T20:19:04.044423+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:19:04.044423+0000","last_undegraded":"2023-09-23T20:19:04.044423+0000","last_fullsized":"2023-09-23T20:19:04.044423+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":203,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056763+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:19:05.056763+0000","last_peered":"2023-09-23T20:19:05.056763+0000","last_clean":"2023-09-23T20:19:05.056763+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:19:05.056763+0000","last_undegraded":"2023-09-23T20:19:05.056763+0000","last_fullsized":"2023-09-23T20:19:05.056763+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056670+0000","last_change":"2023-09-23T20:17:47.266963+0000","last_active":"2023-09-23T20:19:05.056670+0000","last_peered":"2023-09-23T20:19:05.056670+0000","last_clean":"2023-09-23T20:19:05.056670+0000","last_became_active":"2023-09-23T20:17:47.266645+0000","last_became_peered":"2023-09-23T20:17:47.266645+0000","last_unstale":"2023-09-23T20:19:05.056670+0000","last_undegraded":"2023-09-23T20:19:05.056670+0000","last_fullsized":"2023-09-23T20:19:05.056670+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:05.052559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":557,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.046361+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:19:04.046361+0000","last_peered":"2023-09-23T20:19:04.046361+0000","last_clean":"2023-09-23T20:19:04.046361+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:19:04.046361+0000","last_undegraded":"2023-09-23T20:19:04.046361+0000","last_fullsized":"2023-09-23T20:19:04.046361+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":218,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056277+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:19:05.056277+0000","last_peered":"2023-09-23T20:19:05.056277+0000","last_clean":"2023-09-23T20:19:05.056277+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:19:05.056277+0000","last_undegraded":"2023-09-23T20:19:05.056277+0000","last_fullsized":"2023-09-23T20:19:05.056277+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":507,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044637+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:19:04.044637+0000","last_peered":"2023-09-23T20:19:04.044637+0000","last_clean":"2023-09-23T20:19:04.044637+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:19:04.044637+0000","last_undegraded":"2023-09-23T20:19:04.044637+0000","last_fullsized":"2023-09-23T20:19:04.044637+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":568,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.058889+0000","last_change":"2023-09-23T20:17:50.330492+0000","last_active":"2023-09-23T20:19:05.058889+0000","last_peered":"2023-09-23T20:19:05.058889+0000","last_clean":"2023-09-23T20:19:05.058889+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:19:05.058889+0000","last_undegraded":"2023-09-23T20:19:05.058889+0000","last_fullsized":"2023-09-23T20:19:05.058889+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:58.248167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.248167+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":214,"reported_epoch":509,"state":"active+undersized","last_fresh":"2023-09-23T20:19:05.058136+0000","last_change":"2023-09-23T20:18:38.011856+0000","last_active":"2023-09-23T20:19:05.058136+0000","last_peered":"2023-09-23T20:19:05.058136+0000","last_clean":"2023-09-23T20:18:35.260354+0000","last_became_active":"2023-09-23T20:18:38.011856+0000","last_became_peered":"2023-09-23T20:18:38.011856+0000","last_unstale":"2023-09-23T20:19:05.058136+0000","last_undegraded":"2023-09-23T20:19:05.058136+0000","last_fullsized":"2023-09-23T20:18:37.999747+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:58.093947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":499,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.052988+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:19:05.052988+0000","last_peered":"2023-09-23T20:19:05.052988+0000","last_clean":"2023-09-23T20:19:05.052988+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:19:05.052988+0000","last_undegraded":"2023-09-23T20:19:05.052988+0000","last_fullsized":"2023-09-23T20:19:05.052988+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":666,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044776+0000","last_change":"2023-09-23T20:17:50.332129+0000","last_active":"2023-09-23T20:19:04.044776+0000","last_peered":"2023-09-23T20:19:04.044776+0000","last_clean":"2023-09-23T20:19:04.044776+0000","last_became_active":"2023-09-23T20:17:50.331669+0000","last_became_peered":"2023-09-23T20:17:50.331669+0000","last_unstale":"2023-09-23T20:19:04.044776+0000","last_undegraded":"2023-09-23T20:19:04.044776+0000","last_fullsized":"2023-09-23T20:19:04.044776+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:23.893417+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":152,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044346+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:19:04.044346+0000","last_peered":"2023-09-23T20:19:04.044346+0000","last_clean":"2023-09-23T20:19:04.044346+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:19:04.044346+0000","last_undegraded":"2023-09-23T20:19:04.044346+0000","last_fullsized":"2023-09-23T20:19:04.044346+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":161,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056285+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:19:05.056285+0000","last_peered":"2023-09-23T20:19:05.056285+0000","last_clean":"2023-09-23T20:19:05.056285+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:19:05.056285+0000","last_undegraded":"2023-09-23T20:19:05.056285+0000","last_fullsized":"2023-09-23T20:19:05.056285+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"509'6952","reported_seq":6990,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:08.497281+0000","last_change":"2023-09-23T20:16:59.490329+0000","last_active":"2023-09-23T20:19:08.497281+0000","last_peered":"2023-09-23T20:19:08.497281+0000","last_clean":"2023-09-23T20:19:08.497281+0000","last_became_active":"2023-09-23T20:16:59.490069+0000","last_became_peered":"2023-09-23T20:16:59.490069+0000","last_unstale":"2023-09-23T20:19:08.497281+0000","last_undegraded":"2023-09-23T20:19:08.497281+0000","last_fullsized":"2023-09-23T20:19:08.497281+0000","mapping_epoch":479,"log_start":"483'333","ondisk_log_start":"483'333","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.487350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301727744,"num_objects":4604,"num_object_clones":0,"num_object_copies":13812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6952,"num_write_kb":369792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":800,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.065976+0000","last_change":"2023-09-23T20:19:05.065565+0000","last_active":"2023-09-23T20:19:05.065976+0000","last_peered":"2023-09-23T20:19:05.065976+0000","last_clean":"2023-09-23T20:19:05.065976+0000","last_became_active":"2023-09-23T20:19:05.065283+0000","last_became_peered":"2023-09-23T20:19:05.065283+0000","last_unstale":"2023-09-23T20:19:05.065976+0000","last_undegraded":"2023-09-23T20:19:05.065976+0000","last_fullsized":"2023-09-23T20:19:05.065976+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":201,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.059400+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:19:05.059400+0000","last_peered":"2023-09-23T20:19:05.059400+0000","last_clean":"2023-09-23T20:19:05.059400+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:19:05.059400+0000","last_undegraded":"2023-09-23T20:19:05.059400+0000","last_fullsized":"2023-09-23T20:19:05.059400+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":517,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044434+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:19:04.044434+0000","last_peered":"2023-09-23T20:19:04.044434+0000","last_clean":"2023-09-23T20:19:04.044434+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:19:04.044434+0000","last_undegraded":"2023-09-23T20:19:04.044434+0000","last_fullsized":"2023-09-23T20:19:04.044434+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"510'7057","reported_seq":7098,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:11.440308+0000","last_change":"2023-09-23T20:16:59.487459+0000","last_active":"2023-09-23T20:19:11.440308+0000","last_peered":"2023-09-23T20:19:11.440308+0000","last_clean":"2023-09-23T20:19:11.440308+0000","last_became_active":"2023-09-23T20:16:59.487181+0000","last_became_peered":"2023-09-23T20:16:59.487181+0000","last_unstale":"2023-09-23T20:19:11.440308+0000","last_undegraded":"2023-09-23T20:19:11.440308+0000","last_fullsized":"2023-09-23T20:19:11.440308+0000","mapping_epoch":479,"log_start":"484'1962","ondisk_log_start":"484'1962","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5095,"log_dups_size":0,"ondisk_log_size":5095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:34.406760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291045376,"num_objects":4441,"num_object_clones":0,"num_object_copies":13323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7057,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":869,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.089357+0000","last_change":"2023-09-23T20:19:05.086498+0000","last_active":"2023-09-23T20:19:05.089357+0000","last_peered":"2023-09-23T20:19:05.089357+0000","last_clean":"2023-09-23T20:19:05.089357+0000","last_became_active":"2023-09-23T20:19:05.085764+0000","last_became_peered":"2023-09-23T20:19:05.085764+0000","last_unstale":"2023-09-23T20:19:05.089357+0000","last_undegraded":"2023-09-23T20:19:05.089357+0000","last_fullsized":"2023-09-23T20:19:05.089357+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":230,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.059288+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:19:05.059288+0000","last_peered":"2023-09-23T20:19:05.059288+0000","last_clean":"2023-09-23T20:19:05.059288+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:19:05.059288+0000","last_undegraded":"2023-09-23T20:19:05.059288+0000","last_fullsized":"2023-09-23T20:19:05.059288+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056467+0000","last_change":"2023-09-23T20:17:47.264776+0000","last_active":"2023-09-23T20:19:05.056467+0000","last_peered":"2023-09-23T20:19:05.056467+0000","last_clean":"2023-09-23T20:19:05.056467+0000","last_became_active":"2023-09-23T20:17:47.264473+0000","last_became_peered":"2023-09-23T20:17:47.264473+0000","last_unstale":"2023-09-23T20:19:05.056467+0000","last_undegraded":"2023-09-23T20:19:05.056467+0000","last_fullsized":"2023-09-23T20:19:05.056467+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:40.906303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"510'7074","reported_seq":7115,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:10.651056+0000","last_change":"2023-09-23T20:16:59.491310+0000","last_active":"2023-09-23T20:19:10.651056+0000","last_peered":"2023-09-23T20:19:10.651056+0000","last_clean":"2023-09-23T20:19:10.651056+0000","last_became_active":"2023-09-23T20:16:59.491016+0000","last_became_peered":"2023-09-23T20:16:59.491016+0000","last_unstale":"2023-09-23T20:19:10.651056+0000","last_undegraded":"2023-09-23T20:19:10.651056+0000","last_fullsized":"2023-09-23T20:19:10.651056+0000","mapping_epoch":479,"log_start":"484'1728","ondisk_log_start":"484'1728","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5346,"log_dups_size":0,"ondisk_log_size":5346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:59.436079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300023808,"num_objects":4578,"num_object_clones":0,"num_object_copies":13734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7074,"num_write_kb":372864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":581,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044966+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:19:04.044966+0000","last_peered":"2023-09-23T20:19:04.044966+0000","last_clean":"2023-09-23T20:19:04.044966+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:19:04.044966+0000","last_undegraded":"2023-09-23T20:19:04.044966+0000","last_fullsized":"2023-09-23T20:19:04.044966+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":222,"reported_epoch":509,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:19:05.059606+0000","last_change":"2023-09-23T20:18:38.001952+0000","last_active":"2023-09-23T20:19:05.059606+0000","last_peered":"2023-09-23T20:19:05.059606+0000","last_clean":"2023-09-23T20:18:35.257306+0000","last_became_active":"2023-09-23T20:18:38.001952+0000","last_became_peered":"2023-09-23T20:18:38.001952+0000","last_unstale":"2023-09-23T20:19:05.059606+0000","last_undegraded":"2023-09-23T20:18:38.000196+0000","last_fullsized":"2023-09-23T20:18:38.000044+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:17.507792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057650+0000","last_change":"2023-09-23T20:17:47.266977+0000","last_active":"2023-09-23T20:19:05.057650+0000","last_peered":"2023-09-23T20:19:05.057650+0000","last_clean":"2023-09-23T20:19:05.057650+0000","last_became_active":"2023-09-23T20:17:47.266657+0000","last_became_peered":"2023-09-23T20:17:47.266657+0000","last_unstale":"2023-09-23T20:19:05.057650+0000","last_undegraded":"2023-09-23T20:19:05.057650+0000","last_fullsized":"2023-09-23T20:19:05.057650+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:33.774981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"510'7189","reported_seq":7230,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:11.353129+0000","last_change":"2023-09-23T20:16:59.488990+0000","last_active":"2023-09-23T20:19:11.353129+0000","last_peered":"2023-09-23T20:19:11.353129+0000","last_clean":"2023-09-23T20:19:11.353129+0000","last_became_active":"2023-09-23T20:16:59.488614+0000","last_became_peered":"2023-09-23T20:16:59.488614+0000","last_unstale":"2023-09-23T20:19:11.353129+0000","last_undegraded":"2023-09-23T20:19:11.353129+0000","last_fullsized":"2023-09-23T20:19:11.353129+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":7189,"log_dups_size":0,"ondisk_log_size":7189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:11.359750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296943616,"num_objects":4531,"num_object_clones":0,"num_object_copies":13593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7189,"num_write_kb":375040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":723,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.072412+0000","last_change":"2023-09-23T20:19:05.072019+0000","last_active":"2023-09-23T20:19:05.072412+0000","last_peered":"2023-09-23T20:19:05.072412+0000","last_clean":"2023-09-23T20:19:05.072412+0000","last_became_active":"2023-09-23T20:19:05.071700+0000","last_became_peered":"2023-09-23T20:19:05.071700+0000","last_unstale":"2023-09-23T20:19:05.072412+0000","last_undegraded":"2023-09-23T20:19:05.072412+0000","last_fullsized":"2023-09-23T20:19:05.072412+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":567,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044698+0000","last_change":"2023-09-23T20:17:47.267657+0000","last_active":"2023-09-23T20:19:04.044698+0000","last_peered":"2023-09-23T20:19:04.044698+0000","last_clean":"2023-09-23T20:19:04.044698+0000","last_became_active":"2023-09-23T20:17:47.267288+0000","last_became_peered":"2023-09-23T20:17:47.267288+0000","last_unstale":"2023-09-23T20:19:04.044698+0000","last_undegraded":"2023-09-23T20:19:04.044698+0000","last_fullsized":"2023-09-23T20:19:04.044698+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.185253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.058209+0000","last_change":"2023-09-23T20:17:47.280675+0000","last_active":"2023-09-23T20:19:05.058209+0000","last_peered":"2023-09-23T20:19:05.058209+0000","last_clean":"2023-09-23T20:19:05.058209+0000","last_became_active":"2023-09-23T20:17:47.280175+0000","last_became_peered":"2023-09-23T20:17:47.280175+0000","last_unstale":"2023-09-23T20:19:05.058209+0000","last_undegraded":"2023-09-23T20:19:05.058209+0000","last_fullsized":"2023-09-23T20:19:05.058209+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:08.499949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"510'7052","reported_seq":7096,"reported_epoch":510,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:19:10.618257+0000","last_change":"2023-09-23T20:17:48.443980+0000","last_active":"2023-09-23T20:19:10.618257+0000","last_peered":"2023-09-23T20:19:10.618257+0000","last_clean":"2023-09-23T20:17:46.225624+0000","last_became_active":"2023-09-23T20:17:48.428406+0000","last_became_peered":"2023-09-23T20:17:48.428406+0000","last_unstale":"2023-09-23T20:19:10.618257+0000","last_undegraded":"2023-09-23T20:17:48.291655+0000","last_fullsized":"2023-09-23T20:17:48.256072+0000","mapping_epoch":490,"log_start":"484'1747","ondisk_log_start":"484'1747","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:14.276994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294518784,"num_objects":4494,"num_object_clones":0,"num_object_copies":13482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3930,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7052,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[0,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":2651},{"shards":"0,4","objects":564}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":902,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.072418+0000","last_change":"2023-09-23T20:19:05.072169+0000","last_active":"2023-09-23T20:19:05.072418+0000","last_peered":"2023-09-23T20:19:05.072418+0000","last_clean":"2023-09-23T20:19:05.072418+0000","last_became_active":"2023-09-23T20:19:05.071737+0000","last_became_peered":"2023-09-23T20:19:05.071737+0000","last_unstale":"2023-09-23T20:19:05.072418+0000","last_undegraded":"2023-09-23T20:19:05.072418+0000","last_fullsized":"2023-09-23T20:19:05.072418+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":257,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.054407+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:19:05.054407+0000","last_peered":"2023-09-23T20:19:05.054407+0000","last_clean":"2023-09-23T20:19:05.054407+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:19:05.054407+0000","last_undegraded":"2023-09-23T20:19:05.054407+0000","last_fullsized":"2023-09-23T20:19:05.054407+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056710+0000","last_change":"2023-09-23T20:17:47.270450+0000","last_active":"2023-09-23T20:19:05.056710+0000","last_peered":"2023-09-23T20:19:05.056710+0000","last_clean":"2023-09-23T20:19:05.056710+0000","last_became_active":"2023-09-23T20:17:47.270040+0000","last_became_peered":"2023-09-23T20:17:47.270040+0000","last_unstale":"2023-09-23T20:19:05.056710+0000","last_undegraded":"2023-09-23T20:19:05.056710+0000","last_fullsized":"2023-09-23T20:19:05.056710+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:51.572016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.3","version":"510'7035","reported_seq":7076,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:11.305157+0000","last_change":"2023-09-23T20:16:59.488642+0000","last_active":"2023-09-23T20:19:11.305157+0000","last_peered":"2023-09-23T20:19:11.305157+0000","last_clean":"2023-09-23T20:19:11.305157+0000","last_became_active":"2023-09-23T20:16:59.488330+0000","last_became_peered":"2023-09-23T20:16:59.488330+0000","last_unstale":"2023-09-23T20:19:11.305157+0000","last_undegraded":"2023-09-23T20:19:11.305157+0000","last_fullsized":"2023-09-23T20:19:11.305157+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:46.429096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293535744,"num_objects":4479,"num_object_clones":0,"num_object_copies":13437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7035,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":866,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.076033+0000","last_change":"2023-09-23T20:19:05.076033+0000","last_active":"2023-09-23T20:19:05.076033+0000","last_peered":"2023-09-23T20:19:05.076033+0000","last_clean":"2023-09-23T20:19:05.076033+0000","last_became_active":"2023-09-23T20:19:05.075801+0000","last_became_peered":"2023-09-23T20:19:05.075801+0000","last_unstale":"2023-09-23T20:19:05.076033+0000","last_undegraded":"2023-09-23T20:19:05.076033+0000","last_fullsized":"2023-09-23T20:19:05.076033+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057432+0000","last_change":"2023-09-23T20:17:47.266513+0000","last_active":"2023-09-23T20:19:05.057432+0000","last_peered":"2023-09-23T20:19:05.057432+0000","last_clean":"2023-09-23T20:19:05.057432+0000","last_became_active":"2023-09-23T20:17:47.266163+0000","last_became_peered":"2023-09-23T20:17:47.266163+0000","last_unstale":"2023-09-23T20:19:05.057432+0000","last_undegraded":"2023-09-23T20:19:05.057432+0000","last_fullsized":"2023-09-23T20:19:05.057432+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.645381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":421,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.067055+0000","last_change":"2023-09-23T20:17:47.270131+0000","last_active":"2023-09-23T20:19:05.067055+0000","last_peered":"2023-09-23T20:19:05.067055+0000","last_clean":"2023-09-23T20:19:05.067055+0000","last_became_active":"2023-09-23T20:17:47.269818+0000","last_became_peered":"2023-09-23T20:17:47.269818+0000","last_unstale":"2023-09-23T20:19:05.067055+0000","last_undegraded":"2023-09-23T20:19:05.067055+0000","last_fullsized":"2023-09-23T20:19:05.067055+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:40.738070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"510'6993","reported_seq":13697,"reported_epoch":510,"state":"active+recovering+remapped","last_fresh":"2023-09-23T20:19:10.656564+0000","last_change":"2023-09-23T20:17:48.625479+0000","last_active":"2023-09-23T20:19:10.656564+0000","last_peered":"2023-09-23T20:19:10.656564+0000","last_clean":"2023-09-23T20:17:46.197657+0000","last_became_active":"2023-09-23T20:17:48.417981+0000","last_became_peered":"2023-09-23T20:17:48.417981+0000","last_unstale":"2023-09-23T20:19:10.656564+0000","last_undegraded":"2023-09-23T20:19:10.656564+0000","last_fullsized":"2023-09-23T20:19:10.656564+0000","mapping_epoch":490,"log_start":"484'1660","ondisk_log_start":"484'1660","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5333,"log_dups_size":0,"ondisk_log_size":5333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:55.829023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294322176,"num_objects":4491,"num_object_clones":0,"num_object_copies":13473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":536,"num_objects_unfound":0,"num_objects_dirty":4491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6993,"num_write_kb":367488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3329,"num_bytes_recovered":218169344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":3955}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":573,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.091642+0000","last_change":"2023-09-23T20:19:05.088257+0000","last_active":"2023-09-23T20:19:05.091642+0000","last_peered":"2023-09-23T20:19:05.091642+0000","last_clean":"2023-09-23T20:19:05.091642+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:19:05.091642+0000","last_undegraded":"2023-09-23T20:19:05.091642+0000","last_fullsized":"2023-09-23T20:19:05.091642+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":403,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.064421+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:19:05.064421+0000","last_peered":"2023-09-23T20:19:05.064421+0000","last_clean":"2023-09-23T20:19:05.064421+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:19:05.064421+0000","last_undegraded":"2023-09-23T20:19:05.064421+0000","last_fullsized":"2023-09-23T20:19:05.064421+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057047+0000","last_change":"2023-09-23T20:17:47.268811+0000","last_active":"2023-09-23T20:19:05.057047+0000","last_peered":"2023-09-23T20:19:05.057047+0000","last_clean":"2023-09-23T20:19:05.057047+0000","last_became_active":"2023-09-23T20:17:47.268391+0000","last_became_peered":"2023-09-23T20:17:47.268391+0000","last_unstale":"2023-09-23T20:19:05.057047+0000","last_undegraded":"2023-09-23T20:19:05.057047+0000","last_fullsized":"2023-09-23T20:19:05.057047+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:32.388705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"510'7104","reported_seq":7143,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:13.078780+0000","last_change":"2023-09-23T20:16:59.496645+0000","last_active":"2023-09-23T20:19:13.078780+0000","last_peered":"2023-09-23T20:19:13.078780+0000","last_clean":"2023-09-23T20:19:13.078780+0000","last_became_active":"2023-09-23T20:16:59.496430+0000","last_became_peered":"2023-09-23T20:16:59.496430+0000","last_unstale":"2023-09-23T20:19:13.078780+0000","last_undegraded":"2023-09-23T20:19:13.078780+0000","last_fullsized":"2023-09-23T20:19:13.078780+0000","mapping_epoch":479,"log_start":"484'942","ondisk_log_start":"484'942","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6162,"log_dups_size":0,"ondisk_log_size":6162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:59.713976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289144832,"num_objects":4412,"num_object_clones":0,"num_object_copies":13236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7104,"num_write_kb":368512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":860,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:06.795360+0000","last_change":"2023-09-23T20:19:06.795360+0000","last_active":"2023-09-23T20:19:06.795360+0000","last_peered":"2023-09-23T20:19:06.795360+0000","last_clean":"2023-09-23T20:19:06.795360+0000","last_became_active":"2023-09-23T20:19:05.066352+0000","last_became_peered":"2023-09-23T20:19:05.066352+0000","last_unstale":"2023-09-23T20:19:06.795360+0000","last_undegraded":"2023-09-23T20:19:06.795360+0000","last_fullsized":"2023-09-23T20:19:06.795360+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":161,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.051035+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:19:05.051035+0000","last_peered":"2023-09-23T20:19:05.051035+0000","last_clean":"2023-09-23T20:19:05.051035+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:19:05.051035+0000","last_undegraded":"2023-09-23T20:19:05.051035+0000","last_fullsized":"2023-09-23T20:19:05.051035+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056871+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:19:05.056871+0000","last_peered":"2023-09-23T20:19:05.056871+0000","last_clean":"2023-09-23T20:19:05.056871+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:19:05.056871+0000","last_undegraded":"2023-09-23T20:19:05.056871+0000","last_fullsized":"2023-09-23T20:19:05.056871+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"510'7059","reported_seq":7098,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:13.114956+0000","last_change":"2023-09-23T20:16:59.496291+0000","last_active":"2023-09-23T20:19:13.114956+0000","last_peered":"2023-09-23T20:19:13.114956+0000","last_clean":"2023-09-23T20:19:13.114956+0000","last_became_active":"2023-09-23T20:16:59.496000+0000","last_became_peered":"2023-09-23T20:16:59.496000+0000","last_unstale":"2023-09-23T20:19:13.114956+0000","last_undegraded":"2023-09-23T20:19:13.114956+0000","last_fullsized":"2023-09-23T20:19:13.114956+0000","mapping_epoch":479,"log_start":"484'877","ondisk_log_start":"484'877","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6182,"log_dups_size":0,"ondisk_log_size":6182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.808639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292487168,"num_objects":4463,"num_object_clones":0,"num_object_copies":13389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7059,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":802,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.075092+0000","last_change":"2023-09-23T20:19:05.072847+0000","last_active":"2023-09-23T20:19:05.075092+0000","last_peered":"2023-09-23T20:19:05.075092+0000","last_clean":"2023-09-23T20:19:05.075092+0000","last_became_active":"2023-09-23T20:19:05.072187+0000","last_became_peered":"2023-09-23T20:19:05.072187+0000","last_unstale":"2023-09-23T20:19:05.075092+0000","last_undegraded":"2023-09-23T20:19:05.075092+0000","last_fullsized":"2023-09-23T20:19:05.075092+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":460,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056589+0000","last_change":"2023-09-23T20:17:50.311646+0000","last_active":"2023-09-23T20:19:05.056589+0000","last_peered":"2023-09-23T20:19:05.056589+0000","last_clean":"2023-09-23T20:19:05.056589+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:19:05.056589+0000","last_undegraded":"2023-09-23T20:19:05.056589+0000","last_fullsized":"2023-09-23T20:19:05.056589+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":509,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044543+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:19:04.044543+0000","last_peered":"2023-09-23T20:19:04.044543+0000","last_clean":"2023-09-23T20:19:04.044543+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:19:04.044543+0000","last_undegraded":"2023-09-23T20:19:04.044543+0000","last_fullsized":"2023-09-23T20:19:04.044543+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057500+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:19:05.057500+0000","last_peered":"2023-09-23T20:19:05.057500+0000","last_clean":"2023-09-23T20:19:05.057500+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:19:05.057500+0000","last_undegraded":"2023-09-23T20:19:05.057500+0000","last_fullsized":"2023-09-23T20:19:05.057500+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"509'6810","reported_seq":9394,"reported_epoch":509,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:19:08.450869+0000","last_change":"2023-09-23T20:18:56.585041+0000","last_active":"2023-09-23T20:19:08.450869+0000","last_peered":"2023-09-23T20:19:08.450869+0000","last_clean":"2023-09-23T20:19:08.450869+0000","last_became_active":"2023-09-23T20:18:39.050965+0000","last_became_peered":"2023-09-23T20:18:39.050965+0000","last_unstale":"2023-09-23T20:19:08.450869+0000","last_undegraded":"2023-09-23T20:19:08.450869+0000","last_fullsized":"2023-09-23T20:19:08.450869+0000","mapping_epoch":503,"log_start":"483'200","ondisk_log_start":"483'200","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:43.005386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300679168,"num_objects":4588,"num_object_clones":0,"num_object_copies":13764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4588,"num_objects_unfound":0,"num_objects_dirty":4588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6810,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1265,"num_bytes_recovered":82903040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":4588}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":852,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044514+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:19:04.044514+0000","last_peered":"2023-09-23T20:19:04.044514+0000","last_clean":"2023-09-23T20:19:04.044514+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:19:04.044514+0000","last_undegraded":"2023-09-23T20:19:04.044514+0000","last_fullsized":"2023-09-23T20:19:04.044514+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1189,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.045429+0000","last_change":"2023-09-23T20:17:47.305979+0000","last_active":"2023-09-23T20:19:04.045429+0000","last_peered":"2023-09-23T20:19:04.045429+0000","last_clean":"2023-09-23T20:19:04.045429+0000","last_became_active":"2023-09-23T20:17:47.274623+0000","last_became_peered":"2023-09-23T20:17:47.274623+0000","last_unstale":"2023-09-23T20:19:04.045429+0000","last_undegraded":"2023-09-23T20:19:04.045429+0000","last_fullsized":"2023-09-23T20:19:04.045429+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:27.293931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.058094+0000","last_change":"2023-09-23T20:17:47.280498+0000","last_active":"2023-09-23T20:19:05.058094+0000","last_peered":"2023-09-23T20:19:05.058094+0000","last_clean":"2023-09-23T20:19:05.058094+0000","last_became_active":"2023-09-23T20:17:47.279716+0000","last_became_peered":"2023-09-23T20:17:47.279716+0000","last_unstale":"2023-09-23T20:19:05.058094+0000","last_undegraded":"2023-09-23T20:19:05.058094+0000","last_fullsized":"2023-09-23T20:19:05.058094+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:18.625303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056641+0000","last_change":"2023-09-23T20:17:47.276964+0000","last_active":"2023-09-23T20:19:05.056641+0000","last_peered":"2023-09-23T20:19:05.056641+0000","last_clean":"2023-09-23T20:19:05.056641+0000","last_became_active":"2023-09-23T20:17:47.276540+0000","last_became_peered":"2023-09-23T20:17:47.276540+0000","last_unstale":"2023-09-23T20:19:05.056641+0000","last_undegraded":"2023-09-23T20:19:05.056641+0000","last_fullsized":"2023-09-23T20:19:05.056641+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:31.547044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"509'7033","reported_seq":7071,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:08.483166+0000","last_change":"2023-09-23T20:16:59.496065+0000","last_active":"2023-09-23T20:19:08.483166+0000","last_peered":"2023-09-23T20:19:08.483166+0000","last_clean":"2023-09-23T20:19:08.483166+0000","last_became_active":"2023-09-23T20:16:59.495782+0000","last_became_peered":"2023-09-23T20:16:59.495782+0000","last_unstale":"2023-09-23T20:19:08.483166+0000","last_undegraded":"2023-09-23T20:19:08.483166+0000","last_fullsized":"2023-09-23T20:19:08.483166+0000","mapping_epoch":479,"log_start":"483'511","ondisk_log_start":"483'511","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:08.985469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301662208,"num_objects":4603,"num_object_clones":0,"num_object_copies":13809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7033,"num_write_kb":372352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":748,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056358+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:19:05.056358+0000","last_peered":"2023-09-23T20:19:05.056358+0000","last_clean":"2023-09-23T20:19:05.056358+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:19:05.056358+0000","last_undegraded":"2023-09-23T20:19:05.056358+0000","last_fullsized":"2023-09-23T20:19:05.056358+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1149,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.064836+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:19:05.064836+0000","last_peered":"2023-09-23T20:19:05.064836+0000","last_clean":"2023-09-23T20:19:05.064836+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:19:05.064836+0000","last_undegraded":"2023-09-23T20:19:05.064836+0000","last_fullsized":"2023-09-23T20:19:05.064836+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056192+0000","last_change":"2023-09-23T20:17:47.278822+0000","last_active":"2023-09-23T20:19:05.056192+0000","last_peered":"2023-09-23T20:19:05.056192+0000","last_clean":"2023-09-23T20:19:05.056192+0000","last_became_active":"2023-09-23T20:17:47.278407+0000","last_became_peered":"2023-09-23T20:17:47.278407+0000","last_unstale":"2023-09-23T20:19:05.056192+0000","last_undegraded":"2023-09-23T20:19:05.056192+0000","last_fullsized":"2023-09-23T20:19:05.056192+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:01.125231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056952+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:19:05.056952+0000","last_peered":"2023-09-23T20:19:05.056952+0000","last_clean":"2023-09-23T20:19:05.056952+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:19:05.056952+0000","last_undegraded":"2023-09-23T20:19:05.056952+0000","last_fullsized":"2023-09-23T20:19:05.056952+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"510'6855","reported_seq":6899,"reported_epoch":510,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:19:10.636108+0000","last_change":"2023-09-23T20:17:48.438517+0000","last_active":"2023-09-23T20:19:10.636108+0000","last_peered":"2023-09-23T20:19:10.636108+0000","last_clean":"2023-09-23T20:17:46.219689+0000","last_became_active":"2023-09-23T20:17:48.427732+0000","last_became_peered":"2023-09-23T20:17:48.427732+0000","last_unstale":"2023-09-23T20:19:10.636108+0000","last_undegraded":"2023-09-23T20:17:48.289936+0000","last_fullsized":"2023-09-23T20:17:48.256145+0000","mapping_epoch":490,"log_start":"484'1551","ondisk_log_start":"484'1551","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5304,"log_dups_size":0,"ondisk_log_size":5304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:05.919808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293011456,"num_objects":4471,"num_object_clones":0,"num_object_copies":13413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3822,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6855,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":649},{"shards":"0,4,7","objects":2630}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1417,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044855+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:19:04.044855+0000","last_peered":"2023-09-23T20:19:04.044855+0000","last_clean":"2023-09-23T20:19:04.044855+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:19:04.044855+0000","last_undegraded":"2023-09-23T20:19:04.044855+0000","last_fullsized":"2023-09-23T20:19:04.044855+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":725,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.058043+0000","last_change":"2023-09-23T20:17:50.313197+0000","last_active":"2023-09-23T20:19:05.058043+0000","last_peered":"2023-09-23T20:19:05.058043+0000","last_clean":"2023-09-23T20:19:05.058043+0000","last_became_active":"2023-09-23T20:17:50.312838+0000","last_became_peered":"2023-09-23T20:17:50.312838+0000","last_unstale":"2023-09-23T20:19:05.058043+0000","last_undegraded":"2023-09-23T20:19:05.058043+0000","last_fullsized":"2023-09-23T20:19:05.058043+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:31.530167+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":374,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.045604+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:19:04.045604+0000","last_peered":"2023-09-23T20:19:04.045604+0000","last_clean":"2023-09-23T20:19:04.045604+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:19:04.045604+0000","last_undegraded":"2023-09-23T20:19:04.045604+0000","last_fullsized":"2023-09-23T20:19:04.045604+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":307,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.059474+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:19:05.059474+0000","last_peered":"2023-09-23T20:19:05.059474+0000","last_clean":"2023-09-23T20:19:05.059474+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:19:05.059474+0000","last_undegraded":"2023-09-23T20:19:05.059474+0000","last_fullsized":"2023-09-23T20:19:05.059474+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"510'6992","reported_seq":7033,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:11.373081+0000","last_change":"2023-09-23T20:16:59.494039+0000","last_active":"2023-09-23T20:19:11.373081+0000","last_peered":"2023-09-23T20:19:11.373081+0000","last_clean":"2023-09-23T20:19:11.373081+0000","last_became_active":"2023-09-23T20:16:59.493714+0000","last_became_peered":"2023-09-23T20:16:59.493714+0000","last_unstale":"2023-09-23T20:19:11.373081+0000","last_undegraded":"2023-09-23T20:19:11.373081+0000","last_fullsized":"2023-09-23T20:19:11.373081+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:15.171264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294256640,"num_objects":4490,"num_object_clones":0,"num_object_copies":13470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6992,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":520,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.058285+0000","last_change":"2023-09-23T20:17:50.330694+0000","last_active":"2023-09-23T20:19:05.058285+0000","last_peered":"2023-09-23T20:19:05.058285+0000","last_clean":"2023-09-23T20:19:05.058285+0000","last_became_active":"2023-09-23T20:17:50.329940+0000","last_became_peered":"2023-09-23T20:17:50.329940+0000","last_unstale":"2023-09-23T20:19:05.058285+0000","last_undegraded":"2023-09-23T20:19:05.058285+0000","last_fullsized":"2023-09-23T20:19:05.058285+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:55.675445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:55.675445+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1034,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057074+0000","last_change":"2023-09-23T20:17:50.313157+0000","last_active":"2023-09-23T20:19:05.057074+0000","last_peered":"2023-09-23T20:19:05.057074+0000","last_clean":"2023-09-23T20:19:05.057074+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:19:05.057074+0000","last_undegraded":"2023-09-23T20:19:05.057074+0000","last_fullsized":"2023-09-23T20:19:05.057074+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":457,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.051733+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:19:05.051733+0000","last_peered":"2023-09-23T20:19:05.051733+0000","last_clean":"2023-09-23T20:19:05.051733+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:19:05.051733+0000","last_undegraded":"2023-09-23T20:19:05.051733+0000","last_fullsized":"2023-09-23T20:19:05.051733+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057094+0000","last_change":"2023-09-23T20:17:47.278112+0000","last_active":"2023-09-23T20:19:05.057094+0000","last_peered":"2023-09-23T20:19:05.057094+0000","last_clean":"2023-09-23T20:19:05.057094+0000","last_became_active":"2023-09-23T20:17:47.277728+0000","last_became_peered":"2023-09-23T20:17:47.277728+0000","last_unstale":"2023-09-23T20:19:05.057094+0000","last_undegraded":"2023-09-23T20:19:05.057094+0000","last_fullsized":"2023-09-23T20:19:05.057094+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:21.371380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"510'6988","reported_seq":7028,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:11.511756+0000","last_change":"2023-09-23T20:16:59.490420+0000","last_active":"2023-09-23T20:19:11.511756+0000","last_peered":"2023-09-23T20:19:11.511756+0000","last_clean":"2023-09-23T20:19:11.511756+0000","last_became_active":"2023-09-23T20:16:59.490025+0000","last_became_peered":"2023-09-23T20:16:59.490025+0000","last_unstale":"2023-09-23T20:19:11.511756+0000","last_undegraded":"2023-09-23T20:19:11.511756+0000","last_fullsized":"2023-09-23T20:19:11.511756+0000","mapping_epoch":479,"log_start":"484'1879","ondisk_log_start":"484'1879","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5109,"log_dups_size":0,"ondisk_log_size":5109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:03.184763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294977560,"num_objects":4502,"num_object_clones":0,"num_object_copies":13506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4502,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6988,"num_write_kb":367617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":705,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.092845+0000","last_change":"2023-09-23T20:19:05.092371+0000","last_active":"2023-09-23T20:19:05.092845+0000","last_peered":"2023-09-23T20:19:05.092845+0000","last_clean":"2023-09-23T20:19:05.092845+0000","last_became_active":"2023-09-23T20:19:05.092070+0000","last_became_peered":"2023-09-23T20:19:05.092070+0000","last_unstale":"2023-09-23T20:19:05.092845+0000","last_undegraded":"2023-09-23T20:19:05.092845+0000","last_fullsized":"2023-09-23T20:19:05.092845+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":210,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.059718+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:19:05.059718+0000","last_peered":"2023-09-23T20:19:05.059718+0000","last_clean":"2023-09-23T20:19:05.059718+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:19:05.059718+0000","last_undegraded":"2023-09-23T20:19:05.059718+0000","last_fullsized":"2023-09-23T20:19:05.059718+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056562+0000","last_change":"2023-09-23T20:17:47.276790+0000","last_active":"2023-09-23T20:19:05.056562+0000","last_peered":"2023-09-23T20:19:05.056562+0000","last_clean":"2023-09-23T20:19:05.056562+0000","last_became_active":"2023-09-23T20:17:47.276198+0000","last_became_peered":"2023-09-23T20:17:47.276198+0000","last_unstale":"2023-09-23T20:19:05.056562+0000","last_undegraded":"2023-09-23T20:19:05.056562+0000","last_fullsized":"2023-09-23T20:19:05.056562+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:06.124215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":335,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044627+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:19:04.044627+0000","last_peered":"2023-09-23T20:19:04.044627+0000","last_clean":"2023-09-23T20:19:04.044627+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:19:04.044627+0000","last_undegraded":"2023-09-23T20:19:04.044627+0000","last_fullsized":"2023-09-23T20:19:04.044627+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"510'7129","reported_seq":7169,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-09-23T20:19:11.366235+0000","last_change":"2023-09-23T20:16:59.490304+0000","last_active":"2023-09-23T20:19:11.366235+0000","last_peered":"2023-09-23T20:19:11.366235+0000","last_clean":"2023-09-23T20:19:11.366235+0000","last_became_active":"2023-09-23T20:16:59.490026+0000","last_became_peered":"2023-09-23T20:16:59.490026+0000","last_unstale":"2023-09-23T20:19:11.366235+0000","last_undegraded":"2023-09-23T20:19:11.366235+0000","last_fullsized":"2023-09-23T20:19:11.366235+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":7129,"log_dups_size":0,"ondisk_log_size":7129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:59.789512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297730048,"num_objects":4543,"num_object_clones":0,"num_object_copies":13629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7129,"num_write_kb":373504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":751,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.079173+0000","last_change":"2023-09-23T20:19:05.069756+0000","last_active":"2023-09-23T20:19:05.079173+0000","last_peered":"2023-09-23T20:19:05.079173+0000","last_clean":"2023-09-23T20:19:05.079173+0000","last_became_active":"2023-09-23T20:19:05.069124+0000","last_became_peered":"2023-09-23T20:19:05.069124+0000","last_unstale":"2023-09-23T20:19:05.079173+0000","last_undegraded":"2023-09-23T20:19:05.079173+0000","last_fullsized":"2023-09-23T20:19:05.079173+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":352,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.059114+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:19:05.059114+0000","last_peered":"2023-09-23T20:19:05.059114+0000","last_clean":"2023-09-23T20:19:05.059114+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:19:05.059114+0000","last_undegraded":"2023-09-23T20:19:05.059114+0000","last_fullsized":"2023-09-23T20:19:05.059114+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":588,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.057006+0000","last_change":"2023-09-23T20:17:50.311535+0000","last_active":"2023-09-23T20:19:05.057006+0000","last_peered":"2023-09-23T20:19:05.057006+0000","last_clean":"2023-09-23T20:19:05.057006+0000","last_became_active":"2023-09-23T20:17:50.310876+0000","last_became_peered":"2023-09-23T20:17:50.310876+0000","last_unstale":"2023-09-23T20:19:05.057006+0000","last_undegraded":"2023-09-23T20:19:05.057006+0000","last_fullsized":"2023-09-23T20:19:05.057006+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.794641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":143,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.050947+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:19:05.050947+0000","last_peered":"2023-09-23T20:19:05.050947+0000","last_clean":"2023-09-23T20:19:05.050947+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:19:05.050947+0000","last_undegraded":"2023-09-23T20:19:05.050947+0000","last_fullsized":"2023-09-23T20:19:05.050947+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"510'7005","reported_seq":9795,"reported_epoch":510,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:19:11.451537+0000","last_change":"2023-09-23T20:18:39.126760+0000","last_active":"2023-09-23T20:19:11.451537+0000","last_peered":"2023-09-23T20:19:11.451537+0000","last_clean":"2023-09-23T20:18:02.160044+0000","last_became_active":"2023-09-23T20:18:39.117958+0000","last_became_peered":"2023-09-23T20:18:39.117958+0000","last_unstale":"2023-09-23T20:19:11.451537+0000","last_undegraded":"2023-09-23T20:18:39.008966+0000","last_fullsized":"2023-09-23T20:19:11.451537+0000","mapping_epoch":503,"log_start":"484'1826","ondisk_log_start":"484'1826","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5179,"log_dups_size":0,"ondisk_log_size":5179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:45.818039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290390016,"num_objects":4431,"num_object_clones":0,"num_object_copies":13293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1260,"num_objects_misplaced":3171,"num_objects_unfound":0,"num_objects_dirty":4431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7005,"num_write_kb":365952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1370,"num_bytes_recovered":89784320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,4],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1260},{"shards":"2,3,4","objects":3171}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":837,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.054089+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:19:05.054089+0000","last_peered":"2023-09-23T20:19:05.054089+0000","last_clean":"2023-09-23T20:19:05.054089+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:19:05.054089+0000","last_undegraded":"2023-09-23T20:19:05.054089+0000","last_fullsized":"2023-09-23T20:19:05.054089+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":216,"reported_epoch":509,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:19:05.059350+0000","last_change":"2023-09-23T20:18:39.010444+0000","last_active":"2023-09-23T20:19:05.059350+0000","last_peered":"2023-09-23T20:19:05.059350+0000","last_clean":"2023-09-23T20:19:05.059350+0000","last_became_active":"2023-09-23T20:18:39.009999+0000","last_became_peered":"2023-09-23T20:18:39.009999+0000","last_unstale":"2023-09-23T20:19:05.059350+0000","last_undegraded":"2023-09-23T20:19:05.059350+0000","last_fullsized":"2023-09-23T20:19:05.059350+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:53.838719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,3],"avail_no_missing":["7","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":35,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056839+0000","last_change":"2023-09-23T20:17:47.284676+0000","last_active":"2023-09-23T20:19:05.056839+0000","last_peered":"2023-09-23T20:19:05.056839+0000","last_clean":"2023-09-23T20:19:05.056839+0000","last_became_active":"2023-09-23T20:17:47.277321+0000","last_became_peered":"2023-09-23T20:17:47.277321+0000","last_unstale":"2023-09-23T20:19:05.056839+0000","last_undegraded":"2023-09-23T20:19:05.056839+0000","last_fullsized":"2023-09-23T20:19:05.056839+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.082445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":309,"reported_epoch":508,"state":"active+undersized","last_fresh":"2023-09-23T20:19:04.045351+0000","last_change":"2023-09-23T20:18:38.011785+0000","last_active":"2023-09-23T20:19:04.045351+0000","last_peered":"2023-09-23T20:19:04.045351+0000","last_clean":"2023-09-23T20:18:35.261768+0000","last_became_active":"2023-09-23T20:18:38.011785+0000","last_became_peered":"2023-09-23T20:18:38.011785+0000","last_unstale":"2023-09-23T20:19:04.045351+0000","last_undegraded":"2023-09-23T20:19:04.045351+0000","last_fullsized":"2023-09-23T20:18:38.001274+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:00.398059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":901,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.080073+0000","last_change":"2023-09-23T20:19:05.079851+0000","last_active":"2023-09-23T20:19:05.080073+0000","last_peered":"2023-09-23T20:19:05.080073+0000","last_clean":"2023-09-23T20:19:05.080073+0000","last_became_active":"2023-09-23T20:19:05.079569+0000","last_became_peered":"2023-09-23T20:19:05.079569+0000","last_unstale":"2023-09-23T20:19:05.080073+0000","last_undegraded":"2023-09-23T20:19:05.080073+0000","last_fullsized":"2023-09-23T20:19:05.080073+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":288,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.064778+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:19:05.064778+0000","last_peered":"2023-09-23T20:19:05.064778+0000","last_clean":"2023-09-23T20:19:05.064778+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:19:05.064778+0000","last_undegraded":"2023-09-23T20:19:05.064778+0000","last_fullsized":"2023-09-23T20:19:05.064778+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":297,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.064811+0000","last_change":"2023-09-23T20:17:47.281804+0000","last_active":"2023-09-23T20:19:05.064811+0000","last_peered":"2023-09-23T20:19:05.064811+0000","last_clean":"2023-09-23T20:19:05.064811+0000","last_became_active":"2023-09-23T20:17:47.281384+0000","last_became_peered":"2023-09-23T20:17:47.281384+0000","last_unstale":"2023-09-23T20:19:05.064811+0000","last_undegraded":"2023-09-23T20:19:05.064811+0000","last_fullsized":"2023-09-23T20:19:05.064811+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:03.796273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":722,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.072433+0000","last_change":"2023-09-23T20:19:05.072097+0000","last_active":"2023-09-23T20:19:05.072433+0000","last_peered":"2023-09-23T20:19:05.072433+0000","last_clean":"2023-09-23T20:19:05.072433+0000","last_became_active":"2023-09-23T20:19:05.071787+0000","last_became_peered":"2023-09-23T20:19:05.071787+0000","last_unstale":"2023-09-23T20:19:05.072433+0000","last_undegraded":"2023-09-23T20:19:05.072433+0000","last_fullsized":"2023-09-23T20:19:05.072433+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":540,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.044687+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:19:04.044687+0000","last_peered":"2023-09-23T20:19:04.044687+0000","last_clean":"2023-09-23T20:19:04.044687+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:19:04.044687+0000","last_undegraded":"2023-09-23T20:19:04.044687+0000","last_fullsized":"2023-09-23T20:19:04.044687+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":540,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056284+0000","last_change":"2023-09-23T20:17:47.266215+0000","last_active":"2023-09-23T20:19:05.056284+0000","last_peered":"2023-09-23T20:19:05.056284+0000","last_clean":"2023-09-23T20:19:05.056284+0000","last_became_active":"2023-09-23T20:17:47.265890+0000","last_became_peered":"2023-09-23T20:17:47.265890+0000","last_unstale":"2023-09-23T20:19:05.056284+0000","last_undegraded":"2023-09-23T20:19:05.056284+0000","last_fullsized":"2023-09-23T20:19:05.056284+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:59.854975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":530,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.045074+0000","last_change":"2023-09-23T20:17:47.288753+0000","last_active":"2023-09-23T20:19:04.045074+0000","last_peered":"2023-09-23T20:19:04.045074+0000","last_clean":"2023-09-23T20:19:04.045074+0000","last_became_active":"2023-09-23T20:17:47.270488+0000","last_became_peered":"2023-09-23T20:17:47.270488+0000","last_unstale":"2023-09-23T20:19:04.045074+0000","last_undegraded":"2023-09-23T20:19:04.045074+0000","last_fullsized":"2023-09-23T20:19:04.045074+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:35.837184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":506,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.060605+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:19:05.060605+0000","last_peered":"2023-09-23T20:19:05.060605+0000","last_clean":"2023-09-23T20:19:05.060605+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:19:05.060605+0000","last_undegraded":"2023-09-23T20:19:05.060605+0000","last_fullsized":"2023-09-23T20:19:05.060605+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":868,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:06.754431+0000","last_change":"2023-09-23T20:19:05.086585+0000","last_active":"2023-09-23T20:19:06.754431+0000","last_peered":"2023-09-23T20:19:06.754431+0000","last_clean":"2023-09-23T20:19:06.754431+0000","last_became_active":"2023-09-23T20:19:05.086254+0000","last_became_peered":"2023-09-23T20:19:05.086254+0000","last_unstale":"2023-09-23T20:19:06.754431+0000","last_undegraded":"2023-09-23T20:19:06.754431+0000","last_fullsized":"2023-09-23T20:19:06.754431+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":232,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.064085+0000","last_change":"2023-09-23T20:17:47.281777+0000","last_active":"2023-09-23T20:19:05.064085+0000","last_peered":"2023-09-23T20:19:05.064085+0000","last_clean":"2023-09-23T20:19:05.064085+0000","last_became_active":"2023-09-23T20:17:47.281381+0000","last_became_peered":"2023-09-23T20:17:47.281381+0000","last_unstale":"2023-09-23T20:19:05.064085+0000","last_undegraded":"2023-09-23T20:19:05.064085+0000","last_fullsized":"2023-09-23T20:19:05.064085+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.083242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":370,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.064940+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:19:05.064940+0000","last_peered":"2023-09-23T20:19:05.064940+0000","last_clean":"2023-09-23T20:19:05.064940+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:19:05.064940+0000","last_undegraded":"2023-09-23T20:19:05.064940+0000","last_fullsized":"2023-09-23T20:19:05.064940+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":799,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.067665+0000","last_change":"2023-09-23T20:19:05.067447+0000","last_active":"2023-09-23T20:19:05.067665+0000","last_peered":"2023-09-23T20:19:05.067665+0000","last_clean":"2023-09-23T20:19:05.067665+0000","last_became_active":"2023-09-23T20:19:05.067188+0000","last_became_peered":"2023-09-23T20:19:05.067188+0000","last_unstale":"2023-09-23T20:19:05.067665+0000","last_undegraded":"2023-09-23T20:19:05.067665+0000","last_fullsized":"2023-09-23T20:19:05.067665+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":354,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-09-23T20:19:04.045058+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:19:04.045058+0000","last_peered":"2023-09-23T20:19:04.045058+0000","last_clean":"2023-09-23T20:19:04.045058+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:19:04.045058+0000","last_undegraded":"2023-09-23T20:19:04.045058+0000","last_fullsized":"2023-09-23T20:19:04.045058+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-09-23T20:19:05.056417+0000","last_change":"2023-09-23T20:17:47.267673+0000","last_active":"2023-09-23T20:19:05.056417+0000","last_peered":"2023-09-23T20:19:05.056417+0000","last_clean":"2023-09-23T20:19:05.056417+0000","last_became_active":"2023-09-23T20:17:47.267216+0000","last_became_peered":"2023-09-23T20:17:47.267216+0000","last_unstale":"2023-09-23T20:19:05.056417+0000","last_undegraded":"2023-09-23T20:19:05.056417+0000","last_fullsized":"2023-09-23T20:19:05.056417+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:37.855890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4726456344,"num_objects":72121,"num_object_clones":0,"num_object_copies":216363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9012,"num_objects_misplaced":8295,"num_objects_unfound":0,"num_objects_dirty":72121,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112327,"num_write_kb":5902273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5964,"num_bytes_recovered":390856704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14691414016,"data_stored":14691401800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97111,"ondisk_log_size":97111,"up":46,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":10952,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":94,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":30,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":107340,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":8388703,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234457,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2248040,"kb_used_data":1676320,"kb_used_omap":0,"kb_used_meta":571712,"kb_avail":91497112,"statfs":{"total":95995035648,"available":93693042688,"internally_reserved":0,"allocated":1716551680,"data_stored":1715021172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":585433088},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:57 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":5.6399999999999997,"15min":3.218},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":10.135999999999999,"5min":149.25999999999999,"15min":149.25999999999999},"last":1.994},{"interface":"front","average":{"1min":2.8010000000000002,"5min":6.0259999999999998,"15min":3.4329999999999998},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.222},"max":{"1min":9.8409999999999993,"5min":149.11799999999999,"15min":149.11799999999999},"last":2.0529999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:18:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.9710000000000001,"15min":2.5760000000000001},"min":{"1min":0.26300000000000001,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":10.804,"5min":47.197000000000003,"15min":91.335999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.097,"15min":2.8660000000000001},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":10.571999999999999,"5min":47.436999999999998,"15min":91.516999999999996},"last":10.032}]},{"osd":2,"last update":"Sat Sep 23 20:19:04 2023","interfaces":[{"interface":"back","average":{"1min":11.622,"5min":5.1890000000000001,"15min":4.5700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":177.852,"5min":177.852,"15min":177.852},"last":0.48299999999999998},{"interface":"front","average":{"1min":11.391999999999999,"5min":5.3840000000000003,"15min":4.944},"min":{"1min":0.371,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":176.97900000000001,"5min":176.97900000000001,"15min":176.97900000000001},"last":13.470000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:18:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":6.4320000000000004,"15min":4.6619999999999999},"min":{"1min":0.34300000000000003,"5min":0.33400000000000002,"15min":0.29299999999999998},"max":{"1min":11.154,"5min":185.17099999999999,"15min":185.17099999999999},"last":6.9450000000000003},{"interface":"front","average":{"1min":3.024,"5min":6.8890000000000002,"15min":4.7560000000000002},"min":{"1min":0.34300000000000003,"5min":0.32200000000000001,"15min":0.28999999999999998},"max":{"1min":13.887,"5min":184.99100000000001,"15min":184.99100000000001},"last":8.5790000000000006}]},{"osd":4,"last update":"Sat Sep 23 20:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":1.222,"15min":0.78200000000000003},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.23499999999999999},"max":{"1min":2.093,"5min":7.9199999999999999,"15min":7.9199999999999999},"last":1.6439999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":1.379,"15min":0.97499999999999998},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":2.117,"5min":10.882999999999999,"15min":41.822000000000003},"last":1.159}]},{"osd":5,"last update":"Sat Sep 23 20:18:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.8620000000000001,"15min":1.1819999999999999},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":4.6959999999999997,"5min":16.327000000000002,"15min":24.876000000000001},"last":2.0209999999999999},{"interface":"front","average":{"1min":1.042,"5min":1.7310000000000001,"15min":1.073},"min":{"1min":0.33700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":3.5289999999999999,"5min":16.716000000000001,"15min":21.623000000000001},"last":1.679}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.3779999999999999,"15min":1.0569999999999999},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.3420000000000001,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.163,"15min":0.82199999999999995},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1400000000000001,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.436}]}]},{"osd":6,"up_from":251,"seq":1078036791490,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273876,"kb_used_data":233868,"kb_used_omap":6,"kb_used_meta":39993,"kb_avail":93471276,"statfs":{"total":95995035648,"available":95714586624,"internally_reserved":0,"allocated":239480832,"data_stored":238014233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":40953471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":8.9100000000000001,"5min":3.8820000000000001,"15min":2.6890000000000001},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":53.780000000000001,"5min":74.787000000000006,"15min":126.64},"last":0.27000000000000002},{"interface":"front","average":{"1min":9.0090000000000003,"5min":4.0289999999999999,"15min":2.7570000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":53.768000000000001,"5min":74.751999999999995,"15min":126.658},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":4.3120000000000003,"15min":4.0830000000000002},"min":{"1min":0.26500000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":87.864999999999995,"5min":103.545,"15min":156.70699999999999},"last":5.4710000000000001},{"interface":"front","average":{"1min":5.1260000000000003,"5min":4.0279999999999996,"15min":4.0590000000000002},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.23899999999999999},"max":{"1min":87.772000000000006,"5min":100.018,"15min":156.68299999999999},"last":2.8690000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":6.891,"5min":2.4329999999999998,"15min":3.1179999999999999},"min":{"1min":0.315,"5min":0.254,"15min":0.23100000000000001},"max":{"1min":53.911999999999999,"5min":53.911999999999999,"15min":122.149},"last":0.47499999999999998},{"interface":"front","average":{"1min":7.3259999999999996,"5min":2.4129999999999998,"15min":3.0600000000000001},"min":{"1min":0.32400000000000001,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":54.125999999999998,"5min":54.125999999999998,"15min":122.23999999999999},"last":1.7789999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":5.181,"15min":3.8580000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":51.512999999999998,"5min":128.279,"15min":160.27699999999999},"last":4.8179999999999996},{"interface":"front","average":{"1min":3.4249999999999998,"5min":4.4969999999999999,"15min":3.8180000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":51.601999999999997,"5min":127.911,"15min":160.42699999999999},"last":2.9830000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.298,"15min":1.5780000000000001},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.248},"max":{"1min":2.641,"5min":34.534999999999997,"15min":145.29300000000001},"last":2.9140000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.077,"15min":1.538},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.3999999999999999,"5min":34.426000000000002,"15min":145.256},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":4.5880000000000001,"15min":2.972},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.25800000000000001},"max":{"1min":2.5329999999999999,"5min":218.58199999999999,"15min":218.58199999999999},"last":2.9500000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.343,"15min":2.8969999999999998},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.28499999999999998},"max":{"1min":2.4380000000000002,"5min":217.95599999999999,"15min":217.95599999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.1269999999999998,"15min":1.73},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.315},"max":{"1min":4.7380000000000004,"5min":137.63900000000001,"15min":137.63900000000001},"last":1},{"interface":"front","average":{"1min":0.751,"5min":2.8199999999999998,"15min":1.629},"min":{"1min":0.39700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":2.3620000000000001,"5min":136.738,"15min":136.738},"last":0.48799999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762655,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1296796,"kb_used_data":495700,"kb_used_omap":7,"kb_used_meta":801080,"kb_avail":92448356,"statfs":{"total":95995035648,"available":94667116544,"internally_reserved":0,"allocated":507596800,"data_stored":505195298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":820306746},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":4.5659999999999998,"15min":3.7650000000000001},"min":{"1min":0.19600000000000001,"5min":0.185,"15min":0.17100000000000001},"max":{"1min":8.9879999999999995,"5min":145.81299999999999,"15min":145.81299999999999},"last":2.1859999999999999},{"interface":"front","average":{"1min":3.7850000000000001,"5min":4.7999999999999998,"15min":4.1980000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19400000000000001},"max":{"1min":13.798999999999999,"5min":146.83799999999999,"15min":146.83799999999999},"last":1.907}]},{"osd":2,"last update":"Sat Sep 23 20:18:26 2023","interfaces":[{"interface":"back","average":{"1min":10.983000000000001,"5min":4.4349999999999996,"15min":4.359},"min":{"1min":0.23999999999999999,"5min":0.214,"15min":0.20000000000000001},"max":{"1min":152.62,"5min":152.62,"15min":152.62},"last":1.21},{"interface":"front","average":{"1min":10.912000000000001,"5min":3.79,"15min":4.0750000000000002},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.20699999999999999},"max":{"1min":154.84299999999999,"5min":154.84299999999999,"15min":154.84299999999999},"last":2.161}]},{"osd":3,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":4.2199999999999998,"5min":3.194,"15min":3.161},"min":{"1min":0.24399999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":21.056999999999999,"5min":66.527000000000001,"15min":66.527000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.5389999999999999,"5min":2.0499999999999998,"15min":2.7469999999999999},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":5.5599999999999996,"5min":61.854999999999997,"15min":61.854999999999997},"last":1.9399999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.5189999999999999,"15min":1.681},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.23599999999999999},"max":{"1min":6.5359999999999996,"5min":7.7999999999999998,"15min":18.087},"last":3.4199999999999999},{"interface":"front","average":{"1min":1.3169999999999999,"5min":1.403,"15min":1.7470000000000001},"min":{"1min":0.34999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.5910000000000002,"5min":9.8320000000000007,"15min":20.847000000000001},"last":1.9770000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.776,"15min":2.1509999999999998},"min":{"1min":0.313,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.8059999999999992,"5min":9.782,"15min":83.510999999999996},"last":2.0030000000000001},{"interface":"front","average":{"1min":2.1379999999999999,"5min":1.8009999999999999,"15min":2.2770000000000001},"min":{"1min":0.33300000000000002,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":5.5759999999999996,"5min":13.449,"15min":83.353999999999999},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.2390000000000001,"15min":1.665},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":4.5830000000000002,"5min":9.2279999999999998,"15min":18.117999999999999},"last":5.6760000000000002},{"interface":"front","average":{"1min":0.75,"5min":1.325,"15min":1.843},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.0310000000000001,"5min":9.8659999999999997,"15min":20.734999999999999},"last":3.7330000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":3.4820000000000002,"15min":2.589},"min":{"1min":0.41699999999999998,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":5.6070000000000002,"5min":127.03100000000001,"15min":127.03100000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.181,"5min":3.0579999999999998,"15min":2.266},"min":{"1min":0.40200000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.5819999999999999,"5min":125.095,"15min":125.095},"last":2.121}]}]},{"osd":0,"up_from":94,"seq":403726926193,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4184908,"kb_used_data":3201540,"kb_used_omap":8,"kb_used_meta":983351,"kb_avail":89560244,"statfs":{"total":95995035648,"available":91709689856,"internally_reserved":0,"allocated":3278376960,"data_stored":3275996851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1006952033},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:19:09 2023","interfaces":[{"interface":"back","average":{"1min":3.25,"5min":1.867,"15min":2.6440000000000001},"min":{"1min":0.23300000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":17.425000000000001,"5min":29.893000000000001,"15min":72.534000000000006},"last":0.58699999999999997},{"interface":"front","average":{"1min":5.891,"5min":2.3370000000000002,"15min":2.9609999999999999},"min":{"1min":0.26800000000000002,"5min":0.20799999999999999,"15min":0.187},"max":{"1min":21.949000000000002,"5min":23.436,"15min":72.667000000000002},"last":0.94799999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:18:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9409999999999998,"5min":2.911,"15min":3.7320000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":17.475000000000001,"5min":160.542,"15min":162.62899999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":2.992,"5min":3.4079999999999999,"15min":3.9140000000000001},"min":{"1min":0.23599999999999999,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":8.2189999999999994,"5min":160.75999999999999,"15min":162.52500000000001},"last":3.544}]},{"osd":3,"last update":"Sat Sep 23 20:19:09 2023","interfaces":[{"interface":"back","average":{"1min":7.4530000000000003,"5min":3.1259999999999999,"15min":3.6200000000000001},"min":{"1min":0.30199999999999999,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":24.219000000000001,"5min":44.405000000000001,"15min":126.95399999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":7.0679999999999996,"5min":3.1600000000000001,"15min":3.7410000000000001},"min":{"1min":0.35599999999999998,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":22.123000000000001,"5min":43.909999999999997,"15min":127.294},"last":3.2759999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.7609999999999999,"15min":2.0600000000000001},"min":{"1min":0.39000000000000001,"5min":0.316,"15min":0.27600000000000002},"max":{"1min":7.5819999999999999,"5min":8.7360000000000007,"15min":148.20599999999999},"last":6.7140000000000004},{"interface":"front","average":{"1min":2.3199999999999998,"5min":2.3319999999999999,"15min":2.3290000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0150000000000006,"5min":105.282,"15min":148.11000000000001},"last":0.496}]},{"osd":5,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":2.1699999999999999,"15min":1.698},"min":{"1min":0.36499999999999999,"5min":0.35999999999999999,"15min":0.26900000000000002},"max":{"1min":7.6180000000000003,"5min":52.061,"15min":52.061},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.456,"5min":2.351,"15min":1.8660000000000001},"min":{"1min":0.37,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":5.3799999999999999,"5min":51.741,"15min":51.741},"last":6.7670000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6339999999999999,"15min":1.6419999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":7.6500000000000004,"5min":7.6500000000000004,"15min":15.744999999999999},"last":4.1239999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.234,"15min":1.6020000000000001},"min":{"1min":0.44,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":9.8219999999999992,"15min":13.766999999999999},"last":0.82199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.9119999999999999,"15min":1.9910000000000001},"min":{"1min":0.39800000000000002,"5min":0.39200000000000002,"15min":0.377},"max":{"1min":8.0269999999999992,"5min":99.486000000000004,"15min":99.486000000000004},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.069,"5min":3.3740000000000001,"15min":2.3210000000000002},"min":{"1min":0.42599999999999999,"5min":0.40200000000000002,"15min":0.32300000000000001},"max":{"1min":8.048,"5min":99.668999999999997,"15min":99.668999999999997},"last":6.7990000000000004}]}]},{"osd":2,"up_from":104,"seq":446676599131,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3243460,"kb_used_data":2304380,"kb_used_omap":12,"kb_used_meta":939059,"kb_avail":90501692,"statfs":{"total":95995035648,"available":92673732608,"internally_reserved":0,"allocated":2359685120,"data_stored":2357258270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":961597438},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":16.698,"5min":5.444,"15min":3.5150000000000001},"min":{"1min":0.40200000000000002,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":104.93600000000001,"5min":104.93600000000001,"15min":104.93600000000001},"last":4.9189999999999996},{"interface":"front","average":{"1min":18.001999999999999,"5min":5.9480000000000004,"15min":3.851},"min":{"1min":0.23200000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":106.477,"5min":106.477,"15min":106.477},"last":4.1239999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":12.108000000000001,"5min":4.0960000000000001,"15min":3.593},"min":{"1min":0.38700000000000001,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":106.357,"5min":106.357,"15min":106.357},"last":2.4750000000000001},{"interface":"front","average":{"1min":11.811999999999999,"5min":4.2199999999999998,"15min":3.2959999999999998},"min":{"1min":0.371,"5min":0.222,"15min":0.218},"max":{"1min":104.905,"5min":104.905,"15min":104.905},"last":4.7789999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:18:45 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":5.5549999999999997,"15min":4.6159999999999997},"min":{"1min":0.247,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":14.129,"5min":132.82499999999999,"15min":132.82499999999999},"last":5.0220000000000002},{"interface":"front","average":{"1min":3.153,"5min":5.6369999999999996,"15min":3.9079999999999999},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":22.847999999999999,"5min":130.97300000000001,"15min":130.97300000000001},"last":4.8019999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.3009999999999999,"15min":1.8320000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.8220000000000001,"5min":10.079000000000001,"15min":75.608000000000004},"last":4.7539999999999996},{"interface":"front","average":{"1min":2.444,"5min":1.5740000000000001,"15min":2.1970000000000001},"min":{"1min":0.34599999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":12.726000000000001,"5min":12.726000000000001,"15min":76.161000000000001},"last":3.0059999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":2.9039999999999999,"15min":2.9180000000000001},"min":{"1min":0.316,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.8719999999999999,"5min":135.202,"15min":172.505},"last":4.7130000000000001},{"interface":"front","average":{"1min":2.0939999999999999,"5min":1.738,"15min":2.7839999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.778,"5min":12.778,"15min":172.375},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.329,"15min":1.542},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5510000000000002,"5min":11.843,"15min":14.664},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.425,"15min":1.744},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.077,"5min":12.025,"15min":14.279999999999999},"last":4.9660000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4849999999999999,"5min":1.984,"15min":2},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.789999999999999,"5min":12.789999999999999,"15min":13.481999999999999},"last":2.5110000000000001},{"interface":"front","average":{"1min":1.333,"5min":1.635,"15min":1.6399999999999999},"min":{"1min":0.40500000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.8460000000000001,"5min":8.9730000000000008,"15min":14.698},"last":4.8940000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435605,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2973476,"kb_used_data":1986588,"kb_used_omap":4,"kb_used_meta":986875,"kb_avail":90771676,"statfs":{"total":95995035648,"available":92950196224,"internally_reserved":0,"allocated":2034266112,"data_stored":2031567514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1010560109},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":4.3559999999999999,"15min":3.7530000000000001},"min":{"1min":0.20399999999999999,"5min":0.189,"15min":0.186},"max":{"1min":9.1530000000000005,"5min":132.47800000000001,"15min":132.47800000000001},"last":5.3799999999999999},{"interface":"front","average":{"1min":4.3170000000000002,"5min":5.1980000000000004,"15min":4.0270000000000001},"min":{"1min":0.28399999999999997,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":17.539999999999999,"5min":133.22499999999999,"15min":133.22499999999999},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":4.0940000000000003,"5min":4.4820000000000002,"15min":4.5499999999999998},"min":{"1min":0.22600000000000001,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":17.895,"5min":183.87,"15min":183.87},"last":7.4729999999999999},{"interface":"front","average":{"1min":3.2989999999999999,"5min":4.4089999999999998,"15min":4.9459999999999997},"min":{"1min":0.28100000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":12.664999999999999,"5min":185.84899999999999,"15min":185.84899999999999},"last":12.335000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":10.364000000000001,"5min":6.1399999999999997,"15min":5.2450000000000001},"min":{"1min":0.33700000000000002,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":115.47799999999999,"5min":128.369,"15min":163.83500000000001},"last":7.508},{"interface":"front","average":{"1min":9.3520000000000003,"5min":5.6580000000000004,"15min":4.9640000000000004},"min":{"1min":0.28499999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":114.79600000000001,"5min":126.497,"15min":163.92099999999999},"last":5.5810000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.387,"15min":2.5910000000000002},"min":{"1min":0.34899999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":9.1180000000000003,"5min":9.1180000000000003,"15min":109.66800000000001},"last":5.4580000000000002},{"interface":"front","average":{"1min":3.0350000000000001,"5min":2.1749999999999998,"15min":3.0470000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":17.428000000000001,"5min":17.428000000000001,"15min":110.295},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":4.3470000000000004,"15min":3.5830000000000002},"min":{"1min":0.35499999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":17.370999999999999,"5min":200.53200000000001,"15min":200.53200000000001},"last":0.442},{"interface":"front","average":{"1min":3.5760000000000001,"5min":4.4690000000000003,"15min":3.5619999999999998},"min":{"1min":0.374,"5min":0.34100000000000003,"15min":0.26900000000000002},"max":{"1min":18.02,"5min":200.846,"15min":200.846},"last":7.5780000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.593,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":1.5329999999999999,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.3280000000000001,"15min":2.286},"min":{"1min":0.42699999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.2040000000000002,"5min":8.2629999999999999,"15min":82.233000000000004},"last":4.1509999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":3.379,"15min":3.3439999999999999},"min":{"1min":0.434,"5min":0.376,"15min":0.29599999999999999},"max":{"1min":17.986000000000001,"5min":139.465,"15min":139.465},"last":7.6079999999999997},{"interface":"front","average":{"1min":2.5129999999999999,"5min":3.1949999999999998,"15min":3.3919999999999999},"min":{"1min":0.38300000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":9.2289999999999992,"5min":141.56200000000001,"15min":141.56200000000001},"last":5.4290000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019367,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3298792,"kb_used_data":2768672,"kb_used_omap":0,"kb_used_meta":530112,"kb_avail":90446360,"statfs":{"total":95995035648,"available":92617072640,"internally_reserved":0,"allocated":2835120128,"data_stored":2833589632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":542834688},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":14.343,"5min":5.6890000000000001,"15min":3.0819999999999999},"min":{"1min":0.25,"5min":0.224,"15min":0.22},"max":{"1min":232.89500000000001,"5min":232.89500000000001,"15min":232.89500000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":14.416,"5min":5.71,"15min":3.0609999999999999},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":232.851,"5min":232.851,"15min":232.851},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":3.1480000000000001,"15min":5.226},"min":{"1min":0.26700000000000002,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":3.1579999999999999,"5min":114.05200000000001,"15min":161.613},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.1519999999999999,"5min":3.1499999999999999,"15min":5.2380000000000004},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.217},"max":{"1min":3.109,"5min":114.02800000000001,"15min":162.017},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 20:18:21 2023","interfaces":[{"interface":"back","average":{"1min":4.9160000000000004,"5min":4.9770000000000003,"15min":5.9820000000000002},"min":{"1min":0.29899999999999999,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":53.057000000000002,"5min":114.161,"15min":182.96700000000001},"last":39.000999999999998},{"interface":"front","average":{"1min":4.883,"5min":4.9370000000000003,"15min":5.9470000000000001},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":53.796999999999997,"5min":114.26600000000001,"15min":183.244},"last":49.744999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":3.9790000000000001,"15min":5.0960000000000001},"min":{"1min":0.34699999999999998,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":13.426,"5min":115.937,"15min":184.37100000000001},"last":2.6339999999999999},{"interface":"front","average":{"1min":1.9039999999999999,"5min":3.9350000000000001,"15min":5.1559999999999997},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":10.486000000000001,"5min":115.902,"15min":184.22300000000001},"last":2.6560000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":2.2509999999999999,"15min":1.6379999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":2.9670000000000001,"5min":114.788,"15min":176.81899999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.753,"5min":2.363,"15min":1.698},"min":{"1min":0.29699999999999999,"5min":0.29499999999999998,"15min":0.26600000000000001},"max":{"1min":4.0460000000000003,"5min":113.822,"15min":176.172},"last":2.5720000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:17:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.0390000000000001,"15min":1.026},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":2.5110000000000001,"5min":114.232,"15min":114.232},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.3540000000000001,"15min":1.2130000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.247},"max":{"1min":4.085,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":2.4620000000000002,"15min":1.9450000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":1.8500000000000001,"5min":113.861,"15min":113.861},"last":2.6059999999999999},{"interface":"front","average":{"1min":0.73099999999999998,"5min":2.548,"15min":2.0550000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.222},"max":{"1min":2.8839999999999999,"5min":114.32299999999999,"15min":114.32299999999999},"last":0.57899999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757730,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2425808,"kb_used_data":1784904,"kb_used_omap":0,"kb_used_meta":640832,"kb_avail":91319344,"statfs":{"total":95995035648,"available":93511008256,"internally_reserved":0,"allocated":1827741696,"data_stored":1826223738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":656211968},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:35 2023","interfaces":[{"interface":"back","average":{"1min":11.675000000000001,"5min":6.9509999999999996,"15min":3.649},"min":{"1min":0.29699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":166.57900000000001,"5min":166.57900000000001,"15min":166.57900000000001},"last":152.71299999999999},{"interface":"front","average":{"1min":10.929,"5min":6.3719999999999999,"15min":3.6840000000000002},"min":{"1min":0.30599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":165.67500000000001,"5min":165.67500000000001,"15min":165.67500000000001},"last":152.584}]},{"osd":1,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":3.8540000000000001,"15min":4.3979999999999997},"min":{"1min":0.25700000000000001,"5min":0.22,"15min":0.22},"max":{"1min":3.8319999999999999,"5min":150.16999999999999,"15min":171.47300000000001},"last":0.375},{"interface":"front","average":{"1min":2.5459999999999998,"5min":3.7040000000000002,"15min":4.4640000000000004},"min":{"1min":0.36499999999999999,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":10.75,"5min":150.12700000000001,"15min":171.399},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 20:18:12 2023","interfaces":[{"interface":"back","average":{"1min":9.3829999999999991,"5min":3.3450000000000002,"15min":4.7640000000000002},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":161.852,"5min":161.852,"15min":240.57300000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":9.5299999999999994,"5min":3.4279999999999999,"15min":4.8399999999999999},"min":{"1min":0.41799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":161.95599999999999,"5min":161.95599999999999,"15min":240.464},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":9.6489999999999991,"15min":6.7640000000000002},"min":{"1min":0.35899999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":8.8550000000000004,"5min":196.97800000000001,"15min":201.50899999999999},"last":155.97300000000001},{"interface":"front","average":{"1min":2.117,"5min":9.8149999999999995,"15min":6.524},"min":{"1min":0.313,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":8.0090000000000003,"5min":196.875,"15min":201.58099999999999},"last":155.803}]},{"osd":4,"last update":"Sat Sep 23 20:18:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1579999999999999,"15min":0.77800000000000002},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.24099999999999999},"max":{"1min":4.2350000000000003,"5min":8,"15min":9.7240000000000002},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.72899999999999998,"15min":0.60199999999999998},"min":{"1min":0.33700000000000002,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":1.5029999999999999,"5min":3.371,"15min":8.7360000000000007},"last":0.65400000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2490000000000001,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":4.04,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.73299999999999998,"15min":0.60999999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":1.609,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":1.492}]},{"osd":7,"last update":"Sat Sep 23 20:18:12 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":1.6439999999999999,"15min":0.93700000000000006},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":1.877,"5min":11.725,"15min":11.725},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.02,"5min":1.502,"15min":0.92300000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.8499999999999996,"5min":11.041,"15min":11.041},"last":0.35899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3258388480,"data_stored":3258384408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":496762880,"data_stored":496762880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2352021504,"data_stored":2352021504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2021920768,"data_stored":2021916696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2816151552,"data_stored":2816147480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1809645568,"data_stored":1809645568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":237764608,"data_stored":237764608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1698758656,"data_stored":1698758656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:19:16.589 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 16 2023-09-23T20:19:16.621 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:16.707 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:16.798 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:16.801 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:16 smithi135 ceph-mon[213871]: 1.a scrub starts 2023-09-23T20:19:16.801 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:16 smithi135 ceph-mon[221113]: 1.a scrub starts 2023-09-23T20:19:16.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:16.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:16.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:16 smithi144 ceph-mon[194833]: 1.a scrub starts 2023-09-23T20:19:16.903 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:16.931 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:17.342 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:17.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:17.449 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:17.488 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:17.611 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:17.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:17.638 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:17.710 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:19:17.710 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:19:17.718 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:17.737 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:17 smithi144 ceph-mon[194833]: pgmap v1433: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 9013/208071 objects degraded (4.332%); 7556/208071 objects misplaced (3.631%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:19:17.800 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:17.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:17 smithi135 ceph-mon[221113]: pgmap v1433: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 9013/208071 objects degraded (4.332%); 7556/208071 objects misplaced (3.631%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:19:17.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:17 smithi135 ceph-mon[213871]: pgmap v1433: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 9013/208071 objects degraded (4.332%); 7556/208071 objects misplaced (3.631%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:19:18.192 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:18.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:18.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:18.297 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:18.329 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:18.403 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:19:18.413 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:19:18.432 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:18.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:18.537 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:18.569 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:18.623 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:18.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:18 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9013/208071 objects degraded (4.332%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:18 smithi135 ceph-mon[221113]: osdmap e511: 8 total, 7 up, 8 in 2023-09-23T20:19:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:18 smithi135 ceph-mon[221113]: failed to encode map e511 with expected crc 2023-09-23T20:19:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2862343490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:19:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:18 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9013/208071 objects degraded (4.332%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:18 smithi135 ceph-mon[213871]: osdmap e511: 8 total, 7 up, 8 in 2023-09-23T20:19:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:18 smithi135 ceph-mon[213871]: failed to encode map e511 with expected crc 2023-09-23T20:19:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2862343490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:19:18.728 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:18.754 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:18.815 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:18.992 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:18 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9013/208071 objects degraded (4.332%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:18.992 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:18 smithi144 ceph-mon[194833]: osdmap e511: 8 total, 7 up, 8 in 2023-09-23T20:19:18.992 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:18 smithi144 ceph-mon[194833]: failed to encode map e511 with expected crc 2023-09-23T20:19:18.992 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2862343490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:19:19.035 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:19:19.452 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:19.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:19.556 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:19.667 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: 1.8 scrub starts 2023-09-23T20:19:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: failed to encode map e512 with expected crc 2023-09-23T20:19:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2862343490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:19:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: osdmap e512: 8 total, 7 up, 8 in 2023-09-23T20:19:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: failed to encode map e512 with expected crc 2023-09-23T20:19:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[213871]: pgmap v1436: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9013/205083 objects degraded (4.395%); 7367/205083 objects misplaced (3.592%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:19:19.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: failed to encode map e511 with expected crc 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: 1.8 scrub starts 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: failed to encode map e512 with expected crc 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2862343490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: osdmap e512: 8 total, 7 up, 8 in 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: failed to encode map e512 with expected crc 2023-09-23T20:19:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:19 smithi135 ceph-mon[221113]: pgmap v1436: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9013/205083 objects degraded (4.395%); 7367/205083 objects misplaced (3.592%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:19:19.774 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:19.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:19.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:19.881 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:19.978 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:20.122 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: failed to encode map e511 with expected crc 2023-09-23T20:19:20.122 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: failed to encode map e511 with expected crc 2023-09-23T20:19:20.122 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: failed to encode map e511 with expected crc 2023-09-23T20:19:20.122 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: failed to encode map e511 with expected crc 2023-09-23T20:19:20.123 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: failed to encode map e511 with expected crc 2023-09-23T20:19:20.123 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: 1.8 scrub starts 2023-09-23T20:19:20.123 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:20.123 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: failed to encode map e512 with expected crc 2023-09-23T20:19:20.123 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2862343490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:19:20.123 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: osdmap e512: 8 total, 7 up, 8 in 2023-09-23T20:19:20.123 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: failed to encode map e512 with expected crc 2023-09-23T20:19:20.123 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:19 smithi144 ceph-mon[194833]: pgmap v1436: 119 pgs: 1 active+recovering+remapped, 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9013/205083 objects degraded (4.395%); 7367/205083 objects misplaced (3.592%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:19:20.314 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:20.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:20.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:20.377 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:19:20.378 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:19:20.422 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:20.534 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:20.694 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:20.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:20.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:20.733 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:20.799 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:20.869 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:20 smithi135 ceph-mon[221113]: failed to encode map e512 with expected crc 2023-09-23T20:19:20.869 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:20 smithi135 ceph-mon[221113]: failed to encode map e512 with expected crc 2023-09-23T20:19:20.869 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:20 smithi135 ceph-mon[213871]: failed to encode map e512 with expected crc 2023-09-23T20:19:20.869 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:20 smithi135 ceph-mon[213871]: failed to encode map e512 with expected crc 2023-09-23T20:19:20.886 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:21.017 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:21.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:21.093 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:20 smithi144 ceph-mon[194833]: failed to encode map e512 with expected crc 2023-09-23T20:19:21.093 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:20 smithi144 ceph-mon[194833]: failed to encode map e512 with expected crc 2023-09-23T20:19:21.113 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:19:21.127 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:21.209 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:21.279 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:21.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:21.385 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:21.484 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:21.630 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:21.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:21.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:21.663 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:21.753 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:21.755 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 10 pgp_num to 16 2023-09-23T20:19:21.835 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:21.942 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:21.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:21 smithi135 ceph-mon[213871]: pgmap v1437: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 9013/203121 objects degraded (4.437%); 7192/203121 objects misplaced (3.541%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:19:21.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/755806467' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:19:21.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:21 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:19:21.952 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:21.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:21.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:21.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:21 smithi135 ceph-mon[221113]: pgmap v1437: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 9013/203121 objects degraded (4.437%); 7192/203121 objects misplaced (3.541%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:19:21.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/755806467' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:19:21.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:21 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:19:21.957 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:21.959 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:21 smithi144 ceph-mon[194833]: pgmap v1437: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 9013/203121 objects degraded (4.437%); 7192/203121 objects misplaced (3.541%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:19:21.959 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/755806467' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:19:21.959 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:21 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:19:21.959 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:22.055 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:22.102 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:22.477 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:19:22.953 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:22.964 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[213871]: failed to encode map e513 with expected crc 2023-09-23T20:19:22.964 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:19:22.964 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[213871]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T20:19:22.964 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[213871]: failed to encode map e513 with expected crc 2023-09-23T20:19:22.964 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:22.965 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:22 smithi144 ceph-mon[194833]: failed to encode map e513 with expected crc 2023-09-23T20:19:22.965 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:22 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:19:22.965 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:22 smithi144 ceph-mon[194833]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T20:19:22.965 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:22 smithi144 ceph-mon[194833]: failed to encode map e513 with expected crc 2023-09-23T20:19:22.965 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:22.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:22.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:22.966 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[221113]: failed to encode map e513 with expected crc 2023-09-23T20:19:22.967 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:19:22.967 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[221113]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T20:19:22.967 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[221113]: failed to encode map e513 with expected crc 2023-09-23T20:19:22.967 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:23.068 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:23.100 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:23.206 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:23.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:23.315 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:23.358 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:23.440 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:23.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:23.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:23.546 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:23.604 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:23.737 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:23.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:23.753 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:19:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:19:23] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T20:19:23.754 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 9013/203121 objects degraded (4.437%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:23.754 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:23 smithi135 ceph-mon[221113]: pgmap v1439: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 11814/198381 objects degraded (5.955%); 4217/198381 objects misplaced (2.126%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:19:23.761 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 9013/203121 objects degraded (4.437%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:23.761 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:23 smithi135 ceph-mon[213871]: pgmap v1439: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 11814/198381 objects degraded (5.955%); 4217/198381 objects misplaced (2.126%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:19:23.845 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:23.853 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:19:23.853 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:19:23.897 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:23.954 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:24.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 9013/203121 objects degraded (4.437%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:24.133 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:23 smithi144 ceph-mon[194833]: pgmap v1439: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 11814/198381 objects degraded (5.955%); 4217/198381 objects misplaced (2.126%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:19:24.151 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:19:24.165 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:24.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:24.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:24.183 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:19:24.242 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:19:24.277 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:24.298 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:19:24.347 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:19:24.374 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:24.421 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:24.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:24.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:24.521 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:19:24.525 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:24.559 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:19:24.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:19:24.559 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:19:24.560 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:19:24.560 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:19:24.560 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:19:24.560 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:19:24.560 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:19:24.560 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:19:24.609 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:24.853 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:19:25.135 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2232107918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:25.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2232107918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:25.223 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2232107918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:25.668 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:26.246 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:26.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:26.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:26.332 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:26 smithi144 ceph-mon[194833]: pgmap v1440: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 11814/196068 objects degraded (6.025%); 4030/196068 objects misplaced (2.055%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:19:26.352 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:26.415 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:26 smithi135 ceph-mon[213871]: pgmap v1440: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 11814/196068 objects degraded (6.025%); 4030/196068 objects misplaced (2.055%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:19:26.415 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:26 smithi135 ceph-mon[221113]: pgmap v1440: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 11814/196068 objects degraded (6.025%); 4030/196068 objects misplaced (2.055%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:19:26.419 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:26.931 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:26.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:27.035 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:27.058 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:19:27.059 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:19:27.070 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:27.234 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:27.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:27.339 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:27.389 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:27.489 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:27.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:27.594 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:27.723 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:27.831 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:27.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:27.942 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:28.025 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:28.108 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:19:28.108 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:19:28.108 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:19:28.108 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:19:28.129 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:28.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:28.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:28.234 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:28.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:28 smithi135 ceph-mon[213871]: pgmap v1441: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11646/189978 objects degraded (6.130%); 4030/189978 objects misplaced (2.121%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:19:28.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4153021784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:28.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:28 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:28.281 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11646/189978 objects degraded (6.130%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:28.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:28 smithi135 ceph-mon[221113]: pgmap v1441: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11646/189978 objects degraded (6.130%); 4030/189978 objects misplaced (2.121%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:19:28.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4153021784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:28.287 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:28 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:28.288 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11646/189978 objects degraded (6.130%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:28.301 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:19:28.351 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:28.537 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:19:28.537 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:19:28.537 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:19:28.537 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:19:28.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:28 smithi144 ceph-mon[194833]: pgmap v1441: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11646/189978 objects degraded (6.130%); 4030/189978 objects misplaced (2.121%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:19:28.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4153021784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:28.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:28 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:28.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11646/189978 objects degraded (6.130%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:28.657 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:28.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:28.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:28.774 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:28.862 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:28.901 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:19:29.059 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:19:29.383 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[221113]: failed to encode map e514 with expected crc 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[221113]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[221113]: failed to encode map e514 with expected crc 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[213871]: failed to encode map e514 with expected crc 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:29.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[213871]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T20:19:29.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[213871]: failed to encode map e514 with expected crc 2023-09-23T20:19:29.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:29.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:29 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:29.571 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:29 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:29.572 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:29 smithi144 ceph-mon[194833]: failed to encode map e514 with expected crc 2023-09-23T20:19:29.572 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:29 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:29.572 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:29 smithi144 ceph-mon[194833]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T20:19:29.572 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:29 smithi144 ceph-mon[194833]: failed to encode map e514 with expected crc 2023-09-23T20:19:29.572 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:29 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:29.572 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:29 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:29.986 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:30.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:30.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:30.113 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:30.213 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:30.225 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:30.296 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:30 smithi144 ceph-mon[194833]: pgmap v1443: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 11646/187011 objects degraded (6.227%); 3830/187011 objects misplaced (2.048%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:19:30.297 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:30 smithi135 ceph-mon[213871]: pgmap v1443: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 11646/187011 objects degraded (6.227%); 3830/187011 objects misplaced (2.048%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:19:30.326 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:30.332 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:30.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:30.440 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:30.529 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:30.642 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:30 smithi135 ceph-mon[221113]: pgmap v1443: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 11646/187011 objects degraded (6.227%); 3830/187011 objects misplaced (2.048%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:19:30.703 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:30.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:30.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:30.812 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:30.858 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:30.946 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:30.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:31.031 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:19:31.031 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:19:31.031 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:19:31.031 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:19:31.031 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:19:31.050 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:31.142 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:31.758 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:19:32.022 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:19:32.202 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:19:32.249 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:32.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:32.360 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:32.404 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:32 smithi135 ceph-mon[213871]: pgmap v1444: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11646/184767 objects degraded (6.303%); 3830/184767 objects misplaced (2.073%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:19:32.406 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:32 smithi135 ceph-mon[221113]: pgmap v1444: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11646/184767 objects degraded (6.303%); 3830/184767 objects misplaced (2.073%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:19:32.436 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:32.580 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:32 smithi144 ceph-mon[194833]: pgmap v1444: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11646/184767 objects degraded (6.303%); 3830/184767 objects misplaced (2.073%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:19:33.040 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:33.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:33.146 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:33.238 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:33.433 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:33.435 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11646/184767 objects degraded (6.303%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:33.435 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:33 smithi135 ceph-mon[221113]: pgmap v1445: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 11467/179505 objects degraded (6.388%); 3830/179505 objects misplaced (2.134%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:19:33.436 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11646/184767 objects degraded (6.303%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:33.436 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:33 smithi135 ceph-mon[213871]: pgmap v1445: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 11467/179505 objects degraded (6.388%); 3830/179505 objects misplaced (2.134%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:19:33.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:33.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:33.550 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:33.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11646/184767 objects degraded (6.303%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:33.584 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:33 smithi144 ceph-mon[194833]: pgmap v1445: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 11467/179505 objects degraded (6.388%); 3830/179505 objects misplaced (2.134%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:19:33.587 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:33.684 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:33.687 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:19:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:19:33] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T20:19:33.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:33.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:33.703 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:33.789 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:33.882 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71190 9.13 9:cf33cd0d:::benchmark_data_smithi135_549536_object71189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.171993+0000", 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.74825287200002, 2023-09-23T20:19:33.924 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.143627628, 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71190 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:33.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.171993+0000", 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.171993+0000", 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.171993+0000", 2023-09-23T20:19:33.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172033+0000", 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0908999999999997e-05 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172041+0000", 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2049999999999996e-06 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172045+0000", 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4460000000000003e-06 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172091+0000", 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6124000000000001e-05 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172336+0000", 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000244808 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172485+0000", 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000149238 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:33.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.176940+0000", 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0044543389999999999 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.182374+0000", 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054341659999999998 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.182383+0000", 2023-09-23T20:19:33.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9039999999999998e-06 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238107+0000", 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0557239190000001 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238118+0000", 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1071e-05 2023-09-23T20:19:33.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.308271+0000", 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.070152966 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.308949+0000", 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00067792799999999995 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.315503+0000", 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0065539700000000001 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.315621+0000", 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000118136 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.933 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71200 9.13 9:c81b3d8c:::benchmark_data_smithi135_549536_object71199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.177390+0000", 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.742856002, 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1402376740000002, 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:33.934 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71200 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177390+0000", 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177390+0000", 2023-09-23T20:19:33.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177389+0000", 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177431+0000", 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2224000000000001e-05 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177455+0000", 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3859e-05 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177462+0000", 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5350000000000001e-06 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177606+0000", 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014430399999999999 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177723+0000", 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000116592 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177854+0000", 2023-09-23T20:19:33.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000131643 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.180105+0000", 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002250307 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.182419+0000", 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023146220000000001 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.182426+0000", 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7460000000000003e-06 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238592+0000", 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.056166167 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238600+0000", 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3780000000000001e-06 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.309843+0000", 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0712429160000001 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.310503+0000", 2023-09-23T20:19:33.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000659814 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317571+0000", 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070675709999999999 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317628+0000", 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6878e-05 2023-09-23T20:19:33.942 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71205 9.13 9:cb3316ae:::benchmark_data_smithi135_549536_object71204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.186043+0000", 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.73420254000001, 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.132378675, 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:33.943 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71205 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186043+0000", 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.944 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186043+0000", 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186042+0000", 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:33.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186073+0000", 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0552999999999997e-05 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186082+0000", 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7750000000000004e-06 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:33.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186089+0000", 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2949999999999999e-06 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186219+0000", 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013041600000000001 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186245+0000", 2023-09-23T20:19:33.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5329000000000001e-05 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238625+0000", 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0523801159999999 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238629+0000", 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8300000000000003e-06 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.310545+0000", 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.071915256 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.310712+0000", 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016684600000000001 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.310846+0000", 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013424800000000001 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.315448+0000", 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046022349999999997 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317667+0000", 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022188849999999999 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317698+0000", 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1201999999999997e-05 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317726+0000", 2023-09-23T20:19:33.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8175e-05 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.318422+0000", 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000695793 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.952 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71209 9.13 9:cba57e8e:::benchmark_data_smithi135_549536_object71208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.189594+0000", 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.73065153900001, 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1435122980000001, 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71209 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.953 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189594+0000", 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189594+0000", 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189593+0000", 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189628+0000", 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4752000000000002e-05 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189642+0000", 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4295e-05 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189649+0000", 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4209999999999998e-06 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189751+0000", 2023-09-23T20:19:33.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101164 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189760+0000", 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0389999999999993e-06 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238657+0000", 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.048896922 2023-09-23T20:19:33.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238663+0000", 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6460000000000004e-06 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.318462+0000", 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0797985910000001 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.318644+0000", 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018246500000000001 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.318779+0000", 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013493000000000001 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.327867+0000", 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090877849999999993 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331013+0000", 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003146542 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331155+0000", 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000141643 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331220+0000", 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4449000000000006e-05 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.333107+0000", 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018871409999999999 2023-09-23T20:19:33.961 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71233 9.13 9:c9dee9fd:::benchmark_data_smithi135_549536_object71232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.215209+0000", 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.70503710999998, 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1197944849999999, 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:33.962 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71233 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215209+0000", 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215209+0000", 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215208+0000", 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215307+0000", 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9681999999999999e-05 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215319+0000", 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1343e-05 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:33.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215325+0000", 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3090000000000001e-06 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215380+0000", 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4784000000000003e-05 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215389+0000", 2023-09-23T20:19:33.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0459999999999994e-06 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238685+0000", 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.023296427 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238689+0000", 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5209999999999998e-06 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.320194+0000", 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.081505452 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.320322+0000", 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000127627 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:19:33.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.320464+0000", 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000142464 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.327873+0000", 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074090789999999998 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331119+0000", 2023-09-23T20:19:33.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032455510000000002 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.333192+0000", 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020725729999999999 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.333244+0000", 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2225999999999998e-05 2023-09-23T20:19:33.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.335003+0000", 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0017595390000000001 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71281 9.13 9:c9e738f5:::benchmark_data_smithi135_549536_object71280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:19:33.971 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.259961+0000", 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.660285106, 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0788080020000002, 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71281 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.259961+0000", 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.259961+0000", 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:33.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.259960+0000", 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260004+0000", 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4518000000000003e-05 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:33.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260025+0000", 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1234999999999999e-05 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260032+0000", 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7209999999999997e-06 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260115+0000", 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3078000000000007e-05 2023-09-23T20:19:33.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260124+0000", 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9150000000000002e-06 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238714+0000", 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97859040399999997 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238721+0000", 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.392e-06 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.321605+0000", 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0828845419999999 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.321795+0000", 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000189614 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.321900+0000", 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010535900000000001 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:33.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.327879+0000", 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0059790650000000004 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.333156+0000", 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052770200000000003 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.336887+0000", 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037310339999999998 2023-09-23T20:19:33.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.336943+0000", 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5621000000000002e-05 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.338769+0000", 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001825824 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.980 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71322 9.13 9:cf6bc6c4:::benchmark_data_smithi135_549536_object71321:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.300630+0000", 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.619615807, 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0399141589999998, 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:33.981 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71322 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300630+0000", 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:33.982 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300630+0000", 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300630+0000", 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300666+0000", 2023-09-23T20:19:33.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6192000000000002e-05 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300672+0000", 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6390000000000003e-06 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300676+0000", 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7720000000000001e-06 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300721+0000", 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4458e-05 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300729+0000", 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6170000000000007e-06 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238744+0000", 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.93801429700000005 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238747+0000", 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.591e-06 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.323227+0000", 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0844800809999999 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.323358+0000", 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013022399999999999 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:19:33.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.323539+0000", 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018142700000000001 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.330959+0000", 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074204730000000003 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.336842+0000", 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005882473 2023-09-23T20:19:33.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.338832+0000", 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0019901390000000001 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.338896+0000", 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4029999999999995e-05 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.340544+0000", 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001648229 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71336 9.13 9:c90857ba:::benchmark_data_smithi135_549536_object71335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.315133+0000", 2023-09-23T20:19:33.990 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.60511262, 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0271613479999999, 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71336 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:33.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315133+0000", 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315133+0000", 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315132+0000", 2023-09-23T20:19:33.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315249+0000", 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000117068 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315265+0000", 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5707000000000001e-05 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315271+0000", 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.285e-06 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315331+0000", 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0112999999999999e-05 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315341+0000", 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5699999999999999e-06 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238773+0000", 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.92343184199999995 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238779+0000", 2023-09-23T20:19:33.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4250000000000003e-06 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.324804+0000", 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0860250680000001 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.324955+0000", 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000150808 2023-09-23T20:19:33.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.325109+0000", 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015406099999999999 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.330968+0000", 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005858963 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.340608+0000", 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0096397800000000006 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.340660+0000", 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1718000000000002e-05 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:33.998 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.340713+0000", 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3627e-05 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.342295+0000", 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0015814010000000001 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:33.999 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71362 9.13 9:cf72cd0f:::benchmark_data_smithi135_549536_object71361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.342398+0000", 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.57784829299999, 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0014112530000001, 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71362 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.000 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342398+0000", 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342398+0000", 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342397+0000", 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342441+0000", 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3605e-05 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342466+0000", 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5137999999999999e-05 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342471+0000", 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0289999999999996e-06 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342530+0000", 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8743000000000003e-05 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342539+0000", 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3300000000000005e-06 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238819+0000", 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.89628013900000003 2023-09-23T20:19:34.004 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238827+0000", 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0199999999999994e-06 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.326346+0000", 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.087519015 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.326480+0000", 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013393199999999999 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.326631+0000", 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015099699999999999 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331025+0000", 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043942 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.342335+0000", 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011309703000000001 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.342363+0000", 2023-09-23T20:19:34.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7868000000000002e-05 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.342413+0000", 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0590999999999997e-05 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.343809+0000", 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0013954779999999999 2023-09-23T20:19:34.008 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71382 9.13 9:c87487c7:::benchmark_data_smithi135_549536_object71381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.363816+0000", 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 328.55643005399997, 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.981403325, 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.009 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71382 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363816+0000", 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363816+0000", 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363815+0000", 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363902+0000", 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6953999999999995e-05 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363915+0000", 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2771e-05 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363921+0000", 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.127e-06 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363988+0000", 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7515999999999995e-05 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.364001+0000", 2023-09-23T20:19:34.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.257e-05 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238860+0000", 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.87485905600000002 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238876+0000", 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.628e-05 2023-09-23T20:19:34.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.327905+0000", 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.089029228 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.328028+0000", 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012292999999999999 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.328183+0000", 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000154401 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.336849+0000", 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0086667700000000007 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.343845+0000", 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0069960309999999998 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.343880+0000", 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4644000000000002e-05 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.343925+0000", 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4440000000000001e-05 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.345219+0000", 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0012947220000000001 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97522 9.e 9:703747eb:::benchmark_data_smithi135_549536_object97521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.826971+0000", 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 300.09327501000001, 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3623128769999999, 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97522 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.019 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.826971+0000", 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.826971+0000", 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.826969+0000", 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827040+0000", 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0655999999999996e-05 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827061+0000", 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1370999999999999e-05 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827071+0000", 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4169999999999998e-06 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827522+0000", 2023-09-23T20:19:34.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00045141400000000003 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827645+0000", 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000122705 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.828027+0000", 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000382698 2023-09-23T20:19:34.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.831892+0000", 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038651029999999999 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.841958+0000", 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010065960000000001 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861337+0000", 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.01937854 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861344+0000", 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3819999999999998e-06 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854480+0000", 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99313559799999995 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854495+0000", 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4635e-05 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.189169+0000", 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3346740459999999 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.189284+0000", 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115198 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.027 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97523 9.1e 9:7df3908a:::benchmark_data_smithi135_549536_object97522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.827083+0000", 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 300.09316292, 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2868424049999998, 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97523 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.028 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827083+0000", 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827083+0000", 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827082+0000", 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827143+0000", 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0285000000000003e-05 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827152+0000", 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7220000000000005e-06 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827156+0000", 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6059999999999996e-06 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.830286+0000", 2023-09-23T20:19:34.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031297999999999999 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.830437+0000", 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015097799999999999 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.830731+0000", 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000293646 2023-09-23T20:19:34.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.833156+0000", 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0024251110000000002 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.842278+0000", 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0091223970000000008 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861255+0000", 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.018977355000000001 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861264+0000", 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9400000000000008e-06 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.034 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854571+0000", 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99330715300000005 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854579+0000", 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0979999999999996e-06 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.113800+0000", 2023-09-23T20:19:34.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2592216139999999 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.113925+0000", 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000125239 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.036 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97525 9.1e 9:795065da:::benchmark_data_smithi135_549536_object97524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.840530+0000", 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 300.07971645600003, 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3525597380000001, 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97525 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.037 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840530+0000", 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840530+0000", 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840528+0000", 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840584+0000", 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6082000000000003e-05 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840600+0000", 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5248e-05 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840608+0000", 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1680000000000004e-06 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840681+0000", 2023-09-23T20:19:34.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2731999999999994e-05 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840817+0000", 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013614500000000001 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.841107+0000", 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00029028600000000001 2023-09-23T20:19:34.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861290+0000", 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.020182736 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861296+0000", 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8610000000000003e-06 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854618+0000", 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99332210300000001 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854625+0000", 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1439999999999997e-06 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.113961+0000", 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2593362510000001 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.115314+0000", 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001353103 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.193009+0000", 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.077694632 2023-09-23T20:19:34.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.193089+0000", 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0479e-05 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97537 9.e 9:724684c9:::benchmark_data_smithi135_549536_object97536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:19:34.045 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.873780+0000", 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 300.04646645100001, 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3318544399999999, 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97537 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.046 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873780+0000", 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873780+0000", 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873778+0000", 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873818+0000", 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0425e-05 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873836+0000", 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7958999999999999e-05 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873844+0000", 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8939999999999993e-06 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873912+0000", 2023-09-23T20:19:34.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7990000000000005e-05 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873925+0000", 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2796000000000001e-05 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854523+0000", 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98059804800000006 2023-09-23T20:19:34.050 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854534+0000", 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1277000000000001e-05 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.191453+0000", 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3369192000000001 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.051 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.191601+0000", 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014799100000000001 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.192021+0000", 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00041995500000000001 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.052 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.201308+0000", 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0092868169999999993 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.201346+0000", 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8073e-05 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.203702+0000", 2023-09-23T20:19:34.053 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002355518 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.203759+0000", 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7172000000000001e-05 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205634+0000", 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018750539999999999 2023-09-23T20:19:34.054 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97541 9.d 9:b6092fac:::benchmark_data_smithi135_549536_object97540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.879991+0000", 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 300.04025507599999, 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2314555679999999, 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97541 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.879991+0000", 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.056 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.879991+0000", 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.879990+0000", 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.057 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880016+0000", 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.618e-05 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880026+0000", 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6879999999999994e-06 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880031+0000", 2023-09-23T20:19:34.058 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0880000000000002e-06 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880083+0000", 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1801000000000001e-05 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880094+0000", 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1e-05 2023-09-23T20:19:34.059 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.857379+0000", 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97728484299999996 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.857388+0000", 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4949999999999997e-06 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.060 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.073732+0000", 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2163436000000001 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.073897+0000", 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016534899999999999 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.061 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.074331+0000", 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043391399999999998 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.085762+0000", 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011431139 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.062 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.092805+0000", 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070426810000000003 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.109301+0000", 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016496459000000002 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.063 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.109356+0000", 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4648000000000003e-05 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.111446+0000", 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020903089999999998 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.064 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97546 9.e 9:74898ffe:::benchmark_data_smithi135_549536_object97545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.888839+0000", 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 300.03140743199998, 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3184715890000001, 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97546 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.065 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888839+0000", 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888839+0000", 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.066 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888838+0000", 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888899+0000", 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0520000000000003e-05 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.067 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888906+0000", 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.729e-06 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888910+0000", 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3100000000000001e-06 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.068 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888979+0000", 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9623999999999996e-05 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888994+0000", 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4786999999999999e-05 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854637+0000", 2023-09-23T20:19:34.069 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.96564282000000001 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854649+0000", 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1688999999999999e-05 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.193660+0000", 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3390118229999999 2023-09-23T20:19:34.070 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.193809+0000", 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014898999999999999 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.194207+0000", 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00039802099999999998 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.071 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.201380+0000", 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0071722790000000002 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205652+0000", 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0042719180000000004 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.072 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205692+0000", 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0253999999999998e-05 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205746+0000", 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4187999999999997e-05 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.207310+0000", 2023-09-23T20:19:34.073 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001564027 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:96745 10.1 10:8024395c:::benchmark_data_smithi135_602286_object4519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:18:45.899523+0000", 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 48.020723070000003, 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27761313500000001, 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.074 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 96745 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899523+0000", 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.075 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899523+0000", 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899522+0000", 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.076 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899526+0000", 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.515e-06 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899536+0000", 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3130000000000004e-06 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.077 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899541+0000", 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9470000000000002e-06 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899576+0000", 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4932999999999999e-05 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899656+0000", 2023-09-23T20:19:34.078 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9961000000000002e-05 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899722+0000", 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6236999999999996e-05 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.902981+0000", 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032588629999999999 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.079 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.903184+0000", 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020359299999999999 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:46.177032+0000", 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27384766399999999 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.080 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:46.177090+0000", 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8505999999999999e-05 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:46.177136+0000", 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5599999999999997e-05 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.081 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:112760 10.c 10:3b7ab588:::benchmark_data_smithi135_602286_object20534:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:11.524164+0000", 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 22.396082286999999, 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.23737845599999999, 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 112760 2023-09-23T20:19:34.082 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524164+0000", 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524164+0000", 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.083 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524162+0000", 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524169+0000", 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9450000000000004e-06 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.084 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524184+0000", 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4538e-05 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524192+0000", 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7170000000000008e-06 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.085 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524241+0000", 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9531999999999999e-05 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524345+0000", 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000104026 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524440+0000", 2023-09-23T20:19:34.086 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5217000000000007e-05 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.527267+0000", 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028268780000000001 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.089 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.529349+0000", 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020817639999999998 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.761446+0000", 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.232097266 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.761500+0000", 2023-09-23T20:19:34.090 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4014000000000002e-05 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.761542+0000", 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1779000000000001e-05 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:123706 10.1 10:85b26bb1:::benchmark_data_smithi135_602286_object31480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-09-23T20:19:34.091 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:29.317158+0000", 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 4.6030882809999998, 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.22169804300000001, 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 123706 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.092 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317158+0000", 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317158+0000", 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.093 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.094 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317156+0000", 2023-09-23T20:19:34.094 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:19:34.094 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.094 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.094 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317163+0000", 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2809999999999997e-06 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317179+0000", 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5977e-05 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.095 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317185+0000", 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5300000000000002e-06 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317231+0000", 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6217999999999997e-05 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317359+0000", 2023-09-23T20:19:34.096 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012729199999999999 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317491+0000", 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013280900000000001 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.324917+0000", 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074254179999999996 2023-09-23T20:19:34.097 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.329233+0000", 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043163719999999997 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.538756+0000", 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.209522924 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.098 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.538812+0000", 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5584999999999997e-05 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.538856+0000", 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3996999999999999e-05 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.099 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:123708 10.1 10:8b41d22a:::benchmark_data_smithi135_602286_object31482:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:29.317208+0000", 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 4.6030381829999998, 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.22180640700000001, 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 123708 2023-09-23T20:19:34.100 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317208+0000", 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317208+0000", 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:19:34.101 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317207+0000", 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317211+0000", 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9269999999999998e-06 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.102 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317219+0000", 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4719999999999998e-06 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.317223+0000", 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7400000000000004e-06 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:19:34.103 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.321069+0000", 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038460550000000001 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.321207+0000", 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000137766 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.321334+0000", 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000126353 2023-09-23T20:19:34.104 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.326549+0000", 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052152630000000004 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.329331+0000", 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027819640000000001 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.105 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.538919+0000", 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.20958862 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.538973+0000", 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3888999999999999e-05 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:19:34.106 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:19:34.107 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:29.539014+0000", 2023-09-23T20:19:34.107 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0938e-05 2023-09-23T20:19:34.107 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.107 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.107 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.107 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:19:34.107 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:19:34.107 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:19:34.185 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:34.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:34.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:34.244 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:19:34.260 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:19:34.295 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:19:34.300 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:34.334 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:34.357 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:19:34.384 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:34.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:34.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:34.408 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:19:34.488 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:34.489 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:19:34.516 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:34.527 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:19:34.572 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:19:34.611 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:19:34.611 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:19:34.611 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:19:34.611 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:19:34.611 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:19:34.611 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:19:34.611 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:19:34.611 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:19:34.612 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:19:34.689 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:19:34.697 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:34.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:34.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:34.801 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:34.881 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:35.051 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:35.052 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:35 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:35.052 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:35 smithi135 ceph-mon[213871]: from='client.36657 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:19:35.052 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:35 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/924867943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:35.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:35.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:35 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:35.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:35 smithi144 ceph-mon[194833]: from='client.36657 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:19:35.384 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:35 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/924867943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:35 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:35 smithi135 ceph-mon[221113]: from='client.36657 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:19:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:35 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/924867943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:36.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:19:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:36.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:36 smithi144 ceph-mon[194833]: pgmap v1446: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11467/177354 objects degraded (6.466%); 3633/177354 objects misplaced (2.048%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:19:36.391 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:36.391 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:19:36.391 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:36.392 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:36 smithi135 ceph-mon[213871]: pgmap v1446: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11467/177354 objects degraded (6.466%); 3633/177354 objects misplaced (2.048%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:19:36.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:36.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:19:36.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:19:36.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:36 smithi135 ceph-mon[221113]: pgmap v1446: 119 pgs: 2 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 110 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11467/177354 objects degraded (6.466%); 3633/177354 objects misplaced (2.048%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:19:37.091 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:37.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:37.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:37.197 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:37.252 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:19:37.253 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:19:37.322 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1445,"stamp":"2023-09-23T20:19:33.201822+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3924272015,"num_objects":59835,"num_object_clones":0,"num_object_copies":179505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11467,"num_objects_misplaced":3830,"num_objects_unfound":0,"num_objects_dirty":59835,"num_whiteouts":0,"num_read":6011,"num_read_kb":13264,"num_write":125563,"num_write_kb":5915145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7912,"num_bytes_recovered":508743283,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106451,"ondisk_log_size":106451,"up":350,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17833156,"kb_used_data":12303364,"kb_used_omap":38,"kb_used_meta":5529497,"kb_avail":732128060,"statfs":{"total":767960285184,"available":749699133440,"internally_reserved":0,"allocated":12598644736,"data_stored":12582540813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39803,"internal_metadata":5662205061},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,6,3,6,6,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-412352512,"num_objects":-6292,"num_object_clones":0,"num_object_copies":-18876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-347,"num_objects_misplaced":-387,"num_objects_unfound":0,"num_objects_dirty":-6292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6292,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":805,"num_bytes_recovered":52756480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.021278"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":874,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.727727+0000","last_change":"2023-09-23T20:19:13.851887+0000","last_active":"2023-09-23T20:19:21.727727+0000","last_peered":"2023-09-23T20:19:21.727727+0000","last_clean":"2023-09-23T20:19:21.727727+0000","last_became_active":"2023-09-23T20:19:05.089810+0000","last_became_peered":"2023-09-23T20:19:05.089810+0000","last_unstale":"2023-09-23T20:19:21.727727+0000","last_undegraded":"2023-09-23T20:19:21.727727+0000","last_fullsized":"2023-09-23T20:19:21.727727+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.731059+0000","last_change":"2023-09-23T20:17:47.279267+0000","last_active":"2023-09-23T20:19:21.731059+0000","last_peered":"2023-09-23T20:19:21.731059+0000","last_clean":"2023-09-23T20:19:21.731059+0000","last_became_active":"2023-09-23T20:17:47.278920+0000","last_became_peered":"2023-09-23T20:17:47.278920+0000","last_unstale":"2023-09-23T20:19:21.731059+0000","last_undegraded":"2023-09-23T20:19:21.731059+0000","last_fullsized":"2023-09-23T20:19:21.731059+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:16.699236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.731525+0000","last_change":"2023-09-23T20:17:47.270409+0000","last_active":"2023-09-23T20:19:21.731525+0000","last_peered":"2023-09-23T20:19:21.731525+0000","last_clean":"2023-09-23T20:19:21.731525+0000","last_became_active":"2023-09-23T20:17:47.270086+0000","last_became_peered":"2023-09-23T20:17:47.270086+0000","last_unstale":"2023-09-23T20:19:21.731525+0000","last_undegraded":"2023-09-23T20:19:21.731525+0000","last_fullsized":"2023-09-23T20:19:21.731525+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.885372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":577,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730996+0000","last_change":"2023-09-23T20:19:05.090444+0000","last_active":"2023-09-23T20:19:21.730996+0000","last_peered":"2023-09-23T20:19:21.730996+0000","last_clean":"2023-09-23T20:19:21.730996+0000","last_became_active":"2023-09-23T20:19:05.090104+0000","last_became_peered":"2023-09-23T20:19:05.090104+0000","last_unstale":"2023-09-23T20:19:21.730996+0000","last_undegraded":"2023-09-23T20:19:21.730996+0000","last_fullsized":"2023-09-23T20:19:21.730996+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.729005+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:19:21.729005+0000","last_peered":"2023-09-23T20:19:21.729005+0000","last_clean":"2023-09-23T20:19:21.729005+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:19:21.729005+0000","last_undegraded":"2023-09-23T20:19:21.729005+0000","last_fullsized":"2023-09-23T20:19:21.729005+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":222,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.724300+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:19:21.724300+0000","last_peered":"2023-09-23T20:19:21.724300+0000","last_clean":"2023-09-23T20:19:21.724300+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:19:21.724300+0000","last_undegraded":"2023-09-23T20:19:21.724300+0000","last_fullsized":"2023-09-23T20:19:21.724300+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":864,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.727873+0000","last_change":"2023-09-23T20:19:11.849616+0000","last_active":"2023-09-23T20:19:21.727873+0000","last_peered":"2023-09-23T20:19:21.727873+0000","last_clean":"2023-09-23T20:19:21.727873+0000","last_became_active":"2023-09-23T20:19:05.067520+0000","last_became_peered":"2023-09-23T20:19:05.067520+0000","last_unstale":"2023-09-23T20:19:21.727873+0000","last_undegraded":"2023-09-23T20:19:21.727873+0000","last_fullsized":"2023-09-23T20:19:21.727873+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.731861+0000","last_change":"2023-09-23T20:17:47.279385+0000","last_active":"2023-09-23T20:19:21.731861+0000","last_peered":"2023-09-23T20:19:21.731861+0000","last_clean":"2023-09-23T20:19:21.731861+0000","last_became_active":"2023-09-23T20:17:47.279065+0000","last_became_peered":"2023-09-23T20:17:47.279065+0000","last_unstale":"2023-09-23T20:19:21.731861+0000","last_undegraded":"2023-09-23T20:19:21.731861+0000","last_fullsized":"2023-09-23T20:19:21.731861+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:19.638157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":479,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728650+0000","last_change":"2023-09-23T20:17:47.270621+0000","last_active":"2023-09-23T20:19:21.728650+0000","last_peered":"2023-09-23T20:19:21.728650+0000","last_clean":"2023-09-23T20:19:21.728650+0000","last_became_active":"2023-09-23T20:17:47.270375+0000","last_became_peered":"2023-09-23T20:17:47.270375+0000","last_unstale":"2023-09-23T20:19:21.728650+0000","last_undegraded":"2023-09-23T20:19:21.728650+0000","last_fullsized":"2023-09-23T20:19:21.728650+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:16.961271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":755,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.740672+0000","last_change":"2023-09-23T20:19:05.069925+0000","last_active":"2023-09-23T20:19:21.740672+0000","last_peered":"2023-09-23T20:19:21.740672+0000","last_clean":"2023-09-23T20:19:21.740672+0000","last_became_active":"2023-09-23T20:19:05.069486+0000","last_became_peered":"2023-09-23T20:19:05.069486+0000","last_unstale":"2023-09-23T20:19:21.740672+0000","last_undegraded":"2023-09-23T20:19:21.740672+0000","last_fullsized":"2023-09-23T20:19:21.740672+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":440,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728590+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:19:21.728590+0000","last_peered":"2023-09-23T20:19:21.728590+0000","last_clean":"2023-09-23T20:19:21.728590+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:19:21.728590+0000","last_undegraded":"2023-09-23T20:19:21.728590+0000","last_fullsized":"2023-09-23T20:19:21.728590+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":378,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.711038+0000","last_change":"2023-09-23T20:17:47.265452+0000","last_active":"2023-09-23T20:19:21.711038+0000","last_peered":"2023-09-23T20:19:21.711038+0000","last_clean":"2023-09-23T20:19:21.711038+0000","last_became_active":"2023-09-23T20:17:47.265115+0000","last_became_peered":"2023-09-23T20:17:47.265115+0000","last_unstale":"2023-09-23T20:19:21.711038+0000","last_undegraded":"2023-09-23T20:19:21.711038+0000","last_fullsized":"2023-09-23T20:19:21.711038+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:04.898742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":848,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.736548+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:19:21.736548+0000","last_peered":"2023-09-23T20:19:21.736548+0000","last_clean":"2023-09-23T20:19:21.736548+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:19:21.736548+0000","last_undegraded":"2023-09-23T20:19:21.736548+0000","last_fullsized":"2023-09-23T20:19:21.736548+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":533,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728177+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:19:21.728177+0000","last_peered":"2023-09-23T20:19:21.728177+0000","last_clean":"2023-09-23T20:19:21.728177+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:19:21.728177+0000","last_undegraded":"2023-09-23T20:19:21.728177+0000","last_fullsized":"2023-09-23T20:19:21.728177+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":379,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.711155+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:19:21.711155+0000","last_peered":"2023-09-23T20:19:21.711155+0000","last_clean":"2023-09-23T20:19:21.711155+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:19:21.711155+0000","last_undegraded":"2023-09-23T20:19:21.711155+0000","last_fullsized":"2023-09-23T20:19:21.711155+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":806,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.722435+0000","last_change":"2023-09-23T20:19:05.073028+0000","last_active":"2023-09-23T20:19:21.722435+0000","last_peered":"2023-09-23T20:19:21.722435+0000","last_clean":"2023-09-23T20:19:21.722435+0000","last_became_active":"2023-09-23T20:19:05.072575+0000","last_became_peered":"2023-09-23T20:19:05.072575+0000","last_unstale":"2023-09-23T20:19:21.722435+0000","last_undegraded":"2023-09-23T20:19:21.722435+0000","last_fullsized":"2023-09-23T20:19:21.722435+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":207,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723457+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:19:21.723457+0000","last_peered":"2023-09-23T20:19:21.723457+0000","last_clean":"2023-09-23T20:19:21.723457+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:19:21.723457+0000","last_undegraded":"2023-09-23T20:19:21.723457+0000","last_fullsized":"2023-09-23T20:19:21.723457+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730901+0000","last_change":"2023-09-23T20:17:47.278773+0000","last_active":"2023-09-23T20:19:21.730901+0000","last_peered":"2023-09-23T20:19:21.730901+0000","last_clean":"2023-09-23T20:19:21.730901+0000","last_became_active":"2023-09-23T20:17:47.278467+0000","last_became_peered":"2023-09-23T20:17:47.278467+0000","last_unstale":"2023-09-23T20:19:21.730901+0000","last_undegraded":"2023-09-23T20:19:21.730901+0000","last_fullsized":"2023-09-23T20:19:21.730901+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:41.368483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":709,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.719753+0000","last_change":"2023-09-23T20:19:05.092744+0000","last_active":"2023-09-23T20:19:21.719753+0000","last_peered":"2023-09-23T20:19:21.719753+0000","last_clean":"2023-09-23T20:19:21.719753+0000","last_became_active":"2023-09-23T20:19:05.092425+0000","last_became_peered":"2023-09-23T20:19:05.092425+0000","last_unstale":"2023-09-23T20:19:21.719753+0000","last_undegraded":"2023-09-23T20:19:21.719753+0000","last_fullsized":"2023-09-23T20:19:21.719753+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":460,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.724352+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:19:21.724352+0000","last_peered":"2023-09-23T20:19:21.724352+0000","last_clean":"2023-09-23T20:19:21.724352+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:19:21.724352+0000","last_undegraded":"2023-09-23T20:19:21.724352+0000","last_fullsized":"2023-09-23T20:19:21.724352+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":33,"reported_epoch":513,"state":"active+undersized","last_fresh":"2023-09-23T20:19:21.735276+0000","last_change":"2023-09-23T20:18:38.012263+0000","last_active":"2023-09-23T20:19:21.735276+0000","last_peered":"2023-09-23T20:19:21.735276+0000","last_clean":"2023-09-23T20:17:54.133085+0000","last_became_active":"2023-09-23T20:18:38.012263+0000","last_became_peered":"2023-09-23T20:18:38.012263+0000","last_unstale":"2023-09-23T20:19:21.735276+0000","last_undegraded":"2023-09-23T20:19:21.735276+0000","last_fullsized":"2023-09-23T20:18:38.003813+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:47.449647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":873,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728272+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:19:21.728272+0000","last_peered":"2023-09-23T20:19:21.728272+0000","last_clean":"2023-09-23T20:19:21.728272+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:19:21.728272+0000","last_undegraded":"2023-09-23T20:19:21.728272+0000","last_fullsized":"2023-09-23T20:19:21.728272+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":207,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.724711+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:19:21.724711+0000","last_peered":"2023-09-23T20:19:21.724711+0000","last_clean":"2023-09-23T20:19:21.724711+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:19:21.724711+0000","last_undegraded":"2023-09-23T20:19:21.724711+0000","last_fullsized":"2023-09-23T20:19:21.724711+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730328+0000","last_change":"2023-09-23T20:17:47.266963+0000","last_active":"2023-09-23T20:19:21.730328+0000","last_peered":"2023-09-23T20:19:21.730328+0000","last_clean":"2023-09-23T20:19:21.730328+0000","last_became_active":"2023-09-23T20:17:47.266645+0000","last_became_peered":"2023-09-23T20:17:47.266645+0000","last_unstale":"2023-09-23T20:19:21.730328+0000","last_undegraded":"2023-09-23T20:19:21.730328+0000","last_fullsized":"2023-09-23T20:19:21.730328+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:05.052559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":562,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728019+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:19:21.728019+0000","last_peered":"2023-09-23T20:19:21.728019+0000","last_clean":"2023-09-23T20:19:21.728019+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:19:21.728019+0000","last_undegraded":"2023-09-23T20:19:21.728019+0000","last_fullsized":"2023-09-23T20:19:21.728019+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":222,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723103+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:19:21.723103+0000","last_peered":"2023-09-23T20:19:21.723103+0000","last_clean":"2023-09-23T20:19:21.723103+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:19:21.723103+0000","last_undegraded":"2023-09-23T20:19:21.723103+0000","last_fullsized":"2023-09-23T20:19:21.723103+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":512,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728176+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:19:21.728176+0000","last_peered":"2023-09-23T20:19:21.728176+0000","last_clean":"2023-09-23T20:19:21.728176+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:19:21.728176+0000","last_undegraded":"2023-09-23T20:19:21.728176+0000","last_fullsized":"2023-09-23T20:19:21.728176+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":572,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.735406+0000","last_change":"2023-09-23T20:17:50.330492+0000","last_active":"2023-09-23T20:19:21.735406+0000","last_peered":"2023-09-23T20:19:21.735406+0000","last_clean":"2023-09-23T20:19:21.735406+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:19:21.735406+0000","last_undegraded":"2023-09-23T20:19:21.735406+0000","last_fullsized":"2023-09-23T20:19:21.735406+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:58.248167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.248167+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":218,"reported_epoch":513,"state":"active+undersized","last_fresh":"2023-09-23T20:19:21.722724+0000","last_change":"2023-09-23T20:18:38.011856+0000","last_active":"2023-09-23T20:19:21.722724+0000","last_peered":"2023-09-23T20:19:21.722724+0000","last_clean":"2023-09-23T20:18:35.260354+0000","last_became_active":"2023-09-23T20:18:38.011856+0000","last_became_peered":"2023-09-23T20:18:38.011856+0000","last_unstale":"2023-09-23T20:19:21.722724+0000","last_undegraded":"2023-09-23T20:19:21.722724+0000","last_fullsized":"2023-09-23T20:18:37.999747+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:58.093947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":503,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.714436+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:19:21.714436+0000","last_peered":"2023-09-23T20:19:21.714436+0000","last_clean":"2023-09-23T20:19:21.714436+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:19:21.714436+0000","last_undegraded":"2023-09-23T20:19:21.714436+0000","last_fullsized":"2023-09-23T20:19:21.714436+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":671,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.727968+0000","last_change":"2023-09-23T20:17:50.332129+0000","last_active":"2023-09-23T20:19:21.727968+0000","last_peered":"2023-09-23T20:19:21.727968+0000","last_clean":"2023-09-23T20:19:21.727968+0000","last_became_active":"2023-09-23T20:17:50.331669+0000","last_became_peered":"2023-09-23T20:17:50.331669+0000","last_unstale":"2023-09-23T20:19:21.727968+0000","last_undegraded":"2023-09-23T20:19:21.727968+0000","last_fullsized":"2023-09-23T20:19:21.727968+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:23.893417+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":157,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.727752+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:19:21.727752+0000","last_peered":"2023-09-23T20:19:21.727752+0000","last_clean":"2023-09-23T20:19:21.727752+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:19:21.727752+0000","last_undegraded":"2023-09-23T20:19:21.727752+0000","last_fullsized":"2023-09-23T20:19:21.727752+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":165,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.722270+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:19:21.722270+0000","last_peered":"2023-09-23T20:19:21.722270+0000","last_clean":"2023-09-23T20:19:21.722270+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:19:21.722270+0000","last_undegraded":"2023-09-23T20:19:21.722270+0000","last_fullsized":"2023-09-23T20:19:21.722270+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"514'7721","reported_seq":7764,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:28.322453+0000","last_change":"2023-09-23T20:16:59.490329+0000","last_active":"2023-09-23T20:19:28.322453+0000","last_peered":"2023-09-23T20:19:28.322453+0000","last_clean":"2023-09-23T20:19:28.322453+0000","last_became_active":"2023-09-23T20:16:59.490069+0000","last_became_peered":"2023-09-23T20:16:59.490069+0000","last_unstale":"2023-09-23T20:19:28.322453+0000","last_undegraded":"2023-09-23T20:19:28.322453+0000","last_fullsized":"2023-09-23T20:19:28.322453+0000","mapping_epoch":479,"log_start":"484'1133","ondisk_log_start":"484'1133","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6588,"log_dups_size":0,"ondisk_log_size":6588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.487350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251330560,"num_objects":3835,"num_object_clones":0,"num_object_copies":11505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7721,"num_write_kb":369792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":804,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723125+0000","last_change":"2023-09-23T20:19:05.065565+0000","last_active":"2023-09-23T20:19:21.723125+0000","last_peered":"2023-09-23T20:19:21.723125+0000","last_clean":"2023-09-23T20:19:21.723125+0000","last_became_active":"2023-09-23T20:19:05.065283+0000","last_became_peered":"2023-09-23T20:19:05.065283+0000","last_unstale":"2023-09-23T20:19:21.723125+0000","last_undegraded":"2023-09-23T20:19:21.723125+0000","last_fullsized":"2023-09-23T20:19:21.723125+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":205,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723927+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:19:21.723927+0000","last_peered":"2023-09-23T20:19:21.723927+0000","last_clean":"2023-09-23T20:19:21.723927+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:19:21.723927+0000","last_undegraded":"2023-09-23T20:19:21.723927+0000","last_fullsized":"2023-09-23T20:19:21.723927+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":522,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728127+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:19:21.728127+0000","last_peered":"2023-09-23T20:19:21.728127+0000","last_clean":"2023-09-23T20:19:21.728127+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:19:21.728127+0000","last_undegraded":"2023-09-23T20:19:21.728127+0000","last_fullsized":"2023-09-23T20:19:21.728127+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"514'7850","reported_seq":7895,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:31.701169+0000","last_change":"2023-09-23T20:16:59.487459+0000","last_active":"2023-09-23T20:19:31.701169+0000","last_peered":"2023-09-23T20:19:31.701169+0000","last_clean":"2023-09-23T20:19:31.701169+0000","last_became_active":"2023-09-23T20:16:59.487181+0000","last_became_peered":"2023-09-23T20:16:59.487181+0000","last_unstale":"2023-09-23T20:19:31.701169+0000","last_undegraded":"2023-09-23T20:19:31.701169+0000","last_fullsized":"2023-09-23T20:19:31.701169+0000","mapping_epoch":479,"log_start":"485'2762","ondisk_log_start":"485'2762","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5088,"log_dups_size":0,"ondisk_log_size":5088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:34.406760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239075328,"num_objects":3648,"num_object_clones":0,"num_object_copies":10944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7850,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":873,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.711196+0000","last_change":"2023-09-23T20:19:05.086498+0000","last_active":"2023-09-23T20:19:21.711196+0000","last_peered":"2023-09-23T20:19:21.711196+0000","last_clean":"2023-09-23T20:19:21.711196+0000","last_became_active":"2023-09-23T20:19:05.085764+0000","last_became_peered":"2023-09-23T20:19:05.085764+0000","last_unstale":"2023-09-23T20:19:21.711196+0000","last_undegraded":"2023-09-23T20:19:21.711196+0000","last_fullsized":"2023-09-23T20:19:21.711196+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":234,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.724382+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:19:21.724382+0000","last_peered":"2023-09-23T20:19:21.724382+0000","last_clean":"2023-09-23T20:19:21.724382+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:19:21.724382+0000","last_undegraded":"2023-09-23T20:19:21.724382+0000","last_fullsized":"2023-09-23T20:19:21.724382+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730481+0000","last_change":"2023-09-23T20:17:47.264776+0000","last_active":"2023-09-23T20:19:21.730481+0000","last_peered":"2023-09-23T20:19:21.730481+0000","last_clean":"2023-09-23T20:19:21.730481+0000","last_became_active":"2023-09-23T20:17:47.264473+0000","last_became_peered":"2023-09-23T20:17:47.264473+0000","last_unstale":"2023-09-23T20:19:21.730481+0000","last_undegraded":"2023-09-23T20:19:21.730481+0000","last_fullsized":"2023-09-23T20:19:21.730481+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:40.906303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"514'7799","reported_seq":7844,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:30.578930+0000","last_change":"2023-09-23T20:16:59.491310+0000","last_active":"2023-09-23T20:19:30.578930+0000","last_peered":"2023-09-23T20:19:30.578930+0000","last_clean":"2023-09-23T20:19:30.578930+0000","last_became_active":"2023-09-23T20:16:59.491016+0000","last_became_peered":"2023-09-23T20:16:59.491016+0000","last_unstale":"2023-09-23T20:19:30.578930+0000","last_undegraded":"2023-09-23T20:19:30.578930+0000","last_fullsized":"2023-09-23T20:19:30.578930+0000","mapping_epoch":479,"log_start":"485'2528","ondisk_log_start":"485'2528","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5271,"log_dups_size":0,"ondisk_log_size":5271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:59.436079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252510208,"num_objects":3853,"num_object_clones":0,"num_object_copies":11559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7799,"num_write_kb":372864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":586,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728351+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:19:21.728351+0000","last_peered":"2023-09-23T20:19:21.728351+0000","last_clean":"2023-09-23T20:19:21.728351+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:19:21.728351+0000","last_undegraded":"2023-09-23T20:19:21.728351+0000","last_fullsized":"2023-09-23T20:19:21.728351+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":226,"reported_epoch":513,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:19:21.723359+0000","last_change":"2023-09-23T20:18:38.001952+0000","last_active":"2023-09-23T20:19:21.723359+0000","last_peered":"2023-09-23T20:19:21.723359+0000","last_clean":"2023-09-23T20:18:35.257306+0000","last_became_active":"2023-09-23T20:18:38.001952+0000","last_became_peered":"2023-09-23T20:18:38.001952+0000","last_unstale":"2023-09-23T20:19:21.723359+0000","last_undegraded":"2023-09-23T20:18:38.000196+0000","last_fullsized":"2023-09-23T20:18:38.000044+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:17.507792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730978+0000","last_change":"2023-09-23T20:17:47.266977+0000","last_active":"2023-09-23T20:19:21.730978+0000","last_peered":"2023-09-23T20:19:21.730978+0000","last_clean":"2023-09-23T20:19:21.730978+0000","last_became_active":"2023-09-23T20:17:47.266657+0000","last_became_peered":"2023-09-23T20:17:47.266657+0000","last_unstale":"2023-09-23T20:19:21.730978+0000","last_undegraded":"2023-09-23T20:19:21.730978+0000","last_fullsized":"2023-09-23T20:19:21.730978+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:33.774981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"514'7962","reported_seq":8007,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:31.298283+0000","last_change":"2023-09-23T20:16:59.488990+0000","last_active":"2023-09-23T20:19:31.298283+0000","last_peered":"2023-09-23T20:19:31.298283+0000","last_clean":"2023-09-23T20:19:31.298283+0000","last_became_active":"2023-09-23T20:16:59.488614+0000","last_became_peered":"2023-09-23T20:16:59.488614+0000","last_unstale":"2023-09-23T20:19:31.298283+0000","last_undegraded":"2023-09-23T20:19:31.298283+0000","last_fullsized":"2023-09-23T20:19:31.298283+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":7962,"log_dups_size":0,"ondisk_log_size":7962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:11.359750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246284288,"num_objects":3758,"num_object_clones":0,"num_object_copies":11274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7962,"num_write_kb":375040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":727,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.724112+0000","last_change":"2023-09-23T20:19:05.072019+0000","last_active":"2023-09-23T20:19:21.724112+0000","last_peered":"2023-09-23T20:19:21.724112+0000","last_clean":"2023-09-23T20:19:21.724112+0000","last_became_active":"2023-09-23T20:19:05.071700+0000","last_became_peered":"2023-09-23T20:19:05.071700+0000","last_unstale":"2023-09-23T20:19:21.724112+0000","last_undegraded":"2023-09-23T20:19:21.724112+0000","last_fullsized":"2023-09-23T20:19:21.724112+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":572,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.729079+0000","last_change":"2023-09-23T20:17:47.267657+0000","last_active":"2023-09-23T20:19:21.729079+0000","last_peered":"2023-09-23T20:19:21.729079+0000","last_clean":"2023-09-23T20:19:21.729079+0000","last_became_active":"2023-09-23T20:17:47.267288+0000","last_became_peered":"2023-09-23T20:17:47.267288+0000","last_unstale":"2023-09-23T20:19:21.729079+0000","last_undegraded":"2023-09-23T20:19:21.729079+0000","last_fullsized":"2023-09-23T20:19:21.729079+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.185253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.735002+0000","last_change":"2023-09-23T20:17:47.280675+0000","last_active":"2023-09-23T20:19:21.735002+0000","last_peered":"2023-09-23T20:19:21.735002+0000","last_clean":"2023-09-23T20:19:21.735002+0000","last_became_active":"2023-09-23T20:17:47.280175+0000","last_became_peered":"2023-09-23T20:17:47.280175+0000","last_unstale":"2023-09-23T20:19:21.735002+0000","last_undegraded":"2023-09-23T20:19:21.735002+0000","last_fullsized":"2023-09-23T20:19:21.735002+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:08.499949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"514'7867","reported_seq":7913,"reported_epoch":514,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:19:30.619357+0000","last_change":"2023-09-23T20:17:48.443980+0000","last_active":"2023-09-23T20:19:30.619357+0000","last_peered":"2023-09-23T20:19:30.619357+0000","last_clean":"2023-09-23T20:17:46.225624+0000","last_became_active":"2023-09-23T20:17:48.428406+0000","last_became_peered":"2023-09-23T20:17:48.428406+0000","last_unstale":"2023-09-23T20:19:30.619357+0000","last_undegraded":"2023-09-23T20:17:48.291655+0000","last_fullsized":"2023-09-23T20:17:48.256072+0000","mapping_epoch":490,"log_start":"485'2547","ondisk_log_start":"485'2547","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5320,"log_dups_size":0,"ondisk_log_size":5320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:14.276994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241106944,"num_objects":3679,"num_object_clones":0,"num_object_copies":11037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3930,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7867,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[0,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":1836},{"shards":"0,4","objects":-251}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":906,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723469+0000","last_change":"2023-09-23T20:19:05.072169+0000","last_active":"2023-09-23T20:19:21.723469+0000","last_peered":"2023-09-23T20:19:21.723469+0000","last_clean":"2023-09-23T20:19:21.723469+0000","last_became_active":"2023-09-23T20:19:05.071737+0000","last_became_peered":"2023-09-23T20:19:05.071737+0000","last_unstale":"2023-09-23T20:19:21.723469+0000","last_undegraded":"2023-09-23T20:19:21.723469+0000","last_fullsized":"2023-09-23T20:19:21.723469+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":261,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.711457+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:19:21.711457+0000","last_peered":"2023-09-23T20:19:21.711457+0000","last_clean":"2023-09-23T20:19:21.711457+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:19:21.711457+0000","last_undegraded":"2023-09-23T20:19:21.711457+0000","last_fullsized":"2023-09-23T20:19:21.711457+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730391+0000","last_change":"2023-09-23T20:17:47.270450+0000","last_active":"2023-09-23T20:19:21.730391+0000","last_peered":"2023-09-23T20:19:21.730391+0000","last_clean":"2023-09-23T20:19:21.730391+0000","last_became_active":"2023-09-23T20:17:47.270040+0000","last_became_peered":"2023-09-23T20:17:47.270040+0000","last_unstale":"2023-09-23T20:19:21.730391+0000","last_undegraded":"2023-09-23T20:19:21.730391+0000","last_fullsized":"2023-09-23T20:19:21.730391+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:51.572016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.3","version":"514'7837","reported_seq":7882,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:31.289371+0000","last_change":"2023-09-23T20:16:59.488642+0000","last_active":"2023-09-23T20:19:31.289371+0000","last_peered":"2023-09-23T20:19:31.289371+0000","last_clean":"2023-09-23T20:19:31.289371+0000","last_became_active":"2023-09-23T20:16:59.488330+0000","last_became_peered":"2023-09-23T20:16:59.488330+0000","last_unstale":"2023-09-23T20:19:31.289371+0000","last_undegraded":"2023-09-23T20:19:31.289371+0000","last_fullsized":"2023-09-23T20:19:31.289371+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":7837,"log_dups_size":0,"ondisk_log_size":7837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:46.429096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240975872,"num_objects":3677,"num_object_clones":0,"num_object_copies":11031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7837,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":877,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.727836+0000","last_change":"2023-09-23T20:19:18.822783+0000","last_active":"2023-09-23T20:19:21.727836+0000","last_peered":"2023-09-23T20:19:21.727836+0000","last_clean":"2023-09-23T20:19:21.727836+0000","last_became_active":"2023-09-23T20:19:05.075801+0000","last_became_peered":"2023-09-23T20:19:05.075801+0000","last_unstale":"2023-09-23T20:19:21.727836+0000","last_undegraded":"2023-09-23T20:19:21.727836+0000","last_fullsized":"2023-09-23T20:19:21.727836+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:55.017905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:55.017905+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001740154,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730754+0000","last_change":"2023-09-23T20:17:47.266513+0000","last_active":"2023-09-23T20:19:21.730754+0000","last_peered":"2023-09-23T20:19:21.730754+0000","last_clean":"2023-09-23T20:19:21.730754+0000","last_became_active":"2023-09-23T20:17:47.266163+0000","last_became_peered":"2023-09-23T20:17:47.266163+0000","last_unstale":"2023-09-23T20:19:21.730754+0000","last_undegraded":"2023-09-23T20:19:21.730754+0000","last_fullsized":"2023-09-23T20:19:21.730754+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.645381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":425,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.740798+0000","last_change":"2023-09-23T20:17:47.270131+0000","last_active":"2023-09-23T20:19:21.740798+0000","last_peered":"2023-09-23T20:19:21.740798+0000","last_clean":"2023-09-23T20:19:21.740798+0000","last_became_active":"2023-09-23T20:17:47.269818+0000","last_became_peered":"2023-09-23T20:17:47.269818+0000","last_unstale":"2023-09-23T20:19:21.740798+0000","last_undegraded":"2023-09-23T20:19:21.740798+0000","last_fullsized":"2023-09-23T20:19:21.740798+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:40.738070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"514'7696","reported_seq":15476,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:29.190939+0000","last_change":"2023-09-23T20:19:19.046525+0000","last_active":"2023-09-23T20:19:29.190939+0000","last_peered":"2023-09-23T20:19:29.190939+0000","last_clean":"2023-09-23T20:19:29.190939+0000","last_became_active":"2023-09-23T20:19:19.036483+0000","last_became_peered":"2023-09-23T20:19:19.036483+0000","last_unstale":"2023-09-23T20:19:29.190939+0000","last_undegraded":"2023-09-23T20:19:29.190939+0000","last_fullsized":"2023-09-23T20:19:29.190939+0000","mapping_epoch":511,"log_start":"484'1960","ondisk_log_start":"484'1960","created":479,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:37.023435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248250368,"num_objects":3788,"num_object_clones":0,"num_object_copies":11364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7696,"num_write_kb":367488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3865,"num_bytes_recovered":253296640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":577,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.735181+0000","last_change":"2023-09-23T20:19:05.088257+0000","last_active":"2023-09-23T20:19:21.735181+0000","last_peered":"2023-09-23T20:19:21.735181+0000","last_clean":"2023-09-23T20:19:21.735181+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:19:21.735181+0000","last_undegraded":"2023-09-23T20:19:21.735181+0000","last_fullsized":"2023-09-23T20:19:21.735181+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":407,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.735589+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:19:21.735589+0000","last_peered":"2023-09-23T20:19:21.735589+0000","last_clean":"2023-09-23T20:19:21.735589+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:19:21.735589+0000","last_undegraded":"2023-09-23T20:19:21.735589+0000","last_fullsized":"2023-09-23T20:19:21.735589+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.731891+0000","last_change":"2023-09-23T20:17:47.268811+0000","last_active":"2023-09-23T20:19:21.731891+0000","last_peered":"2023-09-23T20:19:21.731891+0000","last_clean":"2023-09-23T20:19:21.731891+0000","last_became_active":"2023-09-23T20:17:47.268391+0000","last_became_peered":"2023-09-23T20:17:47.268391+0000","last_unstale":"2023-09-23T20:19:21.731891+0000","last_undegraded":"2023-09-23T20:19:21.731891+0000","last_fullsized":"2023-09-23T20:19:21.731891+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:32.388705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"514'7867","reported_seq":7910,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:33.063135+0000","last_change":"2023-09-23T20:16:59.496645+0000","last_active":"2023-09-23T20:19:33.063135+0000","last_peered":"2023-09-23T20:19:33.063135+0000","last_clean":"2023-09-23T20:19:33.063135+0000","last_became_active":"2023-09-23T20:16:59.496430+0000","last_became_peered":"2023-09-23T20:16:59.496430+0000","last_unstale":"2023-09-23T20:19:33.063135+0000","last_undegraded":"2023-09-23T20:19:33.063135+0000","last_fullsized":"2023-09-23T20:19:33.063135+0000","mapping_epoch":479,"log_start":"484'1742","ondisk_log_start":"484'1742","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6125,"log_dups_size":0,"ondisk_log_size":6125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:59.713976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239140864,"num_objects":3649,"num_object_clones":0,"num_object_copies":10947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7867,"num_write_kb":368512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":868,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728494+0000","last_change":"2023-09-23T20:19:15.841522+0000","last_active":"2023-09-23T20:19:21.728494+0000","last_peered":"2023-09-23T20:19:21.728494+0000","last_clean":"2023-09-23T20:19:21.728494+0000","last_became_active":"2023-09-23T20:19:05.066352+0000","last_became_peered":"2023-09-23T20:19:05.066352+0000","last_unstale":"2023-09-23T20:19:21.728494+0000","last_undegraded":"2023-09-23T20:19:21.728494+0000","last_fullsized":"2023-09-23T20:19:21.728494+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":165,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.710706+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:19:21.710706+0000","last_peered":"2023-09-23T20:19:21.710706+0000","last_clean":"2023-09-23T20:19:21.710706+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:19:21.710706+0000","last_undegraded":"2023-09-23T20:19:21.710706+0000","last_fullsized":"2023-09-23T20:19:21.710706+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.732156+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:19:21.732156+0000","last_peered":"2023-09-23T20:19:21.732156+0000","last_clean":"2023-09-23T20:19:21.732156+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:19:21.732156+0000","last_undegraded":"2023-09-23T20:19:21.732156+0000","last_fullsized":"2023-09-23T20:19:21.732156+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"514'7902","reported_seq":7945,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:33.077899+0000","last_change":"2023-09-23T20:16:59.496291+0000","last_active":"2023-09-23T20:19:33.077899+0000","last_peered":"2023-09-23T20:19:33.077899+0000","last_clean":"2023-09-23T20:19:33.077899+0000","last_became_active":"2023-09-23T20:16:59.496000+0000","last_became_peered":"2023-09-23T20:16:59.496000+0000","last_unstale":"2023-09-23T20:19:33.077899+0000","last_undegraded":"2023-09-23T20:19:33.077899+0000","last_fullsized":"2023-09-23T20:19:33.077899+0000","mapping_epoch":479,"log_start":"484'1777","ondisk_log_start":"484'1777","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6125,"log_dups_size":0,"ondisk_log_size":6125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.808639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237240320,"num_objects":3620,"num_object_clones":0,"num_object_copies":10860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7902,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":806,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.722354+0000","last_change":"2023-09-23T20:19:05.072847+0000","last_active":"2023-09-23T20:19:21.722354+0000","last_peered":"2023-09-23T20:19:21.722354+0000","last_clean":"2023-09-23T20:19:21.722354+0000","last_became_active":"2023-09-23T20:19:05.072187+0000","last_became_peered":"2023-09-23T20:19:05.072187+0000","last_unstale":"2023-09-23T20:19:21.722354+0000","last_undegraded":"2023-09-23T20:19:21.722354+0000","last_fullsized":"2023-09-23T20:19:21.722354+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:41.333560+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:41.333560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0022841509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":464,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730202+0000","last_change":"2023-09-23T20:17:50.311646+0000","last_active":"2023-09-23T20:19:21.730202+0000","last_peered":"2023-09-23T20:19:21.730202+0000","last_clean":"2023-09-23T20:19:21.730202+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:19:21.730202+0000","last_undegraded":"2023-09-23T20:19:21.730202+0000","last_fullsized":"2023-09-23T20:19:21.730202+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":514,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728085+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:19:21.728085+0000","last_peered":"2023-09-23T20:19:21.728085+0000","last_clean":"2023-09-23T20:19:21.728085+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:19:21.728085+0000","last_undegraded":"2023-09-23T20:19:21.728085+0000","last_fullsized":"2023-09-23T20:19:21.728085+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730836+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:19:21.730836+0000","last_peered":"2023-09-23T20:19:21.730836+0000","last_clean":"2023-09-23T20:19:21.730836+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:19:21.730836+0000","last_undegraded":"2023-09-23T20:19:21.730836+0000","last_fullsized":"2023-09-23T20:19:21.730836+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"514'7568","reported_seq":10157,"reported_epoch":514,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:19:28.399931+0000","last_change":"2023-09-23T20:18:56.585041+0000","last_active":"2023-09-23T20:19:28.399931+0000","last_peered":"2023-09-23T20:19:28.399931+0000","last_clean":"2023-09-23T20:19:28.399931+0000","last_became_active":"2023-09-23T20:18:39.050965+0000","last_became_peered":"2023-09-23T20:18:39.050965+0000","last_unstale":"2023-09-23T20:19:28.399931+0000","last_undegraded":"2023-09-23T20:19:28.399931+0000","last_fullsized":"2023-09-23T20:19:28.399931+0000","mapping_epoch":503,"log_start":"484'1000","ondisk_log_start":"484'1000","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6568,"log_dups_size":0,"ondisk_log_size":6568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:43.005386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251002880,"num_objects":3830,"num_object_clones":0,"num_object_copies":11490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3830,"num_objects_unfound":0,"num_objects_dirty":3830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7568,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1265,"num_bytes_recovered":82903040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":3830}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":857,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728117+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:19:21.728117+0000","last_peered":"2023-09-23T20:19:21.728117+0000","last_clean":"2023-09-23T20:19:21.728117+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:19:21.728117+0000","last_undegraded":"2023-09-23T20:19:21.728117+0000","last_fullsized":"2023-09-23T20:19:21.728117+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1194,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728185+0000","last_change":"2023-09-23T20:17:47.305979+0000","last_active":"2023-09-23T20:19:21.728185+0000","last_peered":"2023-09-23T20:19:21.728185+0000","last_clean":"2023-09-23T20:19:21.728185+0000","last_became_active":"2023-09-23T20:17:47.274623+0000","last_became_peered":"2023-09-23T20:17:47.274623+0000","last_unstale":"2023-09-23T20:19:21.728185+0000","last_undegraded":"2023-09-23T20:19:21.728185+0000","last_fullsized":"2023-09-23T20:19:21.728185+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:27.293931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.737467+0000","last_change":"2023-09-23T20:17:47.280498+0000","last_active":"2023-09-23T20:19:21.737467+0000","last_peered":"2023-09-23T20:19:21.737467+0000","last_clean":"2023-09-23T20:19:21.737467+0000","last_became_active":"2023-09-23T20:17:47.279716+0000","last_became_peered":"2023-09-23T20:17:47.279716+0000","last_unstale":"2023-09-23T20:19:21.737467+0000","last_undegraded":"2023-09-23T20:19:21.737467+0000","last_fullsized":"2023-09-23T20:19:21.737467+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:18.625303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730266+0000","last_change":"2023-09-23T20:17:47.276964+0000","last_active":"2023-09-23T20:19:21.730266+0000","last_peered":"2023-09-23T20:19:21.730266+0000","last_clean":"2023-09-23T20:19:21.730266+0000","last_became_active":"2023-09-23T20:17:47.276540+0000","last_became_peered":"2023-09-23T20:17:47.276540+0000","last_unstale":"2023-09-23T20:19:21.730266+0000","last_undegraded":"2023-09-23T20:19:21.730266+0000","last_fullsized":"2023-09-23T20:19:21.730266+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:31.547044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"514'7790","reported_seq":7833,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:28.413209+0000","last_change":"2023-09-23T20:16:59.496065+0000","last_active":"2023-09-23T20:19:28.413209+0000","last_peered":"2023-09-23T20:19:28.413209+0000","last_clean":"2023-09-23T20:19:28.413209+0000","last_became_active":"2023-09-23T20:16:59.495782+0000","last_became_peered":"2023-09-23T20:16:59.495782+0000","last_unstale":"2023-09-23T20:19:28.413209+0000","last_undegraded":"2023-09-23T20:19:28.413209+0000","last_fullsized":"2023-09-23T20:19:28.413209+0000","mapping_epoch":479,"log_start":"484'1211","ondisk_log_start":"484'1211","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:08.985469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252051456,"num_objects":3846,"num_object_clones":0,"num_object_copies":11538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7790,"num_write_kb":372352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":752,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723185+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:19:21.723185+0000","last_peered":"2023-09-23T20:19:21.723185+0000","last_clean":"2023-09-23T20:19:21.723185+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:19:21.723185+0000","last_undegraded":"2023-09-23T20:19:21.723185+0000","last_fullsized":"2023-09-23T20:19:21.723185+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1154,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:28.287990+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:19:28.287990+0000","last_peered":"2023-09-23T20:19:28.287990+0000","last_clean":"2023-09-23T20:19:28.287990+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:19:28.287990+0000","last_undegraded":"2023-09-23T20:19:28.287990+0000","last_fullsized":"2023-09-23T20:19:28.287990+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.731781+0000","last_change":"2023-09-23T20:17:47.278822+0000","last_active":"2023-09-23T20:19:21.731781+0000","last_peered":"2023-09-23T20:19:21.731781+0000","last_clean":"2023-09-23T20:19:21.731781+0000","last_became_active":"2023-09-23T20:17:47.278407+0000","last_became_peered":"2023-09-23T20:17:47.278407+0000","last_unstale":"2023-09-23T20:19:21.731781+0000","last_undegraded":"2023-09-23T20:19:21.731781+0000","last_fullsized":"2023-09-23T20:19:21.731781+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:01.125231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.732221+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:19:21.732221+0000","last_peered":"2023-09-23T20:19:21.732221+0000","last_clean":"2023-09-23T20:19:21.732221+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:19:21.732221+0000","last_undegraded":"2023-09-23T20:19:21.732221+0000","last_fullsized":"2023-09-23T20:19:21.732221+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"514'7601","reported_seq":7649,"reported_epoch":514,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:19:30.600329+0000","last_change":"2023-09-23T20:17:48.438517+0000","last_active":"2023-09-23T20:19:30.600329+0000","last_peered":"2023-09-23T20:19:30.600329+0000","last_clean":"2023-09-23T20:17:46.219689+0000","last_became_active":"2023-09-23T20:17:48.427732+0000","last_became_peered":"2023-09-23T20:17:48.427732+0000","last_unstale":"2023-09-23T20:19:30.600329+0000","last_undegraded":"2023-09-23T20:17:48.289936+0000","last_fullsized":"2023-09-23T20:17:48.256145+0000","mapping_epoch":490,"log_start":"485'2251","ondisk_log_start":"485'2251","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5350,"log_dups_size":0,"ondisk_log_size":5350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:05.919808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244121600,"num_objects":3725,"num_object_clones":0,"num_object_copies":11175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3822,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7601,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-97},{"shards":"0,4,7","objects":1884}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1422,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728169+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:19:21.728169+0000","last_peered":"2023-09-23T20:19:21.728169+0000","last_clean":"2023-09-23T20:19:21.728169+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:19:21.728169+0000","last_undegraded":"2023-09-23T20:19:21.728169+0000","last_fullsized":"2023-09-23T20:19:21.728169+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":729,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723196+0000","last_change":"2023-09-23T20:17:50.313197+0000","last_active":"2023-09-23T20:19:21.723196+0000","last_peered":"2023-09-23T20:19:21.723196+0000","last_clean":"2023-09-23T20:19:21.723196+0000","last_became_active":"2023-09-23T20:17:50.312838+0000","last_became_peered":"2023-09-23T20:17:50.312838+0000","last_unstale":"2023-09-23T20:19:21.723196+0000","last_undegraded":"2023-09-23T20:19:21.723196+0000","last_fullsized":"2023-09-23T20:19:21.723196+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:31.530167+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":379,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728388+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:19:21.728388+0000","last_peered":"2023-09-23T20:19:21.728388+0000","last_clean":"2023-09-23T20:19:21.728388+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:19:21.728388+0000","last_undegraded":"2023-09-23T20:19:21.728388+0000","last_fullsized":"2023-09-23T20:19:21.728388+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":311,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723206+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:19:21.723206+0000","last_peered":"2023-09-23T20:19:21.723206+0000","last_clean":"2023-09-23T20:19:21.723206+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:19:21.723206+0000","last_undegraded":"2023-09-23T20:19:21.723206+0000","last_fullsized":"2023-09-23T20:19:21.723206+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"514'7719","reported_seq":7764,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:31.289297+0000","last_change":"2023-09-23T20:16:59.494039+0000","last_active":"2023-09-23T20:19:31.289297+0000","last_peered":"2023-09-23T20:19:31.289297+0000","last_clean":"2023-09-23T20:19:31.289297+0000","last_became_active":"2023-09-23T20:16:59.493714+0000","last_became_peered":"2023-09-23T20:16:59.493714+0000","last_unstale":"2023-09-23T20:19:31.289297+0000","last_undegraded":"2023-09-23T20:19:31.289297+0000","last_fullsized":"2023-09-23T20:19:31.289297+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":7719,"log_dups_size":0,"ondisk_log_size":7719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:15.171264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246611968,"num_objects":3763,"num_object_clones":0,"num_object_copies":11289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7719,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":524,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.735099+0000","last_change":"2023-09-23T20:17:50.330694+0000","last_active":"2023-09-23T20:19:21.735099+0000","last_peered":"2023-09-23T20:19:21.735099+0000","last_clean":"2023-09-23T20:19:21.735099+0000","last_became_active":"2023-09-23T20:17:50.329940+0000","last_became_peered":"2023-09-23T20:17:50.329940+0000","last_unstale":"2023-09-23T20:19:21.735099+0000","last_undegraded":"2023-09-23T20:19:21.735099+0000","last_fullsized":"2023-09-23T20:19:21.735099+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:55.675445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:55.675445+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1038,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.722927+0000","last_change":"2023-09-23T20:17:50.313157+0000","last_active":"2023-09-23T20:19:21.722927+0000","last_peered":"2023-09-23T20:19:21.722927+0000","last_clean":"2023-09-23T20:19:21.722927+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:19:21.722927+0000","last_undegraded":"2023-09-23T20:19:21.722927+0000","last_fullsized":"2023-09-23T20:19:21.722927+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":461,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.710837+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:19:21.710837+0000","last_peered":"2023-09-23T20:19:21.710837+0000","last_clean":"2023-09-23T20:19:21.710837+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:19:21.710837+0000","last_undegraded":"2023-09-23T20:19:21.710837+0000","last_fullsized":"2023-09-23T20:19:21.710837+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.731979+0000","last_change":"2023-09-23T20:17:47.278112+0000","last_active":"2023-09-23T20:19:21.731979+0000","last_peered":"2023-09-23T20:19:21.731979+0000","last_clean":"2023-09-23T20:19:21.731979+0000","last_became_active":"2023-09-23T20:17:47.277728+0000","last_became_peered":"2023-09-23T20:17:47.277728+0000","last_unstale":"2023-09-23T20:19:21.731979+0000","last_undegraded":"2023-09-23T20:19:21.731979+0000","last_fullsized":"2023-09-23T20:19:21.731979+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:21.371380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"514'7795","reported_seq":7841,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:31.917177+0000","last_change":"2023-09-23T20:16:59.490420+0000","last_active":"2023-09-23T20:19:31.917177+0000","last_peered":"2023-09-23T20:19:31.917177+0000","last_clean":"2023-09-23T20:19:31.917177+0000","last_became_active":"2023-09-23T20:16:59.490025+0000","last_became_peered":"2023-09-23T20:16:59.490025+0000","last_unstale":"2023-09-23T20:19:31.917177+0000","last_undegraded":"2023-09-23T20:19:31.917177+0000","last_fullsized":"2023-09-23T20:19:31.917177+0000","mapping_epoch":479,"log_start":"485'2679","ondisk_log_start":"485'2679","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5116,"log_dups_size":0,"ondisk_log_size":5116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:03.184763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242090008,"num_objects":3695,"num_object_clones":0,"num_object_copies":11085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3695,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7795,"num_write_kb":367617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":709,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.720185+0000","last_change":"2023-09-23T20:19:05.092371+0000","last_active":"2023-09-23T20:19:21.720185+0000","last_peered":"2023-09-23T20:19:21.720185+0000","last_clean":"2023-09-23T20:19:21.720185+0000","last_became_active":"2023-09-23T20:19:05.092070+0000","last_became_peered":"2023-09-23T20:19:05.092070+0000","last_unstale":"2023-09-23T20:19:21.720185+0000","last_undegraded":"2023-09-23T20:19:21.720185+0000","last_fullsized":"2023-09-23T20:19:21.720185+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":214,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723365+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:19:21.723365+0000","last_peered":"2023-09-23T20:19:21.723365+0000","last_clean":"2023-09-23T20:19:21.723365+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:19:21.723365+0000","last_undegraded":"2023-09-23T20:19:21.723365+0000","last_fullsized":"2023-09-23T20:19:21.723365+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.730132+0000","last_change":"2023-09-23T20:17:47.276790+0000","last_active":"2023-09-23T20:19:21.730132+0000","last_peered":"2023-09-23T20:19:21.730132+0000","last_clean":"2023-09-23T20:19:21.730132+0000","last_became_active":"2023-09-23T20:17:47.276198+0000","last_became_peered":"2023-09-23T20:17:47.276198+0000","last_unstale":"2023-09-23T20:19:21.730132+0000","last_undegraded":"2023-09-23T20:19:21.730132+0000","last_fullsized":"2023-09-23T20:19:21.730132+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:06.124215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":340,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728236+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:19:21.728236+0000","last_peered":"2023-09-23T20:19:21.728236+0000","last_clean":"2023-09-23T20:19:21.728236+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:19:21.728236+0000","last_undegraded":"2023-09-23T20:19:21.728236+0000","last_fullsized":"2023-09-23T20:19:21.728236+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"514'7945","reported_seq":7990,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:19:31.287110+0000","last_change":"2023-09-23T20:16:59.490304+0000","last_active":"2023-09-23T20:19:31.287110+0000","last_peered":"2023-09-23T20:19:31.287110+0000","last_clean":"2023-09-23T20:19:31.287110+0000","last_became_active":"2023-09-23T20:16:59.490026+0000","last_became_peered":"2023-09-23T20:16:59.490026+0000","last_unstale":"2023-09-23T20:19:31.287110+0000","last_undegraded":"2023-09-23T20:19:31.287110+0000","last_fullsized":"2023-09-23T20:19:31.287110+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":7945,"log_dups_size":0,"ondisk_log_size":7945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:59.789512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244252672,"num_objects":3727,"num_object_clones":0,"num_object_copies":11181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7945,"num_write_kb":373504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":755,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.740873+0000","last_change":"2023-09-23T20:19:05.069756+0000","last_active":"2023-09-23T20:19:21.740873+0000","last_peered":"2023-09-23T20:19:21.740873+0000","last_clean":"2023-09-23T20:19:21.740873+0000","last_became_active":"2023-09-23T20:19:05.069124+0000","last_became_peered":"2023-09-23T20:19:05.069124+0000","last_unstale":"2023-09-23T20:19:21.740873+0000","last_undegraded":"2023-09-23T20:19:21.740873+0000","last_fullsized":"2023-09-23T20:19:21.740873+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":356,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723537+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:19:21.723537+0000","last_peered":"2023-09-23T20:19:21.723537+0000","last_clean":"2023-09-23T20:19:21.723537+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:19:21.723537+0000","last_undegraded":"2023-09-23T20:19:21.723537+0000","last_fullsized":"2023-09-23T20:19:21.723537+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":592,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.731819+0000","last_change":"2023-09-23T20:17:50.311535+0000","last_active":"2023-09-23T20:19:21.731819+0000","last_peered":"2023-09-23T20:19:21.731819+0000","last_clean":"2023-09-23T20:19:21.731819+0000","last_became_active":"2023-09-23T20:17:50.310876+0000","last_became_peered":"2023-09-23T20:17:50.310876+0000","last_unstale":"2023-09-23T20:19:21.731819+0000","last_undegraded":"2023-09-23T20:19:21.731819+0000","last_fullsized":"2023-09-23T20:19:21.731819+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.794641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":148,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.777016+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:19:21.777016+0000","last_peered":"2023-09-23T20:19:21.777016+0000","last_clean":"2023-09-23T20:19:21.777016+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:19:21.777016+0000","last_undegraded":"2023-09-23T20:19:21.777016+0000","last_fullsized":"2023-09-23T20:19:21.777016+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"514'7722","reported_seq":12784,"reported_epoch":514,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T20:19:31.917846+0000","last_change":"2023-09-23T20:19:18.121614+0000","last_active":"2023-09-23T20:19:31.917846+0000","last_peered":"2023-09-23T20:19:31.917846+0000","last_clean":"2023-09-23T20:18:02.160044+0000","last_became_active":"2023-09-23T20:18:39.117958+0000","last_became_peered":"2023-09-23T20:18:39.117958+0000","last_unstale":"2023-09-23T20:19:31.917846+0000","last_undegraded":"2023-09-23T20:18:39.008966+0000","last_fullsized":"2023-09-23T20:19:31.917846+0000","mapping_epoch":503,"log_start":"485'2626","ondisk_log_start":"485'2626","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:45.818039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243400704,"num_objects":3714,"num_object_clones":0,"num_object_copies":11142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3714,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7722,"num_write_kb":365952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2503,"num_bytes_recovered":164036608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,4],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":127},{"shards":"2,3,4","objects":3587}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":841,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.711376+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:19:21.711376+0000","last_peered":"2023-09-23T20:19:21.711376+0000","last_clean":"2023-09-23T20:19:21.711376+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:19:21.711376+0000","last_undegraded":"2023-09-23T20:19:21.711376+0000","last_fullsized":"2023-09-23T20:19:21.711376+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":220,"reported_epoch":513,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:19:21.723780+0000","last_change":"2023-09-23T20:18:39.010444+0000","last_active":"2023-09-23T20:19:21.723780+0000","last_peered":"2023-09-23T20:19:21.723780+0000","last_clean":"2023-09-23T20:19:21.723780+0000","last_became_active":"2023-09-23T20:18:39.009999+0000","last_became_peered":"2023-09-23T20:18:39.009999+0000","last_unstale":"2023-09-23T20:19:21.723780+0000","last_undegraded":"2023-09-23T20:19:21.723780+0000","last_fullsized":"2023-09-23T20:19:21.723780+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:53.838719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,3],"avail_no_missing":["7","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":39,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.731686+0000","last_change":"2023-09-23T20:17:47.284676+0000","last_active":"2023-09-23T20:19:21.731686+0000","last_peered":"2023-09-23T20:19:21.731686+0000","last_clean":"2023-09-23T20:19:21.731686+0000","last_became_active":"2023-09-23T20:17:47.277321+0000","last_became_peered":"2023-09-23T20:17:47.277321+0000","last_unstale":"2023-09-23T20:19:21.731686+0000","last_undegraded":"2023-09-23T20:19:21.731686+0000","last_fullsized":"2023-09-23T20:19:21.731686+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.082445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":314,"reported_epoch":513,"state":"active+undersized","last_fresh":"2023-09-23T20:19:21.727707+0000","last_change":"2023-09-23T20:18:38.011785+0000","last_active":"2023-09-23T20:19:21.727707+0000","last_peered":"2023-09-23T20:19:21.727707+0000","last_clean":"2023-09-23T20:18:35.261768+0000","last_became_active":"2023-09-23T20:18:38.011785+0000","last_became_peered":"2023-09-23T20:18:38.011785+0000","last_unstale":"2023-09-23T20:19:21.727707+0000","last_undegraded":"2023-09-23T20:19:21.727707+0000","last_fullsized":"2023-09-23T20:18:38.001274+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:00.398059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":905,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.722886+0000","last_change":"2023-09-23T20:19:05.079851+0000","last_active":"2023-09-23T20:19:21.722886+0000","last_peered":"2023-09-23T20:19:21.722886+0000","last_clean":"2023-09-23T20:19:21.722886+0000","last_became_active":"2023-09-23T20:19:05.079569+0000","last_became_peered":"2023-09-23T20:19:05.079569+0000","last_unstale":"2023-09-23T20:19:21.722886+0000","last_undegraded":"2023-09-23T20:19:21.722886+0000","last_fullsized":"2023-09-23T20:19:21.722886+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:52.668796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T19:52:52.668796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":292,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.736593+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:19:21.736593+0000","last_peered":"2023-09-23T20:19:21.736593+0000","last_clean":"2023-09-23T20:19:21.736593+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:19:21.736593+0000","last_undegraded":"2023-09-23T20:19:21.736593+0000","last_fullsized":"2023-09-23T20:19:21.736593+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":301,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.736777+0000","last_change":"2023-09-23T20:17:47.281804+0000","last_active":"2023-09-23T20:19:21.736777+0000","last_peered":"2023-09-23T20:19:21.736777+0000","last_clean":"2023-09-23T20:19:21.736777+0000","last_became_active":"2023-09-23T20:17:47.281384+0000","last_became_peered":"2023-09-23T20:17:47.281384+0000","last_unstale":"2023-09-23T20:19:21.736777+0000","last_undegraded":"2023-09-23T20:19:21.736777+0000","last_fullsized":"2023-09-23T20:19:21.736777+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:03.796273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":726,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.724021+0000","last_change":"2023-09-23T20:19:05.072097+0000","last_active":"2023-09-23T20:19:21.724021+0000","last_peered":"2023-09-23T20:19:21.724021+0000","last_clean":"2023-09-23T20:19:21.724021+0000","last_became_active":"2023-09-23T20:19:05.071787+0000","last_became_peered":"2023-09-23T20:19:05.071787+0000","last_unstale":"2023-09-23T20:19:21.724021+0000","last_undegraded":"2023-09-23T20:19:21.724021+0000","last_fullsized":"2023-09-23T20:19:21.724021+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":545,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728997+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:19:21.728997+0000","last_peered":"2023-09-23T20:19:21.728997+0000","last_clean":"2023-09-23T20:19:21.728997+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:19:21.728997+0000","last_undegraded":"2023-09-23T20:19:21.728997+0000","last_fullsized":"2023-09-23T20:19:21.728997+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":544,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.711537+0000","last_change":"2023-09-23T20:17:47.266215+0000","last_active":"2023-09-23T20:19:21.711537+0000","last_peered":"2023-09-23T20:19:21.711537+0000","last_clean":"2023-09-23T20:19:21.711537+0000","last_became_active":"2023-09-23T20:17:47.265890+0000","last_became_peered":"2023-09-23T20:17:47.265890+0000","last_unstale":"2023-09-23T20:19:21.711537+0000","last_undegraded":"2023-09-23T20:19:21.711537+0000","last_fullsized":"2023-09-23T20:19:21.711537+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:59.854975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":535,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728252+0000","last_change":"2023-09-23T20:17:47.288753+0000","last_active":"2023-09-23T20:19:21.728252+0000","last_peered":"2023-09-23T20:19:21.728252+0000","last_clean":"2023-09-23T20:19:21.728252+0000","last_became_active":"2023-09-23T20:17:47.270488+0000","last_became_peered":"2023-09-23T20:17:47.270488+0000","last_unstale":"2023-09-23T20:19:21.728252+0000","last_undegraded":"2023-09-23T20:19:21.728252+0000","last_fullsized":"2023-09-23T20:19:21.728252+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:35.837184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":510,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.714023+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:19:21.714023+0000","last_peered":"2023-09-23T20:19:21.714023+0000","last_clean":"2023-09-23T20:19:21.714023+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:19:21.714023+0000","last_undegraded":"2023-09-23T20:19:21.714023+0000","last_fullsized":"2023-09-23T20:19:21.714023+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":872,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.711101+0000","last_change":"2023-09-23T20:19:05.086585+0000","last_active":"2023-09-23T20:19:21.711101+0000","last_peered":"2023-09-23T20:19:21.711101+0000","last_clean":"2023-09-23T20:19:21.711101+0000","last_became_active":"2023-09-23T20:19:05.086254+0000","last_became_peered":"2023-09-23T20:19:05.086254+0000","last_unstale":"2023-09-23T20:19:21.711101+0000","last_undegraded":"2023-09-23T20:19:21.711101+0000","last_fullsized":"2023-09-23T20:19:21.711101+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":236,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.737063+0000","last_change":"2023-09-23T20:17:47.281777+0000","last_active":"2023-09-23T20:19:21.737063+0000","last_peered":"2023-09-23T20:19:21.737063+0000","last_clean":"2023-09-23T20:19:21.737063+0000","last_became_active":"2023-09-23T20:17:47.281381+0000","last_became_peered":"2023-09-23T20:17:47.281381+0000","last_unstale":"2023-09-23T20:19:21.737063+0000","last_undegraded":"2023-09-23T20:19:21.737063+0000","last_fullsized":"2023-09-23T20:19:21.737063+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.083242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":374,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.735794+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:19:21.735794+0000","last_peered":"2023-09-23T20:19:21.735794+0000","last_clean":"2023-09-23T20:19:21.735794+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:19:21.735794+0000","last_undegraded":"2023-09-23T20:19:21.735794+0000","last_fullsized":"2023-09-23T20:19:21.735794+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":803,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.723039+0000","last_change":"2023-09-23T20:19:05.067447+0000","last_active":"2023-09-23T20:19:21.723039+0000","last_peered":"2023-09-23T20:19:21.723039+0000","last_clean":"2023-09-23T20:19:21.723039+0000","last_became_active":"2023-09-23T20:19:05.067188+0000","last_became_peered":"2023-09-23T20:19:05.067188+0000","last_unstale":"2023-09-23T20:19:21.723039+0000","last_undegraded":"2023-09-23T20:19:21.723039+0000","last_fullsized":"2023-09-23T20:19:21.723039+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":359,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.728503+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:19:21.728503+0000","last_peered":"2023-09-23T20:19:21.728503+0000","last_clean":"2023-09-23T20:19:21.728503+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:19:21.728503+0000","last_undegraded":"2023-09-23T20:19:21.728503+0000","last_fullsized":"2023-09-23T20:19:21.728503+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":35,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:19:21.732041+0000","last_change":"2023-09-23T20:17:47.267673+0000","last_active":"2023-09-23T20:19:21.732041+0000","last_peered":"2023-09-23T20:19:21.732041+0000","last_clean":"2023-09-23T20:19:21.732041+0000","last_became_active":"2023-09-23T20:17:47.267216+0000","last_became_peered":"2023-09-23T20:17:47.267216+0000","last_unstale":"2023-09-23T20:19:21.732041+0000","last_undegraded":"2023-09-23T20:19:21.732041+0000","last_fullsized":"2023-09-23T20:19:21.732041+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:37.855890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3919446040,"num_objects":59807,"num_object_clones":0,"num_object_copies":179421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11466,"num_objects_misplaced":3830,"num_objects_unfound":0,"num_objects_dirty":59807,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124641,"num_write_kb":5902273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7633,"num_bytes_recovered":500236288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12477345792,"data_stored":12477333576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100425,"ondisk_log_size":100425,"up":46,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":10952,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":94,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":30,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":107340,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":8388703,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234461,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2005400,"kb_used_data":1435280,"kb_used_omap":0,"kb_used_meta":570112,"kb_avail":91739752,"statfs":{"total":95995035648,"available":93941506048,"internally_reserved":0,"allocated":1469726720,"data_stored":1468179980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":583794688},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:57 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":5.6399999999999997,"15min":3.218},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":10.135999999999999,"5min":149.25999999999999,"15min":149.25999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.8010000000000002,"5min":6.0259999999999998,"15min":3.4329999999999998},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.222},"max":{"1min":9.8409999999999993,"5min":149.11799999999999,"15min":149.11799999999999},"last":0.46899999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:18:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.9710000000000001,"15min":2.5760000000000001},"min":{"1min":0.26300000000000001,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":10.804,"5min":47.197000000000003,"15min":91.335999999999999},"last":61.113999999999997},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.097,"15min":2.8660000000000001},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":10.571999999999999,"5min":47.436999999999998,"15min":91.516999999999996},"last":61.082999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:19:04 2023","interfaces":[{"interface":"back","average":{"1min":11.622,"5min":5.1890000000000001,"15min":4.5700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":177.852,"5min":177.852,"15min":177.852},"last":0.58799999999999997},{"interface":"front","average":{"1min":11.391999999999999,"5min":5.3840000000000003,"15min":4.944},"min":{"1min":0.371,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":176.97900000000001,"5min":176.97900000000001,"15min":176.97900000000001},"last":0.498}]},{"osd":3,"last update":"Sat Sep 23 20:18:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":6.4320000000000004,"15min":4.6619999999999999},"min":{"1min":0.34300000000000003,"5min":0.33400000000000002,"15min":0.29299999999999998},"max":{"1min":11.154,"5min":185.17099999999999,"15min":185.17099999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":3.024,"5min":6.8890000000000002,"15min":4.7560000000000002},"min":{"1min":0.34300000000000003,"5min":0.32200000000000001,"15min":0.28999999999999998},"max":{"1min":13.887,"5min":184.99100000000001,"15min":184.99100000000001},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:19:22 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.133,"15min":0.76200000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":2.6680000000000001,"5min":7.9199999999999999,"15min":7.9199999999999999},"last":83.811999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.361,"15min":0.99199999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.4130000000000003,"5min":10.882999999999999,"15min":41.822000000000003},"last":84.620000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:19:22 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.742,"15min":1.1950000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4,"5min":16.327000000000002,"15min":24.876000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.6799999999999999,"15min":1.0780000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.077,"5min":16.716000000000001,"15min":21.623000000000001},"last":0.34999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.3779999999999999,"15min":1.0569999999999999},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.3420000000000001,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.163,"15min":0.82199999999999995},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1400000000000001,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.436}]}]},{"osd":6,"up_from":251,"seq":1078036791490,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273876,"kb_used_data":233868,"kb_used_omap":6,"kb_used_meta":39993,"kb_avail":93471276,"statfs":{"total":95995035648,"available":95714586624,"internally_reserved":0,"allocated":239480832,"data_stored":238014233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":40953471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":8.9100000000000001,"5min":3.8820000000000001,"15min":2.6890000000000001},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":53.780000000000001,"5min":74.787000000000006,"15min":126.64},"last":0.27000000000000002},{"interface":"front","average":{"1min":9.0090000000000003,"5min":4.0289999999999999,"15min":2.7570000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":53.768000000000001,"5min":74.751999999999995,"15min":126.658},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":4.3120000000000003,"15min":4.0830000000000002},"min":{"1min":0.26500000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":87.864999999999995,"5min":103.545,"15min":156.70699999999999},"last":5.4710000000000001},{"interface":"front","average":{"1min":5.1260000000000003,"5min":4.0279999999999996,"15min":4.0590000000000002},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.23899999999999999},"max":{"1min":87.772000000000006,"5min":100.018,"15min":156.68299999999999},"last":2.8690000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":6.891,"5min":2.4329999999999998,"15min":3.1179999999999999},"min":{"1min":0.315,"5min":0.254,"15min":0.23100000000000001},"max":{"1min":53.911999999999999,"5min":53.911999999999999,"15min":122.149},"last":0.47499999999999998},{"interface":"front","average":{"1min":7.3259999999999996,"5min":2.4129999999999998,"15min":3.0600000000000001},"min":{"1min":0.32400000000000001,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":54.125999999999998,"5min":54.125999999999998,"15min":122.23999999999999},"last":1.7789999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":5.181,"15min":3.8580000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":51.512999999999998,"5min":128.279,"15min":160.27699999999999},"last":4.8179999999999996},{"interface":"front","average":{"1min":3.4249999999999998,"5min":4.4969999999999999,"15min":3.8180000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":51.601999999999997,"5min":127.911,"15min":160.42699999999999},"last":2.9830000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.298,"15min":1.5780000000000001},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.248},"max":{"1min":2.641,"5min":34.534999999999997,"15min":145.29300000000001},"last":2.9140000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.077,"15min":1.538},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.3999999999999999,"5min":34.426000000000002,"15min":145.256},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":4.5880000000000001,"15min":2.972},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.25800000000000001},"max":{"1min":2.5329999999999999,"5min":218.58199999999999,"15min":218.58199999999999},"last":2.9500000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.343,"15min":2.8969999999999998},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.28499999999999998},"max":{"1min":2.4380000000000002,"5min":217.95599999999999,"15min":217.95599999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.1269999999999998,"15min":1.73},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.315},"max":{"1min":4.7380000000000004,"5min":137.63900000000001,"15min":137.63900000000001},"last":1},{"interface":"front","average":{"1min":0.751,"5min":2.8199999999999998,"15min":1.629},"min":{"1min":0.39700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":2.3620000000000001,"5min":136.738,"15min":136.738},"last":0.48799999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762659,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1287012,"kb_used_data":490012,"kb_used_omap":7,"kb_used_meta":796984,"kb_avail":92458140,"statfs":{"total":95995035648,"available":94677135360,"internally_reserved":0,"allocated":501772288,"data_stored":499339184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":816112442},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":4.5659999999999998,"15min":3.7650000000000001},"min":{"1min":0.19600000000000001,"5min":0.185,"15min":0.17100000000000001},"max":{"1min":8.9879999999999995,"5min":145.81299999999999,"15min":145.81299999999999},"last":0.20699999999999999},{"interface":"front","average":{"1min":3.7850000000000001,"5min":4.7999999999999998,"15min":4.1980000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19400000000000001},"max":{"1min":13.798999999999999,"5min":146.83799999999999,"15min":146.83799999999999},"last":4.0629999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:18:26 2023","interfaces":[{"interface":"back","average":{"1min":10.983000000000001,"5min":4.4349999999999996,"15min":4.359},"min":{"1min":0.23999999999999999,"5min":0.214,"15min":0.20000000000000001},"max":{"1min":152.62,"5min":152.62,"15min":152.62},"last":2.9449999999999998},{"interface":"front","average":{"1min":10.912000000000001,"5min":3.79,"15min":4.0750000000000002},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.20699999999999999},"max":{"1min":154.84299999999999,"5min":154.84299999999999,"15min":154.84299999999999},"last":3.875}]},{"osd":3,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":4.2199999999999998,"5min":3.194,"15min":3.161},"min":{"1min":0.24399999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":21.056999999999999,"5min":66.527000000000001,"15min":66.527000000000001},"last":2.6829999999999998},{"interface":"front","average":{"1min":1.5389999999999999,"5min":2.0499999999999998,"15min":2.7469999999999999},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":5.5599999999999996,"5min":61.854999999999997,"15min":61.854999999999997},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.5189999999999999,"15min":1.681},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.23599999999999999},"max":{"1min":6.5359999999999996,"5min":7.7999999999999998,"15min":18.087},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.3169999999999999,"5min":1.403,"15min":1.7470000000000001},"min":{"1min":0.34999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.5910000000000002,"5min":9.8320000000000007,"15min":20.847000000000001},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.776,"15min":2.1509999999999998},"min":{"1min":0.313,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.8059999999999992,"5min":9.782,"15min":83.510999999999996},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.1379999999999999,"5min":1.8009999999999999,"15min":2.2770000000000001},"min":{"1min":0.33300000000000002,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":5.5759999999999996,"5min":13.449,"15min":83.353999999999999},"last":2.73}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.2390000000000001,"15min":1.665},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":4.5830000000000002,"5min":9.2279999999999998,"15min":18.117999999999999},"last":5.6760000000000002},{"interface":"front","average":{"1min":0.75,"5min":1.325,"15min":1.843},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.0310000000000001,"5min":9.8659999999999997,"15min":20.734999999999999},"last":3.7330000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":3.4820000000000002,"15min":2.589},"min":{"1min":0.41699999999999998,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":5.6070000000000002,"5min":127.03100000000001,"15min":127.03100000000001},"last":2.758},{"interface":"front","average":{"1min":2.181,"5min":3.0579999999999998,"15min":2.266},"min":{"1min":0.40200000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.5819999999999999,"5min":125.095,"15min":125.095},"last":0.42099999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926197,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3682116,"kb_used_data":2690428,"kb_used_omap":8,"kb_used_meta":991543,"kb_avail":90063036,"statfs":{"total":95995035648,"available":92224548864,"internally_reserved":0,"allocated":2754998272,"data_stored":2752594380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1015340641},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:19:09 2023","interfaces":[{"interface":"back","average":{"1min":3.25,"5min":1.867,"15min":2.6440000000000001},"min":{"1min":0.23300000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":17.425000000000001,"5min":29.893000000000001,"15min":72.534000000000006},"last":4.0389999999999997},{"interface":"front","average":{"1min":5.891,"5min":2.3370000000000002,"15min":2.9609999999999999},"min":{"1min":0.26800000000000002,"5min":0.20799999999999999,"15min":0.187},"max":{"1min":21.949000000000002,"5min":23.436,"15min":72.667000000000002},"last":8.6709999999999994}]},{"osd":2,"last update":"Sat Sep 23 20:19:20 2023","interfaces":[{"interface":"back","average":{"1min":2.871,"5min":1.73,"15min":3.7639999999999998},"min":{"1min":0.23400000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":17.977,"5min":17.977,"15min":162.62899999999999},"last":4.4429999999999996},{"interface":"front","average":{"1min":4.8319999999999999,"5min":2.5550000000000002,"15min":4.0490000000000004},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.22600000000000001},"max":{"1min":16.521000000000001,"5min":18.433,"15min":162.52500000000001},"last":0.68600000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:19:09 2023","interfaces":[{"interface":"back","average":{"1min":7.4530000000000003,"5min":3.1259999999999999,"15min":3.6200000000000001},"min":{"1min":0.30199999999999999,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":24.219000000000001,"5min":44.405000000000001,"15min":126.95399999999999},"last":25.667000000000002},{"interface":"front","average":{"1min":7.0679999999999996,"5min":3.1600000000000001,"15min":3.7410000000000001},"min":{"1min":0.35599999999999998,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":22.123000000000001,"5min":43.909999999999997,"15min":127.294},"last":25.783999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.7609999999999999,"15min":2.0600000000000001},"min":{"1min":0.39000000000000001,"5min":0.316,"15min":0.27600000000000002},"max":{"1min":7.5819999999999999,"5min":8.7360000000000007,"15min":148.20599999999999},"last":0.436},{"interface":"front","average":{"1min":2.3199999999999998,"5min":2.3319999999999999,"15min":2.3290000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0150000000000006,"5min":105.282,"15min":148.11000000000001},"last":8.7119999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":2.1699999999999999,"15min":1.698},"min":{"1min":0.36499999999999999,"5min":0.35999999999999999,"15min":0.26900000000000002},"max":{"1min":7.6180000000000003,"5min":52.061,"15min":52.061},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.456,"5min":2.351,"15min":1.8660000000000001},"min":{"1min":0.37,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":5.3799999999999999,"5min":51.741,"15min":51.741},"last":0.46200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6339999999999999,"15min":1.6419999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":7.6500000000000004,"5min":7.6500000000000004,"15min":15.744999999999999},"last":4.1239999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.234,"15min":1.6020000000000001},"min":{"1min":0.44,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":9.8219999999999992,"15min":13.766999999999999},"last":0.82199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.9119999999999999,"15min":1.9910000000000001},"min":{"1min":0.39800000000000002,"5min":0.39200000000000002,"15min":0.377},"max":{"1min":8.0269999999999992,"5min":99.486000000000004,"15min":99.486000000000004},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.069,"5min":3.3740000000000001,"15min":2.3210000000000002},"min":{"1min":0.42599999999999999,"5min":0.40200000000000002,"15min":0.32300000000000001},"max":{"1min":8.048,"5min":99.668999999999997,"15min":99.668999999999997},"last":0.497}]}]},{"osd":2,"up_from":104,"seq":446676599135,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2874168,"kb_used_data":1930992,"kb_used_omap":12,"kb_used_meta":943155,"kb_avail":90870984,"statfs":{"total":95995035648,"available":93051887616,"internally_reserved":0,"allocated":1977335808,"data_stored":1974888940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":965791742},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:19:28 2023","interfaces":[{"interface":"back","average":{"1min":15.148,"5min":7.6550000000000002,"15min":4.3819999999999997},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":274.185,"5min":274.185,"15min":274.185},"last":4.1020000000000003},{"interface":"front","average":{"1min":2.3919999999999999,"5min":5.6580000000000004,"15min":3.9129999999999998},"min":{"1min":0.214,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":5.8280000000000003,"5min":106.477,"15min":106.477},"last":3.2930000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:19:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6629999999999998,"5min":3.9220000000000002,"15min":3.4079999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.19600000000000001},"max":{"1min":18.527000000000001,"5min":106.357,"15min":106.357},"last":4.258},{"interface":"front","average":{"1min":2.7530000000000001,"5min":3.8279999999999998,"15min":3.028},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":9.4469999999999992,"5min":104.905,"15min":104.905},"last":9.0730000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:18:45 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":5.5549999999999997,"15min":4.6159999999999997},"min":{"1min":0.247,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":14.129,"5min":132.82499999999999,"15min":132.82499999999999},"last":9.5640000000000001},{"interface":"front","average":{"1min":3.153,"5min":5.6369999999999996,"15min":3.9079999999999999},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":22.847999999999999,"5min":130.97300000000001,"15min":130.97300000000001},"last":9.1270000000000007}]},{"osd":4,"last update":"Sat Sep 23 20:19:28 2023","interfaces":[{"interface":"back","average":{"1min":4.375,"5min":1.964,"15min":2.0409999999999999},"min":{"1min":0.30399999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":43.040999999999997,"5min":43.040999999999997,"15min":75.608000000000004},"last":3.911},{"interface":"front","average":{"1min":5.7999999999999998,"5min":2.5489999999999999,"15min":2.4300000000000002},"min":{"1min":0.29399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":48.951000000000001,"5min":48.951000000000001,"15min":76.161000000000001},"last":4.3209999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:19:28 2023","interfaces":[{"interface":"back","average":{"1min":4.5270000000000001,"5min":2.2869999999999999,"15min":2.5289999999999999},"min":{"1min":0.32900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":43.716000000000001,"5min":43.716000000000001,"15min":135.202},"last":3.859},{"interface":"front","average":{"1min":4.6079999999999997,"5min":2.4889999999999999,"15min":2.4239999999999999},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":43.643999999999998,"5min":43.643999999999998,"15min":63.792000000000002},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.329,"15min":1.542},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5510000000000002,"5min":11.843,"15min":14.664},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.425,"15min":1.744},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.077,"5min":12.025,"15min":14.279999999999999},"last":4.9660000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:19:28 2023","interfaces":[{"interface":"back","average":{"1min":3.1429999999999998,"5min":2.3599999999999999,"15min":2.028},"min":{"1min":0.317,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":18.495999999999999,"5min":18.495999999999999,"15min":18.495999999999999},"last":4.2939999999999996},{"interface":"front","average":{"1min":2.0339999999999998,"5min":1.8120000000000001,"15min":1.6599999999999999},"min":{"1min":0.40799999999999997,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.7519999999999998,"5min":8.9730000000000008,"15min":14.698},"last":3.9390000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435609,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2695072,"kb_used_data":1690072,"kb_used_omap":4,"kb_used_meta":1004923,"kb_avail":91050080,"statfs":{"total":95995035648,"available":93235281920,"internally_reserved":0,"allocated":1730633728,"data_stored":1727907893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1029041261},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":4.3559999999999999,"15min":3.7530000000000001},"min":{"1min":0.20399999999999999,"5min":0.189,"15min":0.186},"max":{"1min":9.1530000000000005,"5min":132.47800000000001,"15min":132.47800000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":4.3170000000000002,"5min":5.1980000000000004,"15min":4.0270000000000001},"min":{"1min":0.28399999999999997,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":17.539999999999999,"5min":133.22499999999999,"15min":133.22499999999999},"last":1.0549999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":4.0940000000000003,"5min":4.4820000000000002,"15min":4.5499999999999998},"min":{"1min":0.22600000000000001,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":17.895,"5min":183.87,"15min":183.87},"last":0.27300000000000002},{"interface":"front","average":{"1min":3.2989999999999999,"5min":4.4089999999999998,"15min":4.9459999999999997},"min":{"1min":0.28100000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":12.664999999999999,"5min":185.84899999999999,"15min":185.84899999999999},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":10.364000000000001,"5min":6.1399999999999997,"15min":5.2450000000000001},"min":{"1min":0.33700000000000002,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":115.47799999999999,"5min":128.369,"15min":163.83500000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":9.3520000000000003,"5min":5.6580000000000004,"15min":4.9640000000000004},"min":{"1min":0.28499999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":114.79600000000001,"5min":126.497,"15min":163.92099999999999},"last":1.821}]},{"osd":4,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.387,"15min":2.5910000000000002},"min":{"1min":0.34899999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":9.1180000000000003,"5min":9.1180000000000003,"15min":109.66800000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":3.0350000000000001,"5min":2.1749999999999998,"15min":3.0470000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":17.428000000000001,"5min":17.428000000000001,"15min":110.295},"last":1.1040000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":4.3470000000000004,"15min":3.5830000000000002},"min":{"1min":0.35499999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":17.370999999999999,"5min":200.53200000000001,"15min":200.53200000000001},"last":1.1359999999999999},{"interface":"front","average":{"1min":3.5760000000000001,"5min":4.4690000000000003,"15min":3.5619999999999998},"min":{"1min":0.374,"5min":0.34100000000000003,"15min":0.26900000000000002},"max":{"1min":18.02,"5min":200.846,"15min":200.846},"last":0.30399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.593,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":1.5329999999999999,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.3280000000000001,"15min":2.286},"min":{"1min":0.42699999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.2040000000000002,"5min":8.2629999999999999,"15min":82.233000000000004},"last":4.1509999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":3.379,"15min":3.3439999999999999},"min":{"1min":0.434,"5min":0.376,"15min":0.29599999999999999},"max":{"1min":17.986000000000001,"5min":139.465,"15min":139.465},"last":0.32800000000000001},{"interface":"front","average":{"1min":2.5129999999999999,"5min":3.1949999999999998,"15min":3.3919999999999999},"min":{"1min":0.38300000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":9.2289999999999992,"5min":141.56200000000001,"15min":141.56200000000001},"last":0.48499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019371,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2889752,"kb_used_data":2351248,"kb_used_omap":0,"kb_used_meta":538496,"kb_avail":90855400,"statfs":{"total":95995035648,"available":93035929600,"internally_reserved":0,"allocated":2407677952,"data_stored":2406131224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":551419904},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":14.343,"5min":5.6890000000000001,"15min":3.0819999999999999},"min":{"1min":0.25,"5min":0.224,"15min":0.22},"max":{"1min":232.89500000000001,"5min":232.89500000000001,"15min":232.89500000000001},"last":11.427},{"interface":"front","average":{"1min":14.416,"5min":5.71,"15min":3.0609999999999999},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":232.851,"5min":232.851,"15min":232.851},"last":11.397}]},{"osd":1,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":3.1480000000000001,"15min":5.226},"min":{"1min":0.26700000000000002,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":3.1579999999999999,"5min":114.05200000000001,"15min":161.613},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.1519999999999999,"5min":3.1499999999999999,"15min":5.2380000000000004},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.217},"max":{"1min":3.109,"5min":114.02800000000001,"15min":162.017},"last":0.64600000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:19:22 2023","interfaces":[{"interface":"back","average":{"1min":12.397,"5min":6.9020000000000001,"15min":5.859},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":120.34699999999999,"5min":120.34699999999999,"15min":182.96700000000001},"last":5.2720000000000002},{"interface":"front","average":{"1min":12.282999999999999,"5min":6.899,"15min":5.79},"min":{"1min":0.28899999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":120.267,"5min":120.267,"15min":183.244},"last":5.4329999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":3.9790000000000001,"15min":5.0960000000000001},"min":{"1min":0.34699999999999998,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":13.426,"5min":115.937,"15min":184.37100000000001},"last":5.032},{"interface":"front","average":{"1min":1.9039999999999999,"5min":3.9350000000000001,"15min":5.1559999999999997},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":10.486000000000001,"5min":115.902,"15min":184.22300000000001},"last":4.9969999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":2.2509999999999999,"15min":1.6379999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":2.9670000000000001,"5min":114.788,"15min":176.81899999999999},"last":0.505},{"interface":"front","average":{"1min":0.753,"5min":2.363,"15min":1.698},"min":{"1min":0.29699999999999999,"5min":0.29499999999999998,"15min":0.26600000000000001},"max":{"1min":4.0460000000000003,"5min":113.822,"15min":176.172},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:17:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.0390000000000001,"15min":1.026},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":2.5110000000000001,"5min":114.232,"15min":114.232},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.3540000000000001,"15min":1.2130000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.247},"max":{"1min":4.085,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:19:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":2.4129999999999998,"15min":1.96},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":4.9050000000000002,"5min":113.861,"15min":113.861},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.4279999999999999,"15min":2.0659999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.222},"max":{"1min":4.4500000000000002,"5min":114.32299999999999,"15min":114.32299999999999},"last":0.47799999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757734,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2125760,"kb_used_data":1481464,"kb_used_omap":0,"kb_used_meta":644288,"kb_avail":91619392,"statfs":{"total":95995035648,"available":93818257408,"internally_reserved":0,"allocated":1517019136,"data_stored":1515484979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":659750912},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:18:35 2023","interfaces":[{"interface":"back","average":{"1min":11.675000000000001,"5min":6.9509999999999996,"15min":3.649},"min":{"1min":0.29699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":166.57900000000001,"5min":166.57900000000001,"15min":166.57900000000001},"last":99.947000000000003},{"interface":"front","average":{"1min":10.929,"5min":6.3719999999999999,"15min":3.6840000000000002},"min":{"1min":0.30599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":165.67500000000001,"5min":165.67500000000001,"15min":165.67500000000001},"last":99.891999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":3.8540000000000001,"15min":4.3979999999999997},"min":{"1min":0.25700000000000001,"5min":0.22,"15min":0.22},"max":{"1min":3.8319999999999999,"5min":150.16999999999999,"15min":171.47300000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.5459999999999998,"5min":3.7040000000000002,"15min":4.4640000000000004},"min":{"1min":0.36499999999999999,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":10.75,"5min":150.12700000000001,"15min":171.399},"last":1.0960000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:19:16 2023","interfaces":[{"interface":"back","average":{"1min":2.8530000000000002,"5min":3.6789999999999998,"15min":4.2359999999999998},"min":{"1min":0.29799999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.7590000000000003,"5min":161.852,"15min":240.57300000000001},"last":96.278000000000006},{"interface":"front","average":{"1min":2.8860000000000001,"5min":3.7370000000000001,"15min":4.3129999999999997},"min":{"1min":0.371,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.7840000000000007,"5min":161.95599999999999,"15min":240.464},"last":96.302999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:18:56 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":9.6489999999999991,"15min":6.7640000000000002},"min":{"1min":0.35899999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":8.8550000000000004,"5min":196.97800000000001,"15min":201.50899999999999},"last":0.432},{"interface":"front","average":{"1min":2.117,"5min":9.8149999999999995,"15min":6.524},"min":{"1min":0.313,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":8.0090000000000003,"5min":196.875,"15min":201.58099999999999},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:18:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1579999999999999,"15min":0.77800000000000002},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.24099999999999999},"max":{"1min":4.2350000000000003,"5min":8,"15min":9.7240000000000002},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.72899999999999998,"15min":0.60199999999999998},"min":{"1min":0.33700000000000002,"5min":0.27100000000000002,"15min":0.255},"max":{"1min":1.5029999999999999,"5min":3.371,"15min":8.7360000000000007},"last":0.36399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2490000000000001,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":4.04,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.73299999999999998,"15min":0.60999999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":1.609,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":1.492}]},{"osd":7,"last update":"Sat Sep 23 20:19:15 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.5900000000000001,"15min":0.95599999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.9489999999999998,"5min":11.725,"15min":11.725},"last":0.314},{"interface":"front","average":{"1min":0.69099999999999995,"5min":1.4419999999999999,"15min":0.93000000000000005},"min":{"1min":0.313,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.5299999999999998,"5min":11.041,"15min":11.041},"last":0.34100000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2734559232,"data_stored":2734555160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":489684992,"data_stored":489684992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1951662080,"data_stored":1951662080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1716457472,"data_stored":1716453400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2393903104,"data_stored":2393899032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1502019584,"data_stored":1502019584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":237764608,"data_stored":237764608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1451294720,"data_stored":1451294720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:19:37.325 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:37.330 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:19:37.331 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-09-23T20:19:37.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:37.423 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:37.626 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:37.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:37.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:37.736 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:37.798 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:37.837 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:37.929 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:37.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:38.039 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:38.141 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:38.268 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:38.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:38.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:38.390 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:38.429 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:38.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:38 smithi135 ceph-mon[213871]: pgmap v1447: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5996/171519 objects degraded (3.496%); 8831/171519 objects misplaced (5.149%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:19:38.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11467/177354 objects degraded (6.466%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:38.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:38 smithi135 ceph-mon[221113]: pgmap v1447: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5996/171519 objects degraded (3.496%); 8831/171519 objects misplaced (5.149%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:19:38.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11467/177354 objects degraded (6.466%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:38.494 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:38.496 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:38.503 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:38 smithi144 ceph-mon[194833]: pgmap v1447: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5996/171519 objects degraded (3.496%); 8831/171519 objects misplaced (5.149%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:19:38.504 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11467/177354 objects degraded (6.466%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:38.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:38.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:38.605 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:38.659 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:38.749 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:38.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:38.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:38.859 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:38.894 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:38.959 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:38.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:39.064 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:39.129 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:39.187 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:39.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:39.228 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:19:39.254 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:19:39.291 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:39.323 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:39.367 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:39.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:39.471 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:39.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3722066058' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:19:39.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:39 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:19:39.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3722066058' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:19:39.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:39 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:19:39.531 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:39.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3722066058' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:19:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:39 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:19:40.114 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:40.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:40.231 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:40.275 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:40.342 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:40.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:40.445 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[221113]: pgmap v1448: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5996/168501 objects degraded (3.558%); 8642/168501 objects misplaced (5.129%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:19:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[221113]: failed to encode map e515 with expected crc 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[221113]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[221113]: failed to encode map e515 with expected crc 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[213871]: pgmap v1448: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5996/168501 objects degraded (3.558%); 8642/168501 objects misplaced (5.129%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[213871]: failed to encode map e515 with expected crc 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[213871]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T20:19:40.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:40 smithi135 ceph-mon[213871]: failed to encode map e515 with expected crc 2023-09-23T20:19:40.537 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:40 smithi144 ceph-mon[194833]: pgmap v1448: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5996/168501 objects degraded (3.558%); 8642/168501 objects misplaced (5.129%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:19:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:40 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:19:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:40 smithi144 ceph-mon[194833]: failed to encode map e515 with expected crc 2023-09-23T20:19:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:40 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:19:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:40 smithi144 ceph-mon[194833]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T20:19:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:40 smithi144 ceph-mon[194833]: failed to encode map e515 with expected crc 2023-09-23T20:19:41.357 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:41.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:41.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:41.463 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:41.495 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:41 smithi144 ceph-mon[194833]: 1.7 scrub starts 2023-09-23T20:19:41.495 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:41 smithi144 ceph-mon[194833]: 1.7 scrub ok 2023-09-23T20:19:41.496 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:41.505 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:41 smithi135 ceph-mon[213871]: 1.7 scrub starts 2023-09-23T20:19:41.505 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:41 smithi135 ceph-mon[213871]: 1.7 scrub ok 2023-09-23T20:19:41.505 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:41 smithi135 ceph-mon[221113]: 1.7 scrub starts 2023-09-23T20:19:41.506 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:41 smithi135 ceph-mon[221113]: 1.7 scrub ok 2023-09-23T20:19:41.546 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:41.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:41.652 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:41.680 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:41.740 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:41.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:41.845 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:41.939 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:42.036 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:42.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:42.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:42.139 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:42.166 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:42.208 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:42.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:42.313 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:42.402 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:42 smithi144 ceph-mon[194833]: 1.c scrub starts 2023-09-23T20:19:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:42 smithi144 ceph-mon[194833]: pgmap v1450: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 6037/166161 objects degraded (3.633%); 8429/166161 objects misplaced (5.073%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:19:42.709 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:42.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:42.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:42.711 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:42 smithi135 ceph-mon[213871]: 1.c scrub starts 2023-09-23T20:19:42.711 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:42 smithi135 ceph-mon[213871]: pgmap v1450: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 6037/166161 objects degraded (3.633%); 8429/166161 objects misplaced (5.073%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:19:42.711 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:42 smithi135 ceph-mon[221113]: 1.c scrub starts 2023-09-23T20:19:42.711 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:42 smithi135 ceph-mon[221113]: pgmap v1450: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 6037/166161 objects degraded (3.633%); 8429/166161 objects misplaced (5.073%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:19:42.771 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:42.812 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:42.901 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:19:42.901 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:19:42.956 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:43.380 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:43.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:43.495 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:43.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:43 smithi144 ceph-mon[194833]: 1.17 scrub starts 2023-09-23T20:19:43.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:43 smithi144 ceph-mon[194833]: 1.17 scrub ok 2023-09-23T20:19:43.529 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:43 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6037/166161 objects degraded (3.633%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:43.533 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:43.545 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:19:43.601 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:43.604 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:43 smithi135 ceph-mon[221113]: 1.17 scrub starts 2023-09-23T20:19:43.605 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:43 smithi135 ceph-mon[221113]: 1.17 scrub ok 2023-09-23T20:19:43.605 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:43 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6037/166161 objects degraded (3.633%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:43.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:43.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:43.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:43 smithi135 ceph-mon[213871]: 1.17 scrub starts 2023-09-23T20:19:43.610 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:43 smithi135 ceph-mon[213871]: 1.17 scrub ok 2023-09-23T20:19:43.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:43 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6037/166161 objects degraded (3.633%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:43.706 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:43.731 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:43.975 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:19:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:19:43] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T20:19:44.503 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:44 smithi135 ceph-mon[213871]: 1.a scrub starts 2023-09-23T20:19:44.504 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:44 smithi135 ceph-mon[213871]: pgmap v1451: 119 pgs: 1 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 6037/160692 objects degraded (3.757%); 8208/160692 objects misplaced (5.108%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:19:44.504 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:44 smithi135 ceph-mon[221113]: 1.a scrub starts 2023-09-23T20:19:44.504 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:44 smithi135 ceph-mon[221113]: pgmap v1451: 119 pgs: 1 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 6037/160692 objects degraded (3.757%); 8208/160692 objects misplaced (5.108%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:19:44.513 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:44.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:44.629 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:44 smithi144 ceph-mon[194833]: 1.a scrub starts 2023-09-23T20:19:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:44 smithi144 ceph-mon[194833]: pgmap v1451: 119 pgs: 1 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 6037/160692 objects degraded (3.757%); 8208/160692 objects misplaced (5.108%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:19:44.720 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:45.542 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:19:45.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:45 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:19:45.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:45 smithi135 ceph-mon[221113]: 1.8 scrub starts 2023-09-23T20:19:45.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:45 smithi135 ceph-mon[221113]: 1.8 scrub ok 2023-09-23T20:19:45.543 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:45 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:19:45.543 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:45 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:19:45.544 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:45 smithi135 ceph-mon[213871]: 1.8 scrub starts 2023-09-23T20:19:45.544 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:45 smithi135 ceph-mon[213871]: 1.8 scrub ok 2023-09-23T20:19:45.544 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:45 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:19:45.606 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:45.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:45.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:45 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:19:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:45 smithi144 ceph-mon[194833]: 1.8 scrub starts 2023-09-23T20:19:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:45 smithi144 ceph-mon[194833]: 1.8 scrub ok 2023-09-23T20:19:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:45 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:19:45.718 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:45.752 INFO:teuthology.orchestra.run.smithi135.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-09-23T20:19:45.852 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:46.058 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:46.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:46.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:46.165 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:46.240 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:46.629 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:46.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:46 smithi144 ceph-mon[194833]: pgmap v1452: 119 pgs: 1 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6037/158262 objects degraded (3.815%); 8002/158262 objects misplaced (5.056%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:19:46.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/18054685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:19:46.636 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:46 smithi135 ceph-mon[213871]: pgmap v1452: 119 pgs: 1 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6037/158262 objects degraded (3.815%); 8002/158262 objects misplaced (5.056%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:19:46.636 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/18054685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:19:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:46.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:46.640 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:46 smithi135 ceph-mon[221113]: pgmap v1452: 119 pgs: 1 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6037/158262 objects degraded (3.815%); 8002/158262 objects misplaced (5.056%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:19:46.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/18054685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:19:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:19:46.738 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:46.788 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:46.841 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:47.062 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:47.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:47.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:47.166 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:47.265 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:47.265 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:47.285 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:19:47.346 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:19:47.395 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:47.396 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:19:47.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:47.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:47.451 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:19:47.473 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:19:47.508 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:19:47.512 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:47.556 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:19:47.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:19:47.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:19:47.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:19:47.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:19:47.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:19:47.556 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:19:47.557 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:19:47.557 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:19:47.575 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:47 smithi135 ceph-mon[221113]: 1.15 scrub starts 2023-09-23T20:19:47.575 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:47 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4031587000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:47.575 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:47 smithi135 ceph-mon[213871]: 1.15 scrub starts 2023-09-23T20:19:47.576 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:47 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4031587000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:47.622 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:47 smithi144 ceph-mon[194833]: 1.15 scrub starts 2023-09-23T20:19:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:47 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4031587000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:19:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[221113]: 1.9 scrub starts 2023-09-23T20:19:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[221113]: 1.9 scrub ok 2023-09-23T20:19:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[221113]: pgmap v1453: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 6061/152376 objects degraded (3.978%); 7633/152376 objects misplaced (5.009%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T20:19:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[221113]: 1.1c scrub starts 2023-09-23T20:19:48.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6061/152376 objects degraded (3.978%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:48.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[213871]: 1.9 scrub starts 2023-09-23T20:19:48.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[213871]: 1.9 scrub ok 2023-09-23T20:19:48.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[213871]: pgmap v1453: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 6061/152376 objects degraded (3.978%); 7633/152376 objects misplaced (5.009%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T20:19:48.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[213871]: 1.1c scrub starts 2023-09-23T20:19:48.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:48 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6061/152376 objects degraded (3.978%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:48.522 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:48.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:48.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:48.627 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:48 smithi144 ceph-mon[194833]: 1.9 scrub starts 2023-09-23T20:19:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:48 smithi144 ceph-mon[194833]: 1.9 scrub ok 2023-09-23T20:19:48.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:48 smithi144 ceph-mon[194833]: pgmap v1453: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 6061/152376 objects degraded (3.978%); 7633/152376 objects misplaced (5.009%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T20:19:48.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:48 smithi144 ceph-mon[194833]: 1.1c scrub starts 2023-09-23T20:19:48.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:48 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6061/152376 objects degraded (3.978%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:48.654 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:49 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:19:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:49 smithi144 ceph-mon[194833]: 1.e scrub starts 2023-09-23T20:19:49.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:49 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:19:49.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:49 smithi135 ceph-mon[213871]: 1.e scrub starts 2023-09-23T20:19:49.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:49 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:19:49.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:49 smithi135 ceph-mon[221113]: 1.e scrub starts 2023-09-23T20:19:50.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:50.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:50.435 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:50 smithi135 ceph-mon[213871]: 1.15 scrub starts 2023-09-23T20:19:50.435 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:50 smithi135 ceph-mon[213871]: pgmap v1454: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 6061/149301 objects degraded (4.060%); 7423/149301 objects misplaced (4.972%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T20:19:50.435 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:50 smithi135 ceph-mon[213871]: 1.1e scrub starts 2023-09-23T20:19:50.435 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:50 smithi135 ceph-mon[221113]: 1.15 scrub starts 2023-09-23T20:19:50.436 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:50 smithi135 ceph-mon[221113]: pgmap v1454: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 6061/149301 objects degraded (4.060%); 7423/149301 objects misplaced (4.972%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T20:19:50.436 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:50 smithi135 ceph-mon[221113]: 1.1e scrub starts 2023-09-23T20:19:50.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:50.537 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:50.569 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:50 smithi144 ceph-mon[194833]: 1.15 scrub starts 2023-09-23T20:19:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:50 smithi144 ceph-mon[194833]: pgmap v1454: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 6061/149301 objects degraded (4.060%); 7423/149301 objects misplaced (4.972%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T20:19:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:50 smithi144 ceph-mon[194833]: 1.1e scrub starts 2023-09-23T20:19:50.637 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:50.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:50.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:50.746 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:19:50.763 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:19:50.781 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:50.834 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:50.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:50.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:50.944 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:50.984 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:51.036 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:51.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:51.142 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:19:51.179 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:51.240 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:19:51.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:51.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:19:51.346 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:51.360 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:51 smithi135 ceph-mon[213871]: 1.c scrub starts 2023-09-23T20:19:51.360 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:51 smithi135 ceph-mon[213871]: 1.1d scrub starts 2023-09-23T20:19:51.360 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:51 smithi135 ceph-mon[213871]: pgmap v1455: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 6106/146832 objects degraded (4.158%); 7211/146832 objects misplaced (4.911%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:51.432 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:51 smithi144 ceph-mon[194833]: 1.c scrub starts 2023-09-23T20:19:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:51 smithi144 ceph-mon[194833]: 1.1d scrub starts 2023-09-23T20:19:51.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:51 smithi144 ceph-mon[194833]: pgmap v1455: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 6106/146832 objects degraded (4.158%); 7211/146832 objects misplaced (4.911%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:51.637 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:51.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:51.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:51.652 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:51 smithi135 ceph-mon[221113]: 1.c scrub starts 2023-09-23T20:19:51.652 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:51 smithi135 ceph-mon[221113]: 1.1d scrub starts 2023-09-23T20:19:51.652 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:51 smithi135 ceph-mon[221113]: pgmap v1455: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 6106/146832 objects degraded (4.158%); 7211/146832 objects misplaced (4.911%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:19:51.705 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:19:51.705 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:19:51.705 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:19:51.705 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-09-23T20:19:51.705 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-09-23T20:19:51.747 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:19:51.747 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:19:51.747 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:19:51.747 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:19:51.747 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:19:51.749 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:51.782 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:51.839 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:51.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:51.945 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:52.029 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:52.143 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:52.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:52.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:52.253 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:52.280 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:52.417 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:19:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:52 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:19:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/45892743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:52.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:52 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:19:52.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/45892743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:52.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:52.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:52.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:52 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:19:52.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/45892743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:52.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:19:52.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:52.764 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:19:52.847 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:52.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:52.954 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:19:52.986 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:53.046 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:19:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:53.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:19:53.154 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:53.231 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:53.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[221113]: 1.1c scrub starts 2023-09-23T20:19:53.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:19:53.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[221113]: failed to encode map e516 with expected crc 2023-09-23T20:19:53.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:53.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[221113]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T20:19:53.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[221113]: failed to encode map e516 with expected crc 2023-09-23T20:19:53.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6106/146832 objects degraded (4.158%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:53.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[221113]: pgmap v1457: 119 pgs: 4 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 6106/141273 objects degraded (4.322%); 7003/141273 objects misplaced (4.957%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:19:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:53 smithi144 ceph-mon[194833]: 1.1c scrub starts 2023-09-23T20:19:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:53 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:19:53.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:53 smithi144 ceph-mon[194833]: failed to encode map e516 with expected crc 2023-09-23T20:19:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:53 smithi144 ceph-mon[194833]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T20:19:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:53 smithi144 ceph-mon[194833]: failed to encode map e516 with expected crc 2023-09-23T20:19:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6106/146832 objects degraded (4.158%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:53 smithi144 ceph-mon[194833]: pgmap v1457: 119 pgs: 4 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 6106/141273 objects degraded (4.322%); 7003/141273 objects misplaced (4.957%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:19:53.633 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[213871]: 1.1c scrub starts 2023-09-23T20:19:53.633 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:19:53.633 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[213871]: failed to encode map e516 with expected crc 2023-09-23T20:19:53.633 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:19:53.634 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[213871]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T20:19:53.634 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[213871]: failed to encode map e516 with expected crc 2023-09-23T20:19:53.634 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6106/146832 objects degraded (4.158%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:53.634 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:53 smithi135 ceph-mon[213871]: pgmap v1457: 119 pgs: 4 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 6106/141273 objects degraded (4.322%); 7003/141273 objects misplaced (4.957%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:19:53.661 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:19:53.869 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:53.883 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:19:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:19:53] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-09-23T20:19:53.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:53.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:53.985 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:54.093 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:54.191 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:54.212 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:54.237 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:54.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:54.364 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:54.394 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:54.458 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:54.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:54.542 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:54 smithi135 ceph-mon[221113]: 1.5 deep-scrub starts 2023-09-23T20:19:54.542 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:54 smithi135 ceph-mon[213871]: 1.5 deep-scrub starts 2023-09-23T20:19:54.565 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:54.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:54 smithi144 ceph-mon[194833]: 1.5 deep-scrub starts 2023-09-23T20:19:54.641 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:54.799 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:54.839 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:54.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:54.966 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:55.052 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:55.491 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:55.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:55.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:55.602 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:19:55.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:55 smithi135 ceph-mon[221113]: 1.15 deep-scrub starts 2023-09-23T20:19:55.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:55 smithi135 ceph-mon[221113]: pgmap v1458: 119 pgs: 4 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6106/138759 objects degraded (4.400%); 6799/138759 objects misplaced (4.900%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:19:55.687 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:55.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:55 smithi135 ceph-mon[213871]: 1.15 deep-scrub starts 2023-09-23T20:19:55.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:55 smithi135 ceph-mon[213871]: pgmap v1458: 119 pgs: 4 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6106/138759 objects degraded (4.400%); 6799/138759 objects misplaced (4.900%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:19:55.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:55 smithi144 ceph-mon[194833]: 1.15 deep-scrub starts 2023-09-23T20:19:55.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:55 smithi144 ceph-mon[194833]: pgmap v1458: 119 pgs: 4 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6106/138759 objects degraded (4.400%); 6799/138759 objects misplaced (4.900%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:19:56.010 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:19:56.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:56 smithi135 ceph-mon[221113]: 1.18 scrub starts 2023-09-23T20:19:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:56 smithi135 ceph-mon[221113]: 1.18 scrub ok 2023-09-23T20:19:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:56 smithi135 ceph-mon[221113]: 1.c scrub starts 2023-09-23T20:19:56.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:56 smithi135 ceph-mon[213871]: 1.18 scrub starts 2023-09-23T20:19:56.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:56 smithi135 ceph-mon[213871]: 1.18 scrub ok 2023-09-23T20:19:56.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:56 smithi135 ceph-mon[213871]: 1.c scrub starts 2023-09-23T20:19:56.754 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:19:56.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:56.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:19:56.861 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:56.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:56 smithi144 ceph-mon[194833]: 1.18 scrub starts 2023-09-23T20:19:56.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:56 smithi144 ceph-mon[194833]: 1.18 scrub ok 2023-09-23T20:19:56.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:56 smithi144 ceph-mon[194833]: 1.c scrub starts 2023-09-23T20:19:56.931 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:57.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:57 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:19:57.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:57 smithi135 ceph-mon[221113]: 1.19 deep-scrub starts 2023-09-23T20:19:57.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:57 smithi135 ceph-mon[221113]: 1.19 deep-scrub ok 2023-09-23T20:19:57.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:57 smithi135 ceph-mon[221113]: pgmap v1459: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 1 active+undersized+degraded, 107 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6132/132798 objects degraded (4.618%); 6391/132798 objects misplaced (4.813%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:19:57.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:57 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:19:57.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:57 smithi135 ceph-mon[213871]: 1.19 deep-scrub starts 2023-09-23T20:19:57.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:57 smithi135 ceph-mon[213871]: 1.19 deep-scrub ok 2023-09-23T20:19:57.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:57 smithi135 ceph-mon[213871]: pgmap v1459: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 1 active+undersized+degraded, 107 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6132/132798 objects degraded (4.618%); 6391/132798 objects misplaced (4.813%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:19:57.653 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:19:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:57.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:19:57.766 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:57.856 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:57.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:57 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:19:57.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:57 smithi144 ceph-mon[194833]: 1.19 deep-scrub starts 2023-09-23T20:19:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:57 smithi144 ceph-mon[194833]: 1.19 deep-scrub ok 2023-09-23T20:19:57.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:57 smithi144 ceph-mon[194833]: pgmap v1459: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 1 active+undersized+degraded, 107 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6132/132798 objects degraded (4.618%); 6391/132798 objects misplaced (4.813%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:19:58.569 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:58.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:58.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:58.688 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:19:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:58 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6132/132798 objects degraded (4.618%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:58.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:58 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6132/132798 objects degraded (4.618%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:58.797 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:58.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:58 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6132/132798 objects degraded (4.618%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:19:58.941 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:19:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:58.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:19:59.060 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:19:59.087 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:59.150 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:19:59.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:59.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:19:59.257 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:19:59.349 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:59.375 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:19:59.533 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:19:59.597 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:19:59.690 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:19:59.709 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:19:59 smithi135 ceph-mon[213871]: pgmap v1460: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6132/129723 objects degraded (4.727%); 6196/129723 objects misplaced (4.776%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:19:59.709 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:19:59 smithi135 ceph-mon[221113]: pgmap v1460: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6132/129723 objects degraded (4.727%); 6196/129723 objects misplaced (4.776%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:19:59.715 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:19:59.726 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:19:59.728 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:19:59.728 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:19:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:19:59.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:19:59.772 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:19:59.813 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:19:59.830 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:19:59.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:19:59 smithi144 ceph-mon[194833]: pgmap v1460: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6132/129723 objects degraded (4.727%); 6196/129723 objects misplaced (4.776%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:19:59.886 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:19:59.886 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:19:59.887 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:19:59.887 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:19:59.887 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:19:59.887 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:19:59.887 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:19:59.887 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:19:59.887 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:19:59.915 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:00.509 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.4 primary-affinity to 0 (802) 2023-09-23T20:20:00.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: 1.15 scrub starts 2023-09-23T20:20:00.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: 1.a deep-scrub starts 2023-09-23T20:20:00.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/850340452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:00.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/415403899' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:20:00.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:20:00.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 6132/129723 objects degraded (4.727%), 3 pgs degraded, 6 pgs undersized 2023-09-23T20:20:00.547 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:20:00.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: osd.6 (root=default,host=smithi144) is down 2023-09-23T20:20:00.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: [WRN] PG_DEGRADED: Degraded data redundancy: 6132/129723 objects degraded (4.727%), 3 pgs degraded, 6 pgs undersized 2023-09-23T20:20:00.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: pg 3.d is stuck undersized for 81s, current state active+undersized+degraded, last acting [7,5] 2023-09-23T20:20:00.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: pg 3.11 is stuck undersized for 81s, current state active+undersized, last acting [7,4] 2023-09-23T20:20:00.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: pg 4.1 is stuck undersized for 81s, current state active+undersized, last acting [5,4] 2023-09-23T20:20:00.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: pg 4.13 is stuck undersized for 81s, current state active+undersized, last acting [1,4] 2023-09-23T20:20:00.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T20:20:00.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[213871]: pg 10.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,4] 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: 1.15 scrub starts 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: 1.a deep-scrub starts 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/850340452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/415403899' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 6132/129723 objects degraded (4.727%), 3 pgs degraded, 6 pgs undersized 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: osd.6 (root=default,host=smithi144) is down 2023-09-23T20:20:00.552 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: [WRN] PG_DEGRADED: Degraded data redundancy: 6132/129723 objects degraded (4.727%), 3 pgs degraded, 6 pgs undersized 2023-09-23T20:20:00.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: pg 3.d is stuck undersized for 81s, current state active+undersized+degraded, last acting [7,5] 2023-09-23T20:20:00.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: pg 3.11 is stuck undersized for 81s, current state active+undersized, last acting [7,4] 2023-09-23T20:20:00.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: pg 4.1 is stuck undersized for 81s, current state active+undersized, last acting [5,4] 2023-09-23T20:20:00.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: pg 4.13 is stuck undersized for 81s, current state active+undersized, last acting [1,4] 2023-09-23T20:20:00.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T20:20:00.553 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:00 smithi135 ceph-mon[221113]: pg 10.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,4] 2023-09-23T20:20:00.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: 1.15 scrub starts 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: 1.a deep-scrub starts 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/850340452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/415403899' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 6132/129723 objects degraded (4.727%), 3 pgs degraded, 6 pgs undersized 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: osd.6 (root=default,host=smithi144) is down 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: [WRN] PG_DEGRADED: Degraded data redundancy: 6132/129723 objects degraded (4.727%), 3 pgs degraded, 6 pgs undersized 2023-09-23T20:20:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: pg 3.d is stuck undersized for 81s, current state active+undersized+degraded, last acting [7,5] 2023-09-23T20:20:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: pg 3.11 is stuck undersized for 81s, current state active+undersized, last acting [7,4] 2023-09-23T20:20:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: pg 4.1 is stuck undersized for 81s, current state active+undersized, last acting [5,4] 2023-09-23T20:20:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: pg 4.13 is stuck undersized for 81s, current state active+undersized, last acting [1,4] 2023-09-23T20:20:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T20:20:00.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:00 smithi144 ceph-mon[194833]: pg 10.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,4] 2023-09-23T20:20:01.415 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:01.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:01.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:01.520 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:01.546 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:01.589 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:01.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:01.694 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:20:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[213871]: osdmap e517: 8 total, 7 up, 8 in 2023-09-23T20:20:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[213871]: failed to encode map e517 with expected crc 2023-09-23T20:20:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[213871]: failed to encode map e517 with expected crc 2023-09-23T20:20:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[213871]: pgmap v1462: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 107 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 6137/127194 objects degraded (4.825%); 5971/127194 objects misplaced (4.694%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:20:01.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:20:01.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[221113]: osdmap e517: 8 total, 7 up, 8 in 2023-09-23T20:20:01.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[221113]: failed to encode map e517 with expected crc 2023-09-23T20:20:01.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[221113]: failed to encode map e517 with expected crc 2023-09-23T20:20:01.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:01 smithi135 ceph-mon[221113]: pgmap v1462: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 107 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 6137/127194 objects degraded (4.825%); 5971/127194 objects misplaced (4.694%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:20:01.798 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:01.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:01 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T20:20:01.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:01 smithi144 ceph-mon[194833]: osdmap e517: 8 total, 7 up, 8 in 2023-09-23T20:20:01.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:01 smithi144 ceph-mon[194833]: failed to encode map e517 with expected crc 2023-09-23T20:20:01.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:01 smithi144 ceph-mon[194833]: failed to encode map e517 with expected crc 2023-09-23T20:20:01.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:01 smithi144 ceph-mon[194833]: pgmap v1462: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 107 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 6137/127194 objects degraded (4.825%); 5971/127194 objects misplaced (4.694%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:20:02.760 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:02.769 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:02 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:20:02.769 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:02 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:20:02.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:02.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:02.871 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:02.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:02 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:20:02.902 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:02.953 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:20:02.965 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:02.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:02.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:03.074 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:03.157 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:20:03.602 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:20:03.781 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:03.789 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:03 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:20:03.789 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:03 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6137/127194 objects degraded (4.825%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:03.790 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:03 smithi135 ceph-mon[213871]: pgmap v1463: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 6137/121569 objects degraded (5.048%); 5773/121569 objects misplaced (4.749%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:20:03.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:03.791 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:20:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:20:03] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-09-23T20:20:03.792 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:03 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:20:03.792 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:03 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6137/127194 objects degraded (4.825%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:03.792 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:03 smithi135 ceph-mon[221113]: pgmap v1463: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 6137/121569 objects degraded (5.048%); 5773/121569 objects misplaced (4.749%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:20:03.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:03 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:20:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:03 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6137/127194 objects degraded (4.825%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:03 smithi144 ceph-mon[194833]: pgmap v1463: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 6137/121569 objects degraded (5.048%); 5773/121569 objects misplaced (4.749%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:20:03.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:03.985 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:04.224 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:04.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:04.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:04.295 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:04.337 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:04.369 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:04.420 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:04.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:04.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:04.527 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:04.592 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:04 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:20:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:04 smithi144 ceph-mon[194833]: 1.e deep-scrub starts 2023-09-23T20:20:04.701 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:04.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:04.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:04.806 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:04.876 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:04 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:20:04.877 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:04 smithi135 ceph-mon[213871]: 1.e deep-scrub starts 2023-09-23T20:20:04.880 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:04.881 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:04 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:20:04.881 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:04 smithi135 ceph-mon[221113]: 1.e deep-scrub starts 2023-09-23T20:20:05.158 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:05.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:05.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:05.267 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:05.387 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:05.484 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:05.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:05.579 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:20:05.593 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:05.603 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:20:05.620 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:05.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:05.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:05.769 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:05.794 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:05 smithi144 ceph-mon[194833]: 1.a deep-scrub starts 2023-09-23T20:20:05.794 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2903287193' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:05.794 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:05 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:05.794 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:05 smithi144 ceph-mon[194833]: pgmap v1464: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6137/119151 objects degraded (5.151%); 5581/119151 objects misplaced (4.684%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:20:05.796 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:05.838 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:05.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:05.942 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:05 smithi135 ceph-mon[213871]: 1.a deep-scrub starts 2023-09-23T20:20:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2903287193' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:05 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:05.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:05 smithi135 ceph-mon[213871]: pgmap v1464: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6137/119151 objects degraded (5.151%); 5581/119151 objects misplaced (4.684%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:20:05.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:05 smithi135 ceph-mon[221113]: 1.a deep-scrub starts 2023-09-23T20:20:05.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2903287193' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:05.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:05 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:05.972 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:05 smithi135 ceph-mon[221113]: pgmap v1464: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6137/119151 objects degraded (5.151%); 5581/119151 objects misplaced (4.684%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:20:06.022 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:06.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[221113]: osdmap e518: 8 total, 7 up, 8 in 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[221113]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[221113]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[221113]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[221113]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:20:06.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[213871]: osdmap e518: 8 total, 7 up, 8 in 2023-09-23T20:20:06.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[213871]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[213871]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[213871]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:06 smithi135 ceph-mon[213871]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:06 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:06 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:20:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:06 smithi144 ceph-mon[194833]: osdmap e518: 8 total, 7 up, 8 in 2023-09-23T20:20:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:06 smithi144 ceph-mon[194833]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:06 smithi144 ceph-mon[194833]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:06 smithi144 ceph-mon[194833]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:06 smithi144 ceph-mon[194833]: failed to encode map e518 with expected crc 2023-09-23T20:20:06.968 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:06.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:06.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:07.074 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:07.110 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:07.175 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:07.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:07.283 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:07.380 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:07.547 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:07.600 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:07.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:07.609 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:07 smithi135 ceph-mon[213871]: pgmap v1466: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 6171/112944 objects degraded (5.464%); 5160/112944 objects misplaced (4.569%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:20:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:07.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:07.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:07.613 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:07 smithi135 ceph-mon[221113]: pgmap v1466: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 6171/112944 objects degraded (5.464%); 5160/112944 objects misplaced (4.569%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:20:07.712 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:07.721 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:20:07.721 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:20:07.752 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:07.811 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:07.817 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:07.817 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:07 smithi144 ceph-mon[194833]: pgmap v1466: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 6171/112944 objects degraded (5.464%); 5160/112944 objects misplaced (4.569%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:20:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:07.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:07.919 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:08.013 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:08 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6171/112944 objects degraded (5.464%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:08.958 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:20:08.958 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:20:08.959 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:20:08.959 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 1.000000 2023-09-23T20:20:08.959 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T20:20:08.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:08 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6171/112944 objects degraded (5.464%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:08.978 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:08 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6171/112944 objects degraded (5.464%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:09.000 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:20:09.002 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:09.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:09.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:09.126 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:09.160 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:20:09.247 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:09.504 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:09.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:09.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:09.597 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:09 smithi135 ceph-mon[213871]: pgmap v1467: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6171/109830 objects degraded (5.619%); 4982/109830 objects misplaced (4.536%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:20:09.618 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:09.708 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:09.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:09 smithi144 ceph-mon[194833]: pgmap v1467: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6171/109830 objects degraded (5.619%); 4982/109830 objects misplaced (4.536%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:20:09.975 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:09 smithi135 ceph-mon[221113]: pgmap v1467: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6171/109830 objects degraded (5.619%); 4982/109830 objects misplaced (4.536%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:20:10.632 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:10.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:10.742 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:10.774 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:10.827 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:10.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:10.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:10.875 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:10.932 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:10.960 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:11.023 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:11.026 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:11.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:11.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:11.052 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:11.132 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:11.160 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:11.213 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:11.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:11.318 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:11.423 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:11.559 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:20:11.619 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:20:11.666 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:20:11.667 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:11.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:11.713 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:20:11.753 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:20:11.779 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:11.786 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:20:11.809 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:11.825 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:20:11.825 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:20:11.825 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:20:11.826 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:20:11.826 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:20:11.826 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:20:11.826 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:20:11.826 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:20:11.826 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:20:11.868 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:11.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:11.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:11.978 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:12.071 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:12.308 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:12 smithi135 ceph-mon[221113]: pgmap v1468: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6209/107265 objects degraded (5.788%); 4748/107265 objects misplaced (4.426%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:12.308 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/861167211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:12.308 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:12 smithi135 ceph-mon[213871]: pgmap v1468: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6209/107265 objects degraded (5.788%); 4748/107265 objects misplaced (4.426%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:12.308 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/861167211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:12.541 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:12.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:12.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:12 smithi144 ceph-mon[194833]: pgmap v1468: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6209/107265 objects degraded (5.788%); 4748/107265 objects misplaced (4.426%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/861167211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:12.655 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:12.734 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:13.603 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:13.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:13.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:13 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6209/107265 objects degraded (5.788%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:13.629 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:13 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6209/107265 objects degraded (5.788%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:13.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:13.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:13 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6209/107265 objects degraded (5.788%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:13.730 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:13.763 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:13.825 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:13.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:13.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:13.925 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:20:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:20:13] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-09-23T20:20:13.934 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:14.048 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:14.585 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:14 smithi135 ceph-mon[221113]: pgmap v1469: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 6209/101589 objects degraded (6.112%); 4552/101589 objects misplaced (4.481%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:14.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:14 smithi135 ceph-mon[213871]: pgmap v1469: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 6209/101589 objects degraded (6.112%); 4552/101589 objects misplaced (4.481%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:14.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:14 smithi144 ceph-mon[194833]: pgmap v1469: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 6209/101589 objects degraded (6.112%); 4552/101589 objects misplaced (4.481%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:20:15.448 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:20:15.589 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:15.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:15.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:15.696 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:15.723 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:15.772 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:15.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:15.878 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:15.903 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:15.944 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:15.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:15.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:16.052 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:16.084 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:16.385 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:16 smithi135 ceph-mon[213871]: pgmap v1470: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 6209/99180 objects degraded (6.260%); 4341/99180 objects misplaced (4.377%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:16.387 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:16 smithi135 ceph-mon[221113]: pgmap v1470: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 6209/99180 objects degraded (6.260%); 4341/99180 objects misplaced (4.377%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:16.468 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:16.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:16.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:16.591 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:16.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:16 smithi144 ceph-mon[194833]: pgmap v1470: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 6209/99180 objects degraded (6.260%); 4341/99180 objects misplaced (4.377%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:20:16.688 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:17.023 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:20:17.023 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:20:17.023 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:20:17.023 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:20:17.023 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:20:17.090 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:17.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:17.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:17.197 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:17.309 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:17.342 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1731878259' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:20:17.342 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:17 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:20:17.356 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T20:20:17.449 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:20:17.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1731878259' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:20:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:17 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:20:17.637 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1731878259' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:20:17.637 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:17 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:20:18.400 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:20:18.450 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:18.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:18.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:18.561 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:18.603 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:18 smithi144 ceph-mon[194833]: pgmap v1471: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 6174/93108 objects degraded (6.631%); 3925/93108 objects misplaced (4.216%); 3.1 MiB/s, 52 objects/s recovering 2023-09-23T20:20:18.604 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:20:18.604 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:18 smithi144 ceph-mon[194833]: osdmap e519: 8 total, 7 up, 8 in 2023-09-23T20:20:18.604 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:18 smithi144 ceph-mon[194833]: failed to encode map e519 with expected crc 2023-09-23T20:20:18.604 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:18 smithi144 ceph-mon[194833]: failed to encode map e519 with expected crc 2023-09-23T20:20:18.604 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2207361123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:18.604 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:18.604 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:18 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 6174/93108 objects degraded (6.631%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:18.606 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:18.668 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:18.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:18.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[221113]: pgmap v1471: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 6174/93108 objects degraded (6.631%); 3925/93108 objects misplaced (4.216%); 3.1 MiB/s, 52 objects/s recovering 2023-09-23T20:20:18.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:20:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[221113]: osdmap e519: 8 total, 7 up, 8 in 2023-09-23T20:20:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[221113]: failed to encode map e519 with expected crc 2023-09-23T20:20:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[221113]: failed to encode map e519 with expected crc 2023-09-23T20:20:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2207361123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 6174/93108 objects degraded (6.631%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:18.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[213871]: pgmap v1471: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 6174/93108 objects degraded (6.631%); 3925/93108 objects misplaced (4.216%); 3.1 MiB/s, 52 objects/s recovering 2023-09-23T20:20:18.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:20:18.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[213871]: osdmap e519: 8 total, 7 up, 8 in 2023-09-23T20:20:18.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[213871]: failed to encode map e519 with expected crc 2023-09-23T20:20:18.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[213871]: failed to encode map e519 with expected crc 2023-09-23T20:20:18.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2207361123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:18.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:18.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:18 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 6174/93108 objects degraded (6.631%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:18.773 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:18.803 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:18.868 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:18.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:18.972 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:18.998 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:19.040 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:19.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:19.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:19.146 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:19.180 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:19.252 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:19.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:19.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:19.280 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:20:19.358 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:19.388 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:19.444 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:19.448 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:19 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:19.449 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:20:19.449 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:19 smithi144 ceph-mon[194833]: osdmap e520: 8 total, 7 up, 8 in 2023-09-23T20:20:19.449 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:19 smithi144 ceph-mon[194833]: failed to encode map e520 with expected crc 2023-09-23T20:20:19.449 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:19 smithi144 ceph-mon[194833]: pgmap v1474: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 6174/90276 objects degraded (6.839%); 3757/90276 objects misplaced (4.162%); 1.6 MiB/s, 30 objects/s recovering 2023-09-23T20:20:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:19.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:19.551 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:19.586 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:19.640 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:19.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:19.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:19.742 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:20:19.743 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[213871]: osdmap e520: 8 total, 7 up, 8 in 2023-09-23T20:20:19.743 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[213871]: failed to encode map e520 with expected crc 2023-09-23T20:20:19.743 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[213871]: pgmap v1474: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 6174/90276 objects degraded (6.839%); 3757/90276 objects misplaced (4.162%); 1.6 MiB/s, 30 objects/s recovering 2023-09-23T20:20:19.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:19.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:20:19.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[221113]: osdmap e520: 8 total, 7 up, 8 in 2023-09-23T20:20:19.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[221113]: failed to encode map e520 with expected crc 2023-09-23T20:20:19.743 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:19 smithi135 ceph-mon[221113]: pgmap v1474: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 6174/90276 objects degraded (6.839%); 3757/90276 objects misplaced (4.162%); 1.6 MiB/s, 30 objects/s recovering 2023-09-23T20:20:19.750 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:19.777 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:19.844 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:19.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:19.956 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:20.036 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:20.254 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:20.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:20.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:20.375 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:20.384 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:20.411 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:20.469 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:20.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:20.492 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:20.576 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:20.604 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:20.665 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:20.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:20.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:20.776 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:20.813 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:20.873 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:20.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:20.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:20.983 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:21.109 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:22.297 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:20:22.499 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:22.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:22.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:22.610 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:22.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:22 smithi144 ceph-mon[194833]: pgmap v1475: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5904/88065 objects degraded (6.704%); 3757/88065 objects misplaced (4.266%); 1.6 MiB/s, 64 objects/s recovering 2023-09-23T20:20:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:22.705 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:22 smithi135 ceph-mon[221113]: pgmap v1475: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5904/88065 objects degraded (6.704%); 3757/88065 objects misplaced (4.266%); 1.6 MiB/s, 64 objects/s recovering 2023-09-23T20:20:22.705 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:22.705 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:22 smithi135 ceph-mon[213871]: pgmap v1475: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5904/88065 objects degraded (6.704%); 3757/88065 objects misplaced (4.266%); 1.6 MiB/s, 64 objects/s recovering 2023-09-23T20:20:22.705 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:22.709 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:23.141 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:23.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:23.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:23.248 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:23.325 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:23.455 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:23.458 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5904/88065 objects degraded (6.704%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:23.458 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5904/88065 objects degraded (6.704%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:23.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:23.549 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:23.559 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:23.583 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5904/88065 objects degraded (6.704%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:23.586 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:23.655 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:23.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:23.714 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:20:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:20:23] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-23T20:20:23.767 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:23.810 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:23.864 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:23.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:23.970 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:24.003 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:24.064 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:24.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:24.113 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:20:24.153 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:20:24.153 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:20:24.153 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:20:24.153 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:20:24.153 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.153 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123384 8.f 8:fe9c53da:::benchmark_data_smithi135_460343_object39151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:20:24.153 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:20:24.153 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 582.24068781100004, 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4434314349999999, 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123384 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.154 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906019+0000", 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906017+0000", 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:20:24.155 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906025+0000", 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6260000000000003e-06 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906041+0000", 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6183000000000001e-05 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.156 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.906048+0000", 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7000000000000008e-06 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.969587+0000", 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.063539010000000007 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:41.974774+0000", 2023-09-23T20:20:24.157 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051870179999999998 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.005518+0000", 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030743594999999999 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.011676+0000", 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061576579999999999 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.158 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.259997+0000", 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2483216180000001 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.274504+0000", 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014506939999999999 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.290807+0000", 2023-09-23T20:20:24.159 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016302543999999999 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.312744+0000", 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021937076999999999 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.314532+0000", 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017877699999999999 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.336416+0000", 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021884023999999998 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.338670+0000", 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0022543989999999998 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.349450+0000", 2023-09-23T20:20:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010779924 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123388 8.f 8:f5171e3d:::benchmark_data_smithi135_460343_object39155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 582.06809041999998, 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5028154900000001, 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.162 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123388 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.163 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078616+0000", 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078615+0000", 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078622+0000", 2023-09-23T20:20:24.164 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.345e-06 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078636+0000", 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3645000000000001e-05 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.078642+0000", 2023-09-23T20:20:24.165 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5810000000000002e-06 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.092915+0000", 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.014273083000000001 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:20:24.166 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.098087+0000", 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051715119999999996 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.137553+0000", 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.039466342000000001 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.146411+0000", 2023-09-23T20:20:24.167 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0088579100000000001 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.513388+0000", 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.366976599 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.518018+0000", 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046298229999999999 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.168 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.527773+0000", 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097548610000000001 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.566237+0000", 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.038464921999999999 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.567647+0000", 2023-09-23T20:20:24.169 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001409684 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.568272+0000", 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00062472500000000002 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.570874+0000", 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026022689999999999 2023-09-23T20:20:24.170 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.581431+0000", 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010557383 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123394 8.f 8:ff01962c:::benchmark_data_smithi135_460343_object39161:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:20:24.171 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 581.94632805100002, 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.413050616, 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123394 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.172 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200378+0000", 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200376+0000", 2023-09-23T20:20:24.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200388+0000", 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2403e-05 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200413+0000", 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4596999999999998e-05 2023-09-23T20:20:24.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.200430+0000", 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6908000000000001e-05 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.221890+0000", 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021460679999999999 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.238665+0000", 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0167742 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.309438+0000", 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.070773321 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.319840+0000", 2023-09-23T20:20:24.176 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010402428 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.545580+0000", 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.225739777 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.550259+0000", 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0046789290000000001 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.177 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.558399+0000", 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081401330000000008 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.583234+0000", 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.024835052999999999 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.178 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.585402+0000", 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0021673310000000002 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.586049+0000", 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00064698200000000005 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.591270+0000", 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052211779999999999 2023-09-23T20:20:24.179 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.613429+0000", 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.022159160000000001 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123402 8.7 8:ec120430:::benchmark_data_smithi135_460343_object39169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:20:24.180 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 581.893578123, 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4055715360000001, 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123402 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:20:24.181 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253128+0000", 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253126+0000", 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:20:24.182 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253138+0000", 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1725e-05 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253158+0000", 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0772999999999999e-05 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.183 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.253169+0000", 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.022e-05 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.321269+0000", 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.068100777000000001 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:20:24.184 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.331911+0000", 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010641866999999999 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.411508+0000", 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.079596497000000002 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.422363+0000", 2023-09-23T20:20:24.185 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010854744 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.612810+0000", 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1904476230000001 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.617814+0000", 2023-09-23T20:20:24.186 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050038449999999998 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.627404+0000", 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0095902439999999995 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.643653+0000", 2023-09-23T20:20:24.187 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.016248987999999999 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.645238+0000", 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00158438 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.645689+0000", 2023-09-23T20:20:24.188 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00045135999999999999 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.647776+0000", 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0020871879999999998 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.658700+0000", 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.010923696 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.189 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123442 8.f 8:f6660517:::benchmark_data_smithi135_460343_object39209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 581.61052133199996, 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4673699600000001, 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.190 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:20:24.191 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:20:24.191 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123442 2023-09-23T20:20:24.191 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.191 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.191 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.191 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.191 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:20:24.191 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.192 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.192 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.192 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.192 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536185+0000", 2023-09-23T20:20:24.192 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.192 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.193 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.193 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536183+0000", 2023-09-23T20:20:24.193 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:20:24.193 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.193 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.193 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.194 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536192+0000", 2023-09-23T20:20:24.194 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.1039999999999996e-06 2023-09-23T20:20:24.194 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.194 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.194 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.194 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536207+0000", 2023-09-23T20:20:24.194 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4834000000000001e-05 2023-09-23T20:20:24.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.195 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.195 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.536219+0000", 2023-09-23T20:20:24.195 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1691999999999999e-05 2023-09-23T20:20:24.195 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.195 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.615218+0000", 2023-09-23T20:20:24.196 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.078999571000000005 2023-09-23T20:20:24.196 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.196 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.196 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:20:24.196 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.620140+0000", 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049211439999999997 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.675848+0000", 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.055708879000000003 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.680973+0000", 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0051250449999999999 2023-09-23T20:20:24.197 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.828876+0000", 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1479029780000001 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.834877+0000", 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0060005409999999999 2023-09-23T20:20:24.198 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.844577+0000", 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097000209999999996 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.858053+0000", 2023-09-23T20:20:24.199 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.013476228 2023-09-23T20:20:24.200 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.200 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.859480+0000", 2023-09-23T20:20:24.200 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001426407 2023-09-23T20:20:24.200 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.200 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.200 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.991498+0000", 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.13201818500000001 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:43.993799+0000", 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023010399999999999 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.201 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.003555+0000", 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097562010000000008 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123449 8.f 8:f357d011:::benchmark_data_smithi135_460343_object39216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 581.57583681999995, 2023-09-23T20:20:24.202 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5039241049999998, 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123449 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.203 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570870+0000", 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570867+0000", 2023-09-23T20:20:24.204 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999971 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570879+0000", 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2435e-05 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570900+0000", 2023-09-23T20:20:24.205 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0509999999999998e-05 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.570919+0000", 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8947999999999999e-05 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.620845+0000", 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.049925803999999997 2023-09-23T20:20:24.206 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.626417+0000", 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0055726200000000004 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.698693+0000", 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.072275866999999994 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.207 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:42.703708+0000", 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0050152590000000002 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.016373+0000", 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.312664507 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.208 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.020480+0000", 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041066089999999998 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.028636+0000", 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0081566039999999996 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.041139+0000", 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.012503377 2023-09-23T20:20:24.209 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.042755+0000", 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0016158360000000001 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.061415+0000", 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.018660039 2023-09-23T20:20:24.210 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.063733+0000", 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023174549999999999 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.074794+0000", 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.011060876000000001 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.211 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.35643.0:123630 8.7 8:e7d430c0:::benchmark_data_smithi135_460343_object39397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 579.82333067000002, 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4124915579999999, 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.35643", 2023-09-23T20:20:24.212 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/4088227555", 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 123630 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323376+0000", 2023-09-23T20:20:24.213 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323374+0000", 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323381+0000", 2023-09-23T20:20:24.214 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8360000000000003e-06 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323393+0000", 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.226e-05 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.323402+0000", 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9330000000000007e-06 2023-09-23T20:20:24.215 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.394929+0000", 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.071526432000000001 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for missing object", 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.399736+0000", 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0048073309999999998 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.216 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.434854+0000", 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.035117795 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for rw locks", 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:44.439795+0000", 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0049409620000000001 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.537860+0000", 2023-09-23T20:20:24.217 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0980649309999999 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.542831+0000", 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004970833 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.551732+0000", 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0089008879999999992 2023-09-23T20:20:24.218 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.719631+0000", 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.16789942299999999 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.721555+0000", 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019236259999999999 2023-09-23T20:20:24.219 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.723874+0000", 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023192669999999999 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.726072+0000", 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0021980229999999999 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.220 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:10:45.735867+0000", 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0097953720000000001 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:124937 10.0 10:0f748a65:::benchmark_data_smithi135_602286_object32711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:19:31.288696+0000", 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 52.858009967999998, 2023-09-23T20:20:24.221 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17698422699999999, 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 124937 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.222 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.288696+0000", 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.288696+0000", 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.288695+0000", 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:20:24.223 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.288702+0000", 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.4110000000000001e-06 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.288712+0000", 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0859999999999999e-05 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.224 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.288720+0000", 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7340000000000008e-06 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.288764+0000", 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3755999999999999e-05 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.225 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.288950+0000", 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000186489 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.289070+0000", 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000119443 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.293592+0000", 2023-09-23T20:20:24.226 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.004522613 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.295538+0000", 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019453750000000001 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.465582+0000", 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.17004469799999999 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.227 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.465634+0000", 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1727999999999997e-05 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:31.465681+0000", 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6474999999999998e-05 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.228 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:125856 10.0 10:00559810:::benchmark_data_smithi135_602286_object33630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:19:32.857213+0000", 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 51.289493726000003, 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.18029335399999999, 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:20:24.229 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 125856 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.857213+0000", 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.230 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.857213+0000", 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.857212+0000", 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.857215+0000", 2023-09-23T20:20:24.231 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.8210000000000002e-06 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.857221+0000", 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3160000000000002e-06 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.857226+0000", 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.955e-06 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.232 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.866763+0000", 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0095363219999999999 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.866928+0000", 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000165713 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:20:24.233 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.867064+0000", 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013540700000000001 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.871580+0000", 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0045164130000000004 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:32.877753+0000", 2023-09-23T20:20:24.234 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0061729899999999997 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:33.037424+0000", 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.159670909 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:33.037463+0000", 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8968999999999999e-05 2023-09-23T20:20:24.235 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:19:33.037506+0000", 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3019000000000003e-05 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:152024 10.2 e518/490 10:4f185401:::benchmark_data_smithi135_602286_object59798:head v 518'9551, mlcod=0'0)", 2023-09-23T20:20:24.236 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.095387+0000", 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 11.051319803, 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.113136535, 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.095387+0000", 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.237 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.095387+0000", 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.095385+0000", 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.238 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.095610+0000", 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00022476500000000001 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.095628+0000", 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7600000000000001e-05 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.239 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.095637+0000", 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.7000000000000003e-06 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.095724+0000", 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6289e-05 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.095751+0000", 2023-09-23T20:20:24.240 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7402e-05 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.208477+0000", 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.112725977 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.208523+0000", 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6100999999999998e-05 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.241 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:152039 10.2 e518/490 10:404459a7:::benchmark_data_smithi135_602286_object59813:head v 518'9552, mlcod=0'0)", 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.102869+0000", 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 11.043837308000001, 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.128061963, 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.242 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.102869+0000", 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.102869+0000", 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.243 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.102868+0000", 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.102878+0000", 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8109999999999997e-06 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.244 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.102889+0000", 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1683e-05 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.102897+0000", 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.8290000000000008e-06 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.245 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.102949+0000", 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1357999999999999e-05 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.102969+0000", 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.0122000000000001e-05 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.230885+0000", 2023-09-23T20:20:24.246 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.12791661800000001 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.230931+0000", 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5676000000000001e-05 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:152036 10.b e518/490 10:d8739be2:::benchmark_data_smithi135_602286_object59810:head v 518'9345, mlcod=0'0)", 2023-09-23T20:20:24.247 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.105308+0000", 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 11.041398056, 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.104835524, 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105308+0000", 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.248 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105308+0000", 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105307+0000", 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.249 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105317+0000", 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0142999999999999e-05 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105331+0000", 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3667e-05 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105339+0000", 2023-09-23T20:20:24.250 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.763e-06 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105393+0000", 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3845999999999999e-05 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105420+0000", 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7509e-05 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.251 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.210102+0000", 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.104682344 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.210144+0000", 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1396999999999997e-05 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.252 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "MOSDPGPush(10.b 518/490 [PushOp(10:da21125b:::benchmark_data_smithi135_602286_object60437:head, version: 488'3716, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:da21125b:::benchmark_data_smithi135_602286_object60437:head@488'3716, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.105354+0000", 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 11.041351958, 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.099718116999999995, 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.253 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105354+0000", 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105354+0000", 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105354+0000", 2023-09-23T20:20:24.254 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967296 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105391+0000", 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6470999999999997e-05 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105405+0000", 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4122e-05 2023-09-23T20:20:24.255 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.256 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105417+0000", 2023-09-23T20:20:24.256 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2632999999999999e-05 2023-09-23T20:20:24.256 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.256 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.256 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.256 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202211+0000", 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.096793279999999995 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202224+0000", 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3645000000000001e-05 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.205073+0000", 2023-09-23T20:20:24.257 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028482009999999999 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46418.0:152048 10.0 10:03479ce6:::benchmark_data_smithi135_602286_object59822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.105678+0000", 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 11.041028007, 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.096530790000000005, 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.258 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46418", 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 152048 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105678+0000", 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.259 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105678+0000", 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105677+0000", 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.260 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105683+0000", 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.0900000000000001e-06 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105700+0000", 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.6835000000000001e-05 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105707+0000", 2023-09-23T20:20:24.261 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4970000000000004e-06 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105759+0000", 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1853e-05 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.105915+0000", 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015615 2023-09-23T20:20:24.262 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.106048+0000", 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000132328 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202005+0000", 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.095957318999999999 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.263 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202024+0000", 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.8872000000000002e-05 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202125+0000", 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000100933 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.264 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202168+0000", 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3334000000000002e-05 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202209+0000", 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1164000000000001e-05 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.265 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.46418.0:152048 10.0 e518/511 ondisk, result = 0)", 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.112017+0000", 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 11.034689266000001, 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.090197309000000003, 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.112017+0000", 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.266 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.112017+0000", 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.112016+0000", 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.267 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.112025+0000", 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.6640000000000004e-06 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202071+0000", 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.090046245999999996 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.268 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202079+0000", 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.0460000000000006e-06 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202112+0000", 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2839000000000002e-05 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202119+0000", 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6040000000000004e-06 2023-09-23T20:20:24.269 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202214+0000", 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.5104000000000003e-05 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop_reply(client.46418.0:152048 10.0 e518/511 ondisk, result = 0)", 2023-09-23T20:20:24.270 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.112715+0000", 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 11.033991702, 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.089296674000000006, 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.112715+0000", 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.271 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.112715+0000", 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.112714+0000", 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.272 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.112721+0000", 2023-09-23T20:20:24.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8190000000000003e-06 2023-09-23T20:20:24.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.273 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.273 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.201866+0000", 2023-09-23T20:20:24.273 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.089145656000000004 2023-09-23T20:20:24.273 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.273 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.201884+0000", 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7910999999999998e-05 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.201989+0000", 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010457 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.274 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.201998+0000", 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9390000000000003e-06 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.202011+0000", 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3559e-05 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.275 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:152051 10.2 e518/490 10:427ef4a1:::benchmark_data_smithi135_602286_object59825:head v 518'9553, mlcod=0'0)", 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.114255+0000", 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 11.032451203999999, 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.11669716199999999, 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.276 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.114255+0000", 2023-09-23T20:20:24.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.277 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.277 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.277 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.277 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.114255+0000", 2023-09-23T20:20:24.277 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.278 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.278 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.114254+0000", 2023-09-23T20:20:24.278 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:20:24.278 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.278 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.279 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.279 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.114263+0000", 2023-09-23T20:20:24.279 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8459999999999997e-06 2023-09-23T20:20:24.279 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.279 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.201871+0000", 2023-09-23T20:20:24.280 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.087607817000000004 2023-09-23T20:20:24.280 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.280 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.280 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.280 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.201885+0000", 2023-09-23T20:20:24.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3689999999999999e-05 2023-09-23T20:20:24.281 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.281 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.281 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.281 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.201934+0000", 2023-09-23T20:20:24.281 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9509999999999999e-05 2023-09-23T20:20:24.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.282 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.282 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.282 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.201966+0000", 2023-09-23T20:20:24.282 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2320000000000002e-05 2023-09-23T20:20:24.282 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.283 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.283 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.230934+0000", 2023-09-23T20:20:24.283 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.028968022999999999 2023-09-23T20:20:24.283 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.283 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.284 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.284 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.230952+0000", 2023-09-23T20:20:24.284 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.7904e-05 2023-09-23T20:20:24.284 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.284 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.285 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.285 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.285 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.285 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:152334 10.2 e518/490 10:40c5d561:::benchmark_data_smithi135_602286_object60108:head v 518'9578, mlcod=0'0)", 2023-09-23T20:20:24.285 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:13.598730+0000", 2023-09-23T20:20:24.285 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 10.547976724, 2023-09-23T20:20:24.285 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.054220625000000001, 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.598730+0000", 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.286 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.598730+0000", 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.598729+0000", 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.598738+0000", 2023-09-23T20:20:24.287 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.2529999999999993e-06 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.598752+0000", 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.385e-05 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.598760+0000", 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.1629999999999996e-06 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.288 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.608588+0000", 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0098281830000000008 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.608619+0000", 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.04e-05 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.289 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.652901+0000", 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.044282187000000001 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:13.652950+0000", 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9509000000000003e-05 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.290 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:153646 10.b e518/490 10:dffb7f31:::benchmark_data_smithi135_602286_object61420:head v 518'9440, mlcod=502'5665)", 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:15.527956+0000", 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 8.6187500749999995, 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.049590625999999999, 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.527956+0000", 2023-09-23T20:20:24.291 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.527956+0000", 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.527956+0000", 2023-09-23T20:20:24.292 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.527964+0000", 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8990000000000007e-06 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.527979+0000", 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4171e-05 2023-09-23T20:20:24.293 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.527986+0000", 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.2409999999999998e-06 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.533956+0000", 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.005970365 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.294 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.295 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:24.296 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:20:24.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.296 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.533988+0000", 2023-09-23T20:20:24.296 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.133e-05 2023-09-23T20:20:24.296 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.296 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.296 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.577502+0000", 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.043514522 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.577547+0000", 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4925000000000001e-05 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.297 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:153656 10.b e518/490 10:dcab2ff8:::benchmark_data_smithi135_602286_object61430:head v 518'9441, mlcod=502'5665)", 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:15.531508+0000", 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 8.6151981180000003, 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.052348927000000003, 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:24.298 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.531508+0000", 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.531508+0000", 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.531507+0000", 2023-09-23T20:20:24.299 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.531515+0000", 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6960000000000005e-06 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.531531+0000", 2023-09-23T20:20:24.300 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.5994999999999998e-05 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.531538+0000", 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.1559999999999998e-06 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.562429+0000", 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030890883000000001 2023-09-23T20:20:24.301 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.562456+0000", 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.7246e-05 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.583808+0000", 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.021351528000000002 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:24.302 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:15.583857+0000", 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9484999999999998e-05 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:24.303 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:20:24.304 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:20:24.304 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:20:24.304 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:20:24.349 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:20:24.377 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:24.379 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:20:24.379 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:20:24.380 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T20:20:24.380 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:20:24.380 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:20:24.380 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T20:20:24.393 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:20:24.394 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:20:24.394 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:20:24.394 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:20:24.394 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:20:24.394 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:20:24.394 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:20:24.394 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:20:24.394 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:20:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:24 smithi135 ceph-mon[221113]: pgmap v1476: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5904/82200 objects degraded (7.182%); 3570/82200 objects misplaced (4.343%); 1.6 MiB/s, 64 objects/s recovering 2023-09-23T20:20:24.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:24 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/786785803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:24.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:24 smithi135 ceph-mon[213871]: pgmap v1476: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5904/82200 objects degraded (7.182%); 3570/82200 objects misplaced (4.343%); 1.6 MiB/s, 64 objects/s recovering 2023-09-23T20:20:24.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:24 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/786785803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:24 smithi144 ceph-mon[194833]: pgmap v1476: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5904/82200 objects degraded (7.182%); 3570/82200 objects misplaced (4.343%); 1.6 MiB/s, 64 objects/s recovering 2023-09-23T20:20:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:24 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/786785803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:24.862 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:20:25.811 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:25.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:25.919 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:26.000 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:26.477 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:20:26.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:26 smithi135 ceph-mon[213871]: pgmap v1477: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5904/79410 objects degraded (7.435%); 3342/79410 objects misplaced (4.209%); 0 B/s, 33 objects/s recovering 2023-09-23T20:20:26.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:26 smithi135 ceph-mon[221113]: pgmap v1477: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5904/79410 objects degraded (7.435%); 3342/79410 objects misplaced (4.209%); 0 B/s, 33 objects/s recovering 2023-09-23T20:20:26.588 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:26.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:26.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:26 smithi144 ceph-mon[194833]: pgmap v1477: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5904/79410 objects degraded (7.435%); 3342/79410 objects misplaced (4.209%); 0 B/s, 33 objects/s recovering 2023-09-23T20:20:26.705 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:26.731 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:27.231 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:20:27.232 INFO:teuthology.orchestra.run.smithi144.stdout: "success": "" 2023-09-23T20:20:27.232 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:20:27.242 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:20:27.242 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:20:27.306 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:27.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:27.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:27.417 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:27.496 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:27.498 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:27.550 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:27.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:27.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:27.655 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:27.716 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:27.998 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:28.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:28.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:28.109 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:28.201 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:28.339 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:28 smithi144 ceph-mon[194833]: pgmap v1478: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5645/73026 objects degraded (7.730%); 3093/73026 objects misplaced (4.235%); 0 B/s, 53 objects/s recovering 2023-09-23T20:20:28.339 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5645/73026 objects degraded (7.730%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:28 smithi135 ceph-mon[213871]: pgmap v1478: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5645/73026 objects degraded (7.730%); 3093/73026 objects misplaced (4.235%); 0 B/s, 53 objects/s recovering 2023-09-23T20:20:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5645/73026 objects degraded (7.730%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:28 smithi135 ceph-mon[221113]: pgmap v1478: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5645/73026 objects degraded (7.730%); 3093/73026 objects misplaced (4.235%); 0 B/s, 53 objects/s recovering 2023-09-23T20:20:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5645/73026 objects degraded (7.730%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:28.531 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:28.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:28.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:28.640 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:28.699 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:28.816 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:28.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:28.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:28.919 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:28.945 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:28.987 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:28.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:28.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:29.090 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:29.171 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:29.243 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:20:29.388 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:20:29.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1288215802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:20:29.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:29 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:20:29.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1288215802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:20:29.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:29 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:20:29.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1288215802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:20:29.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:29 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:20:29.634 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:29.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:29.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:29.742 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:29.772 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:29.815 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:29.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:29.922 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:29.948 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:29.989 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:29.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:29.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:30.093 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:30.132 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:30.189 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:30.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:30.297 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:30 smithi144 ceph-mon[194833]: pgmap v1479: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5645/69963 objects degraded (8.069%); 2862/69963 objects misplaced (4.091%); 0 B/s, 48 objects/s recovering 2023-09-23T20:20:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:30 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:30 smithi144 ceph-mon[194833]: failed to encode map e521 with expected crc 2023-09-23T20:20:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:30 smithi144 ceph-mon[194833]: failed to encode map e521 with expected crc 2023-09-23T20:20:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:30 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:20:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:30 smithi144 ceph-mon[194833]: osdmap e521: 8 total, 7 up, 8 in 2023-09-23T20:20:30.391 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:30.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[221113]: pgmap v1479: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5645/69963 objects degraded (8.069%); 2862/69963 objects misplaced (4.091%); 0 B/s, 48 objects/s recovering 2023-09-23T20:20:30.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:30.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[221113]: failed to encode map e521 with expected crc 2023-09-23T20:20:30.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[221113]: failed to encode map e521 with expected crc 2023-09-23T20:20:30.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:20:30.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[221113]: osdmap e521: 8 total, 7 up, 8 in 2023-09-23T20:20:30.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[213871]: pgmap v1479: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5645/69963 objects degraded (8.069%); 2862/69963 objects misplaced (4.091%); 0 B/s, 48 objects/s recovering 2023-09-23T20:20:30.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:30.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[213871]: failed to encode map e521 with expected crc 2023-09-23T20:20:30.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[213871]: failed to encode map e521 with expected crc 2023-09-23T20:20:30.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:20:30.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:30 smithi135 ceph-mon[213871]: osdmap e521: 8 total, 7 up, 8 in 2023-09-23T20:20:30.703 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:30.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:30.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:30.812 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:30.845 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:30.917 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:30.919 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:30.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:30.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:31.025 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:31.065 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:20:31.065 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:20:31.110 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:31.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:31 smithi144 ceph-mon[194833]: 1.a scrub starts 2023-09-23T20:20:31.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:31 smithi144 ceph-mon[194833]: pgmap v1481: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5338/67719 objects degraded (7.883%); 2862/67719 objects misplaced (4.226%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:31.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:31 smithi135 ceph-mon[213871]: 1.a scrub starts 2023-09-23T20:20:31.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:31 smithi135 ceph-mon[213871]: pgmap v1481: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5338/67719 objects degraded (7.883%); 2862/67719 objects misplaced (4.226%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:31.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:31 smithi135 ceph-mon[221113]: 1.a scrub starts 2023-09-23T20:20:31.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:31 smithi135 ceph-mon[221113]: pgmap v1481: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 110 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5338/67719 objects degraded (7.883%); 2862/67719 objects misplaced (4.226%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:32.010 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:32.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:32.116 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:32.141 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:32.183 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:32.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:32.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:32.288 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:32.315 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:32.363 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:32.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:32.469 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:32.518 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:20:32.536 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:32.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:32 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:20:32.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:32 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:20:32.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:32 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:20:32.923 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:20:32.923 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:20:32.924 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-09-23T20:20:32.924 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:20:33.184 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:33.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:33.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:33.288 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:33.291 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:20:33.408 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5338/67719 objects degraded (7.883%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:33 smithi144 ceph-mon[194833]: pgmap v1482: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5338/62265 objects degraded (8.573%); 2685/62265 objects misplaced (4.312%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:33.663 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:33.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:33.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5338/67719 objects degraded (7.883%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:33.674 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:33 smithi135 ceph-mon[221113]: pgmap v1482: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5338/62265 objects degraded (8.573%); 2685/62265 objects misplaced (4.312%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:33.674 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5338/67719 objects degraded (7.883%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:33.674 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:33 smithi135 ceph-mon[213871]: pgmap v1482: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5338/62265 objects degraded (8.573%); 2685/62265 objects misplaced (4.312%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:33.775 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:33.814 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:33.874 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:33.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:20:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:20:33] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-23T20:20:33.984 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:34.005 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:34.017 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:34.074 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:34.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:34.167 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:20:34.181 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:34.235 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:20:34.239 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:34.246 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:34.290 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:20:34.346 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:20:34.369 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:34.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:34.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:34.404 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:20:34.445 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:20:34.480 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:20:34.480 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:20:34.480 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:20:34.480 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:20:34.481 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:20:34.481 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:20:34.481 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:20:34.481 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:20:34.481 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:20:34.481 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:34.514 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:34 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:20:34.514 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:34 smithi135 ceph-mon[213871]: 1.c scrub starts 2023-09-23T20:20:34.514 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:34 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:20:34.514 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:34 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2553451785' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:34.514 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:34 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:20:34.515 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:34 smithi135 ceph-mon[221113]: 1.c scrub starts 2023-09-23T20:20:34.515 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:34 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:20:34.515 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:34 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2553451785' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:34.558 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:34.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:34 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:20:34.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:34 smithi144 ceph-mon[194833]: 1.c scrub starts 2023-09-23T20:20:34.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:34 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:20:34.884 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:34 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2553451785' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:34.900 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:34.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:34.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:35.011 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:35.039 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:35.101 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:35.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:35.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:35.209 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:35.246 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:35.316 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:35.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:35.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:35.420 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:35.448 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:35.490 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:35.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:35.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:35.597 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:35 smithi144 ceph-mon[194833]: 1.15 scrub starts 2023-09-23T20:20:35.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:35 smithi144 ceph-mon[194833]: 1.a scrub starts 2023-09-23T20:20:35.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:35 smithi144 ceph-mon[194833]: pgmap v1483: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5338/59949 objects degraded (8.904%); 2494/59949 objects misplaced (4.160%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:35.652 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:35.664 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:35 smithi135 ceph-mon[213871]: 1.15 scrub starts 2023-09-23T20:20:35.664 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:35 smithi135 ceph-mon[213871]: 1.a scrub starts 2023-09-23T20:20:35.665 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:35 smithi135 ceph-mon[213871]: pgmap v1483: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5338/59949 objects degraded (8.904%); 2494/59949 objects misplaced (4.160%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:35.665 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:35 smithi135 ceph-mon[221113]: 1.15 scrub starts 2023-09-23T20:20:35.665 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:35 smithi135 ceph-mon[221113]: 1.a scrub starts 2023-09-23T20:20:35.665 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:35 smithi135 ceph-mon[221113]: pgmap v1483: 119 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5338/59949 objects degraded (8.904%); 2494/59949 objects misplaced (4.160%); 0 B/s, 56 objects/s recovering 2023-09-23T20:20:35.723 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:35.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:35.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:35.827 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:35.855 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:36.444 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:20:36.492 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:20:36.526 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:36.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:36.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:36.638 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:36.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:36 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:20:36.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:36 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:36.670 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:36 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:20:36.670 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:36 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:36.670 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:36 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:20:36.670 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:36 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:36.671 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:36.792 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:36.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:36.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:36.902 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:36.953 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:37.122 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:37.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:37.227 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:37.290 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:37.519 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:37.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:37.547 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:37 smithi135 ceph-mon[221113]: 1.a scrub starts 2023-09-23T20:20:37.548 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:37 smithi135 ceph-mon[221113]: 1.e scrub starts 2023-09-23T20:20:37.548 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:37.548 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:37 smithi135 ceph-mon[221113]: pgmap v1484: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 5091/54156 objects degraded (9.401%); 2325/54156 objects misplaced (4.293%); 0 B/s, 55 objects/s recovering 2023-09-23T20:20:37.550 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:37 smithi135 ceph-mon[213871]: 1.a scrub starts 2023-09-23T20:20:37.550 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:37 smithi135 ceph-mon[213871]: 1.e scrub starts 2023-09-23T20:20:37.551 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:37.551 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:37 smithi135 ceph-mon[213871]: pgmap v1484: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 5091/54156 objects degraded (9.401%); 2325/54156 objects misplaced (4.293%); 0 B/s, 55 objects/s recovering 2023-09-23T20:20:37.626 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:37.715 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:37.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:37 smithi144 ceph-mon[194833]: 1.a scrub starts 2023-09-23T20:20:37.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:37 smithi144 ceph-mon[194833]: 1.e scrub starts 2023-09-23T20:20:37.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:37.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:37 smithi144 ceph-mon[194833]: pgmap v1484: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 5091/54156 objects degraded (9.401%); 2325/54156 objects misplaced (4.293%); 0 B/s, 55 objects/s recovering 2023-09-23T20:20:38.080 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:38.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:38.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:38.203 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:38.294 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:38.437 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:38 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:20:38.437 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:38 smithi135 ceph-mon[213871]: 1.1e scrub starts 2023-09-23T20:20:38.437 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:38 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:20:38.438 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:38 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5091/54156 objects degraded (9.401%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:38.492 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:20:38.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:38 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:20:38.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:38 smithi135 ceph-mon[221113]: 1.1e scrub starts 2023-09-23T20:20:38.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:38 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:20:38.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:38 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5091/54156 objects degraded (9.401%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:38 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:20:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:38 smithi144 ceph-mon[194833]: 1.1e scrub starts 2023-09-23T20:20:38.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:38 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:20:38.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:38 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5091/54156 objects degraded (9.401%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:39.015 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:39.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:39.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:39.130 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:39.170 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:39.292 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:39.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:39.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:39.398 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:39.436 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3466974971' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:39.452 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[213871]: pgmap v1485: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 5091/51171 objects degraded (9.949%); 2106/51171 objects misplaced (4.116%); 0 B/s, 55 objects/s recovering 2023-09-23T20:20:39.453 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:39.453 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:39.500 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:39.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:39.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:39.536 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:20:39.606 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:39.641 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:39.700 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:39.704 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3466974971' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:39.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:39 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:39.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:39 smithi144 ceph-mon[194833]: pgmap v1485: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 5091/51171 objects degraded (9.949%); 2106/51171 objects misplaced (4.116%); 0 B/s, 55 objects/s recovering 2023-09-23T20:20:39.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:39.705 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:39.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:39.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:39.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3466974971' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:39.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:20:39.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[221113]: pgmap v1485: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 1.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 5091/51171 objects degraded (9.949%); 2106/51171 objects misplaced (4.116%); 0 B/s, 55 objects/s recovering 2023-09-23T20:20:39.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:39.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:39.732 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:20:39.780 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:20:39.807 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:39.839 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:39.892 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:39.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:39.999 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:40.033 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:40.091 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:40.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:40.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:40.204 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:40.285 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:40.563 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:20:40.563 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:20:40.563 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:20:40.563 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:20:40.563 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:20:40.569 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:20:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[213871]: failed to encode map e522 with expected crc 2023-09-23T20:20:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:20:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[213871]: osdmap e522: 8 total, 7 up, 8 in 2023-09-23T20:20:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[213871]: failed to encode map e522 with expected crc 2023-09-23T20:20:40.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[213871]: from='client.46625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:20:40.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:20:40.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[221113]: failed to encode map e522 with expected crc 2023-09-23T20:20:40.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:20:40.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[221113]: osdmap e522: 8 total, 7 up, 8 in 2023-09-23T20:20:40.730 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[221113]: failed to encode map e522 with expected crc 2023-09-23T20:20:40.731 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:40 smithi135 ceph-mon[221113]: from='client.46625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:20:40.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:40 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:20:40.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:40 smithi144 ceph-mon[194833]: failed to encode map e522 with expected crc 2023-09-23T20:20:40.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:40 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:20:40.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:40 smithi144 ceph-mon[194833]: osdmap e522: 8 total, 7 up, 8 in 2023-09-23T20:20:40.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:40 smithi144 ceph-mon[194833]: failed to encode map e522 with expected crc 2023-09-23T20:20:40.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:40 smithi144 ceph-mon[194833]: from='client.46625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:20:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:41 smithi135 ceph-mon[213871]: 1.a scrub starts 2023-09-23T20:20:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:41 smithi135 ceph-mon[213871]: pgmap v1487: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 1017 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 4811/48690 objects degraded (9.881%); 2106/48690 objects misplaced (4.325%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:41.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:41 smithi135 ceph-mon[221113]: 1.a scrub starts 2023-09-23T20:20:41.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:41 smithi135 ceph-mon[221113]: pgmap v1487: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 1017 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 4811/48690 objects degraded (9.881%); 2106/48690 objects misplaced (4.325%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:41 smithi144 ceph-mon[194833]: 1.a scrub starts 2023-09-23T20:20:41.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:41 smithi144 ceph-mon[194833]: pgmap v1487: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 107 active+clean; 1017 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 4811/48690 objects degraded (9.881%); 2106/48690 objects misplaced (4.325%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:42.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:42 smithi135 ceph-mon[221113]: 1.15 scrub starts 2023-09-23T20:20:42.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:42 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:20:42.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:42 smithi135 ceph-mon[221113]: 1.1c deep-scrub starts 2023-09-23T20:20:42.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:42 smithi135 ceph-mon[213871]: 1.15 scrub starts 2023-09-23T20:20:42.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:42 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:20:42.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:42 smithi135 ceph-mon[213871]: 1.1c deep-scrub starts 2023-09-23T20:20:42.860 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:42.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:42.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:42.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:42 smithi144 ceph-mon[194833]: 1.15 scrub starts 2023-09-23T20:20:42.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:42 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:20:42.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:42 smithi144 ceph-mon[194833]: 1.1c deep-scrub starts 2023-09-23T20:20:42.910 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1486,"stamp":"2023-09-23T20:20:39.472523+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1120773007,"num_objects":17057,"num_object_clones":0,"num_object_copies":51171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5091,"num_objects_misplaced":2106,"num_objects_unfound":0,"num_objects_dirty":17057,"num_whiteouts":0,"num_read":6011,"num_read_kb":13264,"num_write":168341,"num_write_kb":5915145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10701,"num_bytes_recovered":618253939,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107929,"ondisk_log_size":107929,"up":350,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10056652,"kb_used_data":4387916,"kb_used_omap":38,"kb_used_meta":5668185,"kb_avail":739904564,"statfs":{"total":767960285184,"available":757662273536,"internally_reserved":0,"allocated":4493225984,"data_stored":4476873665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39803,"internal_metadata":5804221573},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,7,1,2,5,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-410517504,"num_objects":-6264,"num_object_clones":0,"num_object_copies":-18792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-554,"num_objects_misplaced":-756,"num_objects_unfound":0,"num_objects_dirty":-6264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6264,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":554,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.112881"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":891,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372311+0000","last_change":"2023-09-23T20:19:54.928200+0000","last_active":"2023-09-23T20:20:18.372311+0000","last_peered":"2023-09-23T20:20:18.372311+0000","last_clean":"2023-09-23T20:20:18.372311+0000","last_became_active":"2023-09-23T20:19:05.089810+0000","last_became_peered":"2023-09-23T20:19:05.089810+0000","last_unstale":"2023-09-23T20:20:18.372311+0000","last_undegraded":"2023-09-23T20:20:18.372311+0000","last_fullsized":"2023-09-23T20:20:18.372311+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:54.928103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:54.928103+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:08.492041+0000","scrub_duration":0.011135947,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:19.397966+0000","last_change":"2023-09-23T20:17:47.279267+0000","last_active":"2023-09-23T20:20:19.397966+0000","last_peered":"2023-09-23T20:20:19.397966+0000","last_clean":"2023-09-23T20:20:19.397966+0000","last_became_active":"2023-09-23T20:17:47.278920+0000","last_became_peered":"2023-09-23T20:17:47.278920+0000","last_unstale":"2023-09-23T20:20:19.397966+0000","last_undegraded":"2023-09-23T20:20:19.397966+0000","last_fullsized":"2023-09-23T20:20:19.397966+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:16.699236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:19.398094+0000","last_change":"2023-09-23T20:17:47.270409+0000","last_active":"2023-09-23T20:20:19.398094+0000","last_peered":"2023-09-23T20:20:19.398094+0000","last_clean":"2023-09-23T20:20:19.398094+0000","last_became_active":"2023-09-23T20:17:47.270086+0000","last_became_peered":"2023-09-23T20:17:47.270086+0000","last_unstale":"2023-09-23T20:20:19.398094+0000","last_undegraded":"2023-09-23T20:20:19.398094+0000","last_fullsized":"2023-09-23T20:20:19.398094+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.885372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":584,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369281+0000","last_change":"2023-09-23T20:19:05.090444+0000","last_active":"2023-09-23T20:20:18.369281+0000","last_peered":"2023-09-23T20:20:18.369281+0000","last_clean":"2023-09-23T20:20:18.369281+0000","last_became_active":"2023-09-23T20:19:05.090104+0000","last_became_peered":"2023-09-23T20:19:05.090104+0000","last_unstale":"2023-09-23T20:20:18.369281+0000","last_undegraded":"2023-09-23T20:20:18.369281+0000","last_fullsized":"2023-09-23T20:20:18.369281+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.370317+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:20:18.370317+0000","last_peered":"2023-09-23T20:20:18.370317+0000","last_clean":"2023-09-23T20:20:18.370317+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:20:18.370317+0000","last_undegraded":"2023-09-23T20:20:18.370317+0000","last_fullsized":"2023-09-23T20:20:18.370317+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":229,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387087+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:20:18.387087+0000","last_peered":"2023-09-23T20:20:18.387087+0000","last_clean":"2023-09-23T20:20:18.387087+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:20:18.387087+0000","last_undegraded":"2023-09-23T20:20:18.387087+0000","last_fullsized":"2023-09-23T20:20:18.387087+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":886,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:37.384983+0000","last_change":"2023-09-23T20:20:37.384983+0000","last_active":"2023-09-23T20:20:37.384983+0000","last_peered":"2023-09-23T20:20:37.384983+0000","last_clean":"2023-09-23T20:20:37.384983+0000","last_became_active":"2023-09-23T20:19:05.067520+0000","last_became_peered":"2023-09-23T20:19:05.067520+0000","last_unstale":"2023-09-23T20:20:37.384983+0000","last_undegraded":"2023-09-23T20:20:37.384983+0000","last_fullsized":"2023-09-23T20:20:37.384983+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369361+0000","last_change":"2023-09-23T20:17:47.279385+0000","last_active":"2023-09-23T20:20:18.369361+0000","last_peered":"2023-09-23T20:20:18.369361+0000","last_clean":"2023-09-23T20:20:18.369361+0000","last_became_active":"2023-09-23T20:17:47.279065+0000","last_became_peered":"2023-09-23T20:17:47.279065+0000","last_unstale":"2023-09-23T20:20:18.369361+0000","last_undegraded":"2023-09-23T20:20:18.369361+0000","last_fullsized":"2023-09-23T20:20:18.369361+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:19.638157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":486,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.373633+0000","last_change":"2023-09-23T20:17:47.270621+0000","last_active":"2023-09-23T20:20:18.373633+0000","last_peered":"2023-09-23T20:20:18.373633+0000","last_clean":"2023-09-23T20:20:18.373633+0000","last_became_active":"2023-09-23T20:17:47.270375+0000","last_became_peered":"2023-09-23T20:17:47.270375+0000","last_unstale":"2023-09-23T20:20:18.373633+0000","last_undegraded":"2023-09-23T20:20:18.373633+0000","last_fullsized":"2023-09-23T20:20:18.373633+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:16.961271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":767,"reported_epoch":521,"state":"active+clean+scrubbing","last_fresh":"2023-09-23T20:20:29.374493+0000","last_change":"2023-09-23T20:20:33.708153+0000","last_active":"2023-09-23T20:20:29.374493+0000","last_peered":"2023-09-23T20:20:29.374493+0000","last_clean":"2023-09-23T20:20:29.374493+0000","last_became_active":"2023-09-23T20:19:05.069486+0000","last_became_peered":"2023-09-23T20:19:05.069486+0000","last_unstale":"2023-09-23T20:20:29.374493+0000","last_undegraded":"2023-09-23T20:20:29.374493+0000","last_fullsized":"2023-09-23T20:20:29.374493+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":447,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.373556+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:20:18.373556+0000","last_peered":"2023-09-23T20:20:18.373556+0000","last_clean":"2023-09-23T20:20:18.373556+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:20:18.373556+0000","last_undegraded":"2023-09-23T20:20:18.373556+0000","last_fullsized":"2023-09-23T20:20:18.373556+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":386,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391399+0000","last_change":"2023-09-23T20:17:47.265452+0000","last_active":"2023-09-23T20:20:18.391399+0000","last_peered":"2023-09-23T20:20:18.391399+0000","last_clean":"2023-09-23T20:20:18.391399+0000","last_became_active":"2023-09-23T20:17:47.265115+0000","last_became_peered":"2023-09-23T20:17:47.265115+0000","last_unstale":"2023-09-23T20:20:18.391399+0000","last_undegraded":"2023-09-23T20:20:18.391399+0000","last_fullsized":"2023-09-23T20:20:18.391399+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:04.898742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":856,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.395439+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:20:18.395439+0000","last_peered":"2023-09-23T20:20:18.395439+0000","last_clean":"2023-09-23T20:20:18.395439+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:20:18.395439+0000","last_undegraded":"2023-09-23T20:20:18.395439+0000","last_fullsized":"2023-09-23T20:20:18.395439+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":540,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.373369+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:20:18.373369+0000","last_peered":"2023-09-23T20:20:18.373369+0000","last_clean":"2023-09-23T20:20:18.373369+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:20:18.373369+0000","last_undegraded":"2023-09-23T20:20:18.373369+0000","last_fullsized":"2023-09-23T20:20:18.373369+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":387,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391612+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:20:18.391612+0000","last_peered":"2023-09-23T20:20:18.391612+0000","last_clean":"2023-09-23T20:20:18.391612+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:20:18.391612+0000","last_undegraded":"2023-09-23T20:20:18.391612+0000","last_fullsized":"2023-09-23T20:20:18.391612+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":825,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.392865+0000","last_change":"2023-09-23T20:19:42.182813+0000","last_active":"2023-09-23T20:20:18.392865+0000","last_peered":"2023-09-23T20:20:18.392865+0000","last_clean":"2023-09-23T20:20:18.392865+0000","last_became_active":"2023-09-23T20:19:05.072575+0000","last_became_peered":"2023-09-23T20:19:05.072575+0000","last_unstale":"2023-09-23T20:20:18.392865+0000","last_undegraded":"2023-09-23T20:20:18.392865+0000","last_fullsized":"2023-09-23T20:20:18.392865+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:42.182718+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:42.182718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:00.054669+0000","scrub_duration":0.0014414230000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":214,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387593+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:20:18.387593+0000","last_peered":"2023-09-23T20:20:18.387593+0000","last_clean":"2023-09-23T20:20:18.387593+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:20:18.387593+0000","last_undegraded":"2023-09-23T20:20:18.387593+0000","last_fullsized":"2023-09-23T20:20:18.387593+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.370998+0000","last_change":"2023-09-23T20:17:47.278773+0000","last_active":"2023-09-23T20:20:18.370998+0000","last_peered":"2023-09-23T20:20:18.370998+0000","last_clean":"2023-09-23T20:20:18.370998+0000","last_became_active":"2023-09-23T20:17:47.278467+0000","last_became_peered":"2023-09-23T20:17:47.278467+0000","last_unstale":"2023-09-23T20:20:18.370998+0000","last_undegraded":"2023-09-23T20:20:18.370998+0000","last_fullsized":"2023-09-23T20:20:18.370998+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:41.368483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":716,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.390758+0000","last_change":"2023-09-23T20:19:05.092744+0000","last_active":"2023-09-23T20:20:18.390758+0000","last_peered":"2023-09-23T20:20:18.390758+0000","last_clean":"2023-09-23T20:20:18.390758+0000","last_became_active":"2023-09-23T20:19:05.092425+0000","last_became_peered":"2023-09-23T20:19:05.092425+0000","last_unstale":"2023-09-23T20:20:18.390758+0000","last_undegraded":"2023-09-23T20:20:18.390758+0000","last_fullsized":"2023-09-23T20:20:18.390758+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":467,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.386915+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:20:18.386915+0000","last_peered":"2023-09-23T20:20:18.386915+0000","last_clean":"2023-09-23T20:20:18.386915+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:20:18.386915+0000","last_undegraded":"2023-09-23T20:20:18.386915+0000","last_fullsized":"2023-09-23T20:20:18.386915+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":41,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T20:20:18.369728+0000","last_change":"2023-09-23T20:18:38.012263+0000","last_active":"2023-09-23T20:20:18.369728+0000","last_peered":"2023-09-23T20:20:18.369728+0000","last_clean":"2023-09-23T20:17:54.133085+0000","last_became_active":"2023-09-23T20:18:38.012263+0000","last_became_peered":"2023-09-23T20:18:38.012263+0000","last_unstale":"2023-09-23T20:20:18.369728+0000","last_undegraded":"2023-09-23T20:20:18.369728+0000","last_fullsized":"2023-09-23T20:18:38.003813+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:47.449647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":880,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372422+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:20:18.372422+0000","last_peered":"2023-09-23T20:20:18.372422+0000","last_clean":"2023-09-23T20:20:18.372422+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:20:18.372422+0000","last_undegraded":"2023-09-23T20:20:18.372422+0000","last_fullsized":"2023-09-23T20:20:18.372422+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":214,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387719+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:20:18.387719+0000","last_peered":"2023-09-23T20:20:18.387719+0000","last_clean":"2023-09-23T20:20:18.387719+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:20:18.387719+0000","last_undegraded":"2023-09-23T20:20:18.387719+0000","last_fullsized":"2023-09-23T20:20:18.387719+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371399+0000","last_change":"2023-09-23T20:17:47.266963+0000","last_active":"2023-09-23T20:20:18.371399+0000","last_peered":"2023-09-23T20:20:18.371399+0000","last_clean":"2023-09-23T20:20:18.371399+0000","last_became_active":"2023-09-23T20:17:47.266645+0000","last_became_peered":"2023-09-23T20:17:47.266645+0000","last_unstale":"2023-09-23T20:20:18.371399+0000","last_undegraded":"2023-09-23T20:20:18.371399+0000","last_fullsized":"2023-09-23T20:20:18.371399+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:05.052559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":569,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372511+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:20:18.372511+0000","last_peered":"2023-09-23T20:20:18.372511+0000","last_clean":"2023-09-23T20:20:18.372511+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:20:18.372511+0000","last_undegraded":"2023-09-23T20:20:18.372511+0000","last_fullsized":"2023-09-23T20:20:18.372511+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":229,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.386662+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:20:18.386662+0000","last_peered":"2023-09-23T20:20:18.386662+0000","last_clean":"2023-09-23T20:20:18.386662+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:20:18.386662+0000","last_undegraded":"2023-09-23T20:20:18.386662+0000","last_fullsized":"2023-09-23T20:20:18.386662+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":519,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.373060+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:20:18.373060+0000","last_peered":"2023-09-23T20:20:18.373060+0000","last_clean":"2023-09-23T20:20:18.373060+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:20:18.373060+0000","last_undegraded":"2023-09-23T20:20:18.373060+0000","last_fullsized":"2023-09-23T20:20:18.373060+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":580,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369837+0000","last_change":"2023-09-23T20:17:50.330492+0000","last_active":"2023-09-23T20:20:18.369837+0000","last_peered":"2023-09-23T20:20:18.369837+0000","last_clean":"2023-09-23T20:20:18.369837+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:20:18.369837+0000","last_undegraded":"2023-09-23T20:20:18.369837+0000","last_fullsized":"2023-09-23T20:20:18.369837+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:58.248167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.248167+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":225,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T20:20:18.387763+0000","last_change":"2023-09-23T20:18:38.011856+0000","last_active":"2023-09-23T20:20:18.387763+0000","last_peered":"2023-09-23T20:20:18.387763+0000","last_clean":"2023-09-23T20:18:35.260354+0000","last_became_active":"2023-09-23T20:18:38.011856+0000","last_became_peered":"2023-09-23T20:18:38.011856+0000","last_unstale":"2023-09-23T20:20:18.387763+0000","last_undegraded":"2023-09-23T20:20:18.387763+0000","last_fullsized":"2023-09-23T20:18:37.999747+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:58.093947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":511,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:27.119703+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:20:27.119703+0000","last_peered":"2023-09-23T20:20:27.119703+0000","last_clean":"2023-09-23T20:20:27.119703+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:20:27.119703+0000","last_undegraded":"2023-09-23T20:20:27.119703+0000","last_fullsized":"2023-09-23T20:20:27.119703+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":678,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372602+0000","last_change":"2023-09-23T20:17:50.332129+0000","last_active":"2023-09-23T20:20:18.372602+0000","last_peered":"2023-09-23T20:20:18.372602+0000","last_clean":"2023-09-23T20:20:18.372602+0000","last_became_active":"2023-09-23T20:17:50.331669+0000","last_became_peered":"2023-09-23T20:17:50.331669+0000","last_unstale":"2023-09-23T20:20:18.372602+0000","last_undegraded":"2023-09-23T20:20:18.372602+0000","last_fullsized":"2023-09-23T20:20:18.372602+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:23.893417+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":164,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372363+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:20:18.372363+0000","last_peered":"2023-09-23T20:20:18.372363+0000","last_clean":"2023-09-23T20:20:18.372363+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:20:18.372363+0000","last_undegraded":"2023-09-23T20:20:18.372363+0000","last_fullsized":"2023-09-23T20:20:18.372363+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":172,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.392638+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:20:18.392638+0000","last_peered":"2023-09-23T20:20:18.392638+0000","last_clean":"2023-09-23T20:20:18.392638+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:20:18.392638+0000","last_undegraded":"2023-09-23T20:20:18.392638+0000","last_fullsized":"2023-09-23T20:20:18.392638+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"521'10542","reported_seq":10592,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:38.579642+0000","last_change":"2023-09-23T20:16:59.490329+0000","last_active":"2023-09-23T20:20:38.579642+0000","last_peered":"2023-09-23T20:20:38.579642+0000","last_clean":"2023-09-23T20:20:38.579642+0000","last_became_active":"2023-09-23T20:16:59.490069+0000","last_became_peered":"2023-09-23T20:16:59.490069+0000","last_unstale":"2023-09-23T20:20:38.579642+0000","last_undegraded":"2023-09-23T20:20:38.579642+0000","last_fullsized":"2023-09-23T20:20:38.579642+0000","mapping_epoch":479,"log_start":"492'3933","ondisk_log_start":"492'3933","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.487350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66453504,"num_objects":1014,"num_object_clones":0,"num_object_copies":3042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10542,"num_write_kb":369792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":820,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:33.452164+0000","last_change":"2023-09-23T20:20:33.452164+0000","last_active":"2023-09-23T20:20:33.452164+0000","last_peered":"2023-09-23T20:20:33.452164+0000","last_clean":"2023-09-23T20:20:33.452164+0000","last_became_active":"2023-09-23T20:19:05.065283+0000","last_became_peered":"2023-09-23T20:19:05.065283+0000","last_unstale":"2023-09-23T20:20:33.452164+0000","last_undegraded":"2023-09-23T20:20:33.452164+0000","last_fullsized":"2023-09-23T20:20:33.452164+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":212,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387590+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:20:18.387590+0000","last_peered":"2023-09-23T20:20:18.387590+0000","last_clean":"2023-09-23T20:20:18.387590+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:20:18.387590+0000","last_undegraded":"2023-09-23T20:20:18.387590+0000","last_fullsized":"2023-09-23T20:20:18.387590+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":529,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372411+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:20:18.372411+0000","last_peered":"2023-09-23T20:20:18.372411+0000","last_clean":"2023-09-23T20:20:18.372411+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:20:18.372411+0000","last_undegraded":"2023-09-23T20:20:18.372411+0000","last_fullsized":"2023-09-23T20:20:18.372411+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"521'10443","reported_seq":10495,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:37.046753+0000","last_change":"2023-09-23T20:16:59.487459+0000","last_active":"2023-09-23T20:20:37.046753+0000","last_peered":"2023-09-23T20:20:37.046753+0000","last_clean":"2023-09-23T20:20:37.046753+0000","last_became_active":"2023-09-23T20:16:59.487181+0000","last_became_peered":"2023-09-23T20:16:59.487181+0000","last_unstale":"2023-09-23T20:20:37.046753+0000","last_undegraded":"2023-09-23T20:20:37.046753+0000","last_fullsized":"2023-09-23T20:20:37.046753+0000","mapping_epoch":479,"log_start":"499'5262","ondisk_log_start":"499'5262","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5181,"log_dups_size":0,"ondisk_log_size":5181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:34.406760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10443,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":884,"reported_epoch":521,"state":"active+clean+scrubbing","last_fresh":"2023-09-23T20:20:29.391162+0000","last_change":"2023-09-23T20:20:33.300474+0000","last_active":"2023-09-23T20:20:29.391162+0000","last_peered":"2023-09-23T20:20:29.391162+0000","last_clean":"2023-09-23T20:20:29.391162+0000","last_became_active":"2023-09-23T20:19:05.085764+0000","last_became_peered":"2023-09-23T20:19:05.085764+0000","last_unstale":"2023-09-23T20:20:29.391162+0000","last_undegraded":"2023-09-23T20:20:29.391162+0000","last_fullsized":"2023-09-23T20:20:29.391162+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":241,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387120+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:20:18.387120+0000","last_peered":"2023-09-23T20:20:18.387120+0000","last_clean":"2023-09-23T20:20:18.387120+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:20:18.387120+0000","last_undegraded":"2023-09-23T20:20:18.387120+0000","last_fullsized":"2023-09-23T20:20:18.387120+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.370395+0000","last_change":"2023-09-23T20:17:47.264776+0000","last_active":"2023-09-23T20:20:18.370395+0000","last_peered":"2023-09-23T20:20:18.370395+0000","last_clean":"2023-09-23T20:20:18.370395+0000","last_became_active":"2023-09-23T20:17:47.264473+0000","last_became_peered":"2023-09-23T20:17:47.264473+0000","last_unstale":"2023-09-23T20:20:18.370395+0000","last_undegraded":"2023-09-23T20:20:18.370395+0000","last_fullsized":"2023-09-23T20:20:18.370395+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:40.906303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"521'10561","reported_seq":10613,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:35.956772+0000","last_change":"2023-09-23T20:16:59.491310+0000","last_active":"2023-09-23T20:20:35.956772+0000","last_peered":"2023-09-23T20:20:35.956772+0000","last_clean":"2023-09-23T20:20:35.956772+0000","last_became_active":"2023-09-23T20:16:59.491016+0000","last_became_peered":"2023-09-23T20:16:59.491016+0000","last_unstale":"2023-09-23T20:20:35.956772+0000","last_undegraded":"2023-09-23T20:20:35.956772+0000","last_fullsized":"2023-09-23T20:20:35.956772+0000","mapping_epoch":479,"log_start":"498'5228","ondisk_log_start":"498'5228","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5333,"log_dups_size":0,"ondisk_log_size":5333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:59.436079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71499776,"num_objects":1091,"num_object_clones":0,"num_object_copies":3273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10561,"num_write_kb":372864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":593,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.373479+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:20:18.373479+0000","last_peered":"2023-09-23T20:20:18.373479+0000","last_clean":"2023-09-23T20:20:18.373479+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:20:18.373479+0000","last_undegraded":"2023-09-23T20:20:18.373479+0000","last_fullsized":"2023-09-23T20:20:18.373479+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":233,"reported_epoch":520,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:20:18.387505+0000","last_change":"2023-09-23T20:18:38.001952+0000","last_active":"2023-09-23T20:20:18.387505+0000","last_peered":"2023-09-23T20:20:18.387505+0000","last_clean":"2023-09-23T20:18:35.257306+0000","last_became_active":"2023-09-23T20:18:38.001952+0000","last_became_peered":"2023-09-23T20:18:38.001952+0000","last_unstale":"2023-09-23T20:20:18.387505+0000","last_undegraded":"2023-09-23T20:18:38.000196+0000","last_fullsized":"2023-09-23T20:18:38.000044+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:17.507792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371071+0000","last_change":"2023-09-23T20:17:47.266977+0000","last_active":"2023-09-23T20:20:18.371071+0000","last_peered":"2023-09-23T20:20:18.371071+0000","last_clean":"2023-09-23T20:20:18.371071+0000","last_became_active":"2023-09-23T20:17:47.266657+0000","last_became_peered":"2023-09-23T20:17:47.266657+0000","last_unstale":"2023-09-23T20:20:18.371071+0000","last_undegraded":"2023-09-23T20:20:18.371071+0000","last_fullsized":"2023-09-23T20:20:18.371071+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:33.774981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"521'10632","reported_seq":10684,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:36.987047+0000","last_change":"2023-09-23T20:16:59.488990+0000","last_active":"2023-09-23T20:20:36.987047+0000","last_peered":"2023-09-23T20:20:36.987047+0000","last_clean":"2023-09-23T20:20:36.987047+0000","last_became_active":"2023-09-23T20:16:59.488614+0000","last_became_peered":"2023-09-23T20:16:59.488614+0000","last_unstale":"2023-09-23T20:20:36.987047+0000","last_undegraded":"2023-09-23T20:20:36.987047+0000","last_fullsized":"2023-09-23T20:20:36.987047+0000","mapping_epoch":479,"log_start":"485'2500","ondisk_log_start":"485'2500","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8132,"log_dups_size":0,"ondisk_log_size":8132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:11.359750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71303168,"num_objects":1088,"num_object_clones":0,"num_object_copies":3264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10632,"num_write_kb":375040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":743,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:36.371390+0000","last_change":"2023-09-23T20:20:36.371390+0000","last_active":"2023-09-23T20:20:36.371390+0000","last_peered":"2023-09-23T20:20:36.371390+0000","last_clean":"2023-09-23T20:20:36.371390+0000","last_became_active":"2023-09-23T20:19:05.071700+0000","last_became_peered":"2023-09-23T20:19:05.071700+0000","last_unstale":"2023-09-23T20:20:36.371390+0000","last_undegraded":"2023-09-23T20:20:36.371390+0000","last_fullsized":"2023-09-23T20:20:36.371390+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":579,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.373202+0000","last_change":"2023-09-23T20:17:47.267657+0000","last_active":"2023-09-23T20:20:18.373202+0000","last_peered":"2023-09-23T20:20:18.373202+0000","last_clean":"2023-09-23T20:20:18.373202+0000","last_became_active":"2023-09-23T20:17:47.267288+0000","last_became_peered":"2023-09-23T20:17:47.267288+0000","last_unstale":"2023-09-23T20:20:18.373202+0000","last_undegraded":"2023-09-23T20:20:18.373202+0000","last_fullsized":"2023-09-23T20:20:18.373202+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.185253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369487+0000","last_change":"2023-09-23T20:17:47.280675+0000","last_active":"2023-09-23T20:20:18.369487+0000","last_peered":"2023-09-23T20:20:18.369487+0000","last_clean":"2023-09-23T20:20:18.369487+0000","last_became_active":"2023-09-23T20:17:47.280175+0000","last_became_peered":"2023-09-23T20:17:47.280175+0000","last_unstale":"2023-09-23T20:20:18.369487+0000","last_undegraded":"2023-09-23T20:20:18.369487+0000","last_fullsized":"2023-09-23T20:20:18.369487+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:08.499949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"521'10466","reported_seq":10522,"reported_epoch":521,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:20:35.930520+0000","last_change":"2023-09-23T20:17:48.443980+0000","last_active":"2023-09-23T20:20:35.930520+0000","last_peered":"2023-09-23T20:20:35.930520+0000","last_clean":"2023-09-23T20:17:46.225624+0000","last_became_active":"2023-09-23T20:17:48.428406+0000","last_became_peered":"2023-09-23T20:17:48.428406+0000","last_unstale":"2023-09-23T20:20:35.930520+0000","last_undegraded":"2023-09-23T20:17:48.291655+0000","last_fullsized":"2023-09-23T20:17:48.256072+0000","mapping_epoch":490,"log_start":"498'5147","ondisk_log_start":"498'5147","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5319,"log_dups_size":0,"ondisk_log_size":5319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:14.276994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70778880,"num_objects":1080,"num_object_clones":0,"num_object_copies":3240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3930,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10466,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[0,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,4","objects":-2850}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":923,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387671+0000","last_change":"2023-09-23T20:19:47.182002+0000","last_active":"2023-09-23T20:20:18.387671+0000","last_peered":"2023-09-23T20:20:18.387671+0000","last_clean":"2023-09-23T20:20:18.387671+0000","last_became_active":"2023-09-23T20:19:05.071737+0000","last_became_peered":"2023-09-23T20:19:05.071737+0000","last_unstale":"2023-09-23T20:20:18.387671+0000","last_undegraded":"2023-09-23T20:20:18.387671+0000","last_fullsized":"2023-09-23T20:20:18.387671+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:47.181912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:47.181912+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:28.873453+0000","scrub_duration":0.0074047540000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":268,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391382+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:20:18.391382+0000","last_peered":"2023-09-23T20:20:18.391382+0000","last_clean":"2023-09-23T20:20:18.391382+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:20:18.391382+0000","last_undegraded":"2023-09-23T20:20:18.391382+0000","last_fullsized":"2023-09-23T20:20:18.391382+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371485+0000","last_change":"2023-09-23T20:17:47.270450+0000","last_active":"2023-09-23T20:20:18.371485+0000","last_peered":"2023-09-23T20:20:18.371485+0000","last_clean":"2023-09-23T20:20:18.371485+0000","last_became_active":"2023-09-23T20:17:47.270040+0000","last_became_peered":"2023-09-23T20:17:47.270040+0000","last_unstale":"2023-09-23T20:20:18.371485+0000","last_undegraded":"2023-09-23T20:20:18.371485+0000","last_fullsized":"2023-09-23T20:20:18.371485+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:51.572016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.3","version":"521'10426","reported_seq":10478,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:36.968629+0000","last_change":"2023-09-23T20:16:59.488642+0000","last_active":"2023-09-23T20:20:36.968629+0000","last_peered":"2023-09-23T20:20:36.968629+0000","last_clean":"2023-09-23T20:20:36.968629+0000","last_became_active":"2023-09-23T20:16:59.488330+0000","last_became_peered":"2023-09-23T20:16:59.488330+0000","last_unstale":"2023-09-23T20:20:36.968629+0000","last_undegraded":"2023-09-23T20:20:36.968629+0000","last_fullsized":"2023-09-23T20:20:36.968629+0000","mapping_epoch":479,"log_start":"485'2300","ondisk_log_start":"485'2300","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:46.429096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71303168,"num_objects":1088,"num_object_clones":0,"num_object_copies":3264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10426,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":894,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372451+0000","last_change":"2023-09-23T20:19:43.954300+0000","last_active":"2023-09-23T20:20:18.372451+0000","last_peered":"2023-09-23T20:20:18.372451+0000","last_clean":"2023-09-23T20:20:18.372451+0000","last_became_active":"2023-09-23T20:19:05.075801+0000","last_became_peered":"2023-09-23T20:19:05.075801+0000","last_unstale":"2023-09-23T20:20:18.372451+0000","last_undegraded":"2023-09-23T20:20:18.372451+0000","last_fullsized":"2023-09-23T20:20:18.372451+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:43.954204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:43.954204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:52.060677+0000","scrub_duration":0.0019293979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.370847+0000","last_change":"2023-09-23T20:17:47.266513+0000","last_active":"2023-09-23T20:20:18.370847+0000","last_peered":"2023-09-23T20:20:18.370847+0000","last_clean":"2023-09-23T20:20:18.370847+0000","last_became_active":"2023-09-23T20:17:47.266163+0000","last_became_peered":"2023-09-23T20:17:47.266163+0000","last_unstale":"2023-09-23T20:20:18.370847+0000","last_undegraded":"2023-09-23T20:20:18.370847+0000","last_fullsized":"2023-09-23T20:20:18.370847+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.645381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":432,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.397464+0000","last_change":"2023-09-23T20:17:47.270131+0000","last_active":"2023-09-23T20:20:18.397464+0000","last_peered":"2023-09-23T20:20:18.397464+0000","last_clean":"2023-09-23T20:20:18.397464+0000","last_became_active":"2023-09-23T20:17:47.269818+0000","last_became_peered":"2023-09-23T20:17:47.269818+0000","last_unstale":"2023-09-23T20:20:18.397464+0000","last_undegraded":"2023-09-23T20:20:18.397464+0000","last_fullsized":"2023-09-23T20:20:18.397464+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:40.738070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"521'10321","reported_seq":18108,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:34.404378+0000","last_change":"2023-09-23T20:19:19.046525+0000","last_active":"2023-09-23T20:20:34.404378+0000","last_peered":"2023-09-23T20:20:34.404378+0000","last_clean":"2023-09-23T20:20:34.404378+0000","last_became_active":"2023-09-23T20:19:19.036483+0000","last_became_peered":"2023-09-23T20:19:19.036483+0000","last_unstale":"2023-09-23T20:20:34.404378+0000","last_undegraded":"2023-09-23T20:20:34.404378+0000","last_fullsized":"2023-09-23T20:20:34.404378+0000","mapping_epoch":511,"log_start":"494'4360","ondisk_log_start":"494'4360","created":479,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5961,"log_dups_size":0,"ondisk_log_size":5961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:37.023435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76218368,"num_objects":1163,"num_object_clones":0,"num_object_copies":3489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10321,"num_write_kb":367488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3865,"num_bytes_recovered":253296640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":585,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369643+0000","last_change":"2023-09-23T20:19:05.088257+0000","last_active":"2023-09-23T20:20:18.369643+0000","last_peered":"2023-09-23T20:20:18.369643+0000","last_clean":"2023-09-23T20:20:18.369643+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:20:18.369643+0000","last_undegraded":"2023-09-23T20:20:18.369643+0000","last_fullsized":"2023-09-23T20:20:18.369643+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":415,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:29.374207+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:20:29.374207+0000","last_peered":"2023-09-23T20:20:29.374207+0000","last_clean":"2023-09-23T20:20:29.374207+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:20:29.374207+0000","last_undegraded":"2023-09-23T20:20:29.374207+0000","last_fullsized":"2023-09-23T20:20:29.374207+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371108+0000","last_change":"2023-09-23T20:17:47.268811+0000","last_active":"2023-09-23T20:20:18.371108+0000","last_peered":"2023-09-23T20:20:18.371108+0000","last_clean":"2023-09-23T20:20:18.371108+0000","last_became_active":"2023-09-23T20:17:47.268391+0000","last_became_peered":"2023-09-23T20:17:47.268391+0000","last_unstale":"2023-09-23T20:20:18.371108+0000","last_undegraded":"2023-09-23T20:20:18.371108+0000","last_fullsized":"2023-09-23T20:20:18.371108+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:32.388705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"521'10513","reported_seq":10563,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:38.156110+0000","last_change":"2023-09-23T20:16:59.496645+0000","last_active":"2023-09-23T20:20:38.156110+0000","last_peered":"2023-09-23T20:20:38.156110+0000","last_clean":"2023-09-23T20:20:38.156110+0000","last_became_active":"2023-09-23T20:16:59.496430+0000","last_became_peered":"2023-09-23T20:16:59.496430+0000","last_unstale":"2023-09-23T20:20:38.156110+0000","last_undegraded":"2023-09-23T20:20:38.156110+0000","last_fullsized":"2023-09-23T20:20:38.156110+0000","mapping_epoch":479,"log_start":"494'4342","ondisk_log_start":"494'4342","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6171,"log_dups_size":0,"ondisk_log_size":6171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:59.713976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65732608,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10513,"num_write_kb":368512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":893,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:36.360070+0000","last_change":"2023-09-23T20:20:36.360070+0000","last_active":"2023-09-23T20:20:36.360070+0000","last_peered":"2023-09-23T20:20:36.360070+0000","last_clean":"2023-09-23T20:20:36.360070+0000","last_became_active":"2023-09-23T20:19:05.066352+0000","last_became_peered":"2023-09-23T20:19:05.066352+0000","last_unstale":"2023-09-23T20:20:36.360070+0000","last_undegraded":"2023-09-23T20:20:36.360070+0000","last_fullsized":"2023-09-23T20:20:36.360070+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":173,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391096+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:20:18.391096+0000","last_peered":"2023-09-23T20:20:18.391096+0000","last_clean":"2023-09-23T20:20:18.391096+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:20:18.391096+0000","last_undegraded":"2023-09-23T20:20:18.391096+0000","last_fullsized":"2023-09-23T20:20:18.391096+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371355+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:20:18.371355+0000","last_peered":"2023-09-23T20:20:18.371355+0000","last_clean":"2023-09-23T20:20:18.371355+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:20:18.371355+0000","last_undegraded":"2023-09-23T20:20:18.371355+0000","last_fullsized":"2023-09-23T20:20:18.371355+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"521'10467","reported_seq":10517,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:38.146017+0000","last_change":"2023-09-23T20:16:59.496291+0000","last_active":"2023-09-23T20:20:38.146017+0000","last_peered":"2023-09-23T20:20:38.146017+0000","last_clean":"2023-09-23T20:20:38.146017+0000","last_became_active":"2023-09-23T20:16:59.496000+0000","last_became_peered":"2023-09-23T20:16:59.496000+0000","last_unstale":"2023-09-23T20:20:38.146017+0000","last_undegraded":"2023-09-23T20:20:38.146017+0000","last_fullsized":"2023-09-23T20:20:38.146017+0000","mapping_epoch":479,"log_start":"494'4277","ondisk_log_start":"494'4277","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6190,"log_dups_size":0,"ondisk_log_size":6190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.808639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10467,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":826,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.392721+0000","last_change":"2023-09-23T20:19:40.219448+0000","last_active":"2023-09-23T20:20:18.392721+0000","last_peered":"2023-09-23T20:20:18.392721+0000","last_clean":"2023-09-23T20:20:18.392721+0000","last_became_active":"2023-09-23T20:19:05.072187+0000","last_became_peered":"2023-09-23T20:19:05.072187+0000","last_unstale":"2023-09-23T20:20:18.392721+0000","last_undegraded":"2023-09-23T20:20:18.392721+0000","last_fullsized":"2023-09-23T20:20:18.392721+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:40.219310+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:40.219310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:29.003073+0000","scrub_duration":0.011178762,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":472,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371256+0000","last_change":"2023-09-23T20:17:50.311646+0000","last_active":"2023-09-23T20:20:18.371256+0000","last_peered":"2023-09-23T20:20:18.371256+0000","last_clean":"2023-09-23T20:20:18.371256+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:20:18.371256+0000","last_undegraded":"2023-09-23T20:20:18.371256+0000","last_fullsized":"2023-09-23T20:20:18.371256+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":521,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372984+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:20:18.372984+0000","last_peered":"2023-09-23T20:20:18.372984+0000","last_clean":"2023-09-23T20:20:18.372984+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:20:18.372984+0000","last_undegraded":"2023-09-23T20:20:18.372984+0000","last_fullsized":"2023-09-23T20:20:18.372984+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.370924+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:20:18.370924+0000","last_peered":"2023-09-23T20:20:18.370924+0000","last_clean":"2023-09-23T20:20:18.370924+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:20:18.370924+0000","last_undegraded":"2023-09-23T20:20:18.370924+0000","last_fullsized":"2023-09-23T20:20:18.370924+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"521'10387","reported_seq":12982,"reported_epoch":521,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:20:38.579660+0000","last_change":"2023-09-23T20:18:56.585041+0000","last_active":"2023-09-23T20:20:38.579660+0000","last_peered":"2023-09-23T20:20:38.579660+0000","last_clean":"2023-09-23T20:20:38.579660+0000","last_became_active":"2023-09-23T20:18:39.050965+0000","last_became_peered":"2023-09-23T20:18:39.050965+0000","last_unstale":"2023-09-23T20:20:38.579660+0000","last_undegraded":"2023-09-23T20:20:38.579660+0000","last_fullsized":"2023-09-23T20:20:38.579660+0000","mapping_epoch":503,"log_start":"491'3800","ondisk_log_start":"491'3800","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:43.005386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66256896,"num_objects":1011,"num_object_clones":0,"num_object_copies":3033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1011,"num_objects_unfound":0,"num_objects_dirty":1011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10387,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1265,"num_bytes_recovered":82903040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":1011}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":864,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372467+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:20:18.372467+0000","last_peered":"2023-09-23T20:20:18.372467+0000","last_clean":"2023-09-23T20:20:18.372467+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:20:18.372467+0000","last_undegraded":"2023-09-23T20:20:18.372467+0000","last_fullsized":"2023-09-23T20:20:18.372467+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1201,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372693+0000","last_change":"2023-09-23T20:17:47.305979+0000","last_active":"2023-09-23T20:20:18.372693+0000","last_peered":"2023-09-23T20:20:18.372693+0000","last_clean":"2023-09-23T20:20:18.372693+0000","last_became_active":"2023-09-23T20:17:47.274623+0000","last_became_peered":"2023-09-23T20:17:47.274623+0000","last_unstale":"2023-09-23T20:20:18.372693+0000","last_undegraded":"2023-09-23T20:20:18.372693+0000","last_fullsized":"2023-09-23T20:20:18.372693+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:27.293931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369368+0000","last_change":"2023-09-23T20:17:47.280498+0000","last_active":"2023-09-23T20:20:18.369368+0000","last_peered":"2023-09-23T20:20:18.369368+0000","last_clean":"2023-09-23T20:20:18.369368+0000","last_became_active":"2023-09-23T20:17:47.279716+0000","last_became_peered":"2023-09-23T20:17:47.279716+0000","last_unstale":"2023-09-23T20:20:18.369368+0000","last_undegraded":"2023-09-23T20:20:18.369368+0000","last_fullsized":"2023-09-23T20:20:18.369368+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:18.625303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371328+0000","last_change":"2023-09-23T20:17:47.276964+0000","last_active":"2023-09-23T20:20:18.371328+0000","last_peered":"2023-09-23T20:20:18.371328+0000","last_clean":"2023-09-23T20:20:18.371328+0000","last_became_active":"2023-09-23T20:17:47.276540+0000","last_became_peered":"2023-09-23T20:17:47.276540+0000","last_unstale":"2023-09-23T20:20:18.371328+0000","last_undegraded":"2023-09-23T20:20:18.371328+0000","last_fullsized":"2023-09-23T20:20:18.371328+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:31.547044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"521'10663","reported_seq":10713,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:38.563961+0000","last_change":"2023-09-23T20:16:59.496065+0000","last_active":"2023-09-23T20:20:38.563961+0000","last_peered":"2023-09-23T20:20:38.563961+0000","last_clean":"2023-09-23T20:20:38.563961+0000","last_became_active":"2023-09-23T20:16:59.495782+0000","last_became_peered":"2023-09-23T20:16:59.495782+0000","last_unstale":"2023-09-23T20:20:38.563961+0000","last_undegraded":"2023-09-23T20:20:38.563961+0000","last_fullsized":"2023-09-23T20:20:38.563961+0000","mapping_epoch":479,"log_start":"493'4111","ondisk_log_start":"493'4111","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:08.985469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63766528,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10663,"num_write_kb":372352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":759,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.386764+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:20:18.386764+0000","last_peered":"2023-09-23T20:20:18.386764+0000","last_clean":"2023-09-23T20:20:18.386764+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:20:18.386764+0000","last_undegraded":"2023-09-23T20:20:18.386764+0000","last_fullsized":"2023-09-23T20:20:18.386764+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1161,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.386926+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:20:18.386926+0000","last_peered":"2023-09-23T20:20:18.386926+0000","last_clean":"2023-09-23T20:20:18.386926+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:20:18.386926+0000","last_undegraded":"2023-09-23T20:20:18.386926+0000","last_fullsized":"2023-09-23T20:20:18.386926+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369282+0000","last_change":"2023-09-23T20:17:47.278822+0000","last_active":"2023-09-23T20:20:18.369282+0000","last_peered":"2023-09-23T20:20:18.369282+0000","last_clean":"2023-09-23T20:20:18.369282+0000","last_became_active":"2023-09-23T20:17:47.278407+0000","last_became_peered":"2023-09-23T20:17:47.278407+0000","last_unstale":"2023-09-23T20:20:18.369282+0000","last_undegraded":"2023-09-23T20:20:18.369282+0000","last_fullsized":"2023-09-23T20:20:18.369282+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:01.125231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371421+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:20:18.371421+0000","last_peered":"2023-09-23T20:20:18.371421+0000","last_clean":"2023-09-23T20:20:18.371421+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:20:18.371421+0000","last_undegraded":"2023-09-23T20:20:18.371421+0000","last_fullsized":"2023-09-23T20:20:18.371421+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"521'10229","reported_seq":15608,"reported_epoch":521,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:20:35.956039+0000","last_change":"2023-09-23T20:19:33.457129+0000","last_active":"2023-09-23T20:20:35.956039+0000","last_peered":"2023-09-23T20:20:35.956039+0000","last_clean":"2023-09-23T20:17:46.219689+0000","last_became_active":"2023-09-23T20:17:48.427732+0000","last_became_peered":"2023-09-23T20:17:48.427732+0000","last_unstale":"2023-09-23T20:20:35.956039+0000","last_undegraded":"2023-09-23T20:17:48.289936+0000","last_fullsized":"2023-09-23T20:17:48.256145+0000","mapping_epoch":490,"log_start":"498'4951","ondisk_log_start":"498'4951","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5278,"log_dups_size":0,"ondisk_log_size":5278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:05.919808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1160,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10229,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2662,"num_bytes_recovered":101187584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-63}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1429,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372976+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:20:18.372976+0000","last_peered":"2023-09-23T20:20:18.372976+0000","last_clean":"2023-09-23T20:20:18.372976+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:20:18.372976+0000","last_undegraded":"2023-09-23T20:20:18.372976+0000","last_fullsized":"2023-09-23T20:20:18.372976+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":736,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.389514+0000","last_change":"2023-09-23T20:17:50.313197+0000","last_active":"2023-09-23T20:20:18.389514+0000","last_peered":"2023-09-23T20:20:18.389514+0000","last_clean":"2023-09-23T20:20:18.389514+0000","last_became_active":"2023-09-23T20:17:50.312838+0000","last_became_peered":"2023-09-23T20:17:50.312838+0000","last_unstale":"2023-09-23T20:20:18.389514+0000","last_undegraded":"2023-09-23T20:20:18.389514+0000","last_fullsized":"2023-09-23T20:20:18.389514+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:31.530167+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":386,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372860+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:20:18.372860+0000","last_peered":"2023-09-23T20:20:18.372860+0000","last_clean":"2023-09-23T20:20:18.372860+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:20:18.372860+0000","last_undegraded":"2023-09-23T20:20:18.372860+0000","last_fullsized":"2023-09-23T20:20:18.372860+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":318,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387319+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:20:18.387319+0000","last_peered":"2023-09-23T20:20:18.387319+0000","last_clean":"2023-09-23T20:20:18.387319+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:20:18.387319+0000","last_undegraded":"2023-09-23T20:20:18.387319+0000","last_fullsized":"2023-09-23T20:20:18.387319+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"521'10412","reported_seq":10464,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:36.952267+0000","last_change":"2023-09-23T20:16:59.494039+0000","last_active":"2023-09-23T20:20:36.952267+0000","last_peered":"2023-09-23T20:20:36.952267+0000","last_clean":"2023-09-23T20:20:36.952267+0000","last_became_active":"2023-09-23T20:16:59.493714+0000","last_became_peered":"2023-09-23T20:16:59.493714+0000","last_unstale":"2023-09-23T20:20:36.952267+0000","last_undegraded":"2023-09-23T20:20:36.952267+0000","last_fullsized":"2023-09-23T20:20:36.952267+0000","mapping_epoch":479,"log_start":"485'2300","ondisk_log_start":"485'2300","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8112,"log_dups_size":0,"ondisk_log_size":8112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:15.171264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70123520,"num_objects":1070,"num_object_clones":0,"num_object_copies":3210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10412,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":532,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369569+0000","last_change":"2023-09-23T20:17:50.330694+0000","last_active":"2023-09-23T20:20:18.369569+0000","last_peered":"2023-09-23T20:20:18.369569+0000","last_clean":"2023-09-23T20:20:18.369569+0000","last_became_active":"2023-09-23T20:17:50.329940+0000","last_became_peered":"2023-09-23T20:17:50.329940+0000","last_unstale":"2023-09-23T20:20:18.369569+0000","last_undegraded":"2023-09-23T20:20:18.369569+0000","last_fullsized":"2023-09-23T20:20:18.369569+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:55.675445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:55.675445+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1045,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.389367+0000","last_change":"2023-09-23T20:17:50.313157+0000","last_active":"2023-09-23T20:20:18.389367+0000","last_peered":"2023-09-23T20:20:18.389367+0000","last_clean":"2023-09-23T20:20:18.389367+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:20:18.389367+0000","last_undegraded":"2023-09-23T20:20:18.389367+0000","last_fullsized":"2023-09-23T20:20:18.389367+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":469,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391397+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:20:18.391397+0000","last_peered":"2023-09-23T20:20:18.391397+0000","last_clean":"2023-09-23T20:20:18.391397+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:20:18.391397+0000","last_undegraded":"2023-09-23T20:20:18.391397+0000","last_fullsized":"2023-09-23T20:20:18.391397+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371178+0000","last_change":"2023-09-23T20:17:47.278112+0000","last_active":"2023-09-23T20:20:18.371178+0000","last_peered":"2023-09-23T20:20:18.371178+0000","last_clean":"2023-09-23T20:20:18.371178+0000","last_became_active":"2023-09-23T20:17:47.277728+0000","last_became_peered":"2023-09-23T20:17:47.277728+0000","last_unstale":"2023-09-23T20:20:18.371178+0000","last_undegraded":"2023-09-23T20:20:18.371178+0000","last_fullsized":"2023-09-23T20:20:18.371178+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:21.371380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"521'10405","reported_seq":10458,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:37.043132+0000","last_change":"2023-09-23T20:16:59.490420+0000","last_active":"2023-09-23T20:20:37.043132+0000","last_peered":"2023-09-23T20:20:37.043132+0000","last_clean":"2023-09-23T20:20:37.043132+0000","last_became_active":"2023-09-23T20:16:59.490025+0000","last_became_peered":"2023-09-23T20:16:59.490025+0000","last_unstale":"2023-09-23T20:20:37.043132+0000","last_undegraded":"2023-09-23T20:20:37.043132+0000","last_fullsized":"2023-09-23T20:20:37.043132+0000","mapping_epoch":479,"log_start":"499'5279","ondisk_log_start":"499'5279","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5126,"log_dups_size":0,"ondisk_log_size":5126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:03.184763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71041048,"num_objects":1085,"num_object_clones":0,"num_object_copies":3255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1085,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10405,"num_write_kb":367617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":717,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:25.970313+0000","last_change":"2023-09-23T20:19:05.092371+0000","last_active":"2023-09-23T20:20:25.970313+0000","last_peered":"2023-09-23T20:20:25.970313+0000","last_clean":"2023-09-23T20:20:25.970313+0000","last_became_active":"2023-09-23T20:19:05.092070+0000","last_became_peered":"2023-09-23T20:19:05.092070+0000","last_unstale":"2023-09-23T20:20:25.970313+0000","last_undegraded":"2023-09-23T20:20:25.970313+0000","last_fullsized":"2023-09-23T20:20:25.970313+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":221,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387494+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:20:18.387494+0000","last_peered":"2023-09-23T20:20:18.387494+0000","last_clean":"2023-09-23T20:20:18.387494+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:20:18.387494+0000","last_undegraded":"2023-09-23T20:20:18.387494+0000","last_fullsized":"2023-09-23T20:20:18.387494+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371181+0000","last_change":"2023-09-23T20:17:47.276790+0000","last_active":"2023-09-23T20:20:18.371181+0000","last_peered":"2023-09-23T20:20:18.371181+0000","last_clean":"2023-09-23T20:20:18.371181+0000","last_became_active":"2023-09-23T20:17:47.276198+0000","last_became_peered":"2023-09-23T20:17:47.276198+0000","last_unstale":"2023-09-23T20:20:18.371181+0000","last_undegraded":"2023-09-23T20:20:18.371181+0000","last_fullsized":"2023-09-23T20:20:18.371181+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:06.124215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":347,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372899+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:20:18.372899+0000","last_peered":"2023-09-23T20:20:18.372899+0000","last_clean":"2023-09-23T20:20:18.372899+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:20:18.372899+0000","last_undegraded":"2023-09-23T20:20:18.372899+0000","last_fullsized":"2023-09-23T20:20:18.372899+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"521'10611","reported_seq":10663,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:37.004605+0000","last_change":"2023-09-23T20:16:59.490304+0000","last_active":"2023-09-23T20:20:37.004605+0000","last_peered":"2023-09-23T20:20:37.004605+0000","last_clean":"2023-09-23T20:20:37.004605+0000","last_became_active":"2023-09-23T20:16:59.490026+0000","last_became_peered":"2023-09-23T20:16:59.490026+0000","last_unstale":"2023-09-23T20:20:37.004605+0000","last_undegraded":"2023-09-23T20:20:37.004605+0000","last_fullsized":"2023-09-23T20:20:37.004605+0000","mapping_epoch":479,"log_start":"485'2500","ondisk_log_start":"485'2500","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8111,"log_dups_size":0,"ondisk_log_size":8111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:59.789512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69533696,"num_objects":1061,"num_object_clones":0,"num_object_copies":3183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10611,"num_write_kb":373504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":770,"reported_epoch":521,"state":"active+clean+scrubbing","last_fresh":"2023-09-23T20:20:32.741093+0000","last_change":"2023-09-23T20:20:32.741093+0000","last_active":"2023-09-23T20:20:32.741093+0000","last_peered":"2023-09-23T20:20:32.741093+0000","last_clean":"2023-09-23T20:20:32.741093+0000","last_became_active":"2023-09-23T20:19:05.069124+0000","last_became_peered":"2023-09-23T20:19:05.069124+0000","last_unstale":"2023-09-23T20:20:32.741093+0000","last_undegraded":"2023-09-23T20:20:32.741093+0000","last_fullsized":"2023-09-23T20:20:32.741093+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":363,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387207+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:20:18.387207+0000","last_peered":"2023-09-23T20:20:18.387207+0000","last_clean":"2023-09-23T20:20:18.387207+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:20:18.387207+0000","last_undegraded":"2023-09-23T20:20:18.387207+0000","last_fullsized":"2023-09-23T20:20:18.387207+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":600,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.371034+0000","last_change":"2023-09-23T20:17:50.311535+0000","last_active":"2023-09-23T20:20:18.371034+0000","last_peered":"2023-09-23T20:20:18.371034+0000","last_clean":"2023-09-23T20:20:18.371034+0000","last_became_active":"2023-09-23T20:17:50.310876+0000","last_became_peered":"2023-09-23T20:17:50.310876+0000","last_unstale":"2023-09-23T20:20:18.371034+0000","last_undegraded":"2023-09-23T20:20:18.371034+0000","last_fullsized":"2023-09-23T20:20:18.371034+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.794641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":155,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391008+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:20:18.391008+0000","last_peered":"2023-09-23T20:20:18.391008+0000","last_clean":"2023-09-23T20:20:18.391008+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:20:18.391008+0000","last_undegraded":"2023-09-23T20:20:18.391008+0000","last_fullsized":"2023-09-23T20:20:18.391008+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"521'10341","reported_seq":15667,"reported_epoch":521,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:20:37.077095+0000","last_change":"2023-09-23T20:19:33.456970+0000","last_active":"2023-09-23T20:20:37.077095+0000","last_peered":"2023-09-23T20:20:37.077095+0000","last_clean":"2023-09-23T20:20:37.077095+0000","last_became_active":"2023-09-23T20:18:39.117958+0000","last_became_peered":"2023-09-23T20:18:39.117958+0000","last_unstale":"2023-09-23T20:20:37.077095+0000","last_undegraded":"2023-09-23T20:20:37.077095+0000","last_fullsized":"2023-09-23T20:20:37.077095+0000","mapping_epoch":503,"log_start":"499'5226","ondisk_log_start":"499'5226","created":479,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5115,"log_dups_size":0,"ondisk_log_size":5115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:45.818039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71761920,"num_objects":1095,"num_object_clones":0,"num_object_copies":3285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1095,"num_objects_unfound":0,"num_objects_dirty":1095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10341,"num_write_kb":365952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2630,"num_bytes_recovered":172359680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":1095}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":848,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.392319+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:20:18.392319+0000","last_peered":"2023-09-23T20:20:18.392319+0000","last_clean":"2023-09-23T20:20:18.392319+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:20:18.392319+0000","last_undegraded":"2023-09-23T20:20:18.392319+0000","last_fullsized":"2023-09-23T20:20:18.392319+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":227,"reported_epoch":520,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:20:18.387453+0000","last_change":"2023-09-23T20:18:39.010444+0000","last_active":"2023-09-23T20:20:18.387453+0000","last_peered":"2023-09-23T20:20:18.387453+0000","last_clean":"2023-09-23T20:20:18.387453+0000","last_became_active":"2023-09-23T20:18:39.009999+0000","last_became_peered":"2023-09-23T20:18:39.009999+0000","last_unstale":"2023-09-23T20:20:18.387453+0000","last_undegraded":"2023-09-23T20:20:18.387453+0000","last_fullsized":"2023-09-23T20:20:18.387453+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:53.838719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,3],"avail_no_missing":["7","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":47,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.370899+0000","last_change":"2023-09-23T20:17:47.284676+0000","last_active":"2023-09-23T20:20:18.370899+0000","last_peered":"2023-09-23T20:20:18.370899+0000","last_clean":"2023-09-23T20:20:18.370899+0000","last_became_active":"2023-09-23T20:17:47.277321+0000","last_became_peered":"2023-09-23T20:17:47.277321+0000","last_unstale":"2023-09-23T20:20:18.370899+0000","last_undegraded":"2023-09-23T20:20:18.370899+0000","last_fullsized":"2023-09-23T20:20:18.370899+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.082445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":321,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T20:20:18.372556+0000","last_change":"2023-09-23T20:18:38.011785+0000","last_active":"2023-09-23T20:20:18.372556+0000","last_peered":"2023-09-23T20:20:18.372556+0000","last_clean":"2023-09-23T20:18:35.261768+0000","last_became_active":"2023-09-23T20:18:38.011785+0000","last_became_peered":"2023-09-23T20:18:38.011785+0000","last_unstale":"2023-09-23T20:20:18.372556+0000","last_undegraded":"2023-09-23T20:20:18.372556+0000","last_fullsized":"2023-09-23T20:18:38.001274+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:00.398059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":922,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387932+0000","last_change":"2023-09-23T20:19:56.180311+0000","last_active":"2023-09-23T20:20:18.387932+0000","last_peered":"2023-09-23T20:20:18.387932+0000","last_clean":"2023-09-23T20:20:18.387932+0000","last_became_active":"2023-09-23T20:19:05.079569+0000","last_became_peered":"2023-09-23T20:19:05.079569+0000","last_unstale":"2023-09-23T20:20:18.387932+0000","last_undegraded":"2023-09-23T20:20:18.387932+0000","last_fullsized":"2023-09-23T20:20:18.387932+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_clean_scrub_stamp":"2023-09-23T20:19:56.180203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:19.922964+0000","scrub_duration":0.012799305,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":299,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.397444+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:20:18.397444+0000","last_peered":"2023-09-23T20:20:18.397444+0000","last_clean":"2023-09-23T20:20:18.397444+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:20:18.397444+0000","last_undegraded":"2023-09-23T20:20:18.397444+0000","last_fullsized":"2023-09-23T20:20:18.397444+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":308,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.395663+0000","last_change":"2023-09-23T20:17:47.281804+0000","last_active":"2023-09-23T20:20:18.395663+0000","last_peered":"2023-09-23T20:20:18.395663+0000","last_clean":"2023-09-23T20:20:18.395663+0000","last_became_active":"2023-09-23T20:17:47.281384+0000","last_became_peered":"2023-09-23T20:17:47.281384+0000","last_unstale":"2023-09-23T20:20:18.395663+0000","last_undegraded":"2023-09-23T20:20:18.395663+0000","last_fullsized":"2023-09-23T20:20:18.395663+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:03.796273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":739,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T20:20:37.345353+0000","last_change":"2023-09-23T20:20:37.345353+0000","last_active":"2023-09-23T20:20:37.345353+0000","last_peered":"2023-09-23T20:20:37.345353+0000","last_clean":"2023-09-23T20:20:37.345353+0000","last_became_active":"2023-09-23T20:19:05.071787+0000","last_became_peered":"2023-09-23T20:19:05.071787+0000","last_unstale":"2023-09-23T20:20:37.345353+0000","last_undegraded":"2023-09-23T20:20:37.345353+0000","last_fullsized":"2023-09-23T20:20:37.345353+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":552,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.373158+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:20:18.373158+0000","last_peered":"2023-09-23T20:20:18.373158+0000","last_clean":"2023-09-23T20:20:18.373158+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:20:18.373158+0000","last_undegraded":"2023-09-23T20:20:18.373158+0000","last_fullsized":"2023-09-23T20:20:18.373158+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":551,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391456+0000","last_change":"2023-09-23T20:17:47.266215+0000","last_active":"2023-09-23T20:20:18.391456+0000","last_peered":"2023-09-23T20:20:18.391456+0000","last_clean":"2023-09-23T20:20:18.391456+0000","last_became_active":"2023-09-23T20:17:47.265890+0000","last_became_peered":"2023-09-23T20:17:47.265890+0000","last_unstale":"2023-09-23T20:20:18.391456+0000","last_undegraded":"2023-09-23T20:20:18.391456+0000","last_fullsized":"2023-09-23T20:20:18.391456+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:59.854975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":542,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.372927+0000","last_change":"2023-09-23T20:17:47.288753+0000","last_active":"2023-09-23T20:20:18.372927+0000","last_peered":"2023-09-23T20:20:18.372927+0000","last_clean":"2023-09-23T20:20:18.372927+0000","last_became_active":"2023-09-23T20:17:47.270488+0000","last_became_peered":"2023-09-23T20:17:47.270488+0000","last_unstale":"2023-09-23T20:20:18.372927+0000","last_undegraded":"2023-09-23T20:20:18.372927+0000","last_fullsized":"2023-09-23T20:20:18.372927+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:35.837184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":517,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391083+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:20:18.391083+0000","last_peered":"2023-09-23T20:20:18.391083+0000","last_clean":"2023-09-23T20:20:18.391083+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:20:18.391083+0000","last_undegraded":"2023-09-23T20:20:18.391083+0000","last_fullsized":"2023-09-23T20:20:18.391083+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":881,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.391926+0000","last_change":"2023-09-23T20:19:52.400771+0000","last_active":"2023-09-23T20:20:18.391926+0000","last_peered":"2023-09-23T20:20:18.391926+0000","last_clean":"2023-09-23T20:20:18.391926+0000","last_became_active":"2023-09-23T20:19:05.086254+0000","last_became_peered":"2023-09-23T20:19:05.086254+0000","last_unstale":"2023-09-23T20:20:18.391926+0000","last_undegraded":"2023-09-23T20:20:18.391926+0000","last_fullsized":"2023-09-23T20:20:18.391926+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":244,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.395881+0000","last_change":"2023-09-23T20:17:47.281777+0000","last_active":"2023-09-23T20:20:18.395881+0000","last_peered":"2023-09-23T20:20:18.395881+0000","last_clean":"2023-09-23T20:20:18.395881+0000","last_became_active":"2023-09-23T20:17:47.281381+0000","last_became_peered":"2023-09-23T20:17:47.281381+0000","last_unstale":"2023-09-23T20:20:18.395881+0000","last_undegraded":"2023-09-23T20:20:18.395881+0000","last_fullsized":"2023-09-23T20:20:18.395881+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.083242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":382,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.386996+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:20:18.386996+0000","last_peered":"2023-09-23T20:20:18.386996+0000","last_clean":"2023-09-23T20:20:18.386996+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:20:18.386996+0000","last_undegraded":"2023-09-23T20:20:18.386996+0000","last_fullsized":"2023-09-23T20:20:18.386996+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":813,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.387171+0000","last_change":"2023-09-23T20:19:50.241581+0000","last_active":"2023-09-23T20:20:18.387171+0000","last_peered":"2023-09-23T20:20:18.387171+0000","last_clean":"2023-09-23T20:20:18.387171+0000","last_became_active":"2023-09-23T20:19:05.067188+0000","last_became_peered":"2023-09-23T20:19:05.067188+0000","last_unstale":"2023-09-23T20:20:18.387171+0000","last_undegraded":"2023-09-23T20:20:18.387171+0000","last_fullsized":"2023-09-23T20:20:18.387171+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":366,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.373549+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:20:18.373549+0000","last_peered":"2023-09-23T20:20:18.373549+0000","last_clean":"2023-09-23T20:20:18.373549+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:20:18.373549+0000","last_undegraded":"2023-09-23T20:20:18.373549+0000","last_fullsized":"2023-09-23T20:20:18.373549+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":43,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T20:20:18.369522+0000","last_change":"2023-09-23T20:17:47.267673+0000","last_active":"2023-09-23T20:20:18.369522+0000","last_peered":"2023-09-23T20:20:18.369522+0000","last_clean":"2023-09-23T20:20:18.369522+0000","last_became_active":"2023-09-23T20:17:47.267216+0000","last_became_peered":"2023-09-23T20:17:47.267216+0000","last_unstale":"2023-09-23T20:20:18.369522+0000","last_undegraded":"2023-09-23T20:20:18.369522+0000","last_fullsized":"2023-09-23T20:20:18.369522+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:37.855890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1115947032,"num_objects":17029,"num_object_clones":0,"num_object_copies":51087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5090,"num_objects_misplaced":2106,"num_objects_unfound":0,"num_objects_dirty":17029,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167419,"num_write_kb":5902273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10422,"num_bytes_recovered":609746944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4357763072,"data_stored":4357750856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101903,"ondisk_log_size":101903,"up":46,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":10952,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":94,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":30,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":107340,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":8388703,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234474,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1173420,"kb_used_data":590692,"kb_used_omap":0,"kb_used_meta":582656,"kb_avail":92571732,"statfs":{"total":95995035648,"available":94793453568,"internally_reserved":0,"allocated":604868608,"data_stored":603293462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":596639744},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:19:58 2023","interfaces":[{"interface":"back","average":{"1min":9.7080000000000002,"5min":6.5899999999999999,"15min":3.5990000000000002},"min":{"1min":0.214,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":125.06100000000001,"5min":149.25999999999999,"15min":149.25999999999999},"last":7.6879999999999997},{"interface":"front","average":{"1min":10.789,"5min":7.1710000000000003,"15min":3.851},"min":{"1min":0.28299999999999997,"5min":0.246,"15min":0.222},"max":{"1min":125.029,"5min":149.11799999999999,"15min":149.11799999999999},"last":5.0129999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:19:58 2023","interfaces":[{"interface":"back","average":{"1min":3.9580000000000002,"5min":2.0129999999999999,"15min":2.492},"min":{"1min":0.27300000000000002,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":61.113999999999997,"5min":61.113999999999997,"15min":91.335999999999999},"last":2.617},{"interface":"front","average":{"1min":4.7880000000000003,"5min":2.3439999999999999,"15min":2.7639999999999998},"min":{"1min":0.33200000000000002,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":61.082999999999998,"5min":61.082999999999998,"15min":91.516999999999996},"last":1.1080000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":5.4560000000000004,"15min":4.5750000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.251},"max":{"1min":24.16,"5min":177.852,"15min":177.852},"last":13.025},{"interface":"front","average":{"1min":3.407,"5min":5.7160000000000002,"15min":4.899},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":20.98,"5min":176.97900000000001,"15min":176.97900000000001},"last":2.1040000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:19:58 2023","interfaces":[{"interface":"back","average":{"1min":3.718,"5min":6.4640000000000004,"15min":4.2190000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29299999999999998},"max":{"1min":25.977,"5min":185.17099999999999,"15min":185.17099999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":3.5960000000000001,"5min":6.8799999999999999,"15min":4.2919999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28999999999999998},"max":{"1min":26.481999999999999,"5min":184.99100000000001,"15min":184.99100000000001},"last":1.2030000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0860000000000003,"5min":1.649,"15min":0.995},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":83.811999999999998,"5min":83.811999999999998,"15min":83.811999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":4.0720000000000001,"5min":1.6950000000000001,"15min":1.0900000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":84.620000000000005,"5min":84.620000000000005,"15min":84.620000000000005},"last":0.379}]},{"osd":5,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":1.3340000000000001,"15min":1.1859999999999999},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.794,"5min":16.327000000000002,"15min":24.876000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.495,"5min":1.25,"15min":1.079},"min":{"1min":0.30599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1190000000000002,"5min":16.716000000000001,"15min":21.623000000000001},"last":0.40100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.3779999999999999,"15min":1.0569999999999999},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.3420000000000001,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.163,"15min":0.82199999999999995},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1400000000000001,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.436}]}]},{"osd":6,"up_from":251,"seq":1078036791490,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273876,"kb_used_data":233868,"kb_used_omap":6,"kb_used_meta":39993,"kb_avail":93471276,"statfs":{"total":95995035648,"available":95714586624,"internally_reserved":0,"allocated":239480832,"data_stored":238014233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":40953471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":8.9100000000000001,"5min":3.8820000000000001,"15min":2.6890000000000001},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":53.780000000000001,"5min":74.787000000000006,"15min":126.64},"last":0.27000000000000002},{"interface":"front","average":{"1min":9.0090000000000003,"5min":4.0289999999999999,"15min":2.7570000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":53.768000000000001,"5min":74.751999999999995,"15min":126.658},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":4.3120000000000003,"15min":4.0830000000000002},"min":{"1min":0.26500000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":87.864999999999995,"5min":103.545,"15min":156.70699999999999},"last":5.4710000000000001},{"interface":"front","average":{"1min":5.1260000000000003,"5min":4.0279999999999996,"15min":4.0590000000000002},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.23899999999999999},"max":{"1min":87.772000000000006,"5min":100.018,"15min":156.68299999999999},"last":2.8690000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":6.891,"5min":2.4329999999999998,"15min":3.1179999999999999},"min":{"1min":0.315,"5min":0.254,"15min":0.23100000000000001},"max":{"1min":53.911999999999999,"5min":53.911999999999999,"15min":122.149},"last":0.47499999999999998},{"interface":"front","average":{"1min":7.3259999999999996,"5min":2.4129999999999998,"15min":3.0600000000000001},"min":{"1min":0.32400000000000001,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":54.125999999999998,"5min":54.125999999999998,"15min":122.23999999999999},"last":1.7789999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":5.181,"15min":3.8580000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":51.512999999999998,"5min":128.279,"15min":160.27699999999999},"last":4.8179999999999996},{"interface":"front","average":{"1min":3.4249999999999998,"5min":4.4969999999999999,"15min":3.8180000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":51.601999999999997,"5min":127.911,"15min":160.42699999999999},"last":2.9830000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.298,"15min":1.5780000000000001},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.248},"max":{"1min":2.641,"5min":34.534999999999997,"15min":145.29300000000001},"last":2.9140000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.077,"15min":1.538},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.3999999999999999,"5min":34.426000000000002,"15min":145.256},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":4.5880000000000001,"15min":2.972},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.25800000000000001},"max":{"1min":2.5329999999999999,"5min":218.58199999999999,"15min":218.58199999999999},"last":2.9500000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.343,"15min":2.8969999999999998},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.28499999999999998},"max":{"1min":2.4380000000000002,"5min":217.95599999999999,"15min":217.95599999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.1269999999999998,"15min":1.73},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.315},"max":{"1min":4.7380000000000004,"5min":137.63900000000001,"15min":137.63900000000001},"last":1},{"interface":"front","average":{"1min":0.751,"5min":2.8199999999999998,"15min":1.629},"min":{"1min":0.39700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":2.3620000000000001,"5min":136.738,"15min":136.738},"last":0.48799999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762672,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1062852,"kb_used_data":239548,"kb_used_omap":7,"kb_used_meta":823224,"kb_avail":92682300,"statfs":{"total":95995035648,"available":94906675200,"internally_reserved":0,"allocated":245297152,"data_stored":242824260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":842982202},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:19:36 2023","interfaces":[{"interface":"back","average":{"1min":4.2450000000000001,"5min":5.2009999999999996,"15min":3.8359999999999999},"min":{"1min":0.20599999999999999,"5min":0.185,"15min":0.17100000000000001},"max":{"1min":22.196000000000002,"5min":145.81299999999999,"15min":145.81299999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":4.2450000000000001,"5min":5.4029999999999996,"15min":4.2229999999999999},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.19400000000000001},"max":{"1min":22.341000000000001,"5min":146.83799999999999,"15min":146.83799999999999},"last":6.6159999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:20:32 2023","interfaces":[{"interface":"back","average":{"1min":13.513,"5min":6.0099999999999998,"15min":4.9539999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":113.941,"5min":152.62,"15min":152.62},"last":2.859},{"interface":"front","average":{"1min":14.683,"5min":6.234,"15min":4.702},"min":{"1min":0.318,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":114.34699999999999,"5min":154.84299999999999,"15min":154.84299999999999},"last":3.4729999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:19:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9159999999999999,"5min":3.3599999999999999,"15min":3.177},"min":{"1min":0.251,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.222,"5min":66.527000000000001,"15min":66.527000000000001},"last":6.3700000000000001},{"interface":"front","average":{"1min":2.2999999999999998,"5min":2.2149999999999999,"15min":2.548},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.247},"max":{"1min":10.587999999999999,"5min":61.854999999999997,"15min":61.854999999999997},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 20:19:36 2023","interfaces":[{"interface":"back","average":{"1min":3.077,"5min":1.964,"15min":1.7509999999999999},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.23599999999999999},"max":{"1min":29.791,"5min":29.791,"15min":29.791},"last":0.51100000000000001},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.71,"15min":1.7350000000000001},"min":{"1min":0.34399999999999997,"5min":0.32400000000000001,"15min":0.28100000000000003},"max":{"1min":27.143999999999998,"5min":27.143999999999998,"15min":27.143999999999998},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:19:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8799999999999999,"5min":1.964,"15min":2.1309999999999998},"min":{"1min":0.33800000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":29.280000000000001,"5min":29.280000000000001,"15min":83.510999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":3.7879999999999998,"5min":2.0230000000000001,"15min":2.2850000000000001},"min":{"1min":0.39000000000000001,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":32.692,"5min":32.692,"15min":83.353999999999999},"last":2.9319999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.2390000000000001,"15min":1.665},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":4.5830000000000002,"5min":9.2279999999999998,"15min":18.117999999999999},"last":5.6760000000000002},{"interface":"front","average":{"1min":0.75,"5min":1.325,"15min":1.843},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.0310000000000001,"5min":9.8659999999999997,"15min":20.734999999999999},"last":3.7330000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:19:36 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":2.069,"15min":2.5720000000000001},"min":{"1min":0.39500000000000002,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":10.548,"5min":10.548,"15min":127.03100000000001},"last":2.8929999999999998},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.7430000000000001,"15min":2.234},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.6310000000000002,"5min":10.122,"15min":125.095},"last":0.47799999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926210,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2026984,"kb_used_data":999328,"kb_used_omap":8,"kb_used_meta":1027639,"kb_avail":91718168,"statfs":{"total":95995035648,"available":93919404032,"internally_reserved":0,"allocated":1023311872,"data_stored":1020864406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1052302945},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,3,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:20:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":2.145,"15min":2.5859999999999999},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":7.9299999999999997,"5min":29.893000000000001,"15min":72.534000000000006},"last":9.2699999999999996},{"interface":"front","average":{"1min":3.1949999999999998,"5min":2.7690000000000001,"15min":2.952},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.187},"max":{"1min":8.6709999999999994,"5min":23.436,"15min":72.667000000000002},"last":7.7830000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:20:20 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":2.8069999999999999,"15min":3.4100000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":63.710999999999999,"5min":63.710999999999999,"15min":160.542},"last":9.3119999999999994},{"interface":"front","average":{"1min":6.3049999999999997,"5min":3.5009999999999999,"15min":3.7730000000000001},"min":{"1min":0.255,"5min":0.23599999999999999,"15min":0.22600000000000001},"max":{"1min":60.518999999999998,"5min":60.518999999999998,"15min":160.75999999999999},"last":8.4399999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:20:10 2023","interfaces":[{"interface":"back","average":{"1min":5.1150000000000002,"5min":3.9700000000000002,"15min":3.7010000000000001},"min":{"1min":0.26900000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":25.667000000000002,"5min":44.405000000000001,"15min":126.95399999999999},"last":5.6890000000000001},{"interface":"front","average":{"1min":4.0419999999999998,"5min":3.7490000000000001,"15min":3.7810000000000001},"min":{"1min":0.29599999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":25.783999999999999,"5min":43.909999999999997,"15min":127.294},"last":15.496}]},{"osd":4,"last update":"Sat Sep 23 20:19:34 2023","interfaces":[{"interface":"back","average":{"1min":3.5310000000000001,"5min":2.2559999999999998,"15min":2.2000000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":16.593,"5min":16.593,"15min":148.20599999999999},"last":5.0339999999999998},{"interface":"front","average":{"1min":4.3470000000000004,"5min":3.0449999999999999,"15min":2.468},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":21.899000000000001,"5min":105.282,"15min":148.11000000000001},"last":5.6609999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:19:34 2023","interfaces":[{"interface":"back","average":{"1min":8.4930000000000003,"5min":2.9390000000000001,"15min":2.145},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":145.14500000000001,"5min":145.14500000000001,"15min":145.14500000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":11.028,"5min":3.649,"15min":2.5209999999999999},"min":{"1min":0.33800000000000002,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":145.04499999999999,"5min":145.04499999999999,"15min":145.04499999999999},"last":4.9610000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6339999999999999,"15min":1.6419999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":7.6500000000000004,"5min":7.6500000000000004,"15min":15.744999999999999},"last":4.1239999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.234,"15min":1.6020000000000001},"min":{"1min":0.44,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":9.8219999999999992,"15min":13.766999999999999},"last":0.82199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:19:34 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":2.6629999999999998,"15min":1.9179999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.5880000000000001,"5min":99.486000000000004,"15min":99.486000000000004},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.5979999999999999,"5min":3.6739999999999999,"15min":2.4380000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":16.617000000000001,"5min":99.668999999999997,"15min":99.668999999999997},"last":5.0039999999999996}]}]},{"osd":2,"up_from":104,"seq":446676599149,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1529172,"kb_used_data":563596,"kb_used_omap":12,"kb_used_meta":965299,"kb_avail":92215980,"statfs":{"total":95995035648,"available":94429163520,"internally_reserved":0,"allocated":577122304,"data_stored":574639450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":988467198},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,5,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9009999999999998,"5min":8.0299999999999994,"15min":4.2039999999999997},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":15.4,"5min":274.185,"15min":274.185},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.4729999999999999,"5min":5.8079999999999998,"15min":3.601},"min":{"1min":0.23200000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.8209999999999997,"5min":106.477,"15min":106.477},"last":3.6349999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":4.335,"5min":4.5,"15min":3.5249999999999999},"min":{"1min":0.39600000000000002,"5min":0.20399999999999999,"15min":0.19600000000000001},"max":{"1min":11.939,"5min":106.357,"15min":106.357},"last":0.93799999999999994},{"interface":"front","average":{"1min":4.1619999999999999,"5min":4.2969999999999997,"15min":3.1579999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":15.255000000000001,"5min":104.905,"15min":104.905},"last":0.55600000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:19:47 2023","interfaces":[{"interface":"back","average":{"1min":5.6100000000000003,"5min":6.3650000000000002,"15min":4.7149999999999999},"min":{"1min":0.313,"5min":0.21299999999999999,"15min":0.20799999999999999},"max":{"1min":18.553000000000001,"5min":132.82499999999999,"15min":132.82499999999999},"last":3.758},{"interface":"front","average":{"1min":4.0439999999999996,"5min":5.9669999999999996,"15min":4.0259999999999998},"min":{"1min":0.27400000000000002,"5min":0.23000000000000001,"15min":0.22600000000000001},"max":{"1min":10.387,"5min":130.97300000000001,"15min":130.97300000000001},"last":4.2850000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8450000000000002,"5min":2.2469999999999999,"15min":2.113},"min":{"1min":0.313,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.355,"5min":43.040999999999997,"15min":75.608000000000004},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.2549999999999999,"5min":2.8700000000000001,"15min":2.5470000000000002},"min":{"1min":0.36599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.864000000000001,"5min":48.951000000000001,"15min":76.161000000000001},"last":0.96899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":2.4049999999999998,"15min":2.6349999999999998},"min":{"1min":0.38500000000000001,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":15.378,"5min":43.716000000000001,"15min":135.202},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.585,"5min":2.5990000000000002,"15min":2.4710000000000001},"min":{"1min":0.41499999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.782,"5min":43.643999999999998,"15min":63.792000000000002},"last":3.6829999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.329,"15min":1.542},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5510000000000002,"5min":11.843,"15min":14.664},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.425,"15min":1.744},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.077,"5min":12.025,"15min":14.279999999999999},"last":4.9660000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.4889999999999999,"5min":2.6360000000000001,"15min":2.1560000000000001},"min":{"1min":0.46999999999999997,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":11.91,"5min":18.495999999999999,"15min":18.495999999999999},"last":0.99299999999999999},{"interface":"front","average":{"1min":2.9990000000000001,"5min":2.077,"15min":1.7609999999999999},"min":{"1min":0.41299999999999998,"5min":0.39900000000000002,"15min":0.35299999999999998},"max":{"1min":15.327999999999999,"5min":15.327999999999999,"15min":15.327999999999999},"last":0.53700000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435622,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1693696,"kb_used_data":684664,"kb_used_omap":4,"kb_used_meta":1009019,"kb_avail":92051456,"statfs":{"total":95995035648,"available":94260690944,"internally_reserved":0,"allocated":701095936,"data_stored":698330329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1033235565},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:19:56 2023","interfaces":[{"interface":"back","average":{"1min":9.1760000000000002,"5min":5.9480000000000004,"15min":4.1609999999999996},"min":{"1min":0.193,"5min":0.189,"15min":0.186},"max":{"1min":92.516999999999996,"5min":132.47800000000001,"15min":132.47800000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":9.6229999999999993,"5min":6.7039999999999997,"15min":4.3899999999999997},"min":{"1min":0.22700000000000001,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":92.191000000000003,"5min":133.22499999999999,"15min":133.22499999999999},"last":0.55000000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:19:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":2.8239999999999998,"15min":4.2270000000000003},"min":{"1min":0.22700000000000001,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":9.673,"5min":92.036000000000001,"15min":183.87},"last":1.5089999999999999},{"interface":"front","average":{"1min":2.1899999999999999,"5min":2.5870000000000002,"15min":4.4829999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":12.335000000000001,"5min":92.210999999999999,"15min":185.84899999999999},"last":0.32800000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:19:56 2023","interfaces":[{"interface":"back","average":{"1min":9.4580000000000002,"5min":7.585,"15min":5.3920000000000003},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.23899999999999999},"max":{"1min":96.460999999999999,"5min":128.369,"15min":163.83500000000001},"last":1.962},{"interface":"front","average":{"1min":8.8930000000000007,"5min":7.0869999999999997,"15min":5.3179999999999996},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":92.551000000000002,"5min":126.497,"15min":163.92099999999999},"last":0.35999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:19:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3230000000000004,"5min":1.964,"15min":1.9910000000000001},"min":{"1min":0.40699999999999997,"5min":0.29299999999999998,"15min":0.28999999999999998},"max":{"1min":54.046999999999997,"5min":54.046999999999997,"15min":58.472999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":5.0049999999999999,"5min":2.7709999999999999,"15min":2.4860000000000002},"min":{"1min":0.35199999999999998,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":54.311,"5min":54.311,"15min":58.408000000000001},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:19:56 2023","interfaces":[{"interface":"back","average":{"1min":4.9359999999999999,"5min":2.9460000000000002,"15min":3.444},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":54.139000000000003,"5min":54.139000000000003,"15min":200.53200000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":4.6280000000000001,"5min":2.9830000000000001,"15min":3.4300000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26900000000000002},"max":{"1min":56.457999999999998,"5min":56.457999999999998,"15min":200.846},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.593,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":1.5329999999999999,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.3280000000000001,"15min":2.286},"min":{"1min":0.42699999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.2040000000000002,"5min":8.2629999999999999,"15min":82.233000000000004},"last":4.1509999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.0110000000000001,"15min":2.5779999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29599999999999999},"max":{"1min":7.6079999999999997,"5min":17.986000000000001,"15min":139.465},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.4930000000000001,"5min":1.6879999999999999,"15min":2.617},"min":{"1min":0.38400000000000001,"5min":0.379,"15min":0.32800000000000001},"max":{"1min":5.4290000000000003,"5min":9.2289999999999992,"15min":141.56200000000001},"last":0.53600000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019384,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1245484,"kb_used_data":681700,"kb_used_omap":0,"kb_used_meta":563712,"kb_avail":92499668,"statfs":{"total":95995035648,"available":94719660032,"internally_reserved":0,"allocated":698060800,"data_stored":696485666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":577241088},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:19:34 2023","interfaces":[{"interface":"back","average":{"1min":20.062999999999999,"5min":9.4640000000000004,"15min":4.3609999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":153.108,"5min":232.89500000000001,"15min":232.89500000000001},"last":4.5880000000000001},{"interface":"front","average":{"1min":20.213999999999999,"5min":9.5129999999999999,"15min":4.3499999999999996},"min":{"1min":0.34499999999999997,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":152.495,"5min":232.851,"15min":232.851},"last":4.6040000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:19:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.637,"15min":3.2330000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":4.4560000000000004,"5min":114.05200000000001,"15min":138.47999999999999},"last":2.613},{"interface":"front","average":{"1min":1.0509999999999999,"5min":2.6789999999999998,"15min":3.2189999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.217},"max":{"1min":3.6949999999999998,"5min":114.02800000000001,"15min":138.429},"last":5.0289999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":15.880000000000001,"5min":9.6950000000000003,"15min":6.7859999999999996},"min":{"1min":0.29899999999999999,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":204.345,"5min":204.345,"15min":204.345},"last":0.84699999999999998},{"interface":"front","average":{"1min":15.916,"5min":9.6020000000000003,"15min":6.6429999999999998},"min":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":204.072,"5min":204.072,"15min":204.072},"last":2.0550000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:19:34 2023","interfaces":[{"interface":"back","average":{"1min":20.556999999999999,"5min":7.4340000000000002,"15min":5.5149999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":156.72300000000001,"5min":156.72300000000001,"15min":184.37100000000001},"last":0.999},{"interface":"front","average":{"1min":20.789000000000001,"5min":7.359,"15min":5.6319999999999997},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":155.84800000000001,"5min":155.84800000000001,"15min":184.22300000000001},"last":5.2869999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:19:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":2.1920000000000002,"15min":1.6299999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":2.4420000000000002,"5min":114.788,"15min":176.81899999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.498,"5min":2.3159999999999998,"15min":1.696},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":2.5720000000000001,"5min":113.822,"15min":176.172},"last":0.40799999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:17:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.0390000000000001,"15min":1.026},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":2.5110000000000001,"5min":114.232,"15min":114.232},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.3540000000000001,"15min":1.2130000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.247},"max":{"1min":4.085,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":2.1539999999999999,"15min":1.6719999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63600000000000001,"5min":113.861,"15min":113.861},"last":0.373},{"interface":"front","average":{"1min":0.67600000000000005,"5min":2.258,"15min":1.738},"min":{"1min":0.34499999999999997,"5min":0.30399999999999999,"15min":0.222},"max":{"1min":4.6710000000000003,"5min":114.32299999999999,"15min":114.32299999999999},"last":0.38800000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757748,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1051168,"kb_used_data":394520,"kb_used_omap":0,"kb_used_meta":656640,"kb_avail":92693984,"statfs":{"total":95995035648,"available":94918639616,"internally_reserved":0,"allocated":403988480,"data_stored":402421859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":672399360},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:19:38 2023","interfaces":[{"interface":"back","average":{"1min":16.134,"5min":9.8599999999999994,"15min":4.6870000000000003},"min":{"1min":0.316,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":152.71299999999999,"5min":166.57900000000001,"15min":166.57900000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":15.630000000000001,"5min":9.1880000000000006,"15min":4.6070000000000002},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":152.584,"5min":165.67500000000001,"15min":165.67500000000001},"last":5.1980000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:19:57 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":1.766,"15min":3.7149999999999999},"min":{"1min":0.27600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":6.7140000000000004,"5min":17.010999999999999,"15min":150.16999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.7709999999999999,"5min":1.8720000000000001,"15min":3.73},"min":{"1min":0.27800000000000002,"5min":0.27200000000000002,"15min":0.21199999999999999},"max":{"1min":6.6449999999999996,"5min":18.879000000000001,"15min":150.12700000000001},"last":0.38200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:20:21 2023","interfaces":[{"interface":"back","average":{"1min":17.131,"5min":6.5279999999999996,"15min":5.3019999999999996},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":208.495,"5min":208.495,"15min":240.57300000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":17.167999999999999,"5min":6.5869999999999997,"15min":5.3659999999999997},"min":{"1min":0.33500000000000002,"5min":0.33100000000000002,"15min":0.30299999999999999},"max":{"1min":208.58199999999999,"5min":208.58199999999999,"15min":240.464},"last":0.435}]},{"osd":3,"last update":"Sat Sep 23 20:19:57 2023","interfaces":[{"interface":"back","average":{"1min":10.192,"5min":9.7569999999999997,"15min":6.5990000000000002},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.27100000000000002},"max":{"1min":155.97300000000001,"5min":196.97800000000001,"15min":201.50899999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":10.49,"5min":9.8960000000000008,"15min":6.5359999999999996},"min":{"1min":0.36799999999999999,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":155.803,"5min":196.875,"15min":201.58099999999999},"last":4.9359999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.93400000000000005,"15min":0.77800000000000002},"min":{"1min":0.25600000000000001,"5min":0.25,"15min":0.24099999999999999},"max":{"1min":2.6019999999999999,"5min":8,"15min":9.7240000000000002},"last":0.311},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.63900000000000001,"15min":0.59399999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":1.9510000000000001,"5min":3.371,"15min":8.7360000000000007},"last":0.98499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2490000000000001,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":4.04,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.73299999999999998,"15min":0.60999999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":1.609,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":1.492}]},{"osd":7,"last update":"Sat Sep 23 20:20:21 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.0780000000000001,"15min":0.95799999999999996},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.3609999999999998,"5min":11.725,"15min":11.725},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":1.0449999999999999,"15min":0.90700000000000003},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.58999999999999997,"5min":11.041,"15min":11.041},"last":0.34899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1004146688,"data_stored":1004142616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":226820096,"data_stored":226820096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":562167808,"data_stored":562167808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":679284736,"data_stored":679280664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":670109696,"data_stored":670105624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":394854400,"data_stored":394854400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":237764608,"data_stored":237764608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":582615040,"data_stored":582615040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:20:42.918 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:20:42.918 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-09-23T20:20:42.954 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:20:42.970 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:43.004 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:43.062 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:43.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:43.143 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:20:43.175 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:43.203 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:43.254 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:43.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:43.359 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:43.391 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:43.448 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:43.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:43.557 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:43 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:20:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:43 smithi144 ceph-mon[194833]: 1.c deep-scrub starts 2023-09-23T20:20:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:43 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4811/48690 objects degraded (9.881%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:43 smithi144 ceph-mon[194833]: pgmap v1488: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 896 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 4811/42867 objects degraded (11.223%); 1878/42867 objects misplaced (4.381%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:43.638 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:43.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:43 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:20:43.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:43 smithi135 ceph-mon[221113]: 1.c deep-scrub starts 2023-09-23T20:20:43.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:43 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4811/48690 objects degraded (9.881%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:43.742 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:43 smithi135 ceph-mon[221113]: pgmap v1488: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 896 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 4811/42867 objects degraded (11.223%); 1878/42867 objects misplaced (4.381%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:43.743 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:43 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:20:43.743 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:43 smithi135 ceph-mon[213871]: 1.c deep-scrub starts 2023-09-23T20:20:43.743 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:43 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4811/48690 objects degraded (9.881%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:43.743 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:43 smithi135 ceph-mon[213871]: pgmap v1488: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 896 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 4811/42867 objects degraded (11.223%); 1878/42867 objects misplaced (4.381%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:43.745 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:20:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:20:43] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-09-23T20:20:43.822 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:43.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:43.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:43.945 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:43.978 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:44.046 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:44.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:44.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:44.155 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:44.194 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:44.259 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:44.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:44.364 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:44.449 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:44.634 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:44.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:44.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:44.752 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:44.752 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:44.782 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:44.798 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:44.798 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:44.842 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:44.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:44.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:44.948 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:44.982 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:45.049 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:45.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:45.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:45.155 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:45.303 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:45.606 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:20:45.649 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:20:45.697 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:20:45.748 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:20:45.804 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:20:45.822 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:45.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:45.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:45.846 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:20:45.905 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:20:45.909 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:20:45.909 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:20:45.909 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:20:45.909 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:20:45.909 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:20:45.909 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:20:45.909 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:20:45.909 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:20:45.936 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:45.993 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:46 smithi144 ceph-mon[194833]: pgmap v1489: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 842 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4811/40302 objects degraded (11.937%); 1655/40302 objects misplaced (4.106%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:46 smithi144 ceph-mon[194833]: 1.1c scrub starts 2023-09-23T20:20:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3485028133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:46 smithi135 ceph-mon[221113]: pgmap v1489: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 842 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4811/40302 objects degraded (11.937%); 1655/40302 objects misplaced (4.106%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:46 smithi135 ceph-mon[221113]: 1.1c scrub starts 2023-09-23T20:20:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3485028133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:46 smithi135 ceph-mon[213871]: pgmap v1489: 119 pgs: 2 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 842 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4811/40302 objects degraded (11.937%); 1655/40302 objects misplaced (4.106%); 0 B/s, 52 objects/s recovering 2023-09-23T20:20:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:46 smithi135 ceph-mon[213871]: 1.1c scrub starts 2023-09-23T20:20:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3485028133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:47.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:47 smithi144 ceph-mon[194833]: 1.15 scrub starts 2023-09-23T20:20:47.725 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:47 smithi135 ceph-mon[213871]: 1.15 scrub starts 2023-09-23T20:20:47.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:47 smithi135 ceph-mon[221113]: 1.15 scrub starts 2023-09-23T20:20:47.948 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:47.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:47.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:48.053 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:48.088 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:48.148 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:48.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:48.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:48.257 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:20:48.328 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:48.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:48 smithi144 ceph-mon[194833]: pgmap v1490: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 106 active+clean; 712 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 4467/34050 objects degraded (13.119%); 1433/34050 objects misplaced (4.209%); 0 B/s, 62 objects/s recovering 2023-09-23T20:20:48.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:48 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4467/34050 objects degraded (13.119%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:48.703 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:48 smithi135 ceph-mon[221113]: pgmap v1490: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 106 active+clean; 712 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 4467/34050 objects degraded (13.119%); 1433/34050 objects misplaced (4.209%); 0 B/s, 62 objects/s recovering 2023-09-23T20:20:48.703 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:48 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4467/34050 objects degraded (13.119%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:48.704 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:48 smithi135 ceph-mon[213871]: pgmap v1490: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 106 active+clean; 712 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 4467/34050 objects degraded (13.119%); 1433/34050 objects misplaced (4.209%); 0 B/s, 62 objects/s recovering 2023-09-23T20:20:48.704 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:48 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4467/34050 objects degraded (13.119%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:49.252 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:20:49.252 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:20:49.464 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:20:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:49.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:49 smithi135 ceph-mon[213871]: pgmap v1491: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 646 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4467/30861 objects degraded (14.475%); 1219/30861 objects misplaced (3.950%); 0 B/s, 62 objects/s recovering 2023-09-23T20:20:49.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:49 smithi135 ceph-mon[221113]: pgmap v1491: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 646 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4467/30861 objects degraded (14.475%); 1219/30861 objects misplaced (3.950%); 0 B/s, 62 objects/s recovering 2023-09-23T20:20:49.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:20:49.573 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:49.611 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:49.665 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:49.672 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:49 smithi144 ceph-mon[194833]: pgmap v1491: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 646 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4467/30861 objects degraded (14.475%); 1219/30861 objects misplaced (3.950%); 0 B/s, 62 objects/s recovering 2023-09-23T20:20:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:49.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:49.774 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:49.917 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:50.199 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:50.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:50.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:50.314 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:50.398 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:50.443 INFO:teuthology.orchestra.run.smithi135.stderr:set pool 10 pg_num to 26 2023-09-23T20:20:50.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:50 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:20:50.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1767622094' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:20:50.480 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:50 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:20:50.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:50 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:20:50.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1767622094' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:20:50.491 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:50 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:20:50.658 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:20:50.658 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:20:50.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:50 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:20:50.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1767622094' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:20:50.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:50 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:20:51.253 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:20:51.441 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:20:51.535 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:51.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:51.657 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:51.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:20:51.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: osdmap e523: 8 total, 7 up, 8 in 2023-09-23T20:20:51.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: 1.c scrub starts 2023-09-23T20:20:51.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.725 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2524905660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:51.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:51.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: pgmap v1493: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 106 active+clean; 595 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 4107/28431 objects degraded (14.445%); 1219/28431 objects misplaced (4.288%); 0 B/s, 70 objects/s recovering 2023-09-23T20:20:51.726 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: osdmap e523: 8 total, 7 up, 8 in 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: 1.c scrub starts 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.727 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2524905660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:51.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:51.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: pgmap v1493: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 106 active+clean; 595 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 4107/28431 objects degraded (14.445%); 1219/28431 objects misplaced (4.288%); 0 B/s, 70 objects/s recovering 2023-09-23T20:20:51.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:51 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:20:51.755 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: osdmap e523: 8 total, 7 up, 8 in 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: 1.c scrub starts 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: failed to encode map e523 with expected crc 2023-09-23T20:20:51.757 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2524905660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:51.757 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:20:51.757 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: pgmap v1493: 119 pgs: 3 active+clean+scrubbing, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+clean+scrubbing+deep, 106 active+clean; 595 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 4107/28431 objects degraded (14.445%); 1219/28431 objects misplaced (4.288%); 0 B/s, 70 objects/s recovering 2023-09-23T20:20:51.757 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:51 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:20:51.758 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:51.806 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:51.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:51.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:51.913 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:51.968 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:52.581 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:52.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:52.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:52.687 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:52.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: 1.15 deep-scrub starts 2023-09-23T20:20:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: 1.1d scrub starts 2023-09-23T20:20:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: failed to encode map e524 with expected crc 2023-09-23T20:20:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:20:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:20:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: osdmap e524: 8 total, 7 up, 8 in 2023-09-23T20:20:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: failed to encode map e524 with expected crc 2023-09-23T20:20:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: 1.15 deep-scrub starts 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: 1.1d scrub starts 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: failed to encode map e524 with expected crc 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: osdmap e524: 8 total, 7 up, 8 in 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: failed to encode map e524 with expected crc 2023-09-23T20:20:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:52.727 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:52.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: 1.15 deep-scrub starts 2023-09-23T20:20:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: 1.1d scrub starts 2023-09-23T20:20:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:20:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: failed to encode map e524 with expected crc 2023-09-23T20:20:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:20:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:20:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: osdmap e524: 8 total, 7 up, 8 in 2023-09-23T20:20:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: failed to encode map e524 with expected crc 2023-09-23T20:20:52.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:53.088 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:20:53.317 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:53.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:53.422 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:53.467 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:53.639 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:53.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[213871]: osdmap e525: 8 total, 7 up, 8 in 2023-09-23T20:20:53.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[213871]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[213871]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[213871]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[213871]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[213871]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 4107/28431 objects degraded (14.445%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:53.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[213871]: pgmap v1496: 129 pgs: 4 peering, 9 unknown, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 499 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 4107/23817 objects degraded (17.244%); 1042/23817 objects misplaced (4.375%); 0 B/s, 59 objects/s recovering 2023-09-23T20:20:53.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:53.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:53.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[221113]: osdmap e525: 8 total, 7 up, 8 in 2023-09-23T20:20:53.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[221113]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[221113]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.654 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[221113]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.655 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[221113]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.655 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[221113]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.655 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 4107/28431 objects degraded (14.445%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:53.655 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:53 smithi135 ceph-mon[221113]: pgmap v1496: 129 pgs: 4 peering, 9 unknown, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 499 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 4107/23817 objects degraded (17.244%); 1042/23817 objects misplaced (4.375%); 0 B/s, 59 objects/s recovering 2023-09-23T20:20:53.692 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:53.751 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:20:53.834 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:53 smithi144 ceph-mon[194833]: osdmap e525: 8 total, 7 up, 8 in 2023-09-23T20:20:53.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:53 smithi144 ceph-mon[194833]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:53 smithi144 ceph-mon[194833]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:53 smithi144 ceph-mon[194833]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:53 smithi144 ceph-mon[194833]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:53 smithi144 ceph-mon[194833]: failed to encode map e525 with expected crc 2023-09-23T20:20:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:53 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 4107/28431 objects degraded (14.445%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:53 smithi144 ceph-mon[194833]: pgmap v1496: 129 pgs: 4 peering, 9 unknown, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 499 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 4107/23817 objects degraded (17.244%); 1042/23817 objects misplaced (4.375%); 0 B/s, 59 objects/s recovering 2023-09-23T20:20:53.970 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:20:53.974 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:20:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:20:53] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-09-23T20:20:53.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:53.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:20:54.081 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:54.163 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:54.363 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:54.373 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:54.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:54.481 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:54.548 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:54.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[213871]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[213871]: osdmap e526: 8 total, 7 up, 8 in 2023-09-23T20:20:54.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[213871]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[213871]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:54.726 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:54.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[221113]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[221113]: osdmap e526: 8 total, 7 up, 8 in 2023-09-23T20:20:54.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[221113]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[221113]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:54.727 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:54 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:54 smithi144 ceph-mon[194833]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:54 smithi144 ceph-mon[194833]: osdmap e526: 8 total, 7 up, 8 in 2023-09-23T20:20:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:54 smithi144 ceph-mon[194833]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:54 smithi144 ceph-mon[194833]: failed to encode map e526 with expected crc 2023-09-23T20:20:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:54 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:54 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:55.401 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:55.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:55.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:55.428 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:20:55.512 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:55.541 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:55.592 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:55.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:55.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:55.674 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:55 smithi135 ceph-mon[221113]: pgmap v1498: 129 pgs: 1 active+recovery_wait+degraded, 4 peering, 5 unknown, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 111 active+clean; 456 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 1943/21768 objects degraded (8.926%); 916/21768 objects misplaced (4.208%); 0 B/s, 179 objects/s recovering 2023-09-23T20:20:55.675 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:55 smithi135 ceph-mon[213871]: pgmap v1498: 129 pgs: 1 active+recovery_wait+degraded, 4 peering, 5 unknown, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 111 active+clean; 456 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 1943/21768 objects degraded (8.926%); 916/21768 objects misplaced (4.208%); 0 B/s, 179 objects/s recovering 2023-09-23T20:20:55.699 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:55.727 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:55 smithi144 ceph-mon[194833]: pgmap v1498: 129 pgs: 1 active+recovery_wait+degraded, 4 peering, 5 unknown, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 111 active+clean; 456 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 1943/21768 objects degraded (8.926%); 916/21768 objects misplaced (4.208%); 0 B/s, 179 objects/s recovering 2023-09-23T20:20:55.729 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:55.782 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:55.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:55.885 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:20:55.918 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:55.972 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:20:55.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:55.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:20:56.078 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:56.170 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:56.297 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:56.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:56.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:56.405 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:56.445 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:56.521 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:56.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:56.631 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:56.676 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:56.744 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:56.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:56.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:56.853 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:20:56.936 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:57.198 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:20:57.198 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:20:57.198 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:20:57.198 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:20:57.251 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:20:57.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:57.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:20:57.364 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:20:57.367 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:20:57.392 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:20:57.399 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:57.425 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:20:57.452 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:20:57.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:57.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:20:57.459 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.459 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.459 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:57.459 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:57.460 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.461 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:20:57.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:57.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:57.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:57.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:57.476 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:20:57.507 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:20:57.552 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:20:57.559 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:20:57.591 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:57.598 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:20:57.634 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:20:57.635 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:20:57.635 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:20:57.635 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:20:57.635 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:20:57.635 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:20:57.635 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:20:57.636 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:20:57.636 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:20:57.647 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:20:57.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:57.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:20:57.753 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:57.811 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:20:58.172 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:20:58.172 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:20:58.172 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:58.172 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_repop(client.46418.0:181721 10.5 e526/524)", 2023-09-23T20:20:58.172 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:20:58.170578+0000", 2023-09-23T20:20:58.172 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 0.0015092250000000001, 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001531014, 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "queued for pg", 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:58.170578+0000", 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:20:58.173 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:58.170578+0000", 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:58.170577+0000", 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:58.170587+0000", 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3049999999999998e-06 2023-09-23T20:20:58.174 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:58.170606+0000", 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9140999999999999e-05 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:20:58.170615+0000", 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.0089999999999996e-06 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:20:58.175 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:58.176 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:20:58.176 INFO:teuthology.orchestra.run.smithi135.stdout: ], 2023-09-23T20:20:58.176 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 1 2023-09-23T20:20:58.176 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:20:58.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:58 smithi135 ceph-mon[213871]: pgmap v1499: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 120 active+clean; 343 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1943/16350 objects degraded (11.884%); 680/16350 objects misplaced (4.159%); 0 B/s, 85 objects/s recovering 2023-09-23T20:20:58.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/893782509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:58.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:20:58 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1943/21768 objects degraded (8.926%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:58.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:58 smithi135 ceph-mon[221113]: pgmap v1499: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 120 active+clean; 343 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1943/16350 objects degraded (11.884%); 680/16350 objects misplaced (4.159%); 0 B/s, 85 objects/s recovering 2023-09-23T20:20:58.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/893782509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:58.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:20:58 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1943/21768 objects degraded (8.926%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:58 smithi144 ceph-mon[194833]: pgmap v1499: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 120 active+clean; 343 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1943/16350 objects degraded (11.884%); 680/16350 objects misplaced (4.159%); 0 B/s, 85 objects/s recovering 2023-09-23T20:20:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/893782509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:20:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:20:58 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1943/21768 objects degraded (8.926%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:20:59.509 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:20:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:20:59.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:20:59.626 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:20:59.690 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:00.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:00 smithi135 ceph-mon[213871]: pgmap v1500: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 120 active+clean; 251 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1943/11916 objects degraded (16.306%); 371/11916 objects misplaced (3.113%); 0 B/s, 65 objects/s recovering 2023-09-23T20:21:00.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:00 smithi135 ceph-mon[221113]: pgmap v1500: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 120 active+clean; 251 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1943/11916 objects degraded (16.306%); 371/11916 objects misplaced (3.113%); 0 B/s, 65 objects/s recovering 2023-09-23T20:21:00.615 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:21:00.629 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:21:00.630 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:21:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:00 smithi144 ceph-mon[194833]: pgmap v1500: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 120 active+clean; 251 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1943/11916 objects degraded (16.306%); 371/11916 objects misplaced (3.113%); 0 B/s, 65 objects/s recovering 2023-09-23T20:21:00.806 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:21:00.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:00.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:21:00.915 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:00.951 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:01.005 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:01.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:01.039 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Removed 92223 objects 2023-09-23T20:21:01.040 INFO:tasks.radosbench.radosbench.0.smithi135.stdout:Clean up completed and total clean up time :143.756 2023-09-23T20:21:01.048 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:01.112 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:01.139 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:01.180 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:01.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:01.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:01.285 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:01.312 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:01.353 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:01.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:01.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:01.458 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:01.493 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:01.565 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:02.326 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:21:02.631 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:21:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:02 smithi144 ceph-mon[194833]: pgmap v1501: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 120 active+clean; 173 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 1943/8181 objects degraded (23.750%); 371/8181 objects misplaced (4.535%); 0 B/s, 147 objects/s recovering 2023-09-23T20:21:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2700652332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:02 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:02.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:02 smithi135 ceph-mon[221113]: pgmap v1501: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 120 active+clean; 173 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 1943/8181 objects degraded (23.750%); 371/8181 objects misplaced (4.535%); 0 B/s, 147 objects/s recovering 2023-09-23T20:21:02.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2700652332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:02.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:02 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:02.657 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:02 smithi135 ceph-mon[213871]: pgmap v1501: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+recovering, 1 active+undersized+degraded, 120 active+clean; 173 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 1943/8181 objects degraded (23.750%); 371/8181 objects misplaced (4.535%); 0 B/s, 147 objects/s recovering 2023-09-23T20:21:02.657 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2700652332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:02.657 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:02 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:02.789 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:02.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:02.893 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:21:02.922 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:03.225 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:21:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:03.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:21:03.330 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:03.362 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:03.412 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:03.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:03.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:03.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:03.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[221113]: osdmap e527: 8 total, 7 up, 8 in 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[221113]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[221113]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[221113]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1943/8181 objects degraded (23.750%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[213871]: osdmap e527: 8 total, 7 up, 8 in 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[213871]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[213871]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[213871]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:03 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1943/8181 objects degraded (23.750%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:03.516 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:03.551 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:03.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:03 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:03 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:03 smithi144 ceph-mon[194833]: osdmap e527: 8 total, 7 up, 8 in 2023-09-23T20:21:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:03 smithi144 ceph-mon[194833]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:03 smithi144 ceph-mon[194833]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:03 smithi144 ceph-mon[194833]: failed to encode map e527 with expected crc 2023-09-23T20:21:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:03 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1943/8181 objects degraded (23.750%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:03.806 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:03.808 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:21:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:21:03] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-09-23T20:21:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:03.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:03.822 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:21:03.824 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:21:03.910 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:03.940 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:03.987 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:03.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:04.094 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:21:04.125 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:04.326 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:21:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:04.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:21:04.332 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:04.373 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:04 smithi135 ceph-mon[221113]: pgmap v1503: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 120 active+clean; 53 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 1943/2397 objects degraded (81.060%); 139/2397 objects misplaced (5.799%); 0 B/s, 132 objects/s recovering 2023-09-23T20:21:04.373 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:04 smithi135 ceph-mon[213871]: pgmap v1503: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 120 active+clean; 53 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 1943/2397 objects degraded (81.060%); 139/2397 objects misplaced (5.799%); 0 B/s, 132 objects/s recovering 2023-09-23T20:21:04.433 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:21:04.459 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:04.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:04 smithi144 ceph-mon[194833]: pgmap v1503: 129 pgs: 1 active+recovery_wait+degraded, 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 120 active+clean; 53 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 1943/2397 objects degraded (81.060%); 139/2397 objects misplaced (5.799%); 0 B/s, 132 objects/s recovering 2023-09-23T20:21:04.794 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:21:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:04.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:21:04.898 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:04.929 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:04.979 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:04.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:04.985 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1503,"stamp":"2023-09-23T20:21:03.257091+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":55288695,"num_objects":799,"num_object_clones":0,"num_object_copies":2397,"num_objects_missing_on_primary":2809,"num_objects_missing":2809,"num_objects_degraded":1943,"num_objects_misplaced":139,"num_objects_unfound":0,"num_objects_dirty":799,"num_whiteouts":0,"num_read":6011,"num_read_kb":13264,"num_write":184656,"num_write_kb":5915145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12988,"num_bytes_recovered":618253939,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":175077,"ondisk_log_size":175077,"up":380,"acting":383,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":397,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6917812,"kb_used_data":1122740,"kb_used_omap":38,"kb_used_meta":5794969,"kb_avail":743043404,"statfs":{"total":767960285184,"available":760876445696,"internally_reserved":0,"allocated":1149685760,"data_stored":1133201985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39803,"internal_metadata":5934048389},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,4,6,2,0,0,11],"upper_bound":512},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-467927064,"num_objects":-7140,"num_object_clones":0,"num_object_copies":-21420,"num_objects_missing_on_primary":2809,"num_objects_missing":2809,"num_objects_degraded":-2164,"num_objects_misplaced":-903,"num_objects_unfound":0,"num_objects_dirty":-7140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7197,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.820818"},"pg_stats":[{"pgid":"10.13","version":"526'11289","reported_seq":11353,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.958183+0000","last_change":"2023-09-23T20:20:52.546583+0000","last_active":"2023-09-23T20:21:00.958183+0000","last_peered":"2023-09-23T20:21:00.958183+0000","last_clean":"2023-09-23T20:21:00.958183+0000","last_became_active":"2023-09-23T20:20:52.503982+0000","last_became_peered":"2023-09-23T20:20:52.503982+0000","last_unstale":"2023-09-23T20:21:00.958183+0000","last_undegraded":"2023-09-23T20:21:00.958183+0000","last_fullsized":"2023-09-23T20:21:00.958183+0000","mapping_epoch":524,"log_start":"485'2900","ondisk_log_start":"485'2900","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8389,"log_dups_size":0,"ondisk_log_size":8389,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5769,"num_write_kb":184224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":896,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432098+0000","last_change":"2023-09-23T20:19:54.928200+0000","last_active":"2023-09-23T20:20:52.432098+0000","last_peered":"2023-09-23T20:20:52.432098+0000","last_clean":"2023-09-23T20:20:52.432098+0000","last_became_active":"2023-09-23T20:19:05.089810+0000","last_became_peered":"2023-09-23T20:19:05.089810+0000","last_unstale":"2023-09-23T20:20:52.432098+0000","last_undegraded":"2023-09-23T20:20:52.432098+0000","last_fullsized":"2023-09-23T20:20:52.432098+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:54.928103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:54.928103+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:08.492041+0000","scrub_duration":0.011135947,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.430624+0000","last_change":"2023-09-23T20:17:47.279267+0000","last_active":"2023-09-23T20:20:52.430624+0000","last_peered":"2023-09-23T20:20:52.430624+0000","last_clean":"2023-09-23T20:20:52.430624+0000","last_became_active":"2023-09-23T20:17:47.278920+0000","last_became_peered":"2023-09-23T20:17:47.278920+0000","last_unstale":"2023-09-23T20:20:52.430624+0000","last_undegraded":"2023-09-23T20:20:52.430624+0000","last_fullsized":"2023-09-23T20:20:52.430624+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:16.699236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431529+0000","last_change":"2023-09-23T20:17:47.270409+0000","last_active":"2023-09-23T20:20:52.431529+0000","last_peered":"2023-09-23T20:20:52.431529+0000","last_clean":"2023-09-23T20:20:52.431529+0000","last_became_active":"2023-09-23T20:17:47.270086+0000","last_became_peered":"2023-09-23T20:17:47.270086+0000","last_unstale":"2023-09-23T20:20:52.431529+0000","last_undegraded":"2023-09-23T20:20:52.431529+0000","last_fullsized":"2023-09-23T20:20:52.431529+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.885372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.10","version":"526'11200","reported_seq":19001,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:59.416722+0000","last_change":"2023-09-23T20:20:52.557868+0000","last_active":"2023-09-23T20:20:59.416722+0000","last_peered":"2023-09-23T20:20:59.416722+0000","last_clean":"2023-09-23T20:20:59.416722+0000","last_became_active":"2023-09-23T20:20:52.517403+0000","last_became_peered":"2023-09-23T20:20:52.517403+0000","last_unstale":"2023-09-23T20:20:59.416722+0000","last_undegraded":"2023-09-23T20:20:59.416722+0000","last_fullsized":"2023-09-23T20:20:59.416722+0000","mapping_epoch":524,"log_start":"498'5060","ondisk_log_start":"498'5060","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3407872,"num_objects":52,"num_object_clones":0,"num_object_copies":156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5690,"num_write_kb":183744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1933,"num_bytes_recovered":126648320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":589,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429439+0000","last_change":"2023-09-23T20:19:05.090444+0000","last_active":"2023-09-23T20:20:52.429439+0000","last_peered":"2023-09-23T20:20:52.429439+0000","last_clean":"2023-09-23T20:20:52.429439+0000","last_became_active":"2023-09-23T20:19:05.090104+0000","last_became_peered":"2023-09-23T20:19:05.090104+0000","last_unstale":"2023-09-23T20:20:52.429439+0000","last_undegraded":"2023-09-23T20:20:52.429439+0000","last_fullsized":"2023-09-23T20:20:52.429439+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429243+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:20:52.429243+0000","last_peered":"2023-09-23T20:20:52.429243+0000","last_clean":"2023-09-23T20:20:52.429243+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:20:52.429243+0000","last_undegraded":"2023-09-23T20:20:52.429243+0000","last_fullsized":"2023-09-23T20:20:52.429243+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":235,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449972+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:20:53.449972+0000","last_peered":"2023-09-23T20:20:53.449972+0000","last_clean":"2023-09-23T20:20:53.449972+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:20:53.449972+0000","last_undegraded":"2023-09-23T20:20:53.449972+0000","last_fullsized":"2023-09-23T20:20:53.449972+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.11","version":"526'11307","reported_seq":11373,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.992732+0000","last_change":"2023-09-23T20:20:52.516927+0000","last_active":"2023-09-23T20:21:00.992732+0000","last_peered":"2023-09-23T20:21:00.992732+0000","last_clean":"2023-09-23T20:21:00.992732+0000","last_became_active":"2023-09-23T20:20:52.494120+0000","last_became_peered":"2023-09-23T20:20:52.494120+0000","last_unstale":"2023-09-23T20:21:00.992732+0000","last_undegraded":"2023-09-23T20:21:00.992732+0000","last_fullsized":"2023-09-23T20:21:00.992732+0000","mapping_epoch":524,"log_start":"498'4942","ondisk_log_start":"498'4942","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6365,"log_dups_size":0,"ondisk_log_size":6365,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5774,"num_write_kb":184256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":894,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.433091+0000","last_change":"2023-09-23T20:20:41.315192+0000","last_active":"2023-09-23T20:20:52.433091+0000","last_peered":"2023-09-23T20:20:52.433091+0000","last_clean":"2023-09-23T20:20:52.433091+0000","last_became_active":"2023-09-23T20:19:05.067520+0000","last_became_peered":"2023-09-23T20:19:05.067520+0000","last_unstale":"2023-09-23T20:20:52.433091+0000","last_undegraded":"2023-09-23T20:20:52.433091+0000","last_fullsized":"2023-09-23T20:20:52.433091+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429198+0000","last_change":"2023-09-23T20:17:47.279385+0000","last_active":"2023-09-23T20:20:52.429198+0000","last_peered":"2023-09-23T20:20:52.429198+0000","last_clean":"2023-09-23T20:20:52.429198+0000","last_became_active":"2023-09-23T20:17:47.279065+0000","last_became_peered":"2023-09-23T20:17:47.279065+0000","last_unstale":"2023-09-23T20:20:52.429198+0000","last_undegraded":"2023-09-23T20:20:52.429198+0000","last_fullsized":"2023-09-23T20:20:52.429198+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:19.638157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":491,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.436183+0000","last_change":"2023-09-23T20:17:47.270621+0000","last_active":"2023-09-23T20:20:52.436183+0000","last_peered":"2023-09-23T20:20:52.436183+0000","last_clean":"2023-09-23T20:20:52.436183+0000","last_became_active":"2023-09-23T20:17:47.270375+0000","last_became_peered":"2023-09-23T20:17:47.270375+0000","last_unstale":"2023-09-23T20:20:52.436183+0000","last_undegraded":"2023-09-23T20:20:52.436183+0000","last_fullsized":"2023-09-23T20:20:52.436183+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:16.961271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":775,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.437061+0000","last_change":"2023-09-23T20:20:51.492613+0000","last_active":"2023-09-23T20:20:52.437061+0000","last_peered":"2023-09-23T20:20:52.437061+0000","last_clean":"2023-09-23T20:20:52.437061+0000","last_became_active":"2023-09-23T20:19:05.069486+0000","last_became_peered":"2023-09-23T20:19:05.069486+0000","last_unstale":"2023-09-23T20:20:52.437061+0000","last_undegraded":"2023-09-23T20:20:52.437061+0000","last_fullsized":"2023-09-23T20:20:52.437061+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":452,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.436061+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:20:52.436061+0000","last_peered":"2023-09-23T20:20:52.436061+0000","last_clean":"2023-09-23T20:20:52.436061+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:20:52.436061+0000","last_undegraded":"2023-09-23T20:20:52.436061+0000","last_fullsized":"2023-09-23T20:20:52.436061+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":391,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.426578+0000","last_change":"2023-09-23T20:17:47.265452+0000","last_active":"2023-09-23T20:20:52.426578+0000","last_peered":"2023-09-23T20:20:52.426578+0000","last_clean":"2023-09-23T20:20:52.426578+0000","last_became_active":"2023-09-23T20:17:47.265115+0000","last_became_peered":"2023-09-23T20:17:47.265115+0000","last_unstale":"2023-09-23T20:20:52.426578+0000","last_undegraded":"2023-09-23T20:20:52.426578+0000","last_fullsized":"2023-09-23T20:20:52.426578+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:04.898742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":861,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432321+0000","last_change":"2023-09-23T20:13:25.109316+0000","last_active":"2023-09-23T20:20:52.432321+0000","last_peered":"2023-09-23T20:20:52.432321+0000","last_clean":"2023-09-23T20:20:52.432321+0000","last_became_active":"2023-09-23T20:12:57.739695+0000","last_became_peered":"2023-09-23T20:12:57.739695+0000","last_unstale":"2023-09-23T20:20:52.432321+0000","last_undegraded":"2023-09-23T20:20:52.432321+0000","last_fullsized":"2023-09-23T20:20:52.432321+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:22.784164+0000","scrub_duration":0.0091582439999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":545,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.438565+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:20:52.438565+0000","last_peered":"2023-09-23T20:20:52.438565+0000","last_clean":"2023-09-23T20:20:52.438565+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:20:52.438565+0000","last_undegraded":"2023-09-23T20:20:52.438565+0000","last_fullsized":"2023-09-23T20:20:52.438565+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":392,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.425636+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:20:52.425636+0000","last_peered":"2023-09-23T20:20:52.425636+0000","last_clean":"2023-09-23T20:20:52.425636+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:20:52.425636+0000","last_undegraded":"2023-09-23T20:20:52.425636+0000","last_fullsized":"2023-09-23T20:20:52.425636+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":830,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.427732+0000","last_change":"2023-09-23T20:19:42.182813+0000","last_active":"2023-09-23T20:20:52.427732+0000","last_peered":"2023-09-23T20:20:52.427732+0000","last_clean":"2023-09-23T20:20:52.427732+0000","last_became_active":"2023-09-23T20:19:05.072575+0000","last_became_peered":"2023-09-23T20:19:05.072575+0000","last_unstale":"2023-09-23T20:20:52.427732+0000","last_undegraded":"2023-09-23T20:20:52.427732+0000","last_fullsized":"2023-09-23T20:20:52.427732+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:42.182718+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:42.182718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:00.054669+0000","scrub_duration":0.0014414230000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":220,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449745+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:20:53.449745+0000","last_peered":"2023-09-23T20:20:53.449745+0000","last_clean":"2023-09-23T20:20:53.449745+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:20:53.449745+0000","last_undegraded":"2023-09-23T20:20:53.449745+0000","last_fullsized":"2023-09-23T20:20:53.449745+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429643+0000","last_change":"2023-09-23T20:17:47.278773+0000","last_active":"2023-09-23T20:20:52.429643+0000","last_peered":"2023-09-23T20:20:52.429643+0000","last_clean":"2023-09-23T20:20:52.429643+0000","last_became_active":"2023-09-23T20:17:47.278467+0000","last_became_peered":"2023-09-23T20:17:47.278467+0000","last_unstale":"2023-09-23T20:20:52.429643+0000","last_undegraded":"2023-09-23T20:20:52.429643+0000","last_fullsized":"2023-09-23T20:20:52.429643+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:41.368483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":721,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432001+0000","last_change":"2023-09-23T20:19:05.092744+0000","last_active":"2023-09-23T20:20:52.432001+0000","last_peered":"2023-09-23T20:20:52.432001+0000","last_clean":"2023-09-23T20:20:52.432001+0000","last_became_active":"2023-09-23T20:19:05.092425+0000","last_became_peered":"2023-09-23T20:19:05.092425+0000","last_unstale":"2023-09-23T20:20:52.432001+0000","last_undegraded":"2023-09-23T20:20:52.432001+0000","last_fullsized":"2023-09-23T20:20:52.432001+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":473,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449428+0000","last_change":"2023-09-23T20:13:57.751192+0000","last_active":"2023-09-23T20:20:53.449428+0000","last_peered":"2023-09-23T20:20:53.449428+0000","last_clean":"2023-09-23T20:20:53.449428+0000","last_became_active":"2023-09-23T20:13:57.750848+0000","last_became_peered":"2023-09-23T20:13:57.750848+0000","last_unstale":"2023-09-23T20:20:53.449428+0000","last_undegraded":"2023-09-23T20:20:53.449428+0000","last_fullsized":"2023-09-23T20:20:53.449428+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:39.740501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":46,"reported_epoch":525,"state":"active+undersized","last_fresh":"2023-09-23T20:20:52.430033+0000","last_change":"2023-09-23T20:18:38.012263+0000","last_active":"2023-09-23T20:20:52.430033+0000","last_peered":"2023-09-23T20:20:52.430033+0000","last_clean":"2023-09-23T20:17:54.133085+0000","last_became_active":"2023-09-23T20:18:38.012263+0000","last_became_peered":"2023-09-23T20:18:38.012263+0000","last_unstale":"2023-09-23T20:20:52.430033+0000","last_undegraded":"2023-09-23T20:20:52.430033+0000","last_fullsized":"2023-09-23T20:18:38.003813+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:47.449647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":885,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432678+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:20:52.432678+0000","last_peered":"2023-09-23T20:20:52.432678+0000","last_clean":"2023-09-23T20:20:52.432678+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:20:52.432678+0000","last_undegraded":"2023-09-23T20:20:52.432678+0000","last_fullsized":"2023-09-23T20:20:52.432678+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":220,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449453+0000","last_change":"2023-09-23T20:07:37.072101+0000","last_active":"2023-09-23T20:20:53.449453+0000","last_peered":"2023-09-23T20:20:53.449453+0000","last_clean":"2023-09-23T20:20:53.449453+0000","last_became_active":"2023-09-23T20:07:36.985660+0000","last_became_peered":"2023-09-23T20:07:36.985660+0000","last_unstale":"2023-09-23T20:20:53.449453+0000","last_undegraded":"2023-09-23T20:20:53.449453+0000","last_fullsized":"2023-09-23T20:20:53.449453+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:21.356153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431310+0000","last_change":"2023-09-23T20:17:47.266963+0000","last_active":"2023-09-23T20:20:52.431310+0000","last_peered":"2023-09-23T20:20:52.431310+0000","last_clean":"2023-09-23T20:20:52.431310+0000","last_became_active":"2023-09-23T20:17:47.266645+0000","last_became_peered":"2023-09-23T20:17:47.266645+0000","last_unstale":"2023-09-23T20:20:52.431310+0000","last_undegraded":"2023-09-23T20:20:52.431310+0000","last_fullsized":"2023-09-23T20:20:52.431310+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:05.052559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":574,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432367+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:20:52.432367+0000","last_peered":"2023-09-23T20:20:52.432367+0000","last_clean":"2023-09-23T20:20:52.432367+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:20:52.432367+0000","last_undegraded":"2023-09-23T20:20:52.432367+0000","last_fullsized":"2023-09-23T20:20:52.432367+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":235,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449077+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:20:53.449077+0000","last_peered":"2023-09-23T20:20:53.449077+0000","last_clean":"2023-09-23T20:20:53.449077+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:20:53.449077+0000","last_undegraded":"2023-09-23T20:20:53.449077+0000","last_fullsized":"2023-09-23T20:20:53.449077+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":524,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.435491+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:20:52.435491+0000","last_peered":"2023-09-23T20:20:52.435491+0000","last_clean":"2023-09-23T20:20:52.435491+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:20:52.435491+0000","last_undegraded":"2023-09-23T20:20:52.435491+0000","last_fullsized":"2023-09-23T20:20:52.435491+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.18","version":"526'11246","reported_seq":11316,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.920405+0000","last_change":"2023-09-23T20:20:52.634923+0000","last_active":"2023-09-23T20:21:00.920405+0000","last_peered":"2023-09-23T20:21:00.920405+0000","last_clean":"2023-09-23T20:21:00.920405+0000","last_became_active":"2023-09-23T20:20:52.565451+0000","last_became_peered":"2023-09-23T20:20:52.565451+0000","last_unstale":"2023-09-23T20:21:00.920405+0000","last_undegraded":"2023-09-23T20:21:00.920405+0000","last_fullsized":"2023-09-23T20:21:00.920405+0000","mapping_epoch":524,"log_start":"486'2900","ondisk_log_start":"486'2900","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8346,"log_dups_size":0,"ondisk_log_size":8346,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360448,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5736,"num_write_kb":183712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":585,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.430168+0000","last_change":"2023-09-23T20:17:50.330492+0000","last_active":"2023-09-23T20:20:52.430168+0000","last_peered":"2023-09-23T20:20:52.430168+0000","last_clean":"2023-09-23T20:20:52.430168+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:20:52.430168+0000","last_undegraded":"2023-09-23T20:20:52.430168+0000","last_fullsized":"2023-09-23T20:20:52.430168+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:58.248167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.248167+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":231,"reported_epoch":526,"state":"active+undersized","last_fresh":"2023-09-23T20:20:53.448845+0000","last_change":"2023-09-23T20:18:38.011856+0000","last_active":"2023-09-23T20:20:53.448845+0000","last_peered":"2023-09-23T20:20:53.448845+0000","last_clean":"2023-09-23T20:18:35.260354+0000","last_became_active":"2023-09-23T20:18:38.011856+0000","last_became_peered":"2023-09-23T20:18:38.011856+0000","last_unstale":"2023-09-23T20:20:53.448845+0000","last_undegraded":"2023-09-23T20:20:53.448845+0000","last_fullsized":"2023-09-23T20:18:37.999747+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:58.093947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":516,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.426880+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:20:52.426880+0000","last_peered":"2023-09-23T20:20:52.426880+0000","last_clean":"2023-09-23T20:20:52.426880+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:20:52.426880+0000","last_undegraded":"2023-09-23T20:20:52.426880+0000","last_fullsized":"2023-09-23T20:20:52.426880+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.19","version":"526'11332","reported_seq":11397,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:58.565023+0000","last_change":"2023-09-23T20:20:52.511163+0000","last_active":"2023-09-23T20:20:58.565023+0000","last_peered":"2023-09-23T20:20:58.565023+0000","last_clean":"2023-09-23T20:20:58.565023+0000","last_became_active":"2023-09-23T20:20:52.488847+0000","last_became_peered":"2023-09-23T20:20:52.488847+0000","last_unstale":"2023-09-23T20:20:58.565023+0000","last_undegraded":"2023-09-23T20:20:58.565023+0000","last_fullsized":"2023-09-23T20:20:58.565023+0000","mapping_epoch":524,"log_start":"497'4611","ondisk_log_start":"497'4611","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5373952,"num_objects":82,"num_object_clones":0,"num_object_copies":246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5736,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":683,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.433176+0000","last_change":"2023-09-23T20:17:50.332129+0000","last_active":"2023-09-23T20:20:52.433176+0000","last_peered":"2023-09-23T20:20:52.433176+0000","last_clean":"2023-09-23T20:20:52.433176+0000","last_became_active":"2023-09-23T20:17:50.331669+0000","last_became_peered":"2023-09-23T20:17:50.331669+0000","last_unstale":"2023-09-23T20:20:52.433176+0000","last_undegraded":"2023-09-23T20:20:52.433176+0000","last_fullsized":"2023-09-23T20:20:52.433176+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:23.893417+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":169,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432191+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:20:52.432191+0000","last_peered":"2023-09-23T20:20:52.432191+0000","last_clean":"2023-09-23T20:20:52.432191+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:20:52.432191+0000","last_undegraded":"2023-09-23T20:20:52.432191+0000","last_fullsized":"2023-09-23T20:20:52.432191+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":178,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.448900+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:20:53.448900+0000","last_peered":"2023-09-23T20:20:53.448900+0000","last_clean":"2023-09-23T20:20:53.448900+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:20:53.448900+0000","last_undegraded":"2023-09-23T20:20:53.448900+0000","last_fullsized":"2023-09-23T20:20:53.448900+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"526'11247","reported_seq":11313,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:58.544252+0000","last_change":"2023-09-23T20:20:52.485195+0000","last_active":"2023-09-23T20:20:58.544252+0000","last_peered":"2023-09-23T20:20:58.544252+0000","last_clean":"2023-09-23T20:20:58.544252+0000","last_became_active":"2023-09-23T20:20:52.459902+0000","last_became_peered":"2023-09-23T20:20:52.459902+0000","last_unstale":"2023-09-23T20:20:58.544252+0000","last_undegraded":"2023-09-23T20:20:58.544252+0000","last_fullsized":"2023-09-23T20:20:58.544252+0000","mapping_epoch":479,"log_start":"497'4533","ondisk_log_start":"497'4533","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6714,"log_dups_size":0,"ondisk_log_size":6714,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4849664,"num_objects":74,"num_object_clones":0,"num_object_copies":222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5704,"num_write_kb":184896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":826,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449465+0000","last_change":"2023-09-23T20:20:33.452164+0000","last_active":"2023-09-23T20:20:53.449465+0000","last_peered":"2023-09-23T20:20:53.449465+0000","last_clean":"2023-09-23T20:20:53.449465+0000","last_became_active":"2023-09-23T20:19:05.065283+0000","last_became_peered":"2023-09-23T20:19:05.065283+0000","last_unstale":"2023-09-23T20:20:53.449465+0000","last_undegraded":"2023-09-23T20:20:53.449465+0000","last_fullsized":"2023-09-23T20:20:53.449465+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":218,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.450080+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:20:53.450080+0000","last_peered":"2023-09-23T20:20:53.450080+0000","last_clean":"2023-09-23T20:20:53.450080+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:20:53.450080+0000","last_undegraded":"2023-09-23T20:20:53.450080+0000","last_fullsized":"2023-09-23T20:20:53.450080+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":534,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432245+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:20:52.432245+0000","last_peered":"2023-09-23T20:20:52.432245+0000","last_clean":"2023-09-23T20:20:52.432245+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:20:52.432245+0000","last_undegraded":"2023-09-23T20:20:52.432245+0000","last_fullsized":"2023-09-23T20:20:52.432245+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"526'11286","reported_seq":11353,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:01.019699+0000","last_change":"2023-09-23T20:20:52.463308+0000","last_active":"2023-09-23T20:21:01.019699+0000","last_peered":"2023-09-23T20:21:01.019699+0000","last_clean":"2023-09-23T20:21:01.019699+0000","last_became_active":"2023-09-23T20:20:52.437380+0000","last_became_peered":"2023-09-23T20:20:52.437380+0000","last_unstale":"2023-09-23T20:21:01.019699+0000","last_undegraded":"2023-09-23T20:21:01.019699+0000","last_fullsized":"2023-09-23T20:21:01.019699+0000","mapping_epoch":479,"log_start":"504'5862","ondisk_log_start":"504'5862","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5424,"log_dups_size":0,"ondisk_log_size":5424,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5768,"num_write_kb":183968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":891,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.426161+0000","last_change":"2023-09-23T20:20:51.462474+0000","last_active":"2023-09-23T20:20:52.426161+0000","last_peered":"2023-09-23T20:20:52.426161+0000","last_clean":"2023-09-23T20:20:52.426161+0000","last_became_active":"2023-09-23T20:19:05.085764+0000","last_became_peered":"2023-09-23T20:19:05.085764+0000","last_unstale":"2023-09-23T20:20:52.426161+0000","last_undegraded":"2023-09-23T20:20:52.426161+0000","last_fullsized":"2023-09-23T20:20:52.426161+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":247,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.450019+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:20:53.450019+0000","last_peered":"2023-09-23T20:20:53.450019+0000","last_clean":"2023-09-23T20:20:53.450019+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:20:53.450019+0000","last_undegraded":"2023-09-23T20:20:53.450019+0000","last_fullsized":"2023-09-23T20:20:53.450019+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429317+0000","last_change":"2023-09-23T20:17:47.264776+0000","last_active":"2023-09-23T20:20:52.429317+0000","last_peered":"2023-09-23T20:20:52.429317+0000","last_clean":"2023-09-23T20:20:52.429317+0000","last_became_active":"2023-09-23T20:17:47.264473+0000","last_became_peered":"2023-09-23T20:17:47.264473+0000","last_unstale":"2023-09-23T20:20:52.429317+0000","last_undegraded":"2023-09-23T20:20:52.429317+0000","last_fullsized":"2023-09-23T20:20:52.429317+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:40.906303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"526'11423","reported_seq":11490,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.938091+0000","last_change":"2023-09-23T20:20:52.497170+0000","last_active":"2023-09-23T20:21:00.938091+0000","last_peered":"2023-09-23T20:21:00.938091+0000","last_clean":"2023-09-23T20:21:00.938091+0000","last_became_active":"2023-09-23T20:20:52.465283+0000","last_became_peered":"2023-09-23T20:20:52.465283+0000","last_unstale":"2023-09-23T20:21:00.938091+0000","last_undegraded":"2023-09-23T20:21:00.938091+0000","last_fullsized":"2023-09-23T20:21:00.938091+0000","mapping_epoch":479,"log_start":"502'5828","ondisk_log_start":"502'5828","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5595,"log_dups_size":0,"ondisk_log_size":5595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":186432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":598,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.435848+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:20:52.435848+0000","last_peered":"2023-09-23T20:20:52.435848+0000","last_clean":"2023-09-23T20:20:52.435848+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:20:52.435848+0000","last_undegraded":"2023-09-23T20:20:52.435848+0000","last_fullsized":"2023-09-23T20:20:52.435848+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":239,"reported_epoch":526,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:20:53.449629+0000","last_change":"2023-09-23T20:18:38.001952+0000","last_active":"2023-09-23T20:20:53.449629+0000","last_peered":"2023-09-23T20:20:53.449629+0000","last_clean":"2023-09-23T20:18:35.257306+0000","last_became_active":"2023-09-23T20:18:38.001952+0000","last_became_peered":"2023-09-23T20:18:38.001952+0000","last_unstale":"2023-09-23T20:20:53.449629+0000","last_undegraded":"2023-09-23T20:18:38.000196+0000","last_fullsized":"2023-09-23T20:18:38.000044+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:17.507792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.430572+0000","last_change":"2023-09-23T20:17:47.266977+0000","last_active":"2023-09-23T20:20:52.430572+0000","last_peered":"2023-09-23T20:20:52.430572+0000","last_clean":"2023-09-23T20:20:52.430572+0000","last_became_active":"2023-09-23T20:17:47.266657+0000","last_became_peered":"2023-09-23T20:17:47.266657+0000","last_unstale":"2023-09-23T20:20:52.430572+0000","last_undegraded":"2023-09-23T20:20:52.430572+0000","last_fullsized":"2023-09-23T20:20:52.430572+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:33.774981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"526'11486","reported_seq":11554,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.990586+0000","last_change":"2023-09-23T20:20:52.569258+0000","last_active":"2023-09-23T20:21:00.990586+0000","last_peered":"2023-09-23T20:21:00.990586+0000","last_clean":"2023-09-23T20:21:00.990586+0000","last_became_active":"2023-09-23T20:20:52.487091+0000","last_became_peered":"2023-09-23T20:20:52.487091+0000","last_unstale":"2023-09-23T20:21:00.990586+0000","last_undegraded":"2023-09-23T20:21:00.990586+0000","last_fullsized":"2023-09-23T20:21:00.990586+0000","mapping_epoch":479,"log_start":"487'3100","ondisk_log_start":"487'3100","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8386,"log_dups_size":0,"ondisk_log_size":8386,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5859,"num_write_kb":187520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":749,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449888+0000","last_change":"2023-09-23T20:20:36.371390+0000","last_active":"2023-09-23T20:20:53.449888+0000","last_peered":"2023-09-23T20:20:53.449888+0000","last_clean":"2023-09-23T20:20:53.449888+0000","last_became_active":"2023-09-23T20:19:05.071700+0000","last_became_peered":"2023-09-23T20:19:05.071700+0000","last_unstale":"2023-09-23T20:20:53.449888+0000","last_undegraded":"2023-09-23T20:20:53.449888+0000","last_fullsized":"2023-09-23T20:20:53.449888+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":584,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.434903+0000","last_change":"2023-09-23T20:17:47.267657+0000","last_active":"2023-09-23T20:20:52.434903+0000","last_peered":"2023-09-23T20:20:52.434903+0000","last_clean":"2023-09-23T20:20:52.434903+0000","last_became_active":"2023-09-23T20:17:47.267288+0000","last_became_peered":"2023-09-23T20:17:47.267288+0000","last_unstale":"2023-09-23T20:20:52.434903+0000","last_undegraded":"2023-09-23T20:20:52.434903+0000","last_fullsized":"2023-09-23T20:20:52.434903+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.185253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429646+0000","last_change":"2023-09-23T20:17:47.280675+0000","last_active":"2023-09-23T20:20:52.429646+0000","last_peered":"2023-09-23T20:20:52.429646+0000","last_clean":"2023-09-23T20:20:52.429646+0000","last_became_active":"2023-09-23T20:17:47.280175+0000","last_became_peered":"2023-09-23T20:17:47.280175+0000","last_unstale":"2023-09-23T20:20:52.429646+0000","last_undegraded":"2023-09-23T20:20:52.429646+0000","last_fullsized":"2023-09-23T20:20:52.429646+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:08.499949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"526'11250","reported_seq":11318,"reported_epoch":526,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:20:59.474896+0000","last_change":"2023-09-23T20:20:53.578725+0000","last_active":"2023-09-23T20:20:59.474896+0000","last_peered":"2023-09-23T20:20:59.474896+0000","last_clean":"2023-09-23T20:17:46.225624+0000","last_became_active":"2023-09-23T20:20:53.451770+0000","last_became_peered":"2023-09-23T20:20:53.451770+0000","last_unstale":"2023-09-23T20:20:59.474896+0000","last_undegraded":"2023-09-23T20:20:53.445124+0000","last_fullsized":"2023-09-23T20:20:59.474896+0000","mapping_epoch":525,"log_start":"501'5747","ondisk_log_start":"501'5747","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":1942,"num_objects_missing":1942,"num_objects_degraded":1942,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5709,"num_write_kb":184736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,1,4","objects":64}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":929,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449308+0000","last_change":"2023-09-23T20:19:47.182002+0000","last_active":"2023-09-23T20:20:53.449308+0000","last_peered":"2023-09-23T20:20:53.449308+0000","last_clean":"2023-09-23T20:20:53.449308+0000","last_became_active":"2023-09-23T20:19:05.071737+0000","last_became_peered":"2023-09-23T20:19:05.071737+0000","last_unstale":"2023-09-23T20:20:53.449308+0000","last_undegraded":"2023-09-23T20:20:53.449308+0000","last_fullsized":"2023-09-23T20:20:53.449308+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:47.181912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:47.181912+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:28.873453+0000","scrub_duration":0.0074047540000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":274,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.425819+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:20:52.425819+0000","last_peered":"2023-09-23T20:20:52.425819+0000","last_clean":"2023-09-23T20:20:52.425819+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:20:52.425819+0000","last_undegraded":"2023-09-23T20:20:52.425819+0000","last_fullsized":"2023-09-23T20:20:52.425819+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431381+0000","last_change":"2023-09-23T20:17:47.270450+0000","last_active":"2023-09-23T20:20:52.431381+0000","last_peered":"2023-09-23T20:20:52.431381+0000","last_clean":"2023-09-23T20:20:52.431381+0000","last_became_active":"2023-09-23T20:17:47.270040+0000","last_became_peered":"2023-09-23T20:17:47.270040+0000","last_unstale":"2023-09-23T20:20:52.431381+0000","last_undegraded":"2023-09-23T20:20:52.431381+0000","last_fullsized":"2023-09-23T20:20:52.431381+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:51.572016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.3","version":"526'11265","reported_seq":11329,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.958267+0000","last_change":"2023-09-23T20:20:52.493132+0000","last_active":"2023-09-23T20:21:00.958267+0000","last_peered":"2023-09-23T20:21:00.958267+0000","last_clean":"2023-09-23T20:21:00.958267+0000","last_became_active":"2023-09-23T20:20:52.465350+0000","last_became_peered":"2023-09-23T20:20:52.465350+0000","last_unstale":"2023-09-23T20:21:00.958267+0000","last_undegraded":"2023-09-23T20:21:00.958267+0000","last_fullsized":"2023-09-23T20:21:00.958267+0000","mapping_epoch":479,"log_start":"485'2900","ondisk_log_start":"485'2900","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8365,"log_dups_size":0,"ondisk_log_size":8365,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":786432,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5745,"num_write_kb":184224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":899,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432295+0000","last_change":"2023-09-23T20:19:43.954300+0000","last_active":"2023-09-23T20:20:52.432295+0000","last_peered":"2023-09-23T20:20:52.432295+0000","last_clean":"2023-09-23T20:20:52.432295+0000","last_became_active":"2023-09-23T20:19:05.075801+0000","last_became_peered":"2023-09-23T20:19:05.075801+0000","last_unstale":"2023-09-23T20:20:52.432295+0000","last_undegraded":"2023-09-23T20:20:52.432295+0000","last_fullsized":"2023-09-23T20:20:52.432295+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:43.954204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:43.954204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:52.060677+0000","scrub_duration":0.0019293979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429489+0000","last_change":"2023-09-23T20:17:47.266513+0000","last_active":"2023-09-23T20:20:52.429489+0000","last_peered":"2023-09-23T20:20:52.429489+0000","last_clean":"2023-09-23T20:20:52.429489+0000","last_became_active":"2023-09-23T20:17:47.266163+0000","last_became_peered":"2023-09-23T20:17:47.266163+0000","last_unstale":"2023-09-23T20:20:52.429489+0000","last_undegraded":"2023-09-23T20:20:52.429489+0000","last_fullsized":"2023-09-23T20:20:52.429489+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.645381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":438,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.437233+0000","last_change":"2023-09-23T20:17:47.270131+0000","last_active":"2023-09-23T20:20:52.437233+0000","last_peered":"2023-09-23T20:20:52.437233+0000","last_clean":"2023-09-23T20:20:52.437233+0000","last_became_active":"2023-09-23T20:17:47.269818+0000","last_became_peered":"2023-09-23T20:17:47.269818+0000","last_unstale":"2023-09-23T20:20:52.437233+0000","last_undegraded":"2023-09-23T20:20:52.437233+0000","last_fullsized":"2023-09-23T20:20:52.437233+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:40.738070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"526'11208","reported_seq":19014,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:59.417160+0000","last_change":"2023-09-23T20:20:52.566007+0000","last_active":"2023-09-23T20:20:59.417160+0000","last_peered":"2023-09-23T20:20:59.417160+0000","last_clean":"2023-09-23T20:20:59.417160+0000","last_became_active":"2023-09-23T20:20:52.486153+0000","last_became_peered":"2023-09-23T20:20:52.486153+0000","last_unstale":"2023-09-23T20:20:59.417160+0000","last_undegraded":"2023-09-23T20:20:59.417160+0000","last_fullsized":"2023-09-23T20:20:59.417160+0000","mapping_epoch":511,"log_start":"498'5060","ondisk_log_start":"498'5060","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2883584,"num_objects":44,"num_object_clones":0,"num_object_copies":132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5698,"num_write_kb":183744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1933,"num_bytes_recovered":126648320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":590,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429822+0000","last_change":"2023-09-23T20:19:05.088257+0000","last_active":"2023-09-23T20:20:52.429822+0000","last_peered":"2023-09-23T20:20:52.429822+0000","last_clean":"2023-09-23T20:20:52.429822+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:20:52.429822+0000","last_undegraded":"2023-09-23T20:20:52.429822+0000","last_fullsized":"2023-09-23T20:20:52.429822+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:53.682855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:53.682855+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":420,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.430057+0000","last_change":"2023-09-23T20:13:57.760285+0000","last_active":"2023-09-23T20:20:52.430057+0000","last_peered":"2023-09-23T20:20:52.430057+0000","last_clean":"2023-09-23T20:20:52.430057+0000","last_became_active":"2023-09-23T20:13:57.759919+0000","last_became_peered":"2023-09-23T20:13:57.759919+0000","last_unstale":"2023-09-23T20:20:52.430057+0000","last_undegraded":"2023-09-23T20:20:52.430057+0000","last_fullsized":"2023-09-23T20:20:52.430057+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:22.626096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431688+0000","last_change":"2023-09-23T20:17:47.268811+0000","last_active":"2023-09-23T20:20:52.431688+0000","last_peered":"2023-09-23T20:20:52.431688+0000","last_clean":"2023-09-23T20:20:52.431688+0000","last_became_active":"2023-09-23T20:17:47.268391+0000","last_became_peered":"2023-09-23T20:17:47.268391+0000","last_unstale":"2023-09-23T20:20:52.431688+0000","last_undegraded":"2023-09-23T20:20:52.431688+0000","last_fullsized":"2023-09-23T20:20:52.431688+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:32.388705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"526'11275","reported_seq":11342,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:01.018233+0000","last_change":"2023-09-23T20:20:52.487569+0000","last_active":"2023-09-23T20:21:01.018233+0000","last_peered":"2023-09-23T20:21:01.018233+0000","last_clean":"2023-09-23T20:21:01.018233+0000","last_became_active":"2023-09-23T20:20:52.463959+0000","last_became_peered":"2023-09-23T20:20:52.463959+0000","last_unstale":"2023-09-23T20:21:01.018233+0000","last_undegraded":"2023-09-23T20:21:01.018233+0000","last_fullsized":"2023-09-23T20:21:01.018233+0000","mapping_epoch":479,"log_start":"498'4942","ondisk_log_start":"498'4942","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6333,"log_dups_size":0,"ondisk_log_size":6333,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1048576,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5742,"num_write_kb":184256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":901,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.433697+0000","last_change":"2023-09-23T20:20:40.335301+0000","last_active":"2023-09-23T20:20:52.433697+0000","last_peered":"2023-09-23T20:20:52.433697+0000","last_clean":"2023-09-23T20:20:52.433697+0000","last_became_active":"2023-09-23T20:19:05.066352+0000","last_became_peered":"2023-09-23T20:19:05.066352+0000","last_unstale":"2023-09-23T20:20:52.433697+0000","last_undegraded":"2023-09-23T20:20:52.433697+0000","last_fullsized":"2023-09-23T20:20:52.433697+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T19:52:59.809323+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0055090299999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":178,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.425391+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:20:52.425391+0000","last_peered":"2023-09-23T20:20:52.425391+0000","last_clean":"2023-09-23T20:20:52.425391+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:20:52.425391+0000","last_undegraded":"2023-09-23T20:20:52.425391+0000","last_fullsized":"2023-09-23T20:20:52.425391+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432335+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:20:52.432335+0000","last_peered":"2023-09-23T20:20:52.432335+0000","last_clean":"2023-09-23T20:20:52.432335+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:20:52.432335+0000","last_undegraded":"2023-09-23T20:20:52.432335+0000","last_fullsized":"2023-09-23T20:20:52.432335+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"526'11522","reported_seq":11578,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:01.033720+0000","last_change":"2023-09-23T20:16:59.496291+0000","last_active":"2023-09-23T20:21:01.033720+0000","last_peered":"2023-09-23T20:21:01.033720+0000","last_clean":"2023-09-23T20:21:01.033720+0000","last_became_active":"2023-09-23T20:16:59.496000+0000","last_became_peered":"2023-09-23T20:16:59.496000+0000","last_unstale":"2023-09-23T20:21:01.033720+0000","last_undegraded":"2023-09-23T20:21:01.033720+0000","last_fullsized":"2023-09-23T20:21:01.033720+0000","mapping_epoch":479,"log_start":"502'5768","ondisk_log_start":"502'5768","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5754,"log_dups_size":0,"ondisk_log_size":5754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:18.808639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11522,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":831,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.427572+0000","last_change":"2023-09-23T20:19:40.219448+0000","last_active":"2023-09-23T20:20:52.427572+0000","last_peered":"2023-09-23T20:20:52.427572+0000","last_clean":"2023-09-23T20:20:52.427572+0000","last_became_active":"2023-09-23T20:19:05.072187+0000","last_became_peered":"2023-09-23T20:19:05.072187+0000","last_unstale":"2023-09-23T20:20:52.427572+0000","last_undegraded":"2023-09-23T20:20:52.427572+0000","last_fullsized":"2023-09-23T20:20:52.427572+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:40.219310+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:40.219310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:29.003073+0000","scrub_duration":0.011178762,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":477,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431166+0000","last_change":"2023-09-23T20:17:50.311646+0000","last_active":"2023-09-23T20:20:52.431166+0000","last_peered":"2023-09-23T20:20:52.431166+0000","last_clean":"2023-09-23T20:20:52.431166+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:20:52.431166+0000","last_undegraded":"2023-09-23T20:20:52.431166+0000","last_fullsized":"2023-09-23T20:20:52.431166+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":526,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.435419+0000","last_change":"2023-09-23T19:59:10.223984+0000","last_active":"2023-09-23T20:20:52.435419+0000","last_peered":"2023-09-23T20:20:52.435419+0000","last_clean":"2023-09-23T20:20:52.435419+0000","last_became_active":"2023-09-23T19:59:10.223635+0000","last_became_peered":"2023-09-23T19:59:10.223635+0000","last_unstale":"2023-09-23T20:20:52.435419+0000","last_undegraded":"2023-09-23T20:20:52.435419+0000","last_fullsized":"2023-09-23T20:20:52.435419+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:39.269688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429566+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:20:52.429566+0000","last_peered":"2023-09-23T20:20:52.429566+0000","last_clean":"2023-09-23T20:20:52.429566+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:20:52.429566+0000","last_undegraded":"2023-09-23T20:20:52.429566+0000","last_fullsized":"2023-09-23T20:20:52.429566+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"526'11259","reported_seq":13860,"reported_epoch":526,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:20:58.588949+0000","last_change":"2023-09-23T20:18:56.585041+0000","last_active":"2023-09-23T20:20:58.588949+0000","last_peered":"2023-09-23T20:20:58.588949+0000","last_clean":"2023-09-23T20:20:58.588949+0000","last_became_active":"2023-09-23T20:18:39.050965+0000","last_became_peered":"2023-09-23T20:18:39.050965+0000","last_unstale":"2023-09-23T20:20:58.588949+0000","last_undegraded":"2023-09-23T20:20:58.588949+0000","last_fullsized":"2023-09-23T20:20:58.588949+0000","mapping_epoch":503,"log_start":"499'5233","ondisk_log_start":"499'5233","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:43.005386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9109504,"num_objects":139,"num_object_clones":0,"num_object_copies":417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":139,"num_objects_unfound":0,"num_objects_dirty":139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11259,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1265,"num_bytes_recovered":82903040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":139}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":869,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432764+0000","last_change":"2023-09-23T20:12:54.010761+0000","last_active":"2023-09-23T20:20:52.432764+0000","last_peered":"2023-09-23T20:20:52.432764+0000","last_clean":"2023-09-23T20:20:52.432764+0000","last_became_active":"2023-09-23T20:09:56.042781+0000","last_became_peered":"2023-09-23T20:09:56.042781+0000","last_unstale":"2023-09-23T20:20:52.432764+0000","last_undegraded":"2023-09-23T20:20:52.432764+0000","last_fullsized":"2023-09-23T20:20:52.432764+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:50.227800+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1206,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.433419+0000","last_change":"2023-09-23T20:17:47.305979+0000","last_active":"2023-09-23T20:20:52.433419+0000","last_peered":"2023-09-23T20:20:52.433419+0000","last_clean":"2023-09-23T20:20:52.433419+0000","last_became_active":"2023-09-23T20:17:47.274623+0000","last_became_peered":"2023-09-23T20:17:47.274623+0000","last_unstale":"2023-09-23T20:20:52.433419+0000","last_undegraded":"2023-09-23T20:20:52.433419+0000","last_fullsized":"2023-09-23T20:20:52.433419+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:27.293931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429521+0000","last_change":"2023-09-23T20:17:47.280498+0000","last_active":"2023-09-23T20:20:52.429521+0000","last_peered":"2023-09-23T20:20:52.429521+0000","last_clean":"2023-09-23T20:20:52.429521+0000","last_became_active":"2023-09-23T20:17:47.279716+0000","last_became_peered":"2023-09-23T20:17:47.279716+0000","last_unstale":"2023-09-23T20:20:52.429521+0000","last_undegraded":"2023-09-23T20:20:52.429521+0000","last_fullsized":"2023-09-23T20:20:52.429521+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:18.625303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431238+0000","last_change":"2023-09-23T20:17:47.276964+0000","last_active":"2023-09-23T20:20:52.431238+0000","last_peered":"2023-09-23T20:20:52.431238+0000","last_clean":"2023-09-23T20:20:52.431238+0000","last_became_active":"2023-09-23T20:17:47.276540+0000","last_became_peered":"2023-09-23T20:17:47.276540+0000","last_unstale":"2023-09-23T20:20:52.431238+0000","last_undegraded":"2023-09-23T20:20:52.431238+0000","last_fullsized":"2023-09-23T20:20:52.431238+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:31.547044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"526'11367","reported_seq":11433,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:58.538550+0000","last_change":"2023-09-23T20:20:52.482378+0000","last_active":"2023-09-23T20:20:58.538550+0000","last_peered":"2023-09-23T20:20:58.538550+0000","last_clean":"2023-09-23T20:20:58.538550+0000","last_became_active":"2023-09-23T20:20:52.460860+0000","last_became_peered":"2023-09-23T20:20:52.460860+0000","last_unstale":"2023-09-23T20:20:58.538550+0000","last_undegraded":"2023-09-23T20:20:58.538550+0000","last_fullsized":"2023-09-23T20:20:58.538550+0000","mapping_epoch":479,"log_start":"497'4611","ondisk_log_start":"497'4611","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6756,"log_dups_size":0,"ondisk_log_size":6756,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3080192,"num_objects":47,"num_object_clones":0,"num_object_copies":141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5771,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":765,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449129+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:20:53.449129+0000","last_peered":"2023-09-23T20:20:53.449129+0000","last_clean":"2023-09-23T20:20:53.449129+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:20:53.449129+0000","last_undegraded":"2023-09-23T20:20:53.449129+0000","last_fullsized":"2023-09-23T20:20:53.449129+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":1166,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.430147+0000","last_change":"2023-09-23T20:04:02.271306+0000","last_active":"2023-09-23T20:20:52.430147+0000","last_peered":"2023-09-23T20:20:52.430147+0000","last_clean":"2023-09-23T20:20:52.430147+0000","last_became_active":"2023-09-23T20:04:02.270642+0000","last_became_peered":"2023-09-23T20:04:02.270642+0000","last_unstale":"2023-09-23T20:20:52.430147+0000","last_undegraded":"2023-09-23T20:20:52.430147+0000","last_fullsized":"2023-09-23T20:20:52.430147+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:33.357854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429176+0000","last_change":"2023-09-23T20:17:47.278822+0000","last_active":"2023-09-23T20:20:52.429176+0000","last_peered":"2023-09-23T20:20:52.429176+0000","last_clean":"2023-09-23T20:20:52.429176+0000","last_became_active":"2023-09-23T20:17:47.278407+0000","last_became_peered":"2023-09-23T20:17:47.278407+0000","last_unstale":"2023-09-23T20:20:52.429176+0000","last_undegraded":"2023-09-23T20:20:52.429176+0000","last_fullsized":"2023-09-23T20:20:52.429176+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:01.125231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.432409+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:20:52.432409+0000","last_peered":"2023-09-23T20:20:52.432409+0000","last_clean":"2023-09-23T20:20:52.432409+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:20:52.432409+0000","last_undegraded":"2023-09-23T20:20:52.432409+0000","last_fullsized":"2023-09-23T20:20:52.432409+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"526'11227","reported_seq":18708,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:59.468313+0000","last_change":"2023-09-23T20:20:53.492325+0000","last_active":"2023-09-23T20:20:59.468313+0000","last_peered":"2023-09-23T20:20:59.468313+0000","last_clean":"2023-09-23T20:20:59.468313+0000","last_became_active":"2023-09-23T20:20:53.447416+0000","last_became_peered":"2023-09-23T20:20:53.447416+0000","last_unstale":"2023-09-23T20:20:59.468313+0000","last_undegraded":"2023-09-23T20:20:59.468313+0000","last_fullsized":"2023-09-23T20:20:59.468313+0000","mapping_epoch":525,"log_start":"504'5844","ondisk_log_start":"504'5844","created":479,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5383,"log_dups_size":0,"ondisk_log_size":5383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:08.613029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6488064,"num_objects":99,"num_object_clones":0,"num_object_copies":297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":99,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11227,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3822,"num_bytes_recovered":101187584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1434,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.434995+0000","last_change":"2023-09-23T20:09:08.669054+0000","last_active":"2023-09-23T20:20:52.434995+0000","last_peered":"2023-09-23T20:20:52.434995+0000","last_clean":"2023-09-23T20:20:52.434995+0000","last_became_active":"2023-09-23T20:09:08.668536+0000","last_became_peered":"2023-09-23T20:09:08.668536+0000","last_unstale":"2023-09-23T20:20:52.434995+0000","last_undegraded":"2023-09-23T20:20:52.434995+0000","last_fullsized":"2023-09-23T20:20:52.434995+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:57.175721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":742,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449196+0000","last_change":"2023-09-23T20:17:50.313197+0000","last_active":"2023-09-23T20:20:53.449196+0000","last_peered":"2023-09-23T20:20:53.449196+0000","last_clean":"2023-09-23T20:20:53.449196+0000","last_became_active":"2023-09-23T20:17:50.312838+0000","last_became_peered":"2023-09-23T20:17:50.312838+0000","last_unstale":"2023-09-23T20:20:53.449196+0000","last_undegraded":"2023-09-23T20:20:53.449196+0000","last_fullsized":"2023-09-23T20:20:53.449196+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:31.530167+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":391,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.433562+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:20:52.433562+0000","last_peered":"2023-09-23T20:20:52.433562+0000","last_clean":"2023-09-23T20:20:52.433562+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:20:52.433562+0000","last_undegraded":"2023-09-23T20:20:52.433562+0000","last_fullsized":"2023-09-23T20:20:52.433562+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":324,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449542+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:20:53.449542+0000","last_peered":"2023-09-23T20:20:53.449542+0000","last_clean":"2023-09-23T20:20:53.449542+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:20:53.449542+0000","last_undegraded":"2023-09-23T20:20:53.449542+0000","last_fullsized":"2023-09-23T20:20:53.449542+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"526'11257","reported_seq":11325,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.977956+0000","last_change":"2023-09-23T20:20:52.616464+0000","last_active":"2023-09-23T20:21:00.977956+0000","last_peered":"2023-09-23T20:21:00.977956+0000","last_clean":"2023-09-23T20:21:00.977956+0000","last_became_active":"2023-09-23T20:20:52.485030+0000","last_became_peered":"2023-09-23T20:20:52.485030+0000","last_unstale":"2023-09-23T20:21:00.977956+0000","last_undegraded":"2023-09-23T20:21:00.977956+0000","last_fullsized":"2023-09-23T20:21:00.977956+0000","mapping_epoch":479,"log_start":"486'2900","ondisk_log_start":"486'2900","created":479,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8357,"log_dups_size":0,"ondisk_log_size":8357,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5746,"num_write_kb":183712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":537,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429734+0000","last_change":"2023-09-23T20:17:50.330694+0000","last_active":"2023-09-23T20:20:52.429734+0000","last_peered":"2023-09-23T20:20:52.429734+0000","last_clean":"2023-09-23T20:20:52.429734+0000","last_became_active":"2023-09-23T20:17:50.329940+0000","last_became_peered":"2023-09-23T20:17:50.329940+0000","last_unstale":"2023-09-23T20:20:52.429734+0000","last_undegraded":"2023-09-23T20:20:52.429734+0000","last_fullsized":"2023-09-23T20:20:52.429734+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:55.675445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:55.675445+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0063389099999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1051,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.448996+0000","last_change":"2023-09-23T20:17:50.313157+0000","last_active":"2023-09-23T20:20:53.448996+0000","last_peered":"2023-09-23T20:20:53.448996+0000","last_clean":"2023-09-23T20:20:53.448996+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:20:53.448996+0000","last_undegraded":"2023-09-23T20:20:53.448996+0000","last_fullsized":"2023-09-23T20:20:53.448996+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":474,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.425463+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:20:52.425463+0000","last_peered":"2023-09-23T20:20:52.425463+0000","last_clean":"2023-09-23T20:20:52.425463+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:20:52.425463+0000","last_undegraded":"2023-09-23T20:20:52.425463+0000","last_fullsized":"2023-09-23T20:20:52.425463+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431735+0000","last_change":"2023-09-23T20:17:47.278112+0000","last_active":"2023-09-23T20:20:52.431735+0000","last_peered":"2023-09-23T20:20:52.431735+0000","last_clean":"2023-09-23T20:20:52.431735+0000","last_became_active":"2023-09-23T20:17:47.277728+0000","last_became_peered":"2023-09-23T20:17:47.277728+0000","last_unstale":"2023-09-23T20:20:52.431735+0000","last_undegraded":"2023-09-23T20:20:52.431735+0000","last_fullsized":"2023-09-23T20:20:52.431735+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:21.371380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"526'11490","reported_seq":11548,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.995651+0000","last_change":"2023-09-23T20:16:59.490420+0000","last_active":"2023-09-23T20:21:00.995651+0000","last_peered":"2023-09-23T20:21:00.995651+0000","last_clean":"2023-09-23T20:21:00.995651+0000","last_became_active":"2023-09-23T20:16:59.490025+0000","last_became_peered":"2023-09-23T20:16:59.490025+0000","last_unstale":"2023-09-23T20:21:00.995651+0000","last_undegraded":"2023-09-23T20:21:00.995651+0000","last_fullsized":"2023-09-23T20:21:00.995651+0000","mapping_epoch":479,"log_start":"509'6712","ondisk_log_start":"509'6712","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":4778,"log_dups_size":0,"ondisk_log_size":4778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:03.184763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11490,"num_write_kb":367617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":722,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.433441+0000","last_change":"2023-09-23T20:19:05.092371+0000","last_active":"2023-09-23T20:20:52.433441+0000","last_peered":"2023-09-23T20:20:52.433441+0000","last_clean":"2023-09-23T20:20:52.433441+0000","last_became_active":"2023-09-23T20:19:05.092070+0000","last_became_peered":"2023-09-23T20:19:05.092070+0000","last_unstale":"2023-09-23T20:20:52.433441+0000","last_undegraded":"2023-09-23T20:20:52.433441+0000","last_fullsized":"2023-09-23T20:20:52.433441+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:17:02.526166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:17:02.526166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015505650000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":227,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449700+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:20:53.449700+0000","last_peered":"2023-09-23T20:20:53.449700+0000","last_clean":"2023-09-23T20:20:53.449700+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:20:53.449700+0000","last_undegraded":"2023-09-23T20:20:53.449700+0000","last_fullsized":"2023-09-23T20:20:53.449700+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431086+0000","last_change":"2023-09-23T20:17:47.276790+0000","last_active":"2023-09-23T20:20:52.431086+0000","last_peered":"2023-09-23T20:20:52.431086+0000","last_clean":"2023-09-23T20:20:52.431086+0000","last_became_active":"2023-09-23T20:17:47.276198+0000","last_became_peered":"2023-09-23T20:17:47.276198+0000","last_unstale":"2023-09-23T20:20:52.431086+0000","last_undegraded":"2023-09-23T20:20:52.431086+0000","last_fullsized":"2023-09-23T20:20:52.431086+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:06.124215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":352,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.434910+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:20:52.434910+0000","last_peered":"2023-09-23T20:20:52.434910+0000","last_clean":"2023-09-23T20:20:52.434910+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:20:52.434910+0000","last_undegraded":"2023-09-23T20:20:52.434910+0000","last_fullsized":"2023-09-23T20:20:52.434910+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"526'11672","reported_seq":11729,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:01.020548+0000","last_change":"2023-09-23T20:16:59.490304+0000","last_active":"2023-09-23T20:21:01.020548+0000","last_peered":"2023-09-23T20:21:01.020548+0000","last_clean":"2023-09-23T20:21:01.020548+0000","last_became_active":"2023-09-23T20:16:59.490026+0000","last_became_peered":"2023-09-23T20:16:59.490026+0000","last_unstale":"2023-09-23T20:21:01.020548+0000","last_undegraded":"2023-09-23T20:21:01.020548+0000","last_fullsized":"2023-09-23T20:21:01.020548+0000","mapping_epoch":479,"log_start":"494'4465","ondisk_log_start":"494'4465","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":7207,"log_dups_size":0,"ondisk_log_size":7207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:59.789512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11672,"num_write_kb":373504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":777,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.437314+0000","last_change":"2023-09-23T20:20:50.419406+0000","last_active":"2023-09-23T20:20:52.437314+0000","last_peered":"2023-09-23T20:20:52.437314+0000","last_clean":"2023-09-23T20:20:52.437314+0000","last_became_active":"2023-09-23T20:19:05.069124+0000","last_became_peered":"2023-09-23T20:19:05.069124+0000","last_unstale":"2023-09-23T20:20:52.437314+0000","last_undegraded":"2023-09-23T20:20:52.437314+0000","last_fullsized":"2023-09-23T20:20:52.437314+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:16:56.408578+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:16:56.408578+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015688239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":369,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449931+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:20:53.449931+0000","last_peered":"2023-09-23T20:20:53.449931+0000","last_clean":"2023-09-23T20:20:53.449931+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:20:53.449931+0000","last_undegraded":"2023-09-23T20:20:53.449931+0000","last_fullsized":"2023-09-23T20:20:53.449931+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":605,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431651+0000","last_change":"2023-09-23T20:17:50.311535+0000","last_active":"2023-09-23T20:20:52.431651+0000","last_peered":"2023-09-23T20:20:52.431651+0000","last_clean":"2023-09-23T20:20:52.431651+0000","last_became_active":"2023-09-23T20:17:50.310876+0000","last_became_peered":"2023-09-23T20:17:50.310876+0000","last_unstale":"2023-09-23T20:20:52.431651+0000","last_undegraded":"2023-09-23T20:20:52.431651+0000","last_fullsized":"2023-09-23T20:20:52.431651+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.794641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":160,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.425359+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:20:52.425359+0000","last_peered":"2023-09-23T20:20:52.425359+0000","last_clean":"2023-09-23T20:20:52.425359+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:20:52.425359+0000","last_undegraded":"2023-09-23T20:20:52.425359+0000","last_fullsized":"2023-09-23T20:20:52.425359+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"526'11436","reported_seq":16767,"reported_epoch":526,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:21:01.033656+0000","last_change":"2023-09-23T20:19:33.456970+0000","last_active":"2023-09-23T20:21:01.033656+0000","last_peered":"2023-09-23T20:21:01.033656+0000","last_clean":"2023-09-23T20:21:01.033656+0000","last_became_active":"2023-09-23T20:18:39.117958+0000","last_became_peered":"2023-09-23T20:18:39.117958+0000","last_unstale":"2023-09-23T20:21:01.033656+0000","last_undegraded":"2023-09-23T20:21:01.033656+0000","last_fullsized":"2023-09-23T20:21:01.033656+0000","mapping_epoch":503,"log_start":"507'6681","ondisk_log_start":"507'6681","created":479,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":4755,"log_dups_size":0,"ondisk_log_size":4755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:45.818039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11436,"num_write_kb":365952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2630,"num_bytes_recovered":172359680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":854,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.426465+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:20:52.426465+0000","last_peered":"2023-09-23T20:20:52.426465+0000","last_clean":"2023-09-23T20:20:52.426465+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:20:52.426465+0000","last_undegraded":"2023-09-23T20:20:52.426465+0000","last_fullsized":"2023-09-23T20:20:52.426465+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":233,"reported_epoch":526,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:20:53.450017+0000","last_change":"2023-09-23T20:18:39.010444+0000","last_active":"2023-09-23T20:20:53.450017+0000","last_peered":"2023-09-23T20:20:53.450017+0000","last_clean":"2023-09-23T20:20:53.450017+0000","last_became_active":"2023-09-23T20:18:39.009999+0000","last_became_peered":"2023-09-23T20:18:39.009999+0000","last_unstale":"2023-09-23T20:20:53.450017+0000","last_undegraded":"2023-09-23T20:20:53.450017+0000","last_fullsized":"2023-09-23T20:20:53.450017+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:53.838719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,3],"avail_no_missing":["7","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":52,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431447+0000","last_change":"2023-09-23T20:17:47.284676+0000","last_active":"2023-09-23T20:20:52.431447+0000","last_peered":"2023-09-23T20:20:52.431447+0000","last_clean":"2023-09-23T20:20:52.431447+0000","last_became_active":"2023-09-23T20:17:47.277321+0000","last_became_peered":"2023-09-23T20:17:47.277321+0000","last_unstale":"2023-09-23T20:20:52.431447+0000","last_undegraded":"2023-09-23T20:20:52.431447+0000","last_fullsized":"2023-09-23T20:20:52.431447+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.082445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":326,"reported_epoch":525,"state":"active+undersized","last_fresh":"2023-09-23T20:20:52.432536+0000","last_change":"2023-09-23T20:18:38.011785+0000","last_active":"2023-09-23T20:20:52.432536+0000","last_peered":"2023-09-23T20:20:52.432536+0000","last_clean":"2023-09-23T20:18:35.261768+0000","last_became_active":"2023-09-23T20:18:38.011785+0000","last_became_peered":"2023-09-23T20:18:38.011785+0000","last_unstale":"2023-09-23T20:20:52.432536+0000","last_undegraded":"2023-09-23T20:20:52.432536+0000","last_fullsized":"2023-09-23T20:18:38.001274+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:00.398059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.12","version":"526'11277","reported_seq":13590,"reported_epoch":526,"state":"active+recovering","last_fresh":"2023-09-23T20:20:59.474760+0000","last_change":"2023-09-23T20:20:52.665527+0000","last_active":"2023-09-23T20:20:59.474760+0000","last_peered":"2023-09-23T20:20:59.474760+0000","last_clean":"2023-09-23T20:17:46.225624+0000","last_became_active":"2023-09-23T20:20:52.613218+0000","last_became_peered":"2023-09-23T20:20:52.613218+0000","last_unstale":"2023-09-23T20:20:59.474760+0000","last_undegraded":"2023-09-23T20:20:59.474760+0000","last_fullsized":"2023-09-23T20:20:59.474760+0000","mapping_epoch":524,"log_start":"501'5747","ondisk_log_start":"501'5747","created":524,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5530,"log_dups_size":0,"ondisk_log_size":5530,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2424832,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":867,"num_objects_missing":867,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5736,"num_write_kb":184736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1121,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,1,4","objects":37}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":928,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449016+0000","last_change":"2023-09-23T20:19:56.180311+0000","last_active":"2023-09-23T20:20:53.449016+0000","last_peered":"2023-09-23T20:20:53.449016+0000","last_clean":"2023-09-23T20:20:53.449016+0000","last_became_active":"2023-09-23T20:19:05.079569+0000","last_became_peered":"2023-09-23T20:19:05.079569+0000","last_unstale":"2023-09-23T20:20:53.449016+0000","last_undegraded":"2023-09-23T20:20:53.449016+0000","last_fullsized":"2023-09-23T20:20:53.449016+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_clean_scrub_stamp":"2023-09-23T20:19:56.180203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:19.922964+0000","scrub_duration":0.012799305,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":304,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.433130+0000","last_change":"2023-09-23T20:07:36.991807+0000","last_active":"2023-09-23T20:20:52.433130+0000","last_peered":"2023-09-23T20:20:52.433130+0000","last_clean":"2023-09-23T20:20:52.433130+0000","last_became_active":"2023-09-23T20:07:36.991326+0000","last_became_peered":"2023-09-23T20:07:36.991326+0000","last_unstale":"2023-09-23T20:20:52.433130+0000","last_undegraded":"2023-09-23T20:20:52.433130+0000","last_fullsized":"2023-09-23T20:20:52.433130+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:03.091718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":314,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.498309+0000","last_change":"2023-09-23T20:17:47.281804+0000","last_active":"2023-09-23T20:20:53.498309+0000","last_peered":"2023-09-23T20:20:53.498309+0000","last_clean":"2023-09-23T20:20:53.498309+0000","last_became_active":"2023-09-23T20:17:47.281384+0000","last_became_peered":"2023-09-23T20:17:47.281384+0000","last_unstale":"2023-09-23T20:20:53.498309+0000","last_undegraded":"2023-09-23T20:20:53.498309+0000","last_fullsized":"2023-09-23T20:20:53.498309+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:03.796273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"526'11488","reported_seq":11556,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.990478+0000","last_change":"2023-09-23T20:20:52.627922+0000","last_active":"2023-09-23T20:21:00.990478+0000","last_peered":"2023-09-23T20:21:00.990478+0000","last_clean":"2023-09-23T20:21:00.990478+0000","last_became_active":"2023-09-23T20:20:52.535994+0000","last_became_peered":"2023-09-23T20:20:52.535994+0000","last_unstale":"2023-09-23T20:21:00.990478+0000","last_undegraded":"2023-09-23T20:21:00.990478+0000","last_fullsized":"2023-09-23T20:21:00.990478+0000","mapping_epoch":524,"log_start":"487'3100","ondisk_log_start":"487'3100","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":8388,"log_dups_size":0,"ondisk_log_size":8388,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5861,"num_write_kb":187520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":745,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449358+0000","last_change":"2023-09-23T20:20:37.345353+0000","last_active":"2023-09-23T20:20:53.449358+0000","last_peered":"2023-09-23T20:20:53.449358+0000","last_clean":"2023-09-23T20:20:53.449358+0000","last_became_active":"2023-09-23T20:19:05.071787+0000","last_became_peered":"2023-09-23T20:19:05.071787+0000","last_unstale":"2023-09-23T20:20:53.449358+0000","last_undegraded":"2023-09-23T20:20:53.449358+0000","last_fullsized":"2023-09-23T20:20:53.449358+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.767336+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001761889,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":557,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.434898+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:20:52.434898+0000","last_peered":"2023-09-23T20:20:52.434898+0000","last_clean":"2023-09-23T20:20:52.434898+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:20:52.434898+0000","last_undegraded":"2023-09-23T20:20:52.434898+0000","last_fullsized":"2023-09-23T20:20:52.434898+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":556,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.425892+0000","last_change":"2023-09-23T20:17:47.266215+0000","last_active":"2023-09-23T20:20:52.425892+0000","last_peered":"2023-09-23T20:20:52.425892+0000","last_clean":"2023-09-23T20:20:52.425892+0000","last_became_active":"2023-09-23T20:17:47.265890+0000","last_became_peered":"2023-09-23T20:17:47.265890+0000","last_unstale":"2023-09-23T20:20:52.425892+0000","last_undegraded":"2023-09-23T20:20:52.425892+0000","last_fullsized":"2023-09-23T20:20:52.425892+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:59.854975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.14","version":"526'11417","reported_seq":11484,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.970517+0000","last_change":"2023-09-23T20:20:52.526335+0000","last_active":"2023-09-23T20:21:00.970517+0000","last_peered":"2023-09-23T20:21:00.970517+0000","last_clean":"2023-09-23T20:21:00.970517+0000","last_became_active":"2023-09-23T20:20:52.497417+0000","last_became_peered":"2023-09-23T20:20:52.497417+0000","last_unstale":"2023-09-23T20:21:00.970517+0000","last_undegraded":"2023-09-23T20:21:00.970517+0000","last_fullsized":"2023-09-23T20:21:00.970517+0000","mapping_epoch":524,"log_start":"502'5828","ondisk_log_start":"502'5828","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5589,"log_dups_size":0,"ondisk_log_size":5589,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5822,"num_write_kb":186432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":547,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.435340+0000","last_change":"2023-09-23T20:17:47.288753+0000","last_active":"2023-09-23T20:20:52.435340+0000","last_peered":"2023-09-23T20:20:52.435340+0000","last_clean":"2023-09-23T20:20:52.435340+0000","last_became_active":"2023-09-23T20:17:47.270488+0000","last_became_peered":"2023-09-23T20:17:47.270488+0000","last_unstale":"2023-09-23T20:20:52.435340+0000","last_undegraded":"2023-09-23T20:20:52.435340+0000","last_fullsized":"2023-09-23T20:20:52.435340+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:35.837184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":523,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.440667+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:20:53.440667+0000","last_peered":"2023-09-23T20:20:53.440667+0000","last_clean":"2023-09-23T20:20:53.440667+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:20:53.440667+0000","last_undegraded":"2023-09-23T20:20:53.440667+0000","last_fullsized":"2023-09-23T20:20:53.440667+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"526'11247","reported_seq":11314,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:21:00.859251+0000","last_change":"2023-09-23T20:20:52.534501+0000","last_active":"2023-09-23T20:21:00.859251+0000","last_peered":"2023-09-23T20:21:00.859251+0000","last_clean":"2023-09-23T20:21:00.859251+0000","last_became_active":"2023-09-23T20:20:52.508649+0000","last_became_peered":"2023-09-23T20:20:52.508649+0000","last_unstale":"2023-09-23T20:21:00.859251+0000","last_undegraded":"2023-09-23T20:21:00.859251+0000","last_fullsized":"2023-09-23T20:21:00.859251+0000","mapping_epoch":524,"log_start":"504'5862","ondisk_log_start":"504'5862","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":5385,"log_dups_size":0,"ondisk_log_size":5385,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1277952,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5730,"num_write_kb":183968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":888,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.425718+0000","last_change":"2023-09-23T20:20:50.404686+0000","last_active":"2023-09-23T20:20:52.425718+0000","last_peered":"2023-09-23T20:20:52.425718+0000","last_clean":"2023-09-23T20:20:52.425718+0000","last_became_active":"2023-09-23T20:19:05.086254+0000","last_became_peered":"2023-09-23T20:19:05.086254+0000","last_unstale":"2023-09-23T20:20:52.425718+0000","last_undegraded":"2023-09-23T20:20:52.425718+0000","last_fullsized":"2023-09-23T20:20:52.425718+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:44.420366+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:44.420366+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0041816259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":249,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.433286+0000","last_change":"2023-09-23T20:17:47.281777+0000","last_active":"2023-09-23T20:20:52.433286+0000","last_peered":"2023-09-23T20:20:52.433286+0000","last_clean":"2023-09-23T20:20:52.433286+0000","last_became_active":"2023-09-23T20:17:47.281381+0000","last_became_peered":"2023-09-23T20:17:47.281381+0000","last_unstale":"2023-09-23T20:20:52.433286+0000","last_undegraded":"2023-09-23T20:20:52.433286+0000","last_fullsized":"2023-09-23T20:20:52.433286+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.083242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":387,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.431320+0000","last_change":"2023-09-23T20:08:04.765176+0000","last_active":"2023-09-23T20:20:52.431320+0000","last_peered":"2023-09-23T20:20:52.431320+0000","last_clean":"2023-09-23T20:20:52.431320+0000","last_became_active":"2023-09-23T20:08:04.764828+0000","last_became_peered":"2023-09-23T20:08:04.764828+0000","last_unstale":"2023-09-23T20:20:52.431320+0000","last_undegraded":"2023-09-23T20:20:52.431320+0000","last_fullsized":"2023-09-23T20:20:52.431320+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.725678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.16","version":"526'11247","reported_seq":11312,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:58.468904+0000","last_change":"2023-09-23T20:20:52.504966+0000","last_active":"2023-09-23T20:20:58.468904+0000","last_peered":"2023-09-23T20:20:58.468904+0000","last_clean":"2023-09-23T20:20:58.468904+0000","last_became_active":"2023-09-23T20:20:52.485305+0000","last_became_peered":"2023-09-23T20:20:52.485305+0000","last_unstale":"2023-09-23T20:20:58.468904+0000","last_undegraded":"2023-09-23T20:20:58.468904+0000","last_fullsized":"2023-09-23T20:20:58.468904+0000","mapping_epoch":524,"log_start":"497'4533","ondisk_log_start":"497'4533","created":524,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:16:58.475600+0000","last_clean_scrub_stamp":"2023-09-23T20:16:58.475600+0000","objects_scrubbed":0,"log_size":6714,"log_dups_size":0,"ondisk_log_size":6714,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4849664,"num_objects":74,"num_object_clones":0,"num_object_copies":222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5704,"num_write_kb":184896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":822,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T20:20:53.449378+0000","last_change":"2023-09-23T20:20:51.163673+0000","last_active":"2023-09-23T20:20:53.449378+0000","last_peered":"2023-09-23T20:20:53.449378+0000","last_clean":"2023-09-23T20:20:53.449378+0000","last_became_active":"2023-09-23T20:19:05.067188+0000","last_became_peered":"2023-09-23T20:19:05.067188+0000","last_unstale":"2023-09-23T20:20:53.449378+0000","last_undegraded":"2023-09-23T20:20:53.449378+0000","last_fullsized":"2023-09-23T20:20:53.449378+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":371,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.435993+0000","last_change":"2023-09-23T20:10:52.685670+0000","last_active":"2023-09-23T20:20:52.435993+0000","last_peered":"2023-09-23T20:20:52.435993+0000","last_clean":"2023-09-23T20:20:52.435993+0000","last_became_active":"2023-09-23T20:10:52.685335+0000","last_became_peered":"2023-09-23T20:10:52.685335+0000","last_unstale":"2023-09-23T20:20:52.435993+0000","last_undegraded":"2023-09-23T20:20:52.435993+0000","last_fullsized":"2023-09-23T20:20:52.435993+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:31.956197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":48,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T20:20:52.429282+0000","last_change":"2023-09-23T20:17:47.267673+0000","last_active":"2023-09-23T20:20:52.429282+0000","last_peered":"2023-09-23T20:20:52.429282+0000","last_clean":"2023-09-23T20:20:52.429282+0000","last_became_active":"2023-09-23T20:17:47.267216+0000","last_became_peered":"2023-09-23T20:17:47.267216+0000","last_unstale":"2023-09-23T20:20:52.429282+0000","last_undegraded":"2023-09-23T20:20:52.429282+0000","last_fullsized":"2023-09-23T20:20:52.429282+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:37.855890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":94,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":50462720,"num_objects":771,"num_object_clones":0,"num_object_copies":2313,"num_objects_missing_on_primary":2809,"num_objects_missing":2809,"num_objects_degraded":1942,"num_objects_misplaced":139,"num_objects_unfound":0,"num_objects_dirty":771,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183734,"num_write_kb":5902273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12709,"num_bytes_recovered":609746944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1042550784,"data_stored":1042546712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169051,"ondisk_log_size":169051,"up":76,"acting":78,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":10952,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":94,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":30,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":107340,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":8388703,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":23,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234479,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818984,"kb_used_data":237024,"kb_used_omap":0,"kb_used_meta":581952,"kb_avail":92926168,"statfs":{"total":95995035648,"available":95156396032,"internally_reserved":0,"allocated":242712576,"data_stored":241125204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":595918848},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":17.827999999999999,"5min":7.8849999999999998,"15min":4.6340000000000003},"min":{"1min":0.22700000000000001,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":230.952,"5min":230.952,"15min":230.952},"last":0.40699999999999997},{"interface":"front","average":{"1min":18.509,"5min":8.2889999999999997,"15min":4.8810000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.222},"max":{"1min":231.08699999999999,"5min":231.08699999999999,"15min":231.08699999999999},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":2.0550000000000002,"15min":2.4510000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.21299999999999999},"max":{"1min":13.337,"5min":61.113999999999997,"15min":91.335999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.4219999999999999,"5min":2.234,"15min":2.6909999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.223},"max":{"1min":6.3769999999999998,"5min":61.082999999999998,"15min":91.516999999999996},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":5.4560000000000004,"15min":4.5750000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.251},"max":{"1min":24.16,"5min":177.852,"15min":177.852},"last":0.434},{"interface":"front","average":{"1min":3.407,"5min":5.7160000000000002,"15min":4.899},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":20.98,"5min":176.97900000000001,"15min":176.97900000000001},"last":0.55100000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":4.04,"15min":4.1109999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":19.620000000000001,"5min":112.291,"15min":185.17099999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.5179999999999998,"15min":4.2649999999999997},"min":{"1min":0.33700000000000002,"5min":0.32000000000000001,"15min":0.28999999999999998},"max":{"1min":19.855,"5min":112.887,"15min":184.99100000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0860000000000003,"5min":1.649,"15min":0.995},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":83.811999999999998,"5min":83.811999999999998,"15min":83.811999999999998},"last":0.442},{"interface":"front","average":{"1min":4.0720000000000001,"5min":1.6950000000000001,"15min":1.0900000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":84.620000000000005,"5min":84.620000000000005,"15min":84.620000000000005},"last":0.47399999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":1.3340000000000001,"15min":1.1859999999999999},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.794,"5min":16.327000000000002,"15min":24.876000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.495,"5min":1.25,"15min":1.079},"min":{"1min":0.30599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1190000000000002,"5min":16.716000000000001,"15min":21.623000000000001},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.3779999999999999,"15min":1.0569999999999999},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.3420000000000001,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.163,"15min":0.82199999999999995},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1400000000000001,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.436}]}]},{"osd":6,"up_from":251,"seq":1078036791490,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273876,"kb_used_data":233868,"kb_used_omap":6,"kb_used_meta":39993,"kb_avail":93471276,"statfs":{"total":95995035648,"available":95714586624,"internally_reserved":0,"allocated":239480832,"data_stored":238014233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":40953471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":8.9100000000000001,"5min":3.8820000000000001,"15min":2.6890000000000001},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":53.780000000000001,"5min":74.787000000000006,"15min":126.64},"last":0.27000000000000002},{"interface":"front","average":{"1min":9.0090000000000003,"5min":4.0289999999999999,"15min":2.7570000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":53.768000000000001,"5min":74.751999999999995,"15min":126.658},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":4.3120000000000003,"15min":4.0830000000000002},"min":{"1min":0.26500000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":87.864999999999995,"5min":103.545,"15min":156.70699999999999},"last":5.4710000000000001},{"interface":"front","average":{"1min":5.1260000000000003,"5min":4.0279999999999996,"15min":4.0590000000000002},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.23899999999999999},"max":{"1min":87.772000000000006,"5min":100.018,"15min":156.68299999999999},"last":2.8690000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":6.891,"5min":2.4329999999999998,"15min":3.1179999999999999},"min":{"1min":0.315,"5min":0.254,"15min":0.23100000000000001},"max":{"1min":53.911999999999999,"5min":53.911999999999999,"15min":122.149},"last":0.47499999999999998},{"interface":"front","average":{"1min":7.3259999999999996,"5min":2.4129999999999998,"15min":3.0600000000000001},"min":{"1min":0.32400000000000001,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":54.125999999999998,"5min":54.125999999999998,"15min":122.23999999999999},"last":1.7789999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":5.181,"15min":3.8580000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":51.512999999999998,"5min":128.279,"15min":160.27699999999999},"last":4.8179999999999996},{"interface":"front","average":{"1min":3.4249999999999998,"5min":4.4969999999999999,"15min":3.8180000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":51.601999999999997,"5min":127.911,"15min":160.42699999999999},"last":2.9830000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.298,"15min":1.5780000000000001},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.248},"max":{"1min":2.641,"5min":34.534999999999997,"15min":145.29300000000001},"last":2.9140000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.077,"15min":1.538},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.3999999999999999,"5min":34.426000000000002,"15min":145.256},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":4.5880000000000001,"15min":2.972},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.25800000000000001},"max":{"1min":2.5329999999999999,"5min":218.58199999999999,"15min":218.58199999999999},"last":2.9500000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.343,"15min":2.8969999999999998},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.28499999999999998},"max":{"1min":2.4380000000000002,"5min":217.95599999999999,"15min":217.95599999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.1269999999999998,"15min":1.73},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.315},"max":{"1min":4.7380000000000004,"5min":137.63900000000001,"15min":137.63900000000001},"last":1},{"interface":"front","average":{"1min":0.751,"5min":2.8199999999999998,"15min":1.629},"min":{"1min":0.39700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":2.3620000000000001,"5min":136.738,"15min":136.738},"last":0.48799999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762677,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":872580,"kb_used_data":35388,"kb_used_omap":7,"kb_used_meta":837176,"kb_avail":92872572,"statfs":{"total":95995035648,"available":95101513728,"internally_reserved":0,"allocated":36237312,"data_stored":33741270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":857269050},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":7.1630000000000003,"5min":5.7320000000000002,"15min":3.9820000000000002},"min":{"1min":0.24299999999999999,"5min":0.193,"15min":0.17100000000000001},"max":{"1min":62.710999999999999,"5min":145.81299999999999,"15min":145.81299999999999},"last":0.221},{"interface":"front","average":{"1min":7.6680000000000001,"5min":6.0819999999999999,"15min":4.3010000000000002},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":62.610999999999997,"5min":146.83799999999999,"15min":146.83799999999999},"last":6.3700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:20:32 2023","interfaces":[{"interface":"back","average":{"1min":13.513,"5min":6.0099999999999998,"15min":4.9539999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":113.941,"5min":152.62,"15min":152.62},"last":0.72499999999999998},{"interface":"front","average":{"1min":14.683,"5min":6.234,"15min":4.702},"min":{"1min":0.318,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":114.34699999999999,"5min":154.84299999999999,"15min":154.84299999999999},"last":1.8560000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":4.5830000000000002,"5min":3.1190000000000002,"15min":3.0179999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":22.256,"5min":22.256,"15min":66.527000000000001},"last":1.5740000000000001},{"interface":"front","average":{"1min":4.0469999999999997,"5min":1.9950000000000001,"15min":2.4119999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":22.068000000000001,"5min":22.068000000000001,"15min":61.854999999999997},"last":6.1630000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3740000000000001,"5min":2.226,"15min":1.7370000000000001},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.23599999999999999},"max":{"1min":10.387,"5min":29.791,"15min":29.791},"last":0.26900000000000002},{"interface":"front","average":{"1min":2.3860000000000001,"5min":1.8700000000000001,"15min":1.6739999999999999},"min":{"1min":0.37,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":10.422000000000001,"5min":27.143999999999998,"15min":27.143999999999998},"last":5.9269999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":2.1120000000000001,"15min":2.1040000000000001},"min":{"1min":0.34799999999999998,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":10.461,"5min":29.280000000000001,"15min":83.510999999999996},"last":5.8840000000000003},{"interface":"front","average":{"1min":2.54,"5min":2.258,"15min":2.2959999999999998},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":12.648,"5min":32.692,"15min":83.353999999999999},"last":0.81299999999999994}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.2390000000000001,"15min":1.665},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":4.5830000000000002,"5min":9.2279999999999998,"15min":18.117999999999999},"last":5.6760000000000002},{"interface":"front","average":{"1min":0.75,"5min":1.325,"15min":1.843},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.0310000000000001,"5min":9.8659999999999997,"15min":20.734999999999999},"last":3.7330000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":2.214,"15min":2.5369999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.30399999999999999},"max":{"1min":12.599,"5min":12.599,"15min":127.03100000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":2.7589999999999999,"5min":2.0539999999999998,"15min":2.2389999999999999},"min":{"1min":0.438,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.340999999999999,"5min":10.340999999999999,"15min":125.095},"last":0.312}]}]},{"osd":0,"up_from":94,"seq":403726926215,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1321708,"kb_used_data":274212,"kb_used_omap":8,"kb_used_meta":1047479,"kb_avail":92423444,"statfs":{"total":95995035648,"available":94641606656,"internally_reserved":0,"allocated":280793088,"data_stored":278319018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1072619105},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,11],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:20:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":2.145,"15min":2.5859999999999999},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":7.9299999999999997,"5min":29.893000000000001,"15min":72.534000000000006},"last":0.34499999999999997},{"interface":"front","average":{"1min":3.1949999999999998,"5min":2.7690000000000001,"15min":2.952},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.187},"max":{"1min":8.6709999999999994,"5min":23.436,"15min":72.667000000000002},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:20:20 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":2.8069999999999999,"15min":3.4100000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":63.710999999999999,"5min":63.710999999999999,"15min":160.542},"last":0.443},{"interface":"front","average":{"1min":6.3049999999999997,"5min":3.5009999999999999,"15min":3.7730000000000001},"min":{"1min":0.255,"5min":0.23599999999999999,"15min":0.22600000000000001},"max":{"1min":60.518999999999998,"5min":60.518999999999998,"15min":160.75999999999999},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:20:10 2023","interfaces":[{"interface":"back","average":{"1min":5.1150000000000002,"5min":3.9700000000000002,"15min":3.7010000000000001},"min":{"1min":0.26900000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":25.667000000000002,"5min":44.405000000000001,"15min":126.95399999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":4.0419999999999998,"5min":3.7490000000000001,"15min":3.7810000000000001},"min":{"1min":0.29599999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":25.783999999999999,"5min":43.909999999999997,"15min":127.294},"last":0.77900000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.1440000000000001,"15min":1.736},"min":{"1min":0.28699999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.0339999999999998,"5min":16.593,"15min":16.593},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.9370000000000001,"5min":3.0859999999999999,"15min":2.0009999999999999},"min":{"1min":0.29299999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.1319999999999997,"5min":105.282,"15min":105.282},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.9620000000000002,"15min":2.081},"min":{"1min":0.372,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0670000000000002,"5min":145.14500000000001,"15min":145.14500000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.9390000000000001,"5min":3.5960000000000001,"15min":2.5259999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":4.9610000000000003,"5min":145.04499999999999,"15min":145.04499999999999},"last":0.504}]},{"osd":6,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6339999999999999,"15min":1.6419999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":7.6500000000000004,"5min":7.6500000000000004,"15min":15.744999999999999},"last":4.1239999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.234,"15min":1.6020000000000001},"min":{"1min":0.44,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":9.8219999999999992,"15min":13.766999999999999},"last":0.82199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":2.7829999999999999,"15min":1.875},"min":{"1min":0.36599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.3449999999999998,"5min":99.486000000000004,"15min":99.486000000000004},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.581,"5min":3.6850000000000001,"15min":2.4830000000000001},"min":{"1min":0.36499999999999999,"5min":0.308,"15min":0.308},"max":{"1min":9.5030000000000001,"5min":99.668999999999997,"15min":99.668999999999997},"last":0.47399999999999998}]}]},{"osd":2,"up_from":104,"seq":446676599154,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004120,"kb_used_data":6800,"kb_used_omap":12,"kb_used_meta":997299,"kb_avail":92741032,"statfs":{"total":95995035648,"available":94966816768,"internally_reserved":0,"allocated":6963200,"data_stored":4453742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":1021235198},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9009999999999998,"5min":8.0299999999999994,"15min":4.2039999999999997},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":15.4,"5min":274.185,"15min":274.185},"last":0.96199999999999997},{"interface":"front","average":{"1min":2.4729999999999999,"5min":5.8079999999999998,"15min":3.601},"min":{"1min":0.23200000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.8209999999999997,"5min":106.477,"15min":106.477},"last":1.905}]},{"osd":1,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":4.335,"5min":4.5,"15min":3.5249999999999999},"min":{"1min":0.39600000000000002,"5min":0.20399999999999999,"15min":0.19600000000000001},"max":{"1min":11.939,"5min":106.357,"15min":106.357},"last":1.1439999999999999},{"interface":"front","average":{"1min":4.1619999999999999,"5min":4.2969999999999997,"15min":3.1579999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":15.255000000000001,"5min":104.905,"15min":104.905},"last":0.93600000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:20:50 2023","interfaces":[{"interface":"back","average":{"1min":5.899,"5min":6.0259999999999998,"15min":4.1989999999999998},"min":{"1min":0.52000000000000002,"5min":0.21299999999999999,"15min":0.20799999999999999},"max":{"1min":28.178000000000001,"5min":126.345,"15min":132.82499999999999},"last":1.222},{"interface":"front","average":{"1min":4.5709999999999997,"5min":5.2510000000000003,"15min":3.5419999999999998},"min":{"1min":0.57199999999999995,"5min":0.23000000000000001,"15min":0.22600000000000001},"max":{"1min":15.302,"5min":126.40300000000001,"15min":130.97300000000001},"last":0.68899999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8450000000000002,"5min":2.2469999999999999,"15min":2.113},"min":{"1min":0.313,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.355,"5min":43.040999999999997,"15min":75.608000000000004},"last":0.749},{"interface":"front","average":{"1min":3.2549999999999999,"5min":2.8700000000000001,"15min":2.5470000000000002},"min":{"1min":0.36599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.864000000000001,"5min":48.951000000000001,"15min":76.161000000000001},"last":0.57199999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":2.4049999999999998,"15min":2.6349999999999998},"min":{"1min":0.38500000000000001,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":15.378,"5min":43.716000000000001,"15min":135.202},"last":0.78500000000000003},{"interface":"front","average":{"1min":2.585,"5min":2.5990000000000002,"15min":2.4710000000000001},"min":{"1min":0.41499999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.782,"5min":43.643999999999998,"15min":63.792000000000002},"last":1.4350000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.329,"15min":1.542},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5510000000000002,"5min":11.843,"15min":14.664},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.425,"15min":1.744},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.077,"5min":12.025,"15min":14.279999999999999},"last":4.9660000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.4889999999999999,"5min":2.6360000000000001,"15min":2.1560000000000001},"min":{"1min":0.46999999999999997,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":11.91,"5min":18.495999999999999,"15min":18.495999999999999},"last":0.621},{"interface":"front","average":{"1min":2.9990000000000001,"5min":2.077,"15min":1.7609999999999999},"min":{"1min":0.41299999999999998,"5min":0.39900000000000002,"15min":0.35299999999999998},"max":{"1min":15.327999999999999,"5min":15.327999999999999,"15min":15.327999999999999},"last":0.71799999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435627,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292416,"kb_used_data":258936,"kb_used_omap":4,"kb_used_meta":1033467,"kb_avail":92452736,"statfs":{"total":95995035648,"available":94671601664,"internally_reserved":0,"allocated":265150464,"data_stored":262362325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1058270317},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":14.129,"5min":7.8550000000000004,"15min":4.8159999999999998},"min":{"1min":0.27100000000000002,"5min":0.191,"15min":0.186},"max":{"1min":219.80000000000001,"5min":219.80000000000001,"15min":219.80000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":13.737,"5min":8.3599999999999994,"15min":5.0659999999999998},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":221.36000000000001,"5min":221.36000000000001,"15min":221.36000000000001},"last":0.69399999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6520000000000001,"5min":2.9769999999999999,"15min":4.2160000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20300000000000001},"max":{"1min":9.0830000000000002,"5min":92.036000000000001,"15min":183.87},"last":0.32800000000000001},{"interface":"front","average":{"1min":3.145,"5min":2.972,"15min":4.4349999999999996},"min":{"1min":0.251,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":19.513000000000002,"5min":92.210999999999999,"15min":185.84899999999999},"last":2.4300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":7.6070000000000002,"15min":5.4000000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.7549999999999999,"5min":128.369,"15min":163.83500000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":3.8420000000000001,"5min":7.4020000000000001,"15min":5.2409999999999997},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":15.286,"5min":126.497,"15min":163.92099999999999},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.996,"15min":1.9179999999999999},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.28999999999999998},"max":{"1min":8.8350000000000009,"5min":54.046999999999997,"15min":58.472999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.988,"5min":2.6819999999999999,"15min":2.4470000000000001},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":5.4770000000000003,"5min":54.311,"15min":58.408000000000001},"last":0.34699999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":2.9159999999999999,"15min":3.4089999999999998},"min":{"1min":0.39400000000000002,"5min":0.32000000000000001,"15min":0.30299999999999999},"max":{"1min":7.5179999999999998,"5min":54.139000000000003,"15min":200.53200000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.359,"5min":2.573,"15min":3.4140000000000001},"min":{"1min":0.34899999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.5030000000000001,"5min":56.457999999999998,"15min":200.846},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.593,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":1.5329999999999999,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.3280000000000001,"15min":2.286},"min":{"1min":0.42699999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.2040000000000002,"5min":8.2629999999999999,"15min":82.233000000000004},"last":4.1509999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.706,"15min":2.5529999999999999},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.3940000000000001,"5min":17.986000000000001,"15min":139.465},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.4219999999999999,"5min":1.6299999999999999,"15min":2.5249999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.32800000000000001},"max":{"1min":8.9019999999999992,"5min":9.2289999999999992,"15min":141.56200000000001},"last":0.55800000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019389,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590876,"kb_used_data":5844,"kb_used_omap":0,"kb_used_meta":585024,"kb_avail":93154276,"statfs":{"total":95995035648,"available":95389978624,"internally_reserved":0,"allocated":5984256,"data_stored":4400988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599064576},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":4.5789999999999997,"5min":10.012,"15min":4.46},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.219},"max":{"1min":52.881,"5min":232.89500000000001,"15min":232.89500000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":4.5730000000000004,"5min":10.061,"15min":4.4669999999999996},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":52.890000000000001,"5min":232.851,"15min":232.851},"last":0.441}]},{"osd":1,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":2.6200000000000001,"15min":3.1440000000000001},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":6.9800000000000004,"5min":114.05200000000001,"15min":138.47999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.889,"5min":2.5350000000000001,"15min":3.1890000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.217},"max":{"1min":12.058,"5min":114.02800000000001,"15min":138.429},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":15.880000000000001,"5min":9.6950000000000003,"15min":6.7859999999999996},"min":{"1min":0.29899999999999999,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":204.345,"5min":204.345,"15min":204.345},"last":0.52900000000000003},{"interface":"front","average":{"1min":15.916,"5min":9.6020000000000003,"15min":6.6429999999999998},"min":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":204.072,"5min":204.072,"15min":204.072},"last":0.56000000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":5.585,"5min":8.0199999999999996,"15min":5.7599999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":88.105999999999995,"5min":156.72300000000001,"15min":184.37100000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":5.7809999999999997,"5min":7.9210000000000003,"15min":5.8289999999999997},"min":{"1min":0.33200000000000002,"5min":0.32900000000000001,"15min":0.309},"max":{"1min":87.093999999999994,"5min":155.84800000000001,"15min":184.22300000000001},"last":0.57099999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.9379999999999999,"15min":1.6459999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":4.3970000000000002,"5min":114.788,"15min":176.81899999999999},"last":0.504},{"interface":"front","average":{"1min":0.58599999999999997,"5min":2.0099999999999998,"15min":1.6879999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.5590000000000002,"5min":113.822,"15min":176.172},"last":0.54700000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:17:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.0390000000000001,"15min":1.026},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":2.5110000000000001,"5min":114.232,"15min":114.232},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.3540000000000001,"15min":1.2130000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.247},"max":{"1min":4.085,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":2.1539999999999999,"15min":1.6719999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63600000000000001,"5min":113.861,"15min":113.861},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.67600000000000005,"5min":2.258,"15min":1.738},"min":{"1min":0.34499999999999997,"5min":0.30399999999999999,"15min":0.222},"max":{"1min":4.6710000000000003,"5min":114.32299999999999,"15min":114.32299999999999},"last":0.59299999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757752,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":743252,"kb_used_data":70668,"kb_used_omap":0,"kb_used_meta":672576,"kb_avail":93001900,"statfs":{"total":95995035648,"available":95233945600,"internally_reserved":0,"allocated":72364032,"data_stored":70785205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":688717824},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:42 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":9.9619999999999997,"15min":4.5990000000000002},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":12.819000000000001,"5min":166.57900000000001,"15min":166.57900000000001},"last":0.253},{"interface":"front","average":{"1min":14.805,"5min":11.692,"15min":5.2869999999999999},"min":{"1min":0.22,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":256.52199999999999,"5min":256.52199999999999,"15min":256.52199999999999},"last":2.0030000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:19:57 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":1.766,"15min":3.7149999999999999},"min":{"1min":0.27600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":6.7140000000000004,"5min":17.010999999999999,"15min":150.16999999999999},"last":1.9770000000000001},{"interface":"front","average":{"1min":1.7709999999999999,"5min":1.8720000000000001,"15min":3.73},"min":{"1min":0.27800000000000002,"5min":0.27200000000000002,"15min":0.21199999999999999},"max":{"1min":6.6449999999999996,"5min":18.879000000000001,"15min":150.12700000000001},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:21 2023","interfaces":[{"interface":"back","average":{"1min":17.131,"5min":6.5279999999999996,"15min":5.3019999999999996},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":208.495,"5min":208.495,"15min":240.57300000000001},"last":2.3799999999999999},{"interface":"front","average":{"1min":17.167999999999999,"5min":6.5869999999999997,"15min":5.3659999999999997},"min":{"1min":0.33500000000000002,"5min":0.33100000000000002,"15min":0.30299999999999999},"max":{"1min":208.58199999999999,"5min":208.58199999999999,"15min":240.464},"last":2.4100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:19:57 2023","interfaces":[{"interface":"back","average":{"1min":10.192,"5min":9.7569999999999997,"15min":6.5990000000000002},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.27100000000000002},"max":{"1min":155.97300000000001,"5min":196.97800000000001,"15min":201.50899999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":10.49,"5min":9.8960000000000008,"15min":6.5359999999999996},"min":{"1min":0.36799999999999999,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":155.803,"5min":196.875,"15min":201.58099999999999},"last":6.7080000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.72899999999999998,"15min":0.77700000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.0169999999999999,"5min":4.8159999999999998,"15min":9.7240000000000002},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.60999999999999999,"15min":0.60899999999999999},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":2.6890000000000001,"5min":3.371,"15min":8.7360000000000007},"last":0.32700000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2490000000000001,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":4.04,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.73299999999999998,"15min":0.60999999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":1.609,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":1.492}]},{"osd":7,"last update":"Sat Sep 23 20:20:21 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.0780000000000001,"15min":0.95799999999999996},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.3609999999999998,"5min":11.725,"15min":11.725},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.40699999999999997,"5min":1.0449999999999999,"15min":0.90700000000000003},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.58999999999999997,"5min":11.041,"15min":11.041},"last":0.35699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":241176576,"data_stored":241172504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":19267584,"data_stored":19267584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":257556480,"data_stored":257556480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":55902208,"data_stored":55902208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":237764608,"data_stored":237764608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":230883328,"data_stored":230883328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:21:04.993 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:21:04.993 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-09-23T20:21:05.084 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:05.120 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:05.162 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:21:05.334 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:05 smithi135 ceph-mon[213871]: from='client.36744 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:05.335 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:05 smithi135 ceph-mon[221113]: from='client.36744 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:05.471 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:21:05.471 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:21:05.471 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:21:05.472 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:21:05.472 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:21:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:05 smithi144 ceph-mon[194833]: from='client.36744 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:06.178 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:06.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:06.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:06.282 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:21:06.314 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:06.625 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:21:06.627 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:06 smithi135 ceph-mon[221113]: pgmap v1504: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1/357 objects degraded (0.280%); 0 B/s, 244 objects/s recovering 2023-09-23T20:21:06.627 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:06 smithi135 ceph-mon[213871]: pgmap v1504: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1/357 objects degraded (0.280%); 0 B/s, 244 objects/s recovering 2023-09-23T20:21:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:06.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:21:06.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:06 smithi144 ceph-mon[194833]: pgmap v1504: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1/357 objects degraded (0.280%); 0 B/s, 244 objects/s recovering 2023-09-23T20:21:06.695 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:21:06.729 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:21:06.741 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:06.775 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:06.822 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:21:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:06.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:21:06.839 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:06.926 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:06.960 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:07.010 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:07.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:07.116 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:07.144 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:21:07.147 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:07.166 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:21:07.196 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:21:07.218 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:21:07.238 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:21:07.255 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:07.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:07.262 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:21:07.283 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:21:07.284 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:21:07.284 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:21:07.284 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:21:07.284 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:21:07.284 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:21:07.284 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:21:07.284 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:21:07.284 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:21:07.360 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:07.390 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:07.436 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:07.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:07.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:07.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:07 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/251750248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:07.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:07 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/251750248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:07.544 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:07.574 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:07.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:07 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/251750248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:07.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:08.105 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:08.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:08.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:08.209 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:08.239 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:08.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:08 smithi135 ceph-mon[213871]: pgmap v1505: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 1/354 objects degraded (0.282%); 0 B/s, 243 objects/s recovering 2023-09-23T20:21:08.397 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:08 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1/354 objects degraded (0.282%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:08.397 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:08 smithi135 ceph-mon[221113]: pgmap v1505: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 1/354 objects degraded (0.282%); 0 B/s, 243 objects/s recovering 2023-09-23T20:21:08.397 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:08 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1/354 objects degraded (0.282%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:08.433 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:08.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:08.538 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:08.564 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:08 smithi144 ceph-mon[194833]: pgmap v1505: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 1/354 objects degraded (0.282%); 0 B/s, 243 objects/s recovering 2023-09-23T20:21:08.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:08 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1/354 objects degraded (0.282%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:08.858 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:08.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:08.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:08.961 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:08.987 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:09.181 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:09.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:09.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:09.285 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:09.309 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:21:09.309 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:21:09.317 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:09.366 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:09.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:09.442 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:09.471 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:09.500 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:09.545 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:09.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:09.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:09.652 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:09.682 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:09.732 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:09.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:09.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:09.835 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:09.863 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:09.906 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:09.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:09.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:10.015 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:10.040 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:10.081 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:10.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:10.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:10.097 INFO:teuthology.orchestra.run.smithi135.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-09-23T20:21:10.186 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:10.214 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:10.352 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:21:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[221113]: pgmap v1506: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 1/354 objects degraded (0.282%); 0 B/s, 243 objects/s recovering 2023-09-23T20:21:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/402159602' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/88721284' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:21:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:21:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[213871]: pgmap v1506: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 1/354 objects degraded (0.282%); 0 B/s, 243 objects/s recovering 2023-09-23T20:21:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/402159602' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:10.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:10.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/88721284' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:21:10.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:21:10.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:10 smithi144 ceph-mon[194833]: pgmap v1506: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 1 active+recovering, 121 active+clean; 10 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 1/354 objects degraded (0.282%); 0 B/s, 243 objects/s recovering 2023-09-23T20:21:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/402159602' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:10 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/88721284' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:21:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:21:11.214 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:11.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:11.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:11.241 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:21:11.242 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T20:21:11.242 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T20:21:11.310 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:21:11.320 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:11.352 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:11.400 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:11.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:11 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:11 smithi135 ceph-mon[213871]: failed to encode map e528 with expected crc 2023-09-23T20:21:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:11 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:21:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:11 smithi135 ceph-mon[213871]: osdmap e528: 8 total, 7 up, 8 in 2023-09-23T20:21:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:11 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:11 smithi135 ceph-mon[221113]: failed to encode map e528 with expected crc 2023-09-23T20:21:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:11 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:21:11.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:11 smithi135 ceph-mon[221113]: osdmap e528: 8 total, 7 up, 8 in 2023-09-23T20:21:11.506 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:11.534 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:11.577 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:11.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:11.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:11 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:11 smithi144 ceph-mon[194833]: failed to encode map e528 with expected crc 2023-09-23T20:21:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:11 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:21:11.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:11 smithi144 ceph-mon[194833]: osdmap e528: 8 total, 7 up, 8 in 2023-09-23T20:21:11.683 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:11.711 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:11.977 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:21:11.977 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:21:12.179 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:12.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:12.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:12.283 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:21:12.310 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:12.555 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:21:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:12.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:21:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:12 smithi144 ceph-mon[194833]: pgmap v1508: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 1/354 objects degraded (0.282%); 0 B/s, 314 objects/s recovering 2023-09-23T20:21:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:12 smithi144 ceph-mon[194833]: 1.c scrub starts 2023-09-23T20:21:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:12 smithi144 ceph-mon[194833]: 1.c scrub ok 2023-09-23T20:21:12.659 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:12.691 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:12.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:12 smithi135 ceph-mon[221113]: pgmap v1508: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 1/354 objects degraded (0.282%); 0 B/s, 314 objects/s recovering 2023-09-23T20:21:12.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:12 smithi135 ceph-mon[221113]: 1.c scrub starts 2023-09-23T20:21:12.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:12 smithi135 ceph-mon[221113]: 1.c scrub ok 2023-09-23T20:21:12.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:12 smithi135 ceph-mon[213871]: pgmap v1508: 129 pgs: 3 active+clean+remapped, 3 active+undersized, 1 active+undersized+degraded, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 1/354 objects degraded (0.282%); 0 B/s, 314 objects/s recovering 2023-09-23T20:21:12.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:12 smithi135 ceph-mon[213871]: 1.c scrub starts 2023-09-23T20:21:12.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:12 smithi135 ceph-mon[213871]: 1.c scrub ok 2023-09-23T20:21:12.738 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:12.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:12.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:12.778 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:21:12.846 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:12.868 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:12.875 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:12.919 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:13.000 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:13.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:13.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:13.104 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:13.132 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:13.163 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:21:13.178 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:13.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:13.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:13.283 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:21:13.310 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:13.359 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[221113]: 1.13 scrub starts 2023-09-23T20:21:13.359 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[221113]: 1.13 scrub ok 2023-09-23T20:21:13.359 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[221113]: 1.5 scrub starts 2023-09-23T20:21:13.359 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[221113]: 1.5 scrub ok 2023-09-23T20:21:13.360 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[221113]: 1.a scrub starts 2023-09-23T20:21:13.360 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[221113]: 1.a scrub ok 2023-09-23T20:21:13.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:13 smithi144 ceph-mon[194833]: 1.13 scrub starts 2023-09-23T20:21:13.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:13 smithi144 ceph-mon[194833]: 1.13 scrub ok 2023-09-23T20:21:13.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:13 smithi144 ceph-mon[194833]: 1.5 scrub starts 2023-09-23T20:21:13.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:13 smithi144 ceph-mon[194833]: 1.5 scrub ok 2023-09-23T20:21:13.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:13 smithi144 ceph-mon[194833]: 1.a scrub starts 2023-09-23T20:21:13.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:13 smithi144 ceph-mon[194833]: 1.a scrub ok 2023-09-23T20:21:13.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[213871]: 1.13 scrub starts 2023-09-23T20:21:13.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[213871]: 1.13 scrub ok 2023-09-23T20:21:13.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[213871]: 1.5 scrub starts 2023-09-23T20:21:13.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[213871]: 1.5 scrub ok 2023-09-23T20:21:13.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[213871]: 1.a scrub starts 2023-09-23T20:21:13.685 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:13 smithi135 ceph-mon[213871]: 1.a scrub ok 2023-09-23T20:21:13.816 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:21:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:13.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:21:13.920 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:13.945 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:21:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:21:13] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-09-23T20:21:13.947 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:14.224 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:14.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:14.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:14.327 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:14.355 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:14.527 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:14.529 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:14 smithi135 ceph-mon[221113]: pgmap v1509: 129 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 3 active+undersized, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 1/357 objects degraded (0.280%); 0 B/s, 280 objects/s recovering 2023-09-23T20:21:14.529 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:14 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1/357 objects degraded (0.280%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:14.529 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:14 smithi135 ceph-mon[213871]: pgmap v1509: 129 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 3 active+undersized, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 1/357 objects degraded (0.280%); 0 B/s, 280 objects/s recovering 2023-09-23T20:21:14.529 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:14 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1/357 objects degraded (0.280%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:14.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:14.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:14.552 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:21:14.631 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:14 smithi144 ceph-mon[194833]: pgmap v1509: 129 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 3 active+undersized, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 1/357 objects degraded (0.280%); 0 B/s, 280 objects/s recovering 2023-09-23T20:21:14.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:14 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1/357 objects degraded (0.280%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:14.660 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:14.761 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:14.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:14.865 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:14.901 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:14.950 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:14.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:14.973 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:21:14.992 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:21:15.013 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:21:15.037 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:21:15.054 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:15.056 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:21:15.076 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:21:15.082 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:15.095 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:21:15.096 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:21:15.096 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:21:15.096 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:21:15.096 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:21:15.096 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:21:15.096 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:21:15.096 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:21:15.096 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:21:15.281 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:15.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:15.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:15.386 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:15.413 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:15.457 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:15.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:15.560 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[221113]: 1.15 scrub starts 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[221113]: 1.15 scrub ok 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[221113]: 1.1a scrub ok 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3061379385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[213871]: 1.15 scrub starts 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[213871]: 1.15 scrub ok 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:21:15.586 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[213871]: 1.1a scrub ok 2023-09-23T20:21:15.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3061379385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:15.589 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:15.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:15 smithi144 ceph-mon[194833]: 1.15 scrub starts 2023-09-23T20:21:15.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:15 smithi144 ceph-mon[194833]: 1.15 scrub ok 2023-09-23T20:21:15.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:15 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:21:15.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:15 smithi144 ceph-mon[194833]: 1.1a scrub ok 2023-09-23T20:21:15.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3061379385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:15.767 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:15.786 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:15.787 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:21:15.787 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:21:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:15.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:15.890 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:15.918 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:15.961 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:15.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:16.065 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:16.093 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:16.109 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:21:16.109 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:21:16.109 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:21:16.109 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:21:16.189 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:16.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:16.242 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:21:16.242 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:21:16.243 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:21:16.243 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T20:21:16.243 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T20:21:16.292 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:16.323 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:16 smithi144 ceph-mon[194833]: pgmap v1510: 129 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 3 active+undersized, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/357 objects degraded (0.280%); 0 B/s, 115 objects/s recovering 2023-09-23T20:21:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:16 smithi144 ceph-mon[194833]: 10.9 scrub starts 2023-09-23T20:21:16.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:16 smithi144 ceph-mon[194833]: 10.9 scrub ok 2023-09-23T20:21:16.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2911895321' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:21:16.404 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'unique_pool_5' removed 2023-09-23T20:21:16.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:16.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:16.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:16.530 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:16.554 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:16 smithi135 ceph-mon[221113]: pgmap v1510: 129 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 3 active+undersized, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/357 objects degraded (0.280%); 0 B/s, 115 objects/s recovering 2023-09-23T20:21:16.555 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:16 smithi135 ceph-mon[221113]: 10.9 scrub starts 2023-09-23T20:21:16.555 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:16 smithi135 ceph-mon[221113]: 10.9 scrub ok 2023-09-23T20:21:16.555 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2911895321' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:21:16.555 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:21:16 smithi135 systemd[1]: Stopping Ceph osd.2 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:21:16.555 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:16 smithi135 ceph-mon[213871]: pgmap v1510: 129 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 3 active+undersized, 122 active+clean; 10 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/357 objects degraded (0.280%); 0 B/s, 115 objects/s recovering 2023-09-23T20:21:16.555 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:16 smithi135 ceph-mon[213871]: 10.9 scrub starts 2023-09-23T20:21:16.556 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:16 smithi135 ceph-mon[213871]: 10.9 scrub ok 2023-09-23T20:21:16.556 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2911895321' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:21:16.558 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:16.736 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:21:17.101 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:21:17.317 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:17.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:17.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:17.412 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:21:17.412 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T20:21:17.412 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:21:17.413 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T20:21:17.413 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:21:17.413 DEBUG:teuthology.orchestra.run.smithi135:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:21:17.421 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:17.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:21:17.453 INFO:teuthology.orchestra.run.smithi135.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:21:17.454 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:21:17.522 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:17.536 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:21:17.536 DEBUG:teuthology.orchestra.run.smithi135:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:21:17.593 INFO:tasks.workunit:timeout=3h 2023-09-23T20:21:17.593 INFO:tasks.workunit:cleanup=True 2023-09-23T20:21:17.593 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:21:17.630 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[221113]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.631 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2911895321' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:21:17.631 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[221113]: osdmap e529: 8 total, 7 up, 8 in 2023-09-23T20:21:17.631 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[221113]: 1.1c scrub starts 2023-09-23T20:21:17.631 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[221113]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.631 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[221113]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.631 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[221113]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.631 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[221113]: pgmap v1512: 103 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 97 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:17.631 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:21:17 smithi135 conmon[252159]: 2023-09-23T20:21:17.458+0000 7f97b09de700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:21:17.631 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:21:17 smithi135 conmon[252159]: 2023-09-23T20:21:17.458+0000 7f97b09de700 -1 osd.2 529 *** Got signal Terminated *** 2023-09-23T20:21:17.632 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:21:17 smithi135 conmon[252159]: 2023-09-23T20:21:17.458+0000 7f97b09de700 -1 osd.2 529 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:21:17.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[213871]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2911895321' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:21:17.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[213871]: osdmap e529: 8 total, 7 up, 8 in 2023-09-23T20:21:17.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[213871]: 1.1c scrub starts 2023-09-23T20:21:17.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[213871]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[213871]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.632 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[213871]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.633 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:17 smithi135 ceph-mon[213871]: pgmap v1512: 103 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 97 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:17 smithi144 ceph-mon[194833]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2911895321' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:21:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:17 smithi144 ceph-mon[194833]: osdmap e529: 8 total, 7 up, 8 in 2023-09-23T20:21:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:17 smithi144 ceph-mon[194833]: 1.1c scrub starts 2023-09-23T20:21:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:17 smithi144 ceph-mon[194833]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:17 smithi144 ceph-mon[194833]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:17 smithi144 ceph-mon[194833]: failed to encode map e529 with expected crc 2023-09-23T20:21:17.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:17 smithi144 ceph-mon[194833]: pgmap v1512: 103 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 97 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:17.667 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:17.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:17.682 INFO:tasks.workunit.client.0.smithi135.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T20:21:17.771 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:17.789 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:21:17.800 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:17.849 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:17.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:17.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:17.953 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:17.979 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:18.029 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:18.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:18.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:18.133 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:18.159 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:18.161 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:18.250 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:18.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:18.354 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:18.382 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:18.414 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:21:18.444 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:18.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:18.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:18.533 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:21:18.533 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:21:18.533 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:21:18.534 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:21:18.534 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:21:18.548 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:18.575 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:18.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[221113]: osd.2 marked itself down and dead 2023-09-23T20:21:18.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[221113]: 1.1c scrub starts 2023-09-23T20:21:18.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[221113]: 1.1c scrub ok 2023-09-23T20:21:18.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1769448179' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:18.612 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:18.612 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[213871]: osd.2 marked itself down and dead 2023-09-23T20:21:18.612 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[213871]: 1.1c scrub starts 2023-09-23T20:21:18.613 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[213871]: 1.1c scrub ok 2023-09-23T20:21:18.613 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1769448179' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:18.613 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:18 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:18.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:18 smithi144 ceph-mon[194833]: osd.2 marked itself down and dead 2023-09-23T20:21:18.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:18 smithi144 ceph-mon[194833]: 1.1c scrub starts 2023-09-23T20:21:18.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:18 smithi144 ceph-mon[194833]: 1.1c scrub ok 2023-09-23T20:21:18.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1769448179' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:18.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:18 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:18.668 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:18.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:18.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:18.772 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:18.800 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:18.842 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:18.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:18.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:18.946 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:18.973 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:19.117 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:21:19.129 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:19.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:19.235 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:19.264 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:19.394 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T20:21:19.472 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:21:19.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T20:21:19.668 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: osdmap e530: 8 total, 6 up, 8 in 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: 1.3 scrub starts 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: 1.3 scrub ok 2023-09-23T20:21:19.669 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:19 smithi144 ceph-mon[194833]: pgmap v1514: 103 pgs: 10 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 8 stale+active+clean, 1 active+clean+scrubbing, 3 active+undersized, 79 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:21:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: osdmap e530: 8 total, 6 up, 8 in 2023-09-23T20:21:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: 1.3 scrub starts 2023-09-23T20:21:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: 1.3 scrub ok 2023-09-23T20:21:19.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[213871]: pgmap v1514: 103 pgs: 10 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 8 stale+active+clean, 1 active+clean+scrubbing, 3 active+undersized, 79 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:19.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T20:21:19.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:21:19.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:19.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: osdmap e530: 8 total, 6 up, 8 in 2023-09-23T20:21:19.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: failed to encode map e530 with expected crc 2023-09-23T20:21:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: 1.3 scrub starts 2023-09-23T20:21:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: 1.3 scrub ok 2023-09-23T20:21:19.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:19 smithi135 ceph-mon[221113]: pgmap v1514: 103 pgs: 10 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 8 stale+active+clean, 1 active+clean+scrubbing, 3 active+undersized, 79 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:19.764 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:19.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:19.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:19.874 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:19.903 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:20.220 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:21:19 smithi135 bash[655362]: 4d5310d5a38b331f4ab04398bb045b512e2602d2cc1ffc3f687f9a9a07621b62 2023-09-23T20:21:20.252 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:20.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:20.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:20.356 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:20.384 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:20.480 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:20.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:20.586 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:20.614 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:20 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:21:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:20 smithi144 ceph-mon[194833]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:20 smithi144 ceph-mon[194833]: 1.6 scrub starts 2023-09-23T20:21:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:20 smithi144 ceph-mon[194833]: 1.6 scrub ok 2023-09-23T20:21:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:20 smithi144 ceph-mon[194833]: osdmap e531: 8 total, 6 up, 8 in 2023-09-23T20:21:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:20 smithi144 ceph-mon[194833]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:20 smithi144 ceph-mon[194833]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:20 smithi144 ceph-mon[194833]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.711 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:20.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[221113]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[221113]: 1.6 scrub starts 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[221113]: 1.6 scrub ok 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[221113]: osdmap e531: 8 total, 6 up, 8 in 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[221113]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[221113]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[221113]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:21:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[213871]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[213871]: 1.6 scrub starts 2023-09-23T20:21:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[213871]: 1.6 scrub ok 2023-09-23T20:21:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[213871]: osdmap e531: 8 total, 6 up, 8 in 2023-09-23T20:21:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[213871]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[213871]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:20 smithi135 ceph-mon[213871]: failed to encode map e531 with expected crc 2023-09-23T20:21:20.815 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:20.845 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:20.891 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:20.892 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:21:20.892 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:21:20.893 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:21:20.893 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:21:20.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.893 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199768 9.4 9:22c43d57:::benchmark_data_smithi135_549536_object95913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:21:20.893 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.270504+0000", 2023-09-23T20:21:20.893 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 283.61978521399999, 2023-09-23T20:21:20.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.027306939, 2023-09-23T20:21:20.894 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.894 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.894 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.894 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:21:20.894 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:21:20.895 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199768 2023-09-23T20:21:20.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.895 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270504+0000", 2023-09-23T20:21:20.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270504+0000", 2023-09-23T20:21:20.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270502+0000", 2023-09-23T20:21:20.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:21:20.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270509+0000", 2023-09-23T20:21:20.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6399999999999997e-06 2023-09-23T20:21:20.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270525+0000", 2023-09-23T20:21:20.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5775999999999999e-05 2023-09-23T20:21:20.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.899 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.899 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270532+0000", 2023-09-23T20:21:20.899 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9090000000000003e-06 2023-09-23T20:21:20.899 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.899 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.899 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.899 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270605+0000", 2023-09-23T20:21:20.900 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2836000000000005e-05 2023-09-23T20:21:20.900 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.900 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.900 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:21:20.900 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270615+0000", 2023-09-23T20:21:20.900 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0507e-05 2023-09-23T20:21:20.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.901 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.901 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.290715+0000", 2023-09-23T20:21:20.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0200998189999999 2023-09-23T20:21:20.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.290845+0000", 2023-09-23T20:21:20.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013020100000000001 2023-09-23T20:21:20.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.903 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.291164+0000", 2023-09-23T20:21:20.903 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000318645 2023-09-23T20:21:20.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.903 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.903 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.903 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.296206+0000", 2023-09-23T20:21:20.903 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0050415349999999998 2023-09-23T20:21:20.904 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.904 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297663+0000", 2023-09-23T20:21:20.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014569559999999999 2023-09-23T20:21:20.904 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297758+0000", 2023-09-23T20:21:20.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5564999999999997e-05 2023-09-23T20:21:20.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297786+0000", 2023-09-23T20:21:20.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8314e-05 2023-09-23T20:21:20.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297811+0000", 2023-09-23T20:21:20.907 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4227e-05 2023-09-23T20:21:20.907 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.907 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.907 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.907 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.907 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.907 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199783 9.4 9:2547b74f:::benchmark_data_smithi135_549536_object95928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:21:20.908 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.292081+0000", 2023-09-23T20:21:20.908 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 283.59820770200002, 2023-09-23T20:21:20.908 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0085980670000001, 2023-09-23T20:21:20.908 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.908 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.908 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.908 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:21:20.909 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:21:20.909 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199783 2023-09-23T20:21:20.909 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.909 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.909 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292081+0000", 2023-09-23T20:21:20.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.910 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.910 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.910 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292081+0000", 2023-09-23T20:21:20.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.911 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.911 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.911 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.911 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292080+0000", 2023-09-23T20:21:20.911 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:21:20.911 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.911 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.912 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.912 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292086+0000", 2023-09-23T20:21:20.912 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3169999999999998e-06 2023-09-23T20:21:20.912 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.912 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.912 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.912 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292102+0000", 2023-09-23T20:21:20.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5699999999999999e-05 2023-09-23T20:21:20.913 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.913 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.913 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292114+0000", 2023-09-23T20:21:20.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1664e-05 2023-09-23T20:21:20.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.914 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.914 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292216+0000", 2023-09-23T20:21:20.914 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102756 2023-09-23T20:21:20.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.915 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:21:20.915 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292228+0000", 2023-09-23T20:21:20.915 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1745e-05 2023-09-23T20:21:20.915 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.915 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.916 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.293068+0000", 2023-09-23T20:21:20.916 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0008400669999999 2023-09-23T20:21:20.916 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.916 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.916 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.916 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.293169+0000", 2023-09-23T20:21:20.916 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000100556 2023-09-23T20:21:20.917 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.917 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.293381+0000", 2023-09-23T20:21:20.917 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000212189 2023-09-23T20:21:20.917 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297718+0000", 2023-09-23T20:21:20.918 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043367750000000002 2023-09-23T20:21:20.918 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.918 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297841+0000", 2023-09-23T20:21:20.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000122688 2023-09-23T20:21:20.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.919 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.919 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.919 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300613+0000", 2023-09-23T20:21:20.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027729080000000001 2023-09-23T20:21:20.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.919 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.920 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.920 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300644+0000", 2023-09-23T20:21:20.920 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0125e-05 2023-09-23T20:21:20.920 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.920 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.920 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.920 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300679+0000", 2023-09-23T20:21:20.921 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5787000000000001e-05 2023-09-23T20:21:20.921 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.921 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.921 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.921 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.921 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.921 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199786 9.4 9:22231ef2:::benchmark_data_smithi135_549536_object95931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:21:20.921 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.292128+0000", 2023-09-23T20:21:20.922 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 283.59816060399999, 2023-09-23T20:21:20.922 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.020906723, 2023-09-23T20:21:20.922 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.922 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.922 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.922 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:21:20.922 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:21:20.923 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199786 2023-09-23T20:21:20.923 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.923 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.923 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.923 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.923 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292128+0000", 2023-09-23T20:21:20.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.923 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.924 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.924 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292128+0000", 2023-09-23T20:21:20.924 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.924 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.924 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292128+0000", 2023-09-23T20:21:20.925 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:20.925 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.925 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292131+0000", 2023-09-23T20:21:20.925 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.371e-06 2023-09-23T20:21:20.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292138+0000", 2023-09-23T20:21:20.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9979999999999997e-06 2023-09-23T20:21:20.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292143+0000", 2023-09-23T20:21:20.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4989999999999996e-06 2023-09-23T20:21:20.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292248+0000", 2023-09-23T20:21:20.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105475 2023-09-23T20:21:20.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:21:20.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292255+0000", 2023-09-23T20:21:20.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8399999999999997e-06 2023-09-23T20:21:20.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.294615+0000", 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.002359612 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.294690+0000", 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5012000000000006e-05 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.294967+0000", 2023-09-23T20:21:20.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00027756500000000002 2023-09-23T20:21:20.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.930 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297846+0000", 2023-09-23T20:21:20.930 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028783379999999998 2023-09-23T20:21:20.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300715+0000", 2023-09-23T20:21:20.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028695330000000001 2023-09-23T20:21:20.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.312950+0000", 2023-09-23T20:21:20.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012234910999999999 2023-09-23T20:21:20.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.932 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.312997+0000", 2023-09-23T20:21:20.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6884e-05 2023-09-23T20:21:20.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.933 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.933 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313035+0000", 2023-09-23T20:21:20.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8204000000000002e-05 2023-09-23T20:21:20.933 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.933 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199792 9.4 9:251d5e24:::benchmark_data_smithi135_549536_object95937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.292150+0000", 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 283.59813880799999, 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0209780289999999, 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199792 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292150+0000", 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292150+0000", 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292150+0000", 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292153+0000", 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9979999999999999e-06 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292159+0000", 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6529999999999997e-06 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292164+0000", 2023-09-23T20:21:20.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1389999999999997e-06 2023-09-23T20:21:20.938 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292275+0000", 2023-09-23T20:21:20.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011100299999999999 2023-09-23T20:21:20.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.939 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:21:20.939 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292281+0000", 2023-09-23T20:21:20.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6549999999999998e-06 2023-09-23T20:21:20.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.296248+0000", 2023-09-23T20:21:20.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0039670300000001 2023-09-23T20:21:20.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.296347+0000", 2023-09-23T20:21:20.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8867000000000001e-05 2023-09-23T20:21:20.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.941 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.296539+0000", 2023-09-23T20:21:20.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019160800000000001 2023-09-23T20:21:20.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300566+0000", 2023-09-23T20:21:20.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040269700000000004 2023-09-23T20:21:20.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300753+0000", 2023-09-23T20:21:20.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018712399999999999 2023-09-23T20:21:20.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.943 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313076+0000", 2023-09-23T20:21:20.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012322783 2023-09-23T20:21:20.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.944 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.944 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313105+0000", 2023-09-23T20:21:20.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9830999999999999e-05 2023-09-23T20:21:20.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.945 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313128+0000", 2023-09-23T20:21:20.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2787000000000001e-05 2023-09-23T20:21:20.945 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.945 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.945 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.946 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199801 9.4 9:24182093:::benchmark_data_smithi135_549536_object95946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:21:20.946 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.309108+0000", 2023-09-23T20:21:20.946 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 283.58118103800001, 2023-09-23T20:21:20.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0041302670000001, 2023-09-23T20:21:20.946 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.947 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.947 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.947 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:21:20.947 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:21:20.947 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199801 2023-09-23T20:21:20.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.947 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309108+0000", 2023-09-23T20:21:20.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309108+0000", 2023-09-23T20:21:20.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309106+0000", 2023-09-23T20:21:20.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:21:20.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309114+0000", 2023-09-23T20:21:20.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2629999999999997e-06 2023-09-23T20:21:20.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309130+0000", 2023-09-23T20:21:20.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6359999999999999e-05 2023-09-23T20:21:20.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309137+0000", 2023-09-23T20:21:20.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.939e-06 2023-09-23T20:21:20.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.952 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.952 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309194+0000", 2023-09-23T20:21:20.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6867999999999998e-05 2023-09-23T20:21:20.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.953 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:21:20.953 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309207+0000", 2023-09-23T20:21:20.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2972e-05 2023-09-23T20:21:20.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.953 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297877+0000", 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98867036799999997 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297942+0000", 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4992000000000004e-05 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.298141+0000", 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019871600000000001 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300720+0000", 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025793499999999998 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.303457+0000", 2023-09-23T20:21:20.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027370979999999999 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313162+0000", 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0097042410000000006 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313204+0000", 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2326000000000001e-05 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313238+0000", 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4251999999999997e-05 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199833 9.4 9:23b4d44b:::benchmark_data_smithi135_549536_object95978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.536850+0000", 2023-09-23T20:21:20.957 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 283.35343892999998, 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.78208931299999995, 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199833 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536850+0000", 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536850+0000", 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536848+0000", 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:21:20.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536857+0000", 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2799999999999992e-06 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536888+0000", 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0354000000000001e-05 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536896+0000", 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4409999999999993e-06 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536953+0000", 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6715000000000002e-05 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536960+0000", 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5830000000000001e-06 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.299194+0000", 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.76223389399999997 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.299260+0000", 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6210999999999993e-05 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.299488+0000", 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00022723599999999999 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.302080+0000", 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025918429999999999 2023-09-23T20:21:20.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.306190+0000", 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0041102559999999996 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.318857+0000", 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012667223999999999 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.318905+0000", 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8121000000000002e-05 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.318939+0000", 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4177000000000002e-05 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199843 9.4 9:202c07b4:::benchmark_data_smithi135_549536_object95988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.537102+0000", 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 283.35318674899997, 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.78193986900000001, 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199843 2023-09-23T20:21:20.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537102+0000", 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537102+0000", 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537101+0000", 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537107+0000", 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0020000000000003e-06 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537123+0000", 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5865000000000001e-05 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537130+0000", 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6340000000000003e-06 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.540971+0000", 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038409939999999999 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.540981+0000", 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0468e-05 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300776+0000", 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.759794788 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300844+0000", 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7973999999999997e-05 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.301042+0000", 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019837699999999999 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.306138+0000", 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0050955109999999996 2023-09-23T20:21:20.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.306230+0000", 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2410000000000005e-05 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.318982+0000", 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012751302000000001 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319017+0000", 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5030000000000002e-05 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319042+0000", 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5355999999999999e-05 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.973 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199857 9.4 9:240ce8b0:::benchmark_data_smithi135_549536_object96002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.562071+0000", 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 283.328217537, 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.75710239400000001, 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199857 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.974 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562071+0000", 2023-09-23T20:21:20.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562071+0000", 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562070+0000", 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562078+0000", 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7649999999999993e-06 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562097+0000", 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8938000000000001e-05 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562104+0000", 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5000000000000002e-06 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562164+0000", 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9426999999999999e-05 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562175+0000", 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1391e-05 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.302116+0000", 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.73994101000000001 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.302228+0000", 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000112235 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.302434+0000", 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000205183 2023-09-23T20:21:20.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.306235+0000", 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038017849999999998 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.311586+0000", 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0053509880000000001 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.980 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319090+0000", 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.007503984 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319136+0000", 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5797999999999998e-05 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.981 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319174+0000", 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7664999999999999e-05 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:39763 10.2 10:4a79b0ff:::benchmark_data_smithi135_602286_object39762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:17:26.864094+0000", 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 234.02619488799999, 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21568243400000001, 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.982 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 39763 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864094+0000", 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864094+0000", 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864093+0000", 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864135+0000", 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1718999999999998e-05 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864154+0000", 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9562999999999999e-05 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864162+0000", 2023-09-23T20:21:20.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8159999999999994e-06 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864212+0000", 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9691000000000002e-05 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864351+0000", 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000139431 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864500+0000", 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014922500000000001 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.866289+0000", 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0017887249999999999 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.866811+0000", 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00052236999999999997 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.079697+0000", 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21288605899999999 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.079744+0000", 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6805000000000003e-05 2023-09-23T20:21:20.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.079777+0000", 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2262999999999999e-05 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:39765 10.2 10:41a8f286:::benchmark_data_smithi135_602286_object39764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:17:26.868355+0000", 2023-09-23T20:21:20.989 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 234.02193367199999, 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.214626228, 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 39765 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868355+0000", 2023-09-23T20:21:20.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868355+0000", 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868354+0000", 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868446+0000", 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1996999999999993e-05 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868460+0000", 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4025e-05 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868467+0000", 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8680000000000001e-06 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868510+0000", 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3621999999999997e-05 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868642+0000", 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013202299999999999 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:20.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868782+0000", 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000139572 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.870475+0000", 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001693022 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.873987+0000", 2023-09-23T20:21:20.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035118979999999998 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.082915+0000", 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.208928475 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.082954+0000", 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8390999999999999e-05 2023-09-23T20:21:20.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.082982+0000", 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7738000000000001e-05 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:40985 10.2 10:4d9f6e00:::benchmark_data_smithi135_602286_object40984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:17:27.832431+0000", 2023-09-23T20:21:20.996 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 233.057858307, 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28747203500000001, 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 40985 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:20.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832431+0000", 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832431+0000", 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832430+0000", 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:21:20.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832488+0000", 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8402000000000001e-05 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832502+0000", 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4431e-05 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:20.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832509+0000", 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6499999999999999e-06 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832560+0000", 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0796e-05 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832723+0000", 2023-09-23T20:21:21.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016322900000000001 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832857+0000", 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013411900000000001 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.834290+0000", 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014332259999999999 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.834849+0000", 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00055816899999999996 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:28.119790+0000", 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28494164500000002 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:28.119844+0000", 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3257000000000002e-05 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:28.119903+0000", 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9191000000000003e-05 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:123189 10.f e514/503 10:f5618735:::benchmark_data_smithi135_602286_object30963:head v 514'7613, mlcod=499'5274)", 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:28.403820+0000", 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 112.486468589, 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15784634, 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403820+0000", 2023-09-23T20:21:21.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403820+0000", 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403820+0000", 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403827+0000", 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5320000000000001e-06 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403835+0000", 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2520000000000002e-06 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403839+0000", 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9790000000000004e-06 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403870+0000", 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1056999999999998e-05 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403889+0000", 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9298999999999999e-05 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.561622+0000", 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.157733082 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.561667+0000", 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4728000000000003e-05 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:124933 10.c e514/479 10:3ab62de7:::benchmark_data_smithi135_602286_object32707:head v 514'7830, mlcod=514'7828)", 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.284926+0000", 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 109.605362795, 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17721135599999999, 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284926+0000", 2023-09-23T20:21:21.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284926+0000", 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284926+0000", 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284932+0000", 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3659999999999997e-06 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284940+0000", 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2740000000000007e-06 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284945+0000", 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8779999999999997e-06 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284990+0000", 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4820000000000001e-05 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285008+0000", 2023-09-23T20:21:21.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8425999999999999e-05 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462103+0000", 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17709439900000001 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462138+0000", 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4789000000000002e-05 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.013 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:124930 10.d e514/479 10:b5f42205:::benchmark_data_smithi135_602286_object32704:head v 514'7780, mlcod=514'7778)", 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.285891+0000", 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 109.60439808300001, 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.179623752, 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:21.014 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285891+0000", 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285891+0000", 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285890+0000", 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285896+0000", 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2319999999999998e-06 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285903+0000", 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285907+0000", 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5389999999999999e-06 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285944+0000", 2023-09-23T20:21:21.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6776999999999997e-05 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285957+0000", 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2959000000000001e-05 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.465467+0000", 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17951017699999999 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.465515+0000", 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7811e-05 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "MOSDPGPush(10.f 514/503 [PushOp(10:fca9ccf4:::benchmark_data_smithi135_602286_object89690:head, version: 500'5562, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:fca9ccf4:::benchmark_data_smithi135_602286_object89690:head@500'5562, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.285917+0000", 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 109.60437159999999, 2023-09-23T20:21:21.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17891938700000001, 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285917+0000", 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285917+0000", 2023-09-23T20:21:21.020 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285917+0000", 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967296 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285948+0000", 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0762999999999997e-05 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.021 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285957+0000", 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9120000000000004e-06 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285963+0000", 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8860000000000001e-06 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.022 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462080+0000", 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17611687100000001 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462095+0000", 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5579000000000001e-05 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.464837+0000", 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002741556 2023-09-23T20:21:21.023 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:124931 10.b e514/490 10:da9f67d3:::benchmark_data_smithi135_602286_object32705:head v 514'7623, mlcod=0'0)", 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.285983+0000", 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 109.60430574599999, 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.179452687, 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.024 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285983+0000", 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285983+0000", 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.025 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285983+0000", 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285986+0000", 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3730000000000001e-06 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285990+0000", 2023-09-23T20:21:21.026 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0940000000000001e-06 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285993+0000", 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4899999999999999e-06 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462116+0000", 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17612323599999999 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.027 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462144+0000", 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8013000000000001e-05 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.465401+0000", 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003256802 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.028 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.465436+0000", 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4978000000000003e-05 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:124932 10.4 e514/479 10:2bf1ccf2:::benchmark_data_smithi135_602286_object32706:head v 514'7828, mlcod=514'7826)", 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.286005+0000", 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 109.604284299, 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18492788800000001, 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:21.029 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.286005+0000", 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.286005+0000", 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.030 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.286004+0000", 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967296 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.286007+0000", 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8289999999999999e-06 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462166+0000", 2023-09-23T20:21:21.031 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.1761588 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462173+0000", 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5400000000000001e-06 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462440+0000", 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000267125 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.032 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462460+0000", 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0367999999999999e-05 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.470898+0000", 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0084382220000000004 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.470933+0000", 2023-09-23T20:21:21.033 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4193000000000003e-05 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:125856 10.0 e514/511 10:00559810:::benchmark_data_smithi135_602286_object33630:head v 514'7809, mlcod=514'7806)", 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:32.871097+0000", 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 108.019191896, 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.166058025, 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.034 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871097+0000", 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871097+0000", 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.035 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871096+0000", 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871106+0000", 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9869999999999994e-06 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.036 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871116+0000", 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4970000000000007e-06 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871120+0000", 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1439999999999996e-06 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.037 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871151+0000", 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0666000000000001e-05 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871163+0000", 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2278e-05 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037105+0000", 2023-09-23T20:21:21.038 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16594193300000001 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037155+0000", 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0197000000000001e-05 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:125865 10.1 e514/479 10:822727c4:::benchmark_data_smithi135_602286_object33639:head v 514'7863, mlcod=514'7860)", 2023-09-23T20:21:21.039 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:32.872211+0000", 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 108.018078328, 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.165180407, 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872211+0000", 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.040 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872211+0000", 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872210+0000", 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.041 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872215+0000", 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3140000000000003e-06 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.030919+0000", 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15870314899999999 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.030942+0000", 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3493e-05 2023-09-23T20:21:21.042 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.031058+0000", 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115739 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.031111+0000", 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3075999999999998e-05 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.043 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037357+0000", 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0062462519999999999 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037391+0000", 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3937999999999999e-05 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.044 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:125864 10.6 e514/479 10:6a1ce244:::benchmark_data_smithi135_602286_object33638:head v 514'7883, mlcod=514'7881)", 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:32.872248+0000", 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 108.01804088199999, 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.165563234, 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872248+0000", 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.045 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872248+0000", 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872248+0000", 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.046 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872252+0000", 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9130000000000002e-06 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.030954+0000", 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15870204299999999 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.030976+0000", 2023-09-23T20:21:21.047 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1614e-05 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.031087+0000", 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011085400000000001 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.031143+0000", 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5578999999999998e-05 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.048 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037776+0000", 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0066330169999999997 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037811+0000", 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5790999999999997e-05 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.049 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:21:21.050 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:21:21.050 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:21:21.050 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:21:21.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:21.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:21.066 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:21:21.086 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:21:21.106 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:21:21.125 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:21:21.146 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:21:21.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:21:21.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:21:21.147 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T20:21:21.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:21:21.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:21:21.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:21:21.147 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:21:21.147 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:21:21.152 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:21.178 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:21.221 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:21.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:21.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:21.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:21.358 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:21.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:21 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1229438513' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:21.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:21 smithi135 ceph-mon[221113]: pgmap v1516: 103 pgs: 10 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 5 stale+active+clean, 1 active+clean+scrubbing, 12 active+undersized, 73 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:21 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1229438513' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:21.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:21 smithi135 ceph-mon[213871]: pgmap v1516: 103 pgs: 10 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 5 stale+active+clean, 1 active+clean+scrubbing, 12 active+undersized, 73 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:21.508 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:21:21.788 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:21:21.849 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:21:21.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:21 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1229438513' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:21.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:21 smithi144 ceph-mon[194833]: pgmap v1516: 103 pgs: 10 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 5 stale+active+clean, 1 active+clean+scrubbing, 12 active+undersized, 73 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1/87 objects degraded (1.149%) 2023-09-23T20:21:21.962 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:21.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:22.066 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:22.095 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:22.138 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:22.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:22.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:22.242 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:22.273 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:22.624 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[221113]: 1.b scrub starts 2023-09-23T20:21:22.624 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[221113]: 1.b scrub ok 2023-09-23T20:21:22.624 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[221113]: 1.d scrub ok 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[213871]: 1.b scrub starts 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[213871]: 1.b scrub ok 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[213871]: 1.d scrub ok 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:22.625 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:22.646 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:21:22.646 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:21:22.682 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:22.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:22.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:22.707 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:22.786 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:22.813 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:22 smithi144 ceph-mon[194833]: 1.b scrub starts 2023-09-23T20:21:22.813 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:22 smithi144 ceph-mon[194833]: 1.b scrub ok 2023-09-23T20:21:22.813 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:22 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:21:22.813 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:22 smithi144 ceph-mon[194833]: 1.d scrub ok 2023-09-23T20:21:22.813 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:22.813 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:22.815 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:22.909 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:22.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:22.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:23.014 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:23.042 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:23.138 DEBUG:teuthology.orchestra.run.smithi135:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T20:21:23.139 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:23.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:23.242 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:23.251 INFO:tasks.workunit.client.0.smithi135.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T20:21:23.253 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:21:23.254 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T20:21:23.271 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:23.332 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:21:23.332 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:21:23.367 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:23.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:23.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:23.471 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:23.508 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:23.551 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:23.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:23.605 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T20:21:23.655 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:23 smithi144 ceph-mon[194833]: 1.1b deep-scrub starts 2023-09-23T20:21:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:23 smithi144 ceph-mon[194833]: 1.1b deep-scrub ok 2023-09-23T20:21:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:23 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1/87 objects degraded (1.149%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:23.679 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:23 smithi144 ceph-mon[194833]: pgmap v1517: 103 pgs: 10 peering, 3 active+undersized+degraded, 1 active+undersized+remapped, 19 active+undersized, 70 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 7/84 objects degraded (8.333%); 16 KiB/s, 0 objects/s recovering 2023-09-23T20:21:23.681 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:23.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:23 smithi135 ceph-mon[213871]: 1.1b deep-scrub starts 2023-09-23T20:21:23.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:23 smithi135 ceph-mon[213871]: 1.1b deep-scrub ok 2023-09-23T20:21:23.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:23 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1/87 objects degraded (1.149%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:23.683 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:23 smithi135 ceph-mon[213871]: pgmap v1517: 103 pgs: 10 peering, 3 active+undersized+degraded, 1 active+undersized+remapped, 19 active+undersized, 70 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 7/84 objects degraded (8.333%); 16 KiB/s, 0 objects/s recovering 2023-09-23T20:21:23.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:23 smithi135 ceph-mon[221113]: 1.1b deep-scrub starts 2023-09-23T20:21:23.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:23 smithi135 ceph-mon[221113]: 1.1b deep-scrub ok 2023-09-23T20:21:23.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:23 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1/87 objects degraded (1.149%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:23.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:23 smithi135 ceph-mon[221113]: pgmap v1517: 103 pgs: 10 peering, 3 active+undersized+degraded, 1 active+undersized+remapped, 19 active+undersized, 70 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 7/84 objects degraded (8.333%); 16 KiB/s, 0 objects/s recovering 2023-09-23T20:21:23.712 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T20:21:23.864 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:21:23.865 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:21:23.865 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T20:21:23.920 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:21:23.941 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:21:23.954 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:23.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:23.962 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T20:21:23.963 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T20:21:23.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:21:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:21:23] "GET /metrics HTTP/1.1" 200 34129 "" "Prometheus/2.43.0" 2023-09-23T20:21:23.986 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T20:21:23.987 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T20:21:23.988 DEBUG:teuthology.orchestra.run.smithi135:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T20:21:24.059 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:24.097 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:24.140 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:24.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:24.244 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:24.271 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:24.296 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:21:24.299 INFO:tasks.workunit.client.0.smithi135.stdout:Running main() from gmock_main.cc 2023-09-23T20:21:24.299 INFO:tasks.workunit.client.0.smithi135.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T20:21:24.299 INFO:tasks.workunit.client.0.smithi135.stdout:[----------] Global test environment set-up. 2023-09-23T20:21:24.299 INFO:tasks.workunit.client.0.smithi135.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T20:21:24.316 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:24.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:24.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:24.421 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:24.447 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:24.489 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:24.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:24.593 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:24.619 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:24.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:21:24.816 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:24.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:24.920 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:24.946 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:24.987 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:24.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:25.091 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:25.119 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:25.283 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:25.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:25.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:25 smithi144 ceph-mon[194833]: 1.16 scrub starts 2023-09-23T20:21:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:25 smithi144 ceph-mon[194833]: 1.16 scrub ok 2023-09-23T20:21:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:25 smithi144 ceph-mon[194833]: 1.e scrub starts 2023-09-23T20:21:25.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:25 smithi144 ceph-mon[194833]: 1.e scrub ok 2023-09-23T20:21:25.388 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:25.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:25 smithi135 ceph-mon[221113]: 1.16 scrub starts 2023-09-23T20:21:25.413 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:25 smithi135 ceph-mon[221113]: 1.16 scrub ok 2023-09-23T20:21:25.414 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:25 smithi135 ceph-mon[221113]: 1.e scrub starts 2023-09-23T20:21:25.414 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:25 smithi135 ceph-mon[221113]: 1.e scrub ok 2023-09-23T20:21:25.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:25 smithi135 ceph-mon[213871]: 1.16 scrub starts 2023-09-23T20:21:25.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:25 smithi135 ceph-mon[213871]: 1.16 scrub ok 2023-09-23T20:21:25.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:25 smithi135 ceph-mon[213871]: 1.e scrub starts 2023-09-23T20:21:25.414 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:25 smithi135 ceph-mon[213871]: 1.e scrub ok 2023-09-23T20:21:25.416 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:25.687 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:21:25.688 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:21:26.094 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T20:21:26.120 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_all_features (26 ms) 2023-09-23T20:21:26.120 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T20:21:26.304 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.copyup (184 ms) 2023-09-23T20:21:26.304 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T20:21:26.319 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.sparse_copyup (16 ms) 2023-09-23T20:21:26.320 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T20:21:26.321 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:21:26.327 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-09-23T20:21:26.327 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T20:21:26.337 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2023-09-23T20:21:26.338 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T20:21:26.367 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.directory_methods (30 ms) 2023-09-23T20:21:26.368 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.create 2023-09-23T20:21:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: osdmap e532: 8 total, 6 up, 8 in 2023-09-23T20:21:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1890555918' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:21:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:21:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: pgmap v1519: 135 pgs: 32 unknown, 6 active+undersized+degraded, 1 active+undersized+remapped, 26 active+undersized, 70 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 7.9 KiB/s, 0 objects/s recovering 2023-09-23T20:21:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:26 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:21:26.424 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.create (56 ms) 2023-09-23T20:21:26.424 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T20:21:26.425 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-09-23T20:21:26.425 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T20:21:26.427 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-09-23T20:21:26.427 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T20:21:26.429 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-09-23T20:21:26.429 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T20:21:26.430 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-09-23T20:21:26.430 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T20:21:26.431 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-23T20:21:26.431 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T20:21:26.434 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-09-23T20:21:26.434 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T20:21:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: osdmap e532: 8 total, 6 up, 8 in 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1890555918' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: pgmap v1519: 135 pgs: 32 unknown, 6 active+undersized+degraded, 1 active+undersized+remapped, 26 active+undersized, 70 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 7.9 KiB/s, 0 objects/s recovering 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: osdmap e532: 8 total, 6 up, 8 in 2023-09-23T20:21:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1890555918' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:21:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: failed to encode map e532 with expected crc 2023-09-23T20:21:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:21:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: pgmap v1519: 135 pgs: 32 unknown, 6 active+undersized+degraded, 1 active+undersized+remapped, 26 active+undersized, 70 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 11/84 objects degraded (13.095%); 7.9 KiB/s, 0 objects/s recovering 2023-09-23T20:21:26.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:26 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi135-657432-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:21:26.547 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:26.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:26.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:26.651 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:26.671 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:21:26.678 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:26.706 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T20:21:26.779 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:26.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:26.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:26.799 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:21:26.799 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T20:21:26.823 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:26.886 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:26.915 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:27.010 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:27.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:27.114 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:27.142 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:27.190 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:27.191 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[221113]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.192 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[221113]: osdmap e533: 8 total, 6 up, 8 in 2023-09-23T20:21:27.192 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[221113]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.192 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T20:21:27.192 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[221113]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.192 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[221113]: osdmap e534: 8 total, 6 up, 8 in 2023-09-23T20:21:27.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[213871]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[213871]: osdmap e533: 8 total, 6 up, 8 in 2023-09-23T20:21:27.192 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[213871]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.193 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T20:21:27.193 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[213871]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.193 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:27 smithi135 ceph-mon[213871]: osdmap e534: 8 total, 6 up, 8 in 2023-09-23T20:21:27.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:27.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:27.294 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:27.323 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:27.365 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:27.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:27 smithi144 ceph-mon[194833]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:27 smithi144 ceph-mon[194833]: osdmap e533: 8 total, 6 up, 8 in 2023-09-23T20:21:27.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:27 smithi144 ceph-mon[194833]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:27 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T20:21:27.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:27 smithi144 ceph-mon[194833]: failed to encode map e533 with expected crc 2023-09-23T20:21:27.367 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:27 smithi144 ceph-mon[194833]: osdmap e534: 8 total, 6 up, 8 in 2023-09-23T20:21:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:27.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:27.469 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:27.499 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:27.695 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:27.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:27.798 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:27.825 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:28.100 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:28.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:28.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:28.121 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:21:28.176 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:21:28.176 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:21:28.204 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:28.231 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:28 smithi144 ceph-mon[194833]: failed to encode map e534 with expected crc 2023-09-23T20:21:28.231 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:28 smithi144 ceph-mon[194833]: pgmap v1522: 135 pgs: 27 unknown, 8 active+undersized+degraded, 1 active+undersized+remapped, 27 active+undersized, 72 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 13/90 objects degraded (14.444%); 9.0 KiB/s, 0 objects/s recovering 2023-09-23T20:21:28.231 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:28 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1709272302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:28.231 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:28 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:28.231 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:28.235 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:28.384 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:28.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:28.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:28.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[213871]: failed to encode map e534 with expected crc 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[213871]: pgmap v1522: 135 pgs: 27 unknown, 8 active+undersized+degraded, 1 active+undersized+remapped, 27 active+undersized, 72 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 13/90 objects degraded (14.444%); 9.0 KiB/s, 0 objects/s recovering 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1709272302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[221113]: failed to encode map e534 with expected crc 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[221113]: pgmap v1522: 135 pgs: 27 unknown, 8 active+undersized+degraded, 1 active+undersized+remapped, 27 active+undersized, 72 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 13/90 objects degraded (14.444%); 9.0 KiB/s, 0 objects/s recovering 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1709272302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:28.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:28.487 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:28.513 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:28.714 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:28.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:28.766 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:21:28.818 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:28.845 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:28.886 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:28.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:28.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:28.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:29.022 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:29.116 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[213871]: failed to encode map e534 with expected crc 2023-09-23T20:21:29.116 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:29.116 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[213871]: failed to encode map e535 with expected crc 2023-09-23T20:21:29.117 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:21:29.117 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[213871]: osdmap e535: 8 total, 6 up, 8 in 2023-09-23T20:21:29.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[221113]: failed to encode map e534 with expected crc 2023-09-23T20:21:29.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:29.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[221113]: failed to encode map e535 with expected crc 2023-09-23T20:21:29.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:21:29.117 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:29 smithi135 ceph-mon[221113]: osdmap e535: 8 total, 6 up, 8 in 2023-09-23T20:21:29.118 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:29.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:29.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:29.174 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:21:29.195 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:21:29.216 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:21:29.225 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:29.233 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:29.241 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:21:29.252 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:29.261 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:21:29.283 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:21:29.283 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:21:29.283 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:21:29.283 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T20:21:29.283 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:21:29.283 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:21:29.283 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:21:29.283 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:21:29.284 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:21:29.297 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:29.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:29.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:29.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:29 smithi144 ceph-mon[194833]: failed to encode map e534 with expected crc 2023-09-23T20:21:29.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:29 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:29.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:29 smithi144 ceph-mon[194833]: failed to encode map e535 with expected crc 2023-09-23T20:21:29.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:29 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:21:29.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:29 smithi144 ceph-mon[194833]: osdmap e535: 8 total, 6 up, 8 in 2023-09-23T20:21:29.404 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:29.430 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:29.471 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:29.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:29.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:29.578 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:29.606 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:30.189 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[221113]: 1.1e scrub starts 2023-09-23T20:21:30.189 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[221113]: failed to encode map e535 with expected crc 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[221113]: 1.1e scrub ok 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2510739096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[221113]: pgmap v1524: 135 pgs: 6 unknown, 1 active+undersized+remapped, 10 active+undersized+degraded, 2 undersized+peered, 35 active+undersized, 81 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 987 KiB/s wr, 31 op/s; 15/111 objects degraded (13.514%) 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[221113]: failed to encode map e535 with expected crc 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[213871]: 1.1e scrub starts 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[213871]: failed to encode map e535 with expected crc 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[213871]: 1.1e scrub ok 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2510739096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:30.190 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[213871]: pgmap v1524: 135 pgs: 6 unknown, 1 active+undersized+remapped, 10 active+undersized+degraded, 2 undersized+peered, 35 active+undersized, 81 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 987 KiB/s wr, 31 op/s; 15/111 objects degraded (13.514%) 2023-09-23T20:21:30.191 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:30 smithi135 ceph-mon[213871]: failed to encode map e535 with expected crc 2023-09-23T20:21:30.203 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:21:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:30.207 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:21:30.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:30.308 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:30.338 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:30.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:30 smithi144 ceph-mon[194833]: 1.1e scrub starts 2023-09-23T20:21:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:30 smithi144 ceph-mon[194833]: failed to encode map e535 with expected crc 2023-09-23T20:21:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:30 smithi144 ceph-mon[194833]: 1.1e scrub ok 2023-09-23T20:21:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2510739096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:30 smithi144 ceph-mon[194833]: pgmap v1524: 135 pgs: 6 unknown, 1 active+undersized+remapped, 10 active+undersized+degraded, 2 undersized+peered, 35 active+undersized, 81 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1.9 MiB/s rd, 987 KiB/s wr, 31 op/s; 15/111 objects degraded (13.514%) 2023-09-23T20:21:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:30 smithi144 ceph-mon[194833]: failed to encode map e535 with expected crc 2023-09-23T20:21:30.383 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:30.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:30.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:30.491 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:30.519 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:30.762 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:30.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:30.864 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:21:30.866 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:30.896 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:30.943 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:30.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:31.049 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:31.080 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:31.131 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:31.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:31.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:31.235 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:31.265 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:31.313 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:21:31.313 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:21:31.313 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T20:21:31.313 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:21:31.316 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:31.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:31.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:31.337 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:31.422 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:31.449 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:31.543 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:31.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:31.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:31.674 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:31.718 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:31.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:31.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:31.800 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6, 2] live_osds: [0, 1, 3, 4, 5, 7] 2023-09-23T20:21:31.800 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:21:31.800 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:21:31.800 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:21:31.822 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:31.854 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:31.903 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:31.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:31.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:32.010 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:32.039 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:32.084 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:32.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:32.157 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:21:32.189 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:32.208 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:21:32.214 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:32.256 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:32.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:32.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:32.361 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:32.392 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:32 smithi135 ceph-mon[213871]: pgmap v1525: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1.3 MiB/s rd, 688 KiB/s wr, 22 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:32.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:32 smithi135 ceph-mon[221113]: pgmap v1525: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1.3 MiB/s rd, 688 KiB/s wr, 22 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:32.394 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:32 smithi144 ceph-mon[194833]: pgmap v1525: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1.3 MiB/s rd, 688 KiB/s wr, 22 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:32.863 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:32.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:32.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:32.904 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:32.967 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:32.998 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:33.044 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:33.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:33.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:33.062 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:33.150 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:33.178 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:33.420 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:33.424 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 16/117 objects degraded (13.675%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:33.424 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 16/117 objects degraded (13.675%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:33.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:33.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:33.471 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:33.526 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:33.556 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:33.604 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:33.608 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 16/117 objects degraded (13.675%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:21:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:33.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:33.709 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:33.737 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:21:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:21:33] "GET /metrics HTTP/1.1" 200 34129 "" "Prometheus/2.43.0" 2023-09-23T20:21:34.029 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:34.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:34.134 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:34.164 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:34.212 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:34.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:34.318 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:34.350 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:21:34.353 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:34 smithi144 ceph-mon[194833]: pgmap v1526: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.1 MiB/s rd, 575 KiB/s wr, 19 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:34 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3901215390' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:34 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:34.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:34 smithi135 ceph-mon[213871]: pgmap v1526: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.1 MiB/s rd, 575 KiB/s wr, 19 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:34 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3901215390' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:34.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:34 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:34 smithi135 ceph-mon[221113]: pgmap v1526: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.1 MiB/s rd, 575 KiB/s wr, 19 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:34 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3901215390' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:34.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:34 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:35.394 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:35.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:35.498 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:35.530 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:35.579 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:35.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:35 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:35.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:35 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:35.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:35 smithi144 ceph-mon[194833]: failed to encode map e536 with expected crc 2023-09-23T20:21:35.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:35 smithi144 ceph-mon[194833]: osdmap e536: 8 total, 6 up, 8 in 2023-09-23T20:21:35.586 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:35 smithi144 ceph-mon[194833]: failed to encode map e536 with expected crc 2023-09-23T20:21:35.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:35.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:35.592 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:35.592 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:35.592 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[221113]: failed to encode map e536 with expected crc 2023-09-23T20:21:35.592 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[221113]: osdmap e536: 8 total, 6 up, 8 in 2023-09-23T20:21:35.592 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[221113]: failed to encode map e536 with expected crc 2023-09-23T20:21:35.593 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:35.593 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:35.593 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[213871]: failed to encode map e536 with expected crc 2023-09-23T20:21:35.593 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[213871]: osdmap e536: 8 total, 6 up, 8 in 2023-09-23T20:21:35.593 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:35 smithi135 ceph-mon[213871]: failed to encode map e536 with expected crc 2023-09-23T20:21:35.688 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:35.716 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:35.760 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:35.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:35.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:35.868 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:35.899 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:36.083 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:21:36.086 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:21:36.195 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:36.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:36.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:36.299 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:36.326 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:36.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:36 smithi135 ceph-mon[221113]: pgmap v1528: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s rd, 515 KiB/s wr, 15 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:36 smithi135 ceph-mon[213871]: pgmap v1528: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s rd, 515 KiB/s wr, 15 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:36.619 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:36.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:36.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:36 smithi144 ceph-mon[194833]: pgmap v1528: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s rd, 515 KiB/s wr, 15 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:36.723 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:36.750 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:36.998 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:37.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:37.101 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:37.133 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:37.182 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:37.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:37.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:37.286 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1528,"stamp":"2023-09-23T20:21:35.266470+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4899442,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":6068,"num_read_kb":21498,"num_write":956,"num_write_kb":16853,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":265,"num_bytes_recovered":8508882,"num_keys_recovered":10,"num_objects_omap":25,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6108,"ondisk_log_size":6108,"up":350,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":407,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6529260,"kb_used_data":730284,"kb_used_omap":38,"kb_used_meta":5798873,"kb_avail":743431956,"statfs":{"total":767960285184,"available":761274322944,"internally_reserved":0,"allocated":747810816,"data_stored":731200984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39803,"internal_metadata":5938046085},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":20,"num_objects":9,"num_object_clones":0,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":82,"num_read_kb":8285,"num_write":41,"num_write_kb":4124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002580"},"pg_stats":[{"pgid":"11.12","version":"533'1","reported_seq":10,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.158034+0000","last_change":"2023-09-23T20:21:26.429725+0000","last_active":"2023-09-23T20:21:27.158034+0000","last_peered":"2023-09-23T20:21:27.158034+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116655+0000","last_became_peered":"2023-09-23T20:21:26.116655+0000","last_unstale":"2023-09-23T20:21:27.158034+0000","last_undegraded":"2023-09-23T20:21:26.429489+0000","last_fullsized":"2023-09-23T20:21:26.094804+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:16.512855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":912,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.157896+0000","last_change":"2023-09-23T20:21:19.419918+0000","last_active":"2023-09-23T20:21:27.157896+0000","last_peered":"2023-09-23T20:21:27.157896+0000","last_clean":"2023-09-23T20:21:16.629752+0000","last_became_active":"2023-09-23T20:21:19.419918+0000","last_became_peered":"2023-09-23T20:21:19.419918+0000","last_unstale":"2023-09-23T20:21:27.157896+0000","last_undegraded":"2023-09-23T20:21:19.415497+0000","last_fullsized":"2023-09-23T20:21:19.415369+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:54.928103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:54.928103+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:34.843753+0000","scrub_duration":0.011135947,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":63,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.090053+0000","last_change":"2023-09-23T20:21:19.425270+0000","last_active":"2023-09-23T20:21:27.090053+0000","last_peered":"2023-09-23T20:21:27.090053+0000","last_clean":"2023-09-23T20:21:16.602225+0000","last_became_active":"2023-09-23T20:21:19.425270+0000","last_became_peered":"2023-09-23T20:21:19.425270+0000","last_unstale":"2023-09-23T20:21:27.090053+0000","last_undegraded":"2023-09-23T20:21:27.090053+0000","last_fullsized":"2023-09-23T20:21:19.420753+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:59.536065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088431+0000","last_change":"2023-09-23T20:17:47.270409+0000","last_active":"2023-09-23T20:21:27.088431+0000","last_peered":"2023-09-23T20:21:27.088431+0000","last_clean":"2023-09-23T20:21:27.088431+0000","last_became_active":"2023-09-23T20:17:47.270086+0000","last_became_peered":"2023-09-23T20:17:47.270086+0000","last_unstale":"2023-09-23T20:21:27.088431+0000","last_undegraded":"2023-09-23T20:21:27.088431+0000","last_fullsized":"2023-09-23T20:21:27.088431+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.885372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.11","version":"533'3","reported_seq":14,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.138300+0000","last_change":"2023-09-23T20:21:26.435100+0000","last_active":"2023-09-23T20:21:27.138300+0000","last_peered":"2023-09-23T20:21:27.138300+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.098114+0000","last_became_peered":"2023-09-23T20:21:26.098114+0000","last_unstale":"2023-09-23T20:21:27.138300+0000","last_undegraded":"2023-09-23T20:21:26.434876+0000","last_fullsized":"2023-09-23T20:21:26.092627+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:46.204168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":611,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:21:29.971975+0000","last_change":"2023-09-23T20:21:21.987636+0000","last_active":"2023-09-23T20:21:29.971975+0000","last_peered":"2023-09-23T20:21:29.971975+0000","last_clean":"2023-09-23T20:21:29.971975+0000","last_became_active":"2023-09-23T20:19:05.090104+0000","last_became_peered":"2023-09-23T20:19:05.090104+0000","last_unstale":"2023-09-23T20:21:29.971975+0000","last_undegraded":"2023-09-23T20:21:29.971975+0000","last_fullsized":"2023-09-23T20:21:29.971975+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:21.987455+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:21:21.987455+0000","last_clean_scrub_stamp":"2023-09-23T20:21:21.987455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:43.513652+0000","scrub_duration":0.0017008080000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.090987+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:21:27.090987+0000","last_peered":"2023-09-23T20:21:27.090987+0000","last_clean":"2023-09-23T20:21:27.090987+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:21:27.090987+0000","last_undegraded":"2023-09-23T20:21:27.090987+0000","last_fullsized":"2023-09-23T20:21:27.090987+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":243,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159383+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:21:27.159383+0000","last_peered":"2023-09-23T20:21:27.159383+0000","last_clean":"2023-09-23T20:21:27.159383+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:21:27.159383+0000","last_undegraded":"2023-09-23T20:21:27.159383+0000","last_fullsized":"2023-09-23T20:21:27.159383+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.10","version":"0'0","reported_seq":8,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.157639+0000","last_change":"2023-09-23T20:21:26.116757+0000","last_active":"2023-09-23T20:21:27.157639+0000","last_peered":"2023-09-23T20:21:27.157639+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116757+0000","last_became_peered":"2023-09-23T20:21:26.116757+0000","last_unstale":"2023-09-23T20:21:27.157639+0000","last_undegraded":"2023-09-23T20:21:27.157639+0000","last_fullsized":"2023-09-23T20:21:26.098487+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:15.997588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":920,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.157952+0000","last_change":"2023-09-23T20:21:19.421805+0000","last_active":"2023-09-23T20:21:27.157952+0000","last_peered":"2023-09-23T20:21:27.157952+0000","last_clean":"2023-09-23T20:21:16.629910+0000","last_became_active":"2023-09-23T20:21:19.421805+0000","last_became_peered":"2023-09-23T20:21:19.421805+0000","last_unstale":"2023-09-23T20:21:27.157952+0000","last_undegraded":"2023-09-23T20:21:27.157952+0000","last_fullsized":"2023-09-23T20:21:19.415881+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:14.314819+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T20:21:14.314819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:21.358850+0000","scrub_duration":0.001607596,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088393+0000","last_change":"2023-09-23T20:17:47.279385+0000","last_active":"2023-09-23T20:21:27.088393+0000","last_peered":"2023-09-23T20:21:27.088393+0000","last_clean":"2023-09-23T20:21:27.088393+0000","last_became_active":"2023-09-23T20:17:47.279065+0000","last_became_peered":"2023-09-23T20:17:47.279065+0000","last_unstale":"2023-09-23T20:21:27.088393+0000","last_undegraded":"2023-09-23T20:21:27.088393+0000","last_fullsized":"2023-09-23T20:21:27.088393+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:19.638157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":500,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158808+0000","last_change":"2023-09-23T20:17:47.270621+0000","last_active":"2023-09-23T20:21:27.158808+0000","last_peered":"2023-09-23T20:21:27.158808+0000","last_clean":"2023-09-23T20:21:27.158808+0000","last_became_active":"2023-09-23T20:17:47.270375+0000","last_became_peered":"2023-09-23T20:17:47.270375+0000","last_unstale":"2023-09-23T20:21:27.158808+0000","last_undegraded":"2023-09-23T20:21:27.158808+0000","last_fullsized":"2023-09-23T20:21:27.158808+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:16.961271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1f","version":"0'0","reported_seq":8,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.159703+0000","last_change":"2023-09-23T20:21:26.101606+0000","last_active":"2023-09-23T20:21:27.159703+0000","last_peered":"2023-09-23T20:21:27.159703+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.101606+0000","last_became_peered":"2023-09-23T20:21:26.101606+0000","last_unstale":"2023-09-23T20:21:27.159703+0000","last_undegraded":"2023-09-23T20:21:27.159703+0000","last_fullsized":"2023-09-23T20:21:26.096639+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:40.701088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":495,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158889+0000","last_change":"2023-09-23T20:21:19.421049+0000","last_active":"2023-09-23T20:21:27.158889+0000","last_peered":"2023-09-23T20:21:27.158889+0000","last_clean":"2023-09-23T19:54:37.307730+0000","last_became_active":"2023-09-23T20:21:19.421049+0000","last_became_peered":"2023-09-23T20:21:19.421049+0000","last_unstale":"2023-09-23T20:21:27.158889+0000","last_undegraded":"2023-09-23T20:21:27.158889+0000","last_fullsized":"2023-09-23T20:21:19.416316+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:13.927403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:13.927403+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":461,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158684+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:21:27.158684+0000","last_peered":"2023-09-23T20:21:27.158684+0000","last_clean":"2023-09-23T20:21:27.158684+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:21:27.158684+0000","last_undegraded":"2023-09-23T20:21:27.158684+0000","last_fullsized":"2023-09-23T20:21:27.158684+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":400,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088268+0000","last_change":"2023-09-23T20:17:47.265452+0000","last_active":"2023-09-23T20:21:27.088268+0000","last_peered":"2023-09-23T20:21:27.088268+0000","last_clean":"2023-09-23T20:21:27.088268+0000","last_became_active":"2023-09-23T20:17:47.265115+0000","last_became_peered":"2023-09-23T20:17:47.265115+0000","last_unstale":"2023-09-23T20:21:27.088268+0000","last_undegraded":"2023-09-23T20:21:27.088268+0000","last_fullsized":"2023-09-23T20:21:27.088268+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:04.898742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1e","version":"533'7","reported_seq":16,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.159151+0000","last_change":"2023-09-23T20:21:26.323190+0000","last_active":"2023-09-23T20:21:27.159151+0000","last_peered":"2023-09-23T20:21:27.159151+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.101718+0000","last_became_peered":"2023-09-23T20:21:26.101718+0000","last_unstale":"2023-09-23T20:21:27.159151+0000","last_undegraded":"2023-09-23T20:21:26.322494+0000","last_fullsized":"2023-09-23T20:21:26.096430+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:20.608027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":617,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.159327+0000","last_change":"2023-09-23T20:21:19.426242+0000","last_active":"2023-09-23T20:21:27.159327+0000","last_peered":"2023-09-23T20:21:27.159327+0000","last_clean":"2023-09-23T20:07:24.057948+0000","last_became_active":"2023-09-23T20:21:19.426242+0000","last_became_peered":"2023-09-23T20:21:19.426242+0000","last_unstale":"2023-09-23T20:21:27.159327+0000","last_undegraded":"2023-09-23T20:21:27.159327+0000","last_fullsized":"2023-09-23T20:21:19.422563+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":554,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158654+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:21:27.158654+0000","last_peered":"2023-09-23T20:21:27.158654+0000","last_clean":"2023-09-23T20:21:27.158654+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:21:27.158654+0000","last_undegraded":"2023-09-23T20:21:27.158654+0000","last_fullsized":"2023-09-23T20:21:27.158654+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":401,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089978+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:21:27.089978+0000","last_peered":"2023-09-23T20:21:27.089978+0000","last_clean":"2023-09-23T20:21:27.089978+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:21:27.089978+0000","last_undegraded":"2023-09-23T20:21:27.089978+0000","last_fullsized":"2023-09-23T20:21:27.089978+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1d","version":"0'0","reported_seq":11,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158673+0000","last_change":"2023-09-23T20:21:26.116969+0000","last_active":"2023-09-23T20:21:27.158673+0000","last_peered":"2023-09-23T20:21:27.158673+0000","last_clean":"2023-09-23T20:21:27.158673+0000","last_became_active":"2023-09-23T20:21:26.116655+0000","last_became_peered":"2023-09-23T20:21:26.116655+0000","last_unstale":"2023-09-23T20:21:27.158673+0000","last_undegraded":"2023-09-23T20:21:27.158673+0000","last_fullsized":"2023-09-23T20:21:27.158673+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:22.160495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":840,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:21:28.117001+0000","last_change":"2023-09-23T20:19:42.182813+0000","last_active":"2023-09-23T20:21:28.117001+0000","last_peered":"2023-09-23T20:21:28.117001+0000","last_clean":"2023-09-23T20:21:28.117001+0000","last_became_active":"2023-09-23T20:19:05.072575+0000","last_became_peered":"2023-09-23T20:19:05.072575+0000","last_unstale":"2023-09-23T20:21:28.117001+0000","last_undegraded":"2023-09-23T20:21:28.117001+0000","last_fullsized":"2023-09-23T20:21:28.117001+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:42.182718+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:42.182718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:00.054669+0000","scrub_duration":0.0014414230000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":228,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158730+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:21:27.158730+0000","last_peered":"2023-09-23T20:21:27.158730+0000","last_clean":"2023-09-23T20:21:27.158730+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:21:27.158730+0000","last_undegraded":"2023-09-23T20:21:27.158730+0000","last_fullsized":"2023-09-23T20:21:27.158730+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.089825+0000","last_change":"2023-09-23T20:21:19.424648+0000","last_active":"2023-09-23T20:21:27.089825+0000","last_peered":"2023-09-23T20:21:27.089825+0000","last_clean":"2023-09-23T20:21:16.602032+0000","last_became_active":"2023-09-23T20:21:19.424648+0000","last_became_peered":"2023-09-23T20:21:19.424648+0000","last_unstale":"2023-09-23T20:21:27.089825+0000","last_undegraded":"2023-09-23T20:21:27.089825+0000","last_fullsized":"2023-09-23T20:21:19.420444+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:31.660089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1c","version":"0'0","reported_seq":8,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158900+0000","last_change":"2023-09-23T20:21:26.099470+0000","last_active":"2023-09-23T20:21:27.158900+0000","last_peered":"2023-09-23T20:21:27.158900+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.099470+0000","last_became_peered":"2023-09-23T20:21:26.099470+0000","last_unstale":"2023-09-23T20:21:27.158900+0000","last_undegraded":"2023-09-23T20:21:27.158900+0000","last_fullsized":"2023-09-23T20:21:26.095194+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:45.463006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":743,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:21:29.503957+0000","last_change":"2023-09-23T20:21:23.507116+0000","last_active":"2023-09-23T20:21:29.503957+0000","last_peered":"2023-09-23T20:21:29.503957+0000","last_clean":"2023-09-23T20:21:29.503957+0000","last_became_active":"2023-09-23T20:19:05.092425+0000","last_became_peered":"2023-09-23T20:19:05.092425+0000","last_unstale":"2023-09-23T20:21:29.503957+0000","last_undegraded":"2023-09-23T20:21:29.503957+0000","last_fullsized":"2023-09-23T20:21:29.503957+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:23.507023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:23.507023+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:31.827575+0000","scrub_duration":0.0014042600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":488,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.159144+0000","last_change":"2023-09-23T20:21:19.426310+0000","last_active":"2023-09-23T20:21:27.159144+0000","last_peered":"2023-09-23T20:21:27.159144+0000","last_clean":"2023-09-23T20:21:16.394690+0000","last_became_active":"2023-09-23T20:21:19.426310+0000","last_became_peered":"2023-09-23T20:21:19.426310+0000","last_unstale":"2023-09-23T20:21:27.159144+0000","last_undegraded":"2023-09-23T20:21:19.423095+0000","last_fullsized":"2023-09-23T20:21:19.422996+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:01.978193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":55,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.089109+0000","last_change":"2023-09-23T20:18:38.012263+0000","last_active":"2023-09-23T20:21:27.089109+0000","last_peered":"2023-09-23T20:21:27.089109+0000","last_clean":"2023-09-23T20:17:54.133085+0000","last_became_active":"2023-09-23T20:18:38.012263+0000","last_became_peered":"2023-09-23T20:18:38.012263+0000","last_unstale":"2023-09-23T20:21:27.089109+0000","last_undegraded":"2023-09-23T20:21:27.089109+0000","last_fullsized":"2023-09-23T20:18:38.003813+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:47.449647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1b","version":"0'0","reported_seq":8,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.159435+0000","last_change":"2023-09-23T20:21:26.109483+0000","last_active":"2023-09-23T20:21:27.159435+0000","last_peered":"2023-09-23T20:21:27.159435+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.109483+0000","last_became_peered":"2023-09-23T20:21:26.109483+0000","last_unstale":"2023-09-23T20:21:27.159435+0000","last_undegraded":"2023-09-23T20:21:27.159435+0000","last_fullsized":"2023-09-23T20:21:26.096197+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:21.370916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":894,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.157753+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:21:27.157753+0000","last_peered":"2023-09-23T20:21:27.157753+0000","last_clean":"2023-09-23T20:21:27.157753+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:21:27.157753+0000","last_undegraded":"2023-09-23T20:21:27.157753+0000","last_fullsized":"2023-09-23T20:21:27.157753+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":235,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.159569+0000","last_change":"2023-09-23T20:21:19.426288+0000","last_active":"2023-09-23T20:21:27.159569+0000","last_peered":"2023-09-23T20:21:27.159569+0000","last_clean":"2023-09-23T20:21:16.393528+0000","last_became_active":"2023-09-23T20:21:19.426288+0000","last_became_peered":"2023-09-23T20:21:19.426288+0000","last_unstale":"2023-09-23T20:21:27.159569+0000","last_undegraded":"2023-09-23T20:21:19.421126+0000","last_fullsized":"2023-09-23T20:21:19.421001+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:48.307728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.090531+0000","last_change":"2023-09-23T20:17:47.266963+0000","last_active":"2023-09-23T20:21:27.090531+0000","last_peered":"2023-09-23T20:21:27.090531+0000","last_clean":"2023-09-23T20:21:27.090531+0000","last_became_active":"2023-09-23T20:17:47.266645+0000","last_became_peered":"2023-09-23T20:17:47.266645+0000","last_unstale":"2023-09-23T20:21:27.090531+0000","last_undegraded":"2023-09-23T20:21:27.090531+0000","last_fullsized":"2023-09-23T20:21:27.090531+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:05.052559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1a","version":"0'0","reported_seq":8,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158239+0000","last_change":"2023-09-23T20:21:26.101521+0000","last_active":"2023-09-23T20:21:27.158239+0000","last_peered":"2023-09-23T20:21:27.158239+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.101521+0000","last_became_peered":"2023-09-23T20:21:26.101521+0000","last_unstale":"2023-09-23T20:21:27.158239+0000","last_undegraded":"2023-09-23T20:21:27.158239+0000","last_fullsized":"2023-09-23T20:21:26.098176+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:23.406198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":583,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158517+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:21:27.158517+0000","last_peered":"2023-09-23T20:21:27.158517+0000","last_clean":"2023-09-23T20:21:27.158517+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:21:27.158517+0000","last_undegraded":"2023-09-23T20:21:27.158517+0000","last_fullsized":"2023-09-23T20:21:27.158517+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":243,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158175+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:21:27.158175+0000","last_peered":"2023-09-23T20:21:27.158175+0000","last_clean":"2023-09-23T20:21:27.158175+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:21:27.158175+0000","last_undegraded":"2023-09-23T20:21:27.158175+0000","last_fullsized":"2023-09-23T20:21:27.158175+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":533,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159240+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:21:27.159240+0000","last_peered":"2023-09-23T20:21:27.159240+0000","last_clean":"2023-09-23T20:21:27.159240+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:21:27.159240+0000","last_undegraded":"2023-09-23T20:21:27.159240+0000","last_fullsized":"2023-09-23T20:21:27.159240+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"0'0","reported_seq":8,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158338+0000","last_change":"2023-09-23T20:21:26.116675+0000","last_active":"2023-09-23T20:21:27.158338+0000","last_peered":"2023-09-23T20:21:27.158338+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116675+0000","last_became_peered":"2023-09-23T20:21:26.116675+0000","last_unstale":"2023-09-23T20:21:27.158338+0000","last_undegraded":"2023-09-23T20:21:27.158338+0000","last_fullsized":"2023-09-23T20:21:26.095370+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:11.977313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":606,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089203+0000","last_change":"2023-09-23T20:21:11.930313+0000","last_active":"2023-09-23T20:21:27.089203+0000","last_peered":"2023-09-23T20:21:27.089203+0000","last_clean":"2023-09-23T20:21:27.089203+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:21:27.089203+0000","last_undegraded":"2023-09-23T20:21:27.089203+0000","last_fullsized":"2023-09-23T20:21:27.089203+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:11.930203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.930203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:00.534581+0000","scrub_duration":0.0016982569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":239,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.157599+0000","last_change":"2023-09-23T20:18:38.011856+0000","last_active":"2023-09-23T20:21:27.157599+0000","last_peered":"2023-09-23T20:21:27.157599+0000","last_clean":"2023-09-23T20:18:35.260354+0000","last_became_active":"2023-09-23T20:18:38.011856+0000","last_became_peered":"2023-09-23T20:18:38.011856+0000","last_unstale":"2023-09-23T20:21:27.157599+0000","last_undegraded":"2023-09-23T20:21:27.157599+0000","last_fullsized":"2023-09-23T20:18:37.999747+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:58.093947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":525,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089039+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:21:27.089039+0000","last_peered":"2023-09-23T20:21:27.089039+0000","last_clean":"2023-09-23T20:21:27.089039+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:21:27.089039+0000","last_undegraded":"2023-09-23T20:21:27.089039+0000","last_fullsized":"2023-09-23T20:21:27.089039+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"0'0","reported_seq":8,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158048+0000","last_change":"2023-09-23T20:21:26.110226+0000","last_active":"2023-09-23T20:21:27.158048+0000","last_peered":"2023-09-23T20:21:27.158048+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.110226+0000","last_became_peered":"2023-09-23T20:21:26.110226+0000","last_unstale":"2023-09-23T20:21:27.158048+0000","last_undegraded":"2023-09-23T20:21:27.158048+0000","last_fullsized":"2023-09-23T20:21:26.098141+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:22:02.166653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":692,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158095+0000","last_change":"2023-09-23T20:17:50.332129+0000","last_active":"2023-09-23T20:21:27.158095+0000","last_peered":"2023-09-23T20:21:27.158095+0000","last_clean":"2023-09-23T20:21:27.158095+0000","last_became_active":"2023-09-23T20:17:50.331669+0000","last_became_peered":"2023-09-23T20:17:50.331669+0000","last_unstale":"2023-09-23T20:21:27.158095+0000","last_undegraded":"2023-09-23T20:21:27.158095+0000","last_fullsized":"2023-09-23T20:21:27.158095+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:23.893417+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":178,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158177+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:21:27.158177+0000","last_peered":"2023-09-23T20:21:27.158177+0000","last_clean":"2023-09-23T20:21:27.158177+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:21:27.158177+0000","last_undegraded":"2023-09-23T20:21:27.158177+0000","last_fullsized":"2023-09-23T20:21:27.158177+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":187,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089534+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:21:27.089534+0000","last_peered":"2023-09-23T20:21:27.089534+0000","last_clean":"2023-09-23T20:21:27.089534+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:21:27.089534+0000","last_undegraded":"2023-09-23T20:21:27.089534+0000","last_fullsized":"2023-09-23T20:21:27.089534+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"533'23","reported_seq":55,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158167+0000","last_change":"2023-09-23T20:21:26.108368+0000","last_active":"2023-09-23T20:21:27.158167+0000","last_peered":"2023-09-23T20:21:27.158167+0000","last_clean":"2023-09-23T20:21:27.158167+0000","last_became_active":"2023-09-23T20:21:26.108071+0000","last_became_peered":"2023-09-23T20:21:26.108071+0000","last_unstale":"2023-09-23T20:21:27.158167+0000","last_undegraded":"2023-09-23T20:21:27.158167+0000","last_fullsized":"2023-09-23T20:21:27.158167+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:18.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":40,"num_read_kb":39,"num_write":17,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":844,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158206+0000","last_change":"2023-09-23T20:21:22.116340+0000","last_active":"2023-09-23T20:21:27.158206+0000","last_peered":"2023-09-23T20:21:27.158206+0000","last_clean":"2023-09-23T20:21:27.158206+0000","last_became_active":"2023-09-23T20:19:05.065283+0000","last_became_peered":"2023-09-23T20:19:05.065283+0000","last_unstale":"2023-09-23T20:21:27.158206+0000","last_undegraded":"2023-09-23T20:21:27.158206+0000","last_fullsized":"2023-09-23T20:21:27.158206+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:22.116270+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:21:22.116270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:27.825796+0000","scrub_duration":0.001763922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":226,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159994+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:21:27.159994+0000","last_peered":"2023-09-23T20:21:27.159994+0000","last_clean":"2023-09-23T20:21:27.159994+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:21:27.159994+0000","last_undegraded":"2023-09-23T20:21:27.159994+0000","last_fullsized":"2023-09-23T20:21:27.159994+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":543,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158264+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:21:27.158264+0000","last_peered":"2023-09-23T20:21:27.158264+0000","last_clean":"2023-09-23T20:21:27.158264+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:21:27.158264+0000","last_undegraded":"2023-09-23T20:21:27.158264+0000","last_fullsized":"2023-09-23T20:21:27.158264+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"533'1","reported_seq":11,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:29.485589+0000","last_change":"2023-09-23T20:21:26.430890+0000","last_active":"2023-09-23T20:21:29.485589+0000","last_peered":"2023-09-23T20:21:29.485589+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116526+0000","last_became_peered":"2023-09-23T20:21:26.116526+0000","last_unstale":"2023-09-23T20:21:29.485589+0000","last_undegraded":"2023-09-23T20:21:26.430638+0000","last_fullsized":"2023-09-23T20:21:26.096111+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:20.265662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":913,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088348+0000","last_change":"2023-09-23T20:21:11.597505+0000","last_active":"2023-09-23T20:21:27.088348+0000","last_peered":"2023-09-23T20:21:27.088348+0000","last_clean":"2023-09-23T20:21:27.088348+0000","last_became_active":"2023-09-23T20:19:05.085764+0000","last_became_peered":"2023-09-23T20:19:05.085764+0000","last_unstale":"2023-09-23T20:21:27.088348+0000","last_undegraded":"2023-09-23T20:21:27.088348+0000","last_fullsized":"2023-09-23T20:21:27.088348+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:21:11.597355+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.597355+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:24.991370+0000","scrub_duration":0.0021599240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":255,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159465+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:21:27.159465+0000","last_peered":"2023-09-23T20:21:27.159465+0000","last_clean":"2023-09-23T20:21:27.159465+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:21:27.159465+0000","last_undegraded":"2023-09-23T20:21:27.159465+0000","last_fullsized":"2023-09-23T20:21:27.159465+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.091033+0000","last_change":"2023-09-23T20:17:47.264776+0000","last_active":"2023-09-23T20:21:27.091033+0000","last_peered":"2023-09-23T20:21:27.091033+0000","last_clean":"2023-09-23T20:21:27.091033+0000","last_became_active":"2023-09-23T20:17:47.264473+0000","last_became_peered":"2023-09-23T20:17:47.264473+0000","last_unstale":"2023-09-23T20:21:27.091033+0000","last_undegraded":"2023-09-23T20:21:27.091033+0000","last_fullsized":"2023-09-23T20:21:27.091033+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:40.906303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":11,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158441+0000","last_change":"2023-09-23T20:21:26.102464+0000","last_active":"2023-09-23T20:21:27.158441+0000","last_peered":"2023-09-23T20:21:27.158441+0000","last_clean":"2023-09-23T20:21:27.158441+0000","last_became_active":"2023-09-23T20:21:26.102159+0000","last_became_peered":"2023-09-23T20:21:26.102159+0000","last_unstale":"2023-09-23T20:21:27.158441+0000","last_undegraded":"2023-09-23T20:21:27.158441+0000","last_fullsized":"2023-09-23T20:21:27.158441+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:39.141820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":607,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158322+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:21:27.158322+0000","last_peered":"2023-09-23T20:21:27.158322+0000","last_clean":"2023-09-23T20:21:27.158322+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:21:27.158322+0000","last_undegraded":"2023-09-23T20:21:27.158322+0000","last_fullsized":"2023-09-23T20:21:27.158322+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":247,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.158540+0000","last_change":"2023-09-23T20:18:38.001952+0000","last_active":"2023-09-23T20:21:27.158540+0000","last_peered":"2023-09-23T20:21:27.158540+0000","last_clean":"2023-09-23T20:18:35.257306+0000","last_became_active":"2023-09-23T20:18:38.001952+0000","last_became_peered":"2023-09-23T20:18:38.001952+0000","last_unstale":"2023-09-23T20:21:27.158540+0000","last_undegraded":"2023-09-23T20:18:38.000196+0000","last_fullsized":"2023-09-23T20:18:38.000044+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:17.507792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089948+0000","last_change":"2023-09-23T20:17:47.266977+0000","last_active":"2023-09-23T20:21:27.089948+0000","last_peered":"2023-09-23T20:21:27.089948+0000","last_clean":"2023-09-23T20:21:27.089948+0000","last_became_active":"2023-09-23T20:17:47.266657+0000","last_became_peered":"2023-09-23T20:17:47.266657+0000","last_unstale":"2023-09-23T20:21:27.089948+0000","last_undegraded":"2023-09-23T20:21:27.089948+0000","last_fullsized":"2023-09-23T20:21:27.089948+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:33.774981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":7,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T20:21:27.158771+0000","last_change":"2023-09-23T20:21:26.096160+0000","last_active":"2023-09-23T20:21:25.081362+0000","last_peered":"2023-09-23T20:21:27.158771+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T20:21:26.096160+0000","last_unstale":"2023-09-23T20:21:27.158771+0000","last_undegraded":"2023-09-23T20:21:27.158771+0000","last_fullsized":"2023-09-23T20:21:26.095059+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:05.996546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":767,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159038+0000","last_change":"2023-09-23T20:21:24.112801+0000","last_active":"2023-09-23T20:21:27.159038+0000","last_peered":"2023-09-23T20:21:27.159038+0000","last_clean":"2023-09-23T20:21:27.159038+0000","last_became_active":"2023-09-23T20:19:05.071700+0000","last_became_peered":"2023-09-23T20:19:05.071700+0000","last_unstale":"2023-09-23T20:21:27.159038+0000","last_undegraded":"2023-09-23T20:21:27.159038+0000","last_fullsized":"2023-09-23T20:21:27.159038+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:24.112725+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:21:24.112725+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:07.289445+0000","scrub_duration":0.0014886809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":593,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159046+0000","last_change":"2023-09-23T20:17:47.267657+0000","last_active":"2023-09-23T20:21:27.159046+0000","last_peered":"2023-09-23T20:21:27.159046+0000","last_clean":"2023-09-23T20:21:27.159046+0000","last_became_active":"2023-09-23T20:17:47.267288+0000","last_became_peered":"2023-09-23T20:17:47.267288+0000","last_unstale":"2023-09-23T20:21:27.159046+0000","last_undegraded":"2023-09-23T20:21:27.159046+0000","last_fullsized":"2023-09-23T20:21:27.159046+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.185253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088970+0000","last_change":"2023-09-23T20:17:47.280675+0000","last_active":"2023-09-23T20:21:27.088970+0000","last_peered":"2023-09-23T20:21:27.088970+0000","last_clean":"2023-09-23T20:21:27.088970+0000","last_became_active":"2023-09-23T20:17:47.280175+0000","last_became_peered":"2023-09-23T20:17:47.280175+0000","last_unstale":"2023-09-23T20:21:27.088970+0000","last_undegraded":"2023-09-23T20:21:27.088970+0000","last_fullsized":"2023-09-23T20:21:27.088970+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:08.499949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":11,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158963+0000","last_change":"2023-09-23T20:21:26.101105+0000","last_active":"2023-09-23T20:21:27.158963+0000","last_peered":"2023-09-23T20:21:27.158963+0000","last_clean":"2023-09-23T20:21:27.158963+0000","last_became_active":"2023-09-23T20:21:26.100814+0000","last_became_peered":"2023-09-23T20:21:26.100814+0000","last_unstale":"2023-09-23T20:21:27.158963+0000","last_undegraded":"2023-09-23T20:21:27.158963+0000","last_fullsized":"2023-09-23T20:21:27.158963+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:46.990251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":944,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158198+0000","last_change":"2023-09-23T20:21:19.424707+0000","last_active":"2023-09-23T20:21:27.158198+0000","last_peered":"2023-09-23T20:21:27.158198+0000","last_clean":"2023-09-23T20:21:16.394694+0000","last_became_active":"2023-09-23T20:21:19.424707+0000","last_became_peered":"2023-09-23T20:21:19.424707+0000","last_unstale":"2023-09-23T20:21:27.158198+0000","last_undegraded":"2023-09-23T20:21:27.158198+0000","last_fullsized":"2023-09-23T20:21:19.420966+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:47.181912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:47.181912+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.799364+0000","scrub_duration":0.0074047540000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":283,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.090292+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:21:27.090292+0000","last_peered":"2023-09-23T20:21:27.090292+0000","last_clean":"2023-09-23T20:21:27.090292+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:21:27.090292+0000","last_undegraded":"2023-09-23T20:21:27.090292+0000","last_fullsized":"2023-09-23T20:21:27.090292+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.090609+0000","last_change":"2023-09-23T20:17:47.270450+0000","last_active":"2023-09-23T20:21:27.090609+0000","last_peered":"2023-09-23T20:21:27.090609+0000","last_clean":"2023-09-23T20:21:27.090609+0000","last_became_active":"2023-09-23T20:17:47.270040+0000","last_became_peered":"2023-09-23T20:17:47.270040+0000","last_unstale":"2023-09-23T20:21:27.090609+0000","last_undegraded":"2023-09-23T20:21:27.090609+0000","last_fullsized":"2023-09-23T20:21:27.090609+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:51.572016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.2","version":"533'1","reported_seq":11,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.138371+0000","last_change":"2023-09-23T20:21:26.120563+0000","last_active":"2023-09-23T20:21:27.138371+0000","last_peered":"2023-09-23T20:21:27.138371+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116639+0000","last_became_peered":"2023-09-23T20:21:26.116639+0000","last_unstale":"2023-09-23T20:21:27.138371+0000","last_undegraded":"2023-09-23T20:21:26.120189+0000","last_fullsized":"2023-09-23T20:21:26.091682+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:16.130580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":915,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158363+0000","last_change":"2023-09-23T20:21:19.423194+0000","last_active":"2023-09-23T20:21:27.158363+0000","last_peered":"2023-09-23T20:21:27.158363+0000","last_clean":"2023-09-23T20:21:16.629850+0000","last_became_active":"2023-09-23T20:21:19.423194+0000","last_became_peered":"2023-09-23T20:21:19.423194+0000","last_unstale":"2023-09-23T20:21:27.158363+0000","last_undegraded":"2023-09-23T20:21:27.158363+0000","last_fullsized":"2023-09-23T20:21:19.417294+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:43.954204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:43.954204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:15.279707+0000","scrub_duration":0.0019293979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089655+0000","last_change":"2023-09-23T20:17:47.266513+0000","last_active":"2023-09-23T20:21:27.089655+0000","last_peered":"2023-09-23T20:21:27.089655+0000","last_clean":"2023-09-23T20:21:27.089655+0000","last_became_active":"2023-09-23T20:17:47.266163+0000","last_became_peered":"2023-09-23T20:17:47.266163+0000","last_unstale":"2023-09-23T20:21:27.089655+0000","last_undegraded":"2023-09-23T20:21:27.089655+0000","last_fullsized":"2023-09-23T20:21:27.089655+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.645381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":404,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.088488+0000","last_change":"2023-09-23T20:21:19.425194+0000","last_active":"2023-09-23T20:21:27.088488+0000","last_peered":"2023-09-23T20:21:27.088488+0000","last_clean":"2023-09-23T20:17:42.212478+0000","last_became_active":"2023-09-23T20:21:19.425194+0000","last_became_peered":"2023-09-23T20:21:19.425194+0000","last_unstale":"2023-09-23T20:21:27.088488+0000","last_undegraded":"2023-09-23T20:21:27.088488+0000","last_fullsized":"2023-09-23T20:21:19.419364+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:01.812348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1","version":"533'15","reported_seq":24,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.138189+0000","last_change":"2023-09-23T20:21:26.422339+0000","last_active":"2023-09-23T20:21:27.138189+0000","last_peered":"2023-09-23T20:21:27.138189+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116852+0000","last_became_peered":"2023-09-23T20:21:26.116852+0000","last_unstale":"2023-09-23T20:21:27.138189+0000","last_undegraded":"2023-09-23T20:21:27.138189+0000","last_fullsized":"2023-09-23T20:21:26.091379+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:33.974639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":611,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089057+0000","last_change":"2023-09-23T20:21:20.952512+0000","last_active":"2023-09-23T20:21:27.089057+0000","last_peered":"2023-09-23T20:21:27.089057+0000","last_clean":"2023-09-23T20:21:27.089057+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:21:27.089057+0000","last_undegraded":"2023-09-23T20:21:27.089057+0000","last_fullsized":"2023-09-23T20:21:27.089057+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:20.952352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:20.952352+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:28.266826+0000","scrub_duration":0.00171708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":320,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158007+0000","last_change":"2023-09-23T20:21:19.426288+0000","last_active":"2023-09-23T20:21:27.158007+0000","last_peered":"2023-09-23T20:21:27.158007+0000","last_clean":"2023-09-23T20:07:24.062735+0000","last_became_active":"2023-09-23T20:21:19.426288+0000","last_became_peered":"2023-09-23T20:21:19.426288+0000","last_unstale":"2023-09-23T20:21:27.158007+0000","last_undegraded":"2023-09-23T20:21:27.158007+0000","last_fullsized":"2023-09-23T20:21:19.421386+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:00.204744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089890+0000","last_change":"2023-09-23T20:17:47.268811+0000","last_active":"2023-09-23T20:21:27.089890+0000","last_peered":"2023-09-23T20:21:27.089890+0000","last_clean":"2023-09-23T20:21:27.089890+0000","last_became_active":"2023-09-23T20:17:47.268391+0000","last_became_peered":"2023-09-23T20:17:47.268391+0000","last_unstale":"2023-09-23T20:21:27.089890+0000","last_undegraded":"2023-09-23T20:21:27.089890+0000","last_fullsized":"2023-09-23T20:21:27.089890+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:32.388705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":12,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:29.485304+0000","last_change":"2023-09-23T20:21:26.101824+0000","last_active":"2023-09-23T20:21:29.485304+0000","last_peered":"2023-09-23T20:21:29.485304+0000","last_clean":"2023-09-23T20:21:29.485304+0000","last_became_active":"2023-09-23T20:21:26.101337+0000","last_became_peered":"2023-09-23T20:21:26.101337+0000","last_unstale":"2023-09-23T20:21:29.485304+0000","last_undegraded":"2023-09-23T20:21:29.485304+0000","last_fullsized":"2023-09-23T20:21:29.485304+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:24.734547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":927,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.158745+0000","last_change":"2023-09-23T20:21:19.421445+0000","last_active":"2023-09-23T20:21:27.158745+0000","last_peered":"2023-09-23T20:21:27.158745+0000","last_clean":"2023-09-23T20:21:16.631116+0000","last_became_active":"2023-09-23T20:21:19.421445+0000","last_became_peered":"2023-09-23T20:21:19.421445+0000","last_unstale":"2023-09-23T20:21:27.158745+0000","last_undegraded":"2023-09-23T20:21:27.158745+0000","last_fullsized":"2023-09-23T20:21:19.416634+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:12.314623+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T20:21:12.314623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:35.040895+0000","scrub_duration":0.001743334,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":187,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.087977+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:21:27.087977+0000","last_peered":"2023-09-23T20:21:27.087977+0000","last_clean":"2023-09-23T20:21:27.087977+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:21:27.087977+0000","last_undegraded":"2023-09-23T20:21:27.087977+0000","last_fullsized":"2023-09-23T20:21:27.087977+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.090969+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:21:27.090969+0000","last_peered":"2023-09-23T20:21:27.090969+0000","last_clean":"2023-09-23T20:21:27.090969+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:21:27.090969+0000","last_undegraded":"2023-09-23T20:21:27.090969+0000","last_fullsized":"2023-09-23T20:21:27.090969+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":7,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T20:21:27.226087+0000","last_change":"2023-09-23T20:21:26.107994+0000","last_active":"2023-09-23T20:21:25.081362+0000","last_peered":"2023-09-23T20:21:27.226087+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T20:21:26.107994+0000","last_unstale":"2023-09-23T20:21:27.226087+0000","last_undegraded":"2023-09-23T20:21:27.226087+0000","last_fullsized":"2023-09-23T20:21:26.106806+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:34.085552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":840,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089617+0000","last_change":"2023-09-23T20:19:40.219448+0000","last_active":"2023-09-23T20:21:27.089617+0000","last_peered":"2023-09-23T20:21:27.089617+0000","last_clean":"2023-09-23T20:21:27.089617+0000","last_became_active":"2023-09-23T20:19:05.072187+0000","last_became_peered":"2023-09-23T20:19:05.072187+0000","last_unstale":"2023-09-23T20:21:27.089617+0000","last_undegraded":"2023-09-23T20:21:27.089617+0000","last_fullsized":"2023-09-23T20:21:27.089617+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:40.219310+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:40.219310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:29.003073+0000","scrub_duration":0.011178762,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":486,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.090414+0000","last_change":"2023-09-23T20:17:50.311646+0000","last_active":"2023-09-23T20:21:27.090414+0000","last_peered":"2023-09-23T20:21:27.090414+0000","last_clean":"2023-09-23T20:21:27.090414+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:21:27.090414+0000","last_undegraded":"2023-09-23T20:21:27.090414+0000","last_fullsized":"2023-09-23T20:21:27.090414+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":542,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.159124+0000","last_change":"2023-09-23T20:21:19.422507+0000","last_active":"2023-09-23T20:21:27.159124+0000","last_peered":"2023-09-23T20:21:27.159124+0000","last_clean":"2023-09-23T20:21:16.630306+0000","last_became_active":"2023-09-23T20:21:19.422507+0000","last_became_peered":"2023-09-23T20:21:19.422507+0000","last_unstale":"2023-09-23T20:21:27.159124+0000","last_undegraded":"2023-09-23T20:21:27.159124+0000","last_fullsized":"2023-09-23T20:21:19.416760+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:35.522169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089732+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:21:27.089732+0000","last_peered":"2023-09-23T20:21:27.089732+0000","last_clean":"2023-09-23T20:21:27.089732+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:21:27.089732+0000","last_undegraded":"2023-09-23T20:21:27.089732+0000","last_fullsized":"2023-09-23T20:21:27.089732+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":12,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.138435+0000","last_change":"2023-09-23T20:21:26.117398+0000","last_active":"2023-09-23T20:21:27.138435+0000","last_peered":"2023-09-23T20:21:27.138435+0000","last_clean":"2023-09-23T20:21:27.138435+0000","last_became_active":"2023-09-23T20:21:26.117088+0000","last_became_peered":"2023-09-23T20:21:26.117088+0000","last_unstale":"2023-09-23T20:21:27.138435+0000","last_undegraded":"2023-09-23T20:21:27.138435+0000","last_fullsized":"2023-09-23T20:21:27.138435+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:42.026338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":885,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.157869+0000","last_change":"2023-09-23T20:21:19.422275+0000","last_active":"2023-09-23T20:21:27.157869+0000","last_peered":"2023-09-23T20:21:27.157869+0000","last_clean":"2023-09-23T20:21:16.630920+0000","last_became_active":"2023-09-23T20:21:19.422275+0000","last_became_peered":"2023-09-23T20:21:19.422275+0000","last_unstale":"2023-09-23T20:21:27.157869+0000","last_undegraded":"2023-09-23T20:21:27.157869+0000","last_fullsized":"2023-09-23T20:21:19.415540+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:29.088569+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1215,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158360+0000","last_change":"2023-09-23T20:17:47.305979+0000","last_active":"2023-09-23T20:21:27.158360+0000","last_peered":"2023-09-23T20:21:27.158360+0000","last_clean":"2023-09-23T20:21:27.158360+0000","last_became_active":"2023-09-23T20:17:47.274623+0000","last_became_peered":"2023-09-23T20:17:47.274623+0000","last_unstale":"2023-09-23T20:21:27.158360+0000","last_undegraded":"2023-09-23T20:21:27.158360+0000","last_fullsized":"2023-09-23T20:21:27.158360+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:27.293931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088893+0000","last_change":"2023-09-23T20:17:47.280498+0000","last_active":"2023-09-23T20:21:27.088893+0000","last_peered":"2023-09-23T20:21:27.088893+0000","last_clean":"2023-09-23T20:21:27.088893+0000","last_became_active":"2023-09-23T20:17:47.279716+0000","last_became_peered":"2023-09-23T20:17:47.279716+0000","last_unstale":"2023-09-23T20:21:27.088893+0000","last_undegraded":"2023-09-23T20:21:27.088893+0000","last_fullsized":"2023-09-23T20:21:27.088893+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:18.625303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.090494+0000","last_change":"2023-09-23T20:21:19.421772+0000","last_active":"2023-09-23T20:21:27.090494+0000","last_peered":"2023-09-23T20:21:27.090494+0000","last_clean":"2023-09-23T20:21:16.602164+0000","last_became_active":"2023-09-23T20:21:19.421772+0000","last_became_peered":"2023-09-23T20:21:19.421772+0000","last_unstale":"2023-09-23T20:21:27.090494+0000","last_undegraded":"2023-09-23T20:21:27.090494+0000","last_fullsized":"2023-09-23T20:21:19.418648+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:32.788234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"533'12","reported_seq":28,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.157785+0000","last_change":"2023-09-23T20:21:26.117274+0000","last_active":"2023-09-23T20:21:27.157785+0000","last_peered":"2023-09-23T20:21:27.157785+0000","last_clean":"2023-09-23T20:21:27.157785+0000","last_became_active":"2023-09-23T20:21:26.116768+0000","last_became_peered":"2023-09-23T20:21:26.116768+0000","last_unstale":"2023-09-23T20:21:27.157785+0000","last_undegraded":"2023-09-23T20:21:27.157785+0000","last_fullsized":"2023-09-23T20:21:27.157785+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:45.952246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":18,"num_read_kb":18,"num_write":9,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":773,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158262+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:21:27.158262+0000","last_peered":"2023-09-23T20:21:27.158262+0000","last_clean":"2023-09-23T20:21:27.158262+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:21:27.158262+0000","last_undegraded":"2023-09-23T20:21:27.158262+0000","last_fullsized":"2023-09-23T20:21:27.158262+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":876,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.089682+0000","last_change":"2023-09-23T20:21:19.422746+0000","last_active":"2023-09-23T20:21:27.089682+0000","last_peered":"2023-09-23T20:21:27.089682+0000","last_clean":"2023-09-23T20:03:53.495528+0000","last_became_active":"2023-09-23T20:21:19.422746+0000","last_became_peered":"2023-09-23T20:21:19.422746+0000","last_unstale":"2023-09-23T20:21:27.089682+0000","last_undegraded":"2023-09-23T20:21:27.089682+0000","last_fullsized":"2023-09-23T20:21:19.415128+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:15.902841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":63,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.088290+0000","last_change":"2023-09-23T20:21:19.422296+0000","last_active":"2023-09-23T20:21:27.088290+0000","last_peered":"2023-09-23T20:21:27.088290+0000","last_clean":"2023-09-23T20:21:16.601818+0000","last_became_active":"2023-09-23T20:21:19.422296+0000","last_became_peered":"2023-09-23T20:21:19.422296+0000","last_unstale":"2023-09-23T20:21:27.088290+0000","last_undegraded":"2023-09-23T20:21:27.088290+0000","last_fullsized":"2023-09-23T20:21:19.419475+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:26.118095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.091017+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:21:27.091017+0000","last_peered":"2023-09-23T20:21:27.091017+0000","last_clean":"2023-09-23T20:21:27.091017+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:21:27.091017+0000","last_undegraded":"2023-09-23T20:21:27.091017+0000","last_fullsized":"2023-09-23T20:21:27.091017+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":11,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158512+0000","last_change":"2023-09-23T20:21:26.101788+0000","last_active":"2023-09-23T20:21:27.158512+0000","last_peered":"2023-09-23T20:21:27.158512+0000","last_clean":"2023-09-23T20:21:27.158512+0000","last_became_active":"2023-09-23T20:21:26.101484+0000","last_became_peered":"2023-09-23T20:21:26.101484+0000","last_unstale":"2023-09-23T20:21:27.158512+0000","last_undegraded":"2023-09-23T20:21:27.158512+0000","last_fullsized":"2023-09-23T20:21:27.158512+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:55.459652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1450,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.159564+0000","last_change":"2023-09-23T20:21:19.424613+0000","last_active":"2023-09-23T20:21:27.159564+0000","last_peered":"2023-09-23T20:21:27.159564+0000","last_clean":"2023-09-23T20:21:16.630598+0000","last_became_active":"2023-09-23T20:21:19.424613+0000","last_became_peered":"2023-09-23T20:21:19.424613+0000","last_unstale":"2023-09-23T20:21:27.159564+0000","last_undegraded":"2023-09-23T20:21:19.416004+0000","last_fullsized":"2023-09-23T20:21:19.415888+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:37.709441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":750,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.157908+0000","last_change":"2023-09-23T20:17:50.313197+0000","last_active":"2023-09-23T20:21:27.157908+0000","last_peered":"2023-09-23T20:21:27.157908+0000","last_clean":"2023-09-23T20:21:27.157908+0000","last_became_active":"2023-09-23T20:17:50.312838+0000","last_became_peered":"2023-09-23T20:17:50.312838+0000","last_unstale":"2023-09-23T20:21:27.157908+0000","last_undegraded":"2023-09-23T20:21:27.157908+0000","last_fullsized":"2023-09-23T20:21:27.157908+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:31.530167+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":400,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158594+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:21:27.158594+0000","last_peered":"2023-09-23T20:21:27.158594+0000","last_clean":"2023-09-23T20:21:27.158594+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:21:27.158594+0000","last_undegraded":"2023-09-23T20:21:27.158594+0000","last_fullsized":"2023-09-23T20:21:27.158594+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":332,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158297+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:21:27.158297+0000","last_peered":"2023-09-23T20:21:27.158297+0000","last_clean":"2023-09-23T20:21:27.158297+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:21:27.158297+0000","last_undegraded":"2023-09-23T20:21:27.158297+0000","last_fullsized":"2023-09-23T20:21:27.158297+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":12,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.138244+0000","last_change":"2023-09-23T20:21:26.117225+0000","last_active":"2023-09-23T20:21:27.138244+0000","last_peered":"2023-09-23T20:21:27.138244+0000","last_clean":"2023-09-23T20:21:27.138244+0000","last_became_active":"2023-09-23T20:21:26.116915+0000","last_became_peered":"2023-09-23T20:21:26.116915+0000","last_unstale":"2023-09-23T20:21:27.138244+0000","last_undegraded":"2023-09-23T20:21:27.138244+0000","last_fullsized":"2023-09-23T20:21:27.138244+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:18.770524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":558,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089014+0000","last_change":"2023-09-23T20:21:18.875984+0000","last_active":"2023-09-23T20:21:27.089014+0000","last_peered":"2023-09-23T20:21:27.089014+0000","last_clean":"2023-09-23T20:21:27.089014+0000","last_became_active":"2023-09-23T20:17:50.329940+0000","last_became_peered":"2023-09-23T20:17:50.329940+0000","last_unstale":"2023-09-23T20:21:27.089014+0000","last_undegraded":"2023-09-23T20:21:27.089014+0000","last_fullsized":"2023-09-23T20:21:27.089014+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:18.875863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:18.875863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:17.358094+0000","scrub_duration":0.0015811600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1059,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159708+0000","last_change":"2023-09-23T20:17:50.313157+0000","last_active":"2023-09-23T20:21:27.159708+0000","last_peered":"2023-09-23T20:21:27.159708+0000","last_clean":"2023-09-23T20:21:27.159708+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:21:27.159708+0000","last_undegraded":"2023-09-23T20:21:27.159708+0000","last_fullsized":"2023-09-23T20:21:27.159708+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":483,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089548+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:21:27.089548+0000","last_peered":"2023-09-23T20:21:27.089548+0000","last_clean":"2023-09-23T20:21:27.089548+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:21:27.089548+0000","last_undegraded":"2023-09-23T20:21:27.089548+0000","last_fullsized":"2023-09-23T20:21:27.089548+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":63,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.089981+0000","last_change":"2023-09-23T20:21:19.422717+0000","last_active":"2023-09-23T20:21:27.089981+0000","last_peered":"2023-09-23T20:21:27.089981+0000","last_clean":"2023-09-23T20:21:16.602848+0000","last_became_active":"2023-09-23T20:21:19.422717+0000","last_became_peered":"2023-09-23T20:21:19.422717+0000","last_unstale":"2023-09-23T20:21:27.089981+0000","last_undegraded":"2023-09-23T20:21:27.089981+0000","last_fullsized":"2023-09-23T20:21:19.420290+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:44.277154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":9,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:29.485463+0000","last_change":"2023-09-23T20:21:26.109485+0000","last_active":"2023-09-23T20:21:29.485463+0000","last_peered":"2023-09-23T20:21:29.485463+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.109485+0000","last_became_peered":"2023-09-23T20:21:26.109485+0000","last_unstale":"2023-09-23T20:21:29.485463+0000","last_undegraded":"2023-09-23T20:21:29.485463+0000","last_fullsized":"2023-09-23T20:21:26.095138+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:15.508535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":743,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.156020+0000","last_change":"2023-09-23T20:21:19.413132+0000","last_active":"2023-09-23T20:21:27.156020+0000","last_peered":"2023-09-23T20:21:27.156020+0000","last_clean":"2023-09-23T20:21:27.156020+0000","last_became_active":"2023-09-23T20:19:05.092070+0000","last_became_peered":"2023-09-23T20:19:05.092070+0000","last_unstale":"2023-09-23T20:21:27.156020+0000","last_undegraded":"2023-09-23T20:21:27.156020+0000","last_fullsized":"2023-09-23T20:21:27.156020+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:19.413043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:19.413043+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:11.511055+0000","scrub_duration":0.0014842639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":235,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158594+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:21:27.158594+0000","last_peered":"2023-09-23T20:21:27.158594+0000","last_clean":"2023-09-23T20:21:27.158594+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:21:27.158594+0000","last_undegraded":"2023-09-23T20:21:27.158594+0000","last_fullsized":"2023-09-23T20:21:27.158594+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":63,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.090352+0000","last_change":"2023-09-23T20:21:19.422083+0000","last_active":"2023-09-23T20:21:27.090352+0000","last_peered":"2023-09-23T20:21:27.090352+0000","last_clean":"2023-09-23T20:21:16.602031+0000","last_became_active":"2023-09-23T20:21:19.422083+0000","last_became_peered":"2023-09-23T20:21:19.422083+0000","last_unstale":"2023-09-23T20:21:27.090352+0000","last_undegraded":"2023-09-23T20:21:27.090352+0000","last_fullsized":"2023-09-23T20:21:19.418680+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:45.222502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":361,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159499+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:21:27.159499+0000","last_peered":"2023-09-23T20:21:27.159499+0000","last_clean":"2023-09-23T20:21:27.159499+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:21:27.159499+0000","last_undegraded":"2023-09-23T20:21:27.159499+0000","last_fullsized":"2023-09-23T20:21:27.159499+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"533'7","reported_seq":18,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.159782+0000","last_change":"2023-09-23T20:21:27.159782+0000","last_active":"2023-09-23T20:21:27.159782+0000","last_peered":"2023-09-23T20:21:27.159782+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.099528+0000","last_became_peered":"2023-09-23T20:21:26.099528+0000","last_unstale":"2023-09-23T20:21:27.159782+0000","last_undegraded":"2023-09-23T20:21:27.159782+0000","last_fullsized":"2023-09-23T20:21:26.096941+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:38.049209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":6,"num_read_kb":8192,"num_write":4,"num_write_kb":4096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":495,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.159411+0000","last_change":"2023-09-23T20:21:19.421286+0000","last_active":"2023-09-23T20:21:27.159411+0000","last_peered":"2023-09-23T20:21:27.159411+0000","last_clean":"2023-09-23T19:54:37.307730+0000","last_became_active":"2023-09-23T20:21:19.421286+0000","last_became_peered":"2023-09-23T20:21:19.421286+0000","last_unstale":"2023-09-23T20:21:27.159411+0000","last_undegraded":"2023-09-23T20:21:27.159411+0000","last_fullsized":"2023-09-23T20:21:19.417211+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:11.941739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.941739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":377,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159887+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:21:27.159887+0000","last_peered":"2023-09-23T20:21:27.159887+0000","last_clean":"2023-09-23T20:21:27.159887+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:21:27.159887+0000","last_undegraded":"2023-09-23T20:21:27.159887+0000","last_fullsized":"2023-09-23T20:21:27.159887+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":614,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089791+0000","last_change":"2023-09-23T20:17:50.311535+0000","last_active":"2023-09-23T20:21:27.089791+0000","last_peered":"2023-09-23T20:21:27.089791+0000","last_clean":"2023-09-23T20:21:27.089791+0000","last_became_active":"2023-09-23T20:17:50.310876+0000","last_became_peered":"2023-09-23T20:17:50.310876+0000","last_unstale":"2023-09-23T20:21:27.089791+0000","last_undegraded":"2023-09-23T20:21:27.089791+0000","last_fullsized":"2023-09-23T20:21:27.089791+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.794641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":170,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.087866+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:21:27.087866+0000","last_peered":"2023-09-23T20:21:27.087866+0000","last_clean":"2023-09-23T20:21:27.087866+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:21:27.087866+0000","last_undegraded":"2023-09-23T20:21:27.087866+0000","last_fullsized":"2023-09-23T20:21:27.087866+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"533'1","reported_seq":14,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:29.485633+0000","last_change":"2023-09-23T20:21:26.108186+0000","last_active":"2023-09-23T20:21:29.485633+0000","last_peered":"2023-09-23T20:21:29.485633+0000","last_clean":"2023-09-23T20:21:29.485633+0000","last_became_active":"2023-09-23T20:21:26.107787+0000","last_became_peered":"2023-09-23T20:21:26.107787+0000","last_unstale":"2023-09-23T20:21:29.485633+0000","last_undegraded":"2023-09-23T20:21:29.485633+0000","last_fullsized":"2023-09-23T20:21:29.485633+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:13.846369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":863,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088649+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:21:27.088649+0000","last_peered":"2023-09-23T20:21:27.088649+0000","last_clean":"2023-09-23T20:21:27.088649+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:21:27.088649+0000","last_undegraded":"2023-09-23T20:21:27.088649+0000","last_fullsized":"2023-09-23T20:21:27.088649+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":248,"reported_epoch":534,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:21:27.159992+0000","last_change":"2023-09-23T20:21:19.424950+0000","last_active":"2023-09-23T20:21:27.159992+0000","last_peered":"2023-09-23T20:21:27.159992+0000","last_clean":"2023-09-23T20:21:16.394241+0000","last_became_active":"2023-09-23T20:21:19.424950+0000","last_became_peered":"2023-09-23T20:21:19.424950+0000","last_unstale":"2023-09-23T20:21:27.159992+0000","last_undegraded":"2023-09-23T20:21:27.159992+0000","last_fullsized":"2023-09-23T20:21:19.422321+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:28.720349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":61,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.089712+0000","last_change":"2023-09-23T20:17:47.284676+0000","last_active":"2023-09-23T20:21:27.089712+0000","last_peered":"2023-09-23T20:21:27.089712+0000","last_clean":"2023-09-23T20:21:27.089712+0000","last_became_active":"2023-09-23T20:17:47.277321+0000","last_became_peered":"2023-09-23T20:17:47.277321+0000","last_unstale":"2023-09-23T20:21:27.089712+0000","last_undegraded":"2023-09-23T20:21:27.089712+0000","last_fullsized":"2023-09-23T20:21:27.089712+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.082445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":335,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.157599+0000","last_change":"2023-09-23T20:18:38.011785+0000","last_active":"2023-09-23T20:21:27.157599+0000","last_peered":"2023-09-23T20:21:27.157599+0000","last_clean":"2023-09-23T20:18:35.261768+0000","last_became_active":"2023-09-23T20:18:38.011785+0000","last_became_peered":"2023-09-23T20:18:38.011785+0000","last_unstale":"2023-09-23T20:21:27.157599+0000","last_undegraded":"2023-09-23T20:21:27.157599+0000","last_fullsized":"2023-09-23T20:18:38.001274+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:00.398059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"0'0","reported_seq":11,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159462+0000","last_change":"2023-09-23T20:21:26.117122+0000","last_active":"2023-09-23T20:21:27.159462+0000","last_peered":"2023-09-23T20:21:27.159462+0000","last_clean":"2023-09-23T20:21:27.159462+0000","last_became_active":"2023-09-23T20:21:26.116794+0000","last_became_peered":"2023-09-23T20:21:26.116794+0000","last_unstale":"2023-09-23T20:21:27.159462+0000","last_undegraded":"2023-09-23T20:21:27.159462+0000","last_fullsized":"2023-09-23T20:21:27.159462+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:56.978661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":943,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.157843+0000","last_change":"2023-09-23T20:21:19.425177+0000","last_active":"2023-09-23T20:21:27.157843+0000","last_peered":"2023-09-23T20:21:27.157843+0000","last_clean":"2023-09-23T20:21:16.393190+0000","last_became_active":"2023-09-23T20:21:19.425177+0000","last_became_peered":"2023-09-23T20:21:19.425177+0000","last_unstale":"2023-09-23T20:21:27.157843+0000","last_undegraded":"2023-09-23T20:21:27.157843+0000","last_fullsized":"2023-09-23T20:21:19.421689+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_clean_scrub_stamp":"2023-09-23T20:19:56.180203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:52.985095+0000","scrub_duration":0.012799305,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":101,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.159072+0000","last_change":"2023-09-23T20:21:19.425051+0000","last_active":"2023-09-23T20:21:27.159072+0000","last_peered":"2023-09-23T20:21:27.159072+0000","last_clean":"2023-09-23T20:07:24.087089+0000","last_became_active":"2023-09-23T20:21:19.425051+0000","last_became_peered":"2023-09-23T20:21:19.425051+0000","last_unstale":"2023-09-23T20:21:27.159072+0000","last_undegraded":"2023-09-23T20:21:27.159072+0000","last_fullsized":"2023-09-23T20:21:19.421743+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:23.902489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":280,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.090237+0000","last_change":"2023-09-23T20:21:19.422404+0000","last_active":"2023-09-23T20:21:27.090237+0000","last_peered":"2023-09-23T20:21:27.090237+0000","last_clean":"2023-09-23T20:17:42.211994+0000","last_became_active":"2023-09-23T20:21:19.422404+0000","last_became_peered":"2023-09-23T20:21:19.422404+0000","last_unstale":"2023-09-23T20:21:27.090237+0000","last_undegraded":"2023-09-23T20:21:27.090237+0000","last_fullsized":"2023-09-23T20:21:19.418975+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:54.318949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.14","version":"0'0","reported_seq":9,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:29.485429+0000","last_change":"2023-09-23T20:21:26.107890+0000","last_active":"2023-09-23T20:21:29.485429+0000","last_peered":"2023-09-23T20:21:29.485429+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.107890+0000","last_became_peered":"2023-09-23T20:21:26.107890+0000","last_unstale":"2023-09-23T20:21:29.485429+0000","last_undegraded":"2023-09-23T20:21:29.485429+0000","last_fullsized":"2023-09-23T20:21:26.095766+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:47.598260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":763,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:29.073836+0000","last_change":"2023-09-23T20:21:29.073767+0000","last_active":"2023-09-23T20:21:29.073836+0000","last_peered":"2023-09-23T20:21:29.073836+0000","last_clean":"2023-09-23T20:21:29.073836+0000","last_became_active":"2023-09-23T20:19:05.071787+0000","last_became_peered":"2023-09-23T20:19:05.071787+0000","last_unstale":"2023-09-23T20:21:29.073836+0000","last_undegraded":"2023-09-23T20:21:29.073836+0000","last_fullsized":"2023-09-23T20:21:29.073836+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:29.073665+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:21:29.073665+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:22.713699+0000","scrub_duration":0.00139634,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":566,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158966+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:21:27.158966+0000","last_peered":"2023-09-23T20:21:27.158966+0000","last_clean":"2023-09-23T20:21:27.158966+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:21:27.158966+0000","last_undegraded":"2023-09-23T20:21:27.158966+0000","last_fullsized":"2023-09-23T20:21:27.158966+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":566,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:21:28.116117+0000","last_change":"2023-09-23T20:17:47.266215+0000","last_active":"2023-09-23T20:21:28.116117+0000","last_peered":"2023-09-23T20:21:28.116117+0000","last_clean":"2023-09-23T20:21:28.116117+0000","last_became_active":"2023-09-23T20:17:47.265890+0000","last_became_peered":"2023-09-23T20:17:47.265890+0000","last_unstale":"2023-09-23T20:21:28.116117+0000","last_undegraded":"2023-09-23T20:21:28.116117+0000","last_fullsized":"2023-09-23T20:21:28.116117+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:59.854975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"0'0","reported_seq":9,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:29.485559+0000","last_change":"2023-09-23T20:21:26.107729+0000","last_active":"2023-09-23T20:21:29.485559+0000","last_peered":"2023-09-23T20:21:29.485559+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.107729+0000","last_became_peered":"2023-09-23T20:21:26.107729+0000","last_unstale":"2023-09-23T20:21:29.485559+0000","last_undegraded":"2023-09-23T20:21:29.485559+0000","last_fullsized":"2023-09-23T20:21:26.095358+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:35.064604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":556,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159032+0000","last_change":"2023-09-23T20:17:47.288753+0000","last_active":"2023-09-23T20:21:27.159032+0000","last_peered":"2023-09-23T20:21:27.159032+0000","last_clean":"2023-09-23T20:21:27.159032+0000","last_became_active":"2023-09-23T20:17:47.270488+0000","last_became_peered":"2023-09-23T20:17:47.270488+0000","last_unstale":"2023-09-23T20:21:27.159032+0000","last_undegraded":"2023-09-23T20:21:27.159032+0000","last_fullsized":"2023-09-23T20:21:27.159032+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:35.837184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":531,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.225907+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:21:27.225907+0000","last_peered":"2023-09-23T20:21:27.225907+0000","last_clean":"2023-09-23T20:21:27.225907+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:21:27.225907+0000","last_undegraded":"2023-09-23T20:21:27.225907+0000","last_fullsized":"2023-09-23T20:21:27.225907+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"533'11","reported_seq":27,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158567+0000","last_change":"2023-09-23T20:21:26.101586+0000","last_active":"2023-09-23T20:21:27.158567+0000","last_peered":"2023-09-23T20:21:27.158567+0000","last_clean":"2023-09-23T20:21:27.158567+0000","last_became_active":"2023-09-23T20:21:26.101294+0000","last_became_peered":"2023-09-23T20:21:26.101294+0000","last_unstale":"2023-09-23T20:21:27.158567+0000","last_undegraded":"2023-09-23T20:21:27.158567+0000","last_fullsized":"2023-09-23T20:21:27.158567+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:53.208791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":29,"num_write":7,"num_write_kb":10,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":910,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088248+0000","last_change":"2023-09-23T20:21:17.621262+0000","last_active":"2023-09-23T20:21:27.088248+0000","last_peered":"2023-09-23T20:21:27.088248+0000","last_clean":"2023-09-23T20:21:27.088248+0000","last_became_active":"2023-09-23T20:19:05.086254+0000","last_became_peered":"2023-09-23T20:19:05.086254+0000","last_unstale":"2023-09-23T20:21:27.088248+0000","last_undegraded":"2023-09-23T20:21:27.088248+0000","last_fullsized":"2023-09-23T20:21:27.088248+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:21:17.621169+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:21:17.621169+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:20.261133+0000","scrub_duration":0.0018371310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":215,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.089514+0000","last_change":"2023-09-23T20:21:19.422426+0000","last_active":"2023-09-23T20:21:27.089514+0000","last_peered":"2023-09-23T20:21:27.089514+0000","last_clean":"2023-09-23T20:17:42.212603+0000","last_became_active":"2023-09-23T20:21:19.422426+0000","last_became_peered":"2023-09-23T20:21:19.422426+0000","last_unstale":"2023-09-23T20:21:27.089514+0000","last_undegraded":"2023-09-23T20:21:27.089514+0000","last_fullsized":"2023-09-23T20:21:19.419497+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:47.408072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":193,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T20:21:27.089809+0000","last_change":"2023-09-23T20:21:19.422350+0000","last_active":"2023-09-23T20:21:27.089809+0000","last_peered":"2023-09-23T20:21:27.089809+0000","last_clean":"2023-09-23T19:49:51.368969+0000","last_became_active":"2023-09-23T20:21:19.422350+0000","last_became_peered":"2023-09-23T20:21:19.422350+0000","last_unstale":"2023-09-23T20:21:27.089809+0000","last_undegraded":"2023-09-23T20:21:27.089809+0000","last_fullsized":"2023-09-23T20:21:19.415245+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:28.648556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"0'0","reported_seq":11,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.159770+0000","last_change":"2023-09-23T20:21:26.103532+0000","last_active":"2023-09-23T20:21:27.159770+0000","last_peered":"2023-09-23T20:21:27.159770+0000","last_clean":"2023-09-23T20:21:27.159770+0000","last_became_active":"2023-09-23T20:21:26.103284+0000","last_became_peered":"2023-09-23T20:21:26.103284+0000","last_unstale":"2023-09-23T20:21:27.159770+0000","last_undegraded":"2023-09-23T20:21:27.159770+0000","last_fullsized":"2023-09-23T20:21:27.159770+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:07.217311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":830,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.158118+0000","last_change":"2023-09-23T20:20:51.163673+0000","last_active":"2023-09-23T20:21:27.158118+0000","last_peered":"2023-09-23T20:21:27.158118+0000","last_clean":"2023-09-23T20:21:27.158118+0000","last_became_active":"2023-09-23T20:19:05.067188+0000","last_became_peered":"2023-09-23T20:19:05.067188+0000","last_unstale":"2023-09-23T20:21:27.158118+0000","last_undegraded":"2023-09-23T20:21:27.158118+0000","last_fullsized":"2023-09-23T20:21:27.158118+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:25.898063+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.898063+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":387,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:27.158563+0000","last_change":"2023-09-23T20:21:19.422811+0000","last_active":"2023-09-23T20:21:27.158563+0000","last_peered":"2023-09-23T20:21:27.158563+0000","last_clean":"2023-09-23T20:21:16.633070+0000","last_became_active":"2023-09-23T20:21:19.422811+0000","last_became_peered":"2023-09-23T20:21:19.422811+0000","last_unstale":"2023-09-23T20:21:27.158563+0000","last_undegraded":"2023-09-23T20:21:19.416170+0000","last_fullsized":"2023-09-23T20:21:19.416064+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:41.667749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:21:27.088530+0000","last_change":"2023-09-23T20:17:47.267673+0000","last_active":"2023-09-23T20:21:27.088530+0000","last_peered":"2023-09-23T20:21:27.088530+0000","last_clean":"2023-09-23T20:21:27.088530+0000","last_became_active":"2023-09-23T20:17:47.267216+0000","last_became_peered":"2023-09-23T20:17:47.267216+0000","last_unstale":"2023-09-23T20:21:27.088530+0000","last_undegraded":"2023-09-23T20:21:27.088530+0000","last_fullsized":"2023-09-23T20:21:27.088530+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:37.855890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":32,"stat_sum":{"num_bytes":20,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":88,"num_read_kb":8291,"num_write":49,"num_write_kb":4128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":9,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":40,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":75,"acting":75,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":10952,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":84,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":699595,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":83,"acting":83,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":8388703,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234485,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":763248,"kb_used_data":179560,"kb_used_omap":0,"kb_used_meta":583680,"kb_avail":92981904,"statfs":{"total":95995035648,"available":95213469696,"internally_reserved":0,"allocated":183869440,"data_stored":182272631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":597688320},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":17.827999999999999,"5min":7.8849999999999998,"15min":4.6340000000000003},"min":{"1min":0.22700000000000001,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":230.952,"5min":230.952,"15min":230.952},"last":0.45700000000000002},{"interface":"front","average":{"1min":18.509,"5min":8.2889999999999997,"15min":4.8810000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.222},"max":{"1min":231.08699999999999,"5min":231.08699999999999,"15min":231.08699999999999},"last":0.42799999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":2.0550000000000002,"15min":2.4510000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.21299999999999999},"max":{"1min":13.337,"5min":61.113999999999997,"15min":91.335999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.4219999999999999,"5min":2.234,"15min":2.6909999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.223},"max":{"1min":6.3769999999999998,"5min":61.082999999999998,"15min":91.516999999999996},"last":0.498}]},{"osd":2,"last update":"Sat Sep 23 20:21:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":5.585,"15min":4.6130000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":18.73,"5min":177.852,"15min":177.852},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.6629999999999998,"5min":5.7220000000000004,"15min":4.827},"min":{"1min":0.27800000000000002,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":19.018000000000001,"5min":176.97900000000001,"15min":176.97900000000001},"last":0.38700000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":4.04,"15min":4.1109999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":19.620000000000001,"5min":112.291,"15min":185.17099999999999},"last":0.439},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.5179999999999998,"15min":4.2649999999999997},"min":{"1min":0.33700000000000002,"5min":0.32000000000000001,"15min":0.28999999999999998},"max":{"1min":19.855,"5min":112.887,"15min":184.99100000000001},"last":0.46800000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.466,"15min":0.98899999999999999},"min":{"1min":0.26900000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.55800000000000005,"5min":83.811999999999998,"15min":83.811999999999998},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.532,"15min":1.095},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":2.851,"5min":84.620000000000005,"15min":84.620000000000005},"last":0.32500000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.99399999999999999,"15min":1.179},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.6279999999999999,"5min":16.327000000000002,"15min":24.876000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":1.0609999999999999,"15min":1.073},"min":{"1min":0.314,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.57499999999999996,"5min":16.716000000000001,"15min":21.623000000000001},"last":0.315}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.3779999999999999,"15min":1.0569999999999999},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.3420000000000001,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.163,"15min":0.82199999999999995},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1400000000000001,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.436}]}]},{"osd":6,"up_from":251,"seq":1078036791490,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273876,"kb_used_data":233868,"kb_used_omap":6,"kb_used_meta":39993,"kb_avail":93471276,"statfs":{"total":95995035648,"available":95714586624,"internally_reserved":0,"allocated":239480832,"data_stored":238014233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":40953471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":8.9100000000000001,"5min":3.8820000000000001,"15min":2.6890000000000001},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":53.780000000000001,"5min":74.787000000000006,"15min":126.64},"last":0.27000000000000002},{"interface":"front","average":{"1min":9.0090000000000003,"5min":4.0289999999999999,"15min":2.7570000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":53.768000000000001,"5min":74.751999999999995,"15min":126.658},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":4.3120000000000003,"15min":4.0830000000000002},"min":{"1min":0.26500000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":87.864999999999995,"5min":103.545,"15min":156.70699999999999},"last":5.4710000000000001},{"interface":"front","average":{"1min":5.1260000000000003,"5min":4.0279999999999996,"15min":4.0590000000000002},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.23899999999999999},"max":{"1min":87.772000000000006,"5min":100.018,"15min":156.68299999999999},"last":2.8690000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":6.891,"5min":2.4329999999999998,"15min":3.1179999999999999},"min":{"1min":0.315,"5min":0.254,"15min":0.23100000000000001},"max":{"1min":53.911999999999999,"5min":53.911999999999999,"15min":122.149},"last":0.47499999999999998},{"interface":"front","average":{"1min":7.3259999999999996,"5min":2.4129999999999998,"15min":3.0600000000000001},"min":{"1min":0.32400000000000001,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":54.125999999999998,"5min":54.125999999999998,"15min":122.23999999999999},"last":1.7789999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":5.181,"15min":3.8580000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":51.512999999999998,"5min":128.279,"15min":160.27699999999999},"last":4.8179999999999996},{"interface":"front","average":{"1min":3.4249999999999998,"5min":4.4969999999999999,"15min":3.8180000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":51.601999999999997,"5min":127.911,"15min":160.42699999999999},"last":2.9830000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.298,"15min":1.5780000000000001},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.248},"max":{"1min":2.641,"5min":34.534999999999997,"15min":145.29300000000001},"last":2.9140000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.077,"15min":1.538},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.3999999999999999,"5min":34.426000000000002,"15min":145.256},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":4.5880000000000001,"15min":2.972},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.25800000000000001},"max":{"1min":2.5329999999999999,"5min":218.58199999999999,"15min":218.58199999999999},"last":2.9500000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.343,"15min":2.8969999999999998},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.28499999999999998},"max":{"1min":2.4380000000000002,"5min":217.95599999999999,"15min":217.95599999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.1269999999999998,"15min":1.73},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.315},"max":{"1min":4.7380000000000004,"5min":137.63900000000001,"15min":137.63900000000001},"last":1},{"interface":"front","average":{"1min":0.751,"5min":2.8199999999999998,"15min":1.629},"min":{"1min":0.39700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":2.3620000000000001,"5min":136.738,"15min":136.738},"last":0.48799999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762684,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856116,"kb_used_data":10732,"kb_used_omap":7,"kb_used_meta":845368,"kb_avail":92889036,"statfs":{"total":95995035648,"available":95118372864,"internally_reserved":0,"allocated":10989568,"data_stored":8455852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":865657658},"hb_peers":[0,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":7.1630000000000003,"5min":5.7320000000000002,"15min":3.9820000000000002},"min":{"1min":0.24299999999999999,"5min":0.193,"15min":0.17100000000000001},"max":{"1min":62.710999999999999,"5min":145.81299999999999,"15min":145.81299999999999},"last":0.26500000000000001},{"interface":"front","average":{"1min":7.6680000000000001,"5min":6.0819999999999999,"15min":4.3010000000000002},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":62.610999999999997,"5min":146.83799999999999,"15min":146.83799999999999},"last":0.25600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:20:32 2023","interfaces":[{"interface":"back","average":{"1min":13.513,"5min":6.0099999999999998,"15min":4.9539999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":113.941,"5min":152.62,"15min":152.62},"last":0.33800000000000002},{"interface":"front","average":{"1min":14.683,"5min":6.234,"15min":4.702},"min":{"1min":0.318,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":114.34699999999999,"5min":154.84299999999999,"15min":154.84299999999999},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":4.5830000000000002,"5min":3.1190000000000002,"15min":3.0179999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":22.256,"5min":22.256,"15min":66.527000000000001},"last":0.31},{"interface":"front","average":{"1min":4.0469999999999997,"5min":1.9950000000000001,"15min":2.4119999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":22.068000000000001,"5min":22.068000000000001,"15min":61.854999999999997},"last":0.28299999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3740000000000001,"5min":2.226,"15min":1.7370000000000001},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.23599999999999999},"max":{"1min":10.387,"5min":29.791,"15min":29.791},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.3860000000000001,"5min":1.8700000000000001,"15min":1.6739999999999999},"min":{"1min":0.37,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":10.422000000000001,"5min":27.143999999999998,"15min":27.143999999999998},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":2.1120000000000001,"15min":2.1040000000000001},"min":{"1min":0.34799999999999998,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":10.461,"5min":29.280000000000001,"15min":83.510999999999996},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.54,"5min":2.258,"15min":2.2959999999999998},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":12.648,"5min":32.692,"15min":83.353999999999999},"last":0.40799999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.2390000000000001,"15min":1.665},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":4.5830000000000002,"5min":9.2279999999999998,"15min":18.117999999999999},"last":5.6760000000000002},{"interface":"front","average":{"1min":0.75,"5min":1.325,"15min":1.843},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.0310000000000001,"5min":9.8659999999999997,"15min":20.734999999999999},"last":3.7330000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:20:36 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":2.214,"15min":2.5369999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.30399999999999999},"max":{"1min":12.599,"5min":12.599,"15min":127.03100000000001},"last":0.437},{"interface":"front","average":{"1min":2.7589999999999999,"5min":2.0539999999999998,"15min":2.2389999999999999},"min":{"1min":0.438,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.340999999999999,"5min":10.340999999999999,"15min":125.095},"last":0.44800000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926221,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181328,"kb_used_data":133832,"kb_used_omap":8,"kb_used_meta":1047479,"kb_avail":92563824,"statfs":{"total":95995035648,"available":94785355776,"internally_reserved":0,"allocated":137043968,"data_stored":134543504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1072619105},"hb_peers":[1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:21:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3740000000000001,"5min":2.1400000000000001,"15min":2.3159999999999998},"min":{"1min":0.26700000000000002,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":9.3529999999999998,"5min":17.425000000000001,"15min":72.534000000000006},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.774,"5min":2.9289999999999998,"15min":2.754},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":7.7830000000000004,"5min":21.949000000000002,"15min":72.667000000000002},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:20 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":2.8069999999999999,"15min":3.4100000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":63.710999999999999,"5min":63.710999999999999,"15min":160.542},"last":0.33100000000000002},{"interface":"front","average":{"1min":6.3049999999999997,"5min":3.5009999999999999,"15min":3.7730000000000001},"min":{"1min":0.255,"5min":0.23599999999999999,"15min":0.22600000000000001},"max":{"1min":60.518999999999998,"5min":60.518999999999998,"15min":160.75999999999999},"last":0.36599999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:21:15 2023","interfaces":[{"interface":"back","average":{"1min":12.856,"5min":6.2640000000000002,"15min":4.0860000000000003},"min":{"1min":0.29299999999999998,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":201.655,"5min":201.655,"15min":201.655},"last":0.28299999999999997},{"interface":"front","average":{"1min":13.087,"5min":6.0709999999999997,"15min":4.1840000000000002},"min":{"1min":0.33000000000000002,"5min":0.27800000000000002,"15min":0.27400000000000002},"max":{"1min":201.69399999999999,"5min":201.69399999999999,"15min":201.69399999999999},"last":0.31900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.1440000000000001,"15min":1.736},"min":{"1min":0.28699999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.0339999999999998,"5min":16.593,"15min":16.593},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.9370000000000001,"5min":3.0859999999999999,"15min":2.0009999999999999},"min":{"1min":0.29299999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.1319999999999997,"5min":105.282,"15min":105.282},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.9620000000000002,"15min":2.081},"min":{"1min":0.372,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0670000000000002,"5min":145.14500000000001,"15min":145.14500000000001},"last":0.434},{"interface":"front","average":{"1min":1.9390000000000001,"5min":3.5960000000000001,"15min":2.5259999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":4.9610000000000003,"5min":145.04499999999999,"15min":145.04499999999999},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6339999999999999,"15min":1.6419999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":7.6500000000000004,"5min":7.6500000000000004,"15min":15.744999999999999},"last":4.1239999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.234,"15min":1.6020000000000001},"min":{"1min":0.44,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":9.8219999999999992,"15min":13.766999999999999},"last":0.82199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":2.7829999999999999,"15min":1.875},"min":{"1min":0.36599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.3449999999999998,"5min":99.486000000000004,"15min":99.486000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.581,"5min":3.6850000000000001,"15min":2.4830000000000001},"min":{"1min":0.36499999999999999,"5min":0.308,"15min":0.308},"max":{"1min":9.5030000000000001,"5min":99.668999999999997,"15min":99.668999999999997},"last":0.40100000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599157,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004136,"kb_used_data":6816,"kb_used_omap":12,"kb_used_meta":997299,"kb_avail":92741016,"statfs":{"total":95995035648,"available":94966800384,"internally_reserved":0,"allocated":6979584,"data_stored":4470086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":1021235198},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9009999999999998,"5min":8.0299999999999994,"15min":4.2039999999999997},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":15.4,"5min":274.185,"15min":274.185},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.4729999999999999,"5min":5.8079999999999998,"15min":3.601},"min":{"1min":0.23200000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.8209999999999997,"5min":106.477,"15min":106.477},"last":0.26400000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":4.335,"5min":4.5,"15min":3.5249999999999999},"min":{"1min":0.39600000000000002,"5min":0.20399999999999999,"15min":0.19600000000000001},"max":{"1min":11.939,"5min":106.357,"15min":106.357},"last":0.27800000000000002},{"interface":"front","average":{"1min":4.1619999999999999,"5min":4.2969999999999997,"15min":3.1579999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":15.255000000000001,"5min":104.905,"15min":104.905},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 20:20:50 2023","interfaces":[{"interface":"back","average":{"1min":5.899,"5min":6.0259999999999998,"15min":4.1989999999999998},"min":{"1min":0.52000000000000002,"5min":0.21299999999999999,"15min":0.20799999999999999},"max":{"1min":28.178000000000001,"5min":126.345,"15min":132.82499999999999},"last":0.224},{"interface":"front","average":{"1min":4.5709999999999997,"5min":5.2510000000000003,"15min":3.5419999999999998},"min":{"1min":0.57199999999999995,"5min":0.23000000000000001,"15min":0.22600000000000001},"max":{"1min":15.302,"5min":126.40300000000001,"15min":130.97300000000001},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8450000000000002,"5min":2.2469999999999999,"15min":2.113},"min":{"1min":0.313,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.355,"5min":43.040999999999997,"15min":75.608000000000004},"last":0.377},{"interface":"front","average":{"1min":3.2549999999999999,"5min":2.8700000000000001,"15min":2.5470000000000002},"min":{"1min":0.36599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.864000000000001,"5min":48.951000000000001,"15min":76.161000000000001},"last":0.38400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":2.4049999999999998,"15min":2.6349999999999998},"min":{"1min":0.38500000000000001,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":15.378,"5min":43.716000000000001,"15min":135.202},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.585,"5min":2.5990000000000002,"15min":2.4710000000000001},"min":{"1min":0.41499999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.782,"5min":43.643999999999998,"15min":63.792000000000002},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.329,"15min":1.542},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5510000000000002,"5min":11.843,"15min":14.664},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.425,"15min":1.744},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.077,"5min":12.025,"15min":14.279999999999999},"last":4.9660000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.4889999999999999,"5min":2.6360000000000001,"15min":2.1560000000000001},"min":{"1min":0.46999999999999997,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":11.91,"5min":18.495999999999999,"15min":18.495999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.9990000000000001,"5min":2.077,"15min":1.7609999999999999},"min":{"1min":0.41299999999999998,"5min":0.39900000000000002,"15min":0.35299999999999998},"max":{"1min":15.327999999999999,"5min":15.327999999999999,"15min":15.327999999999999},"last":0.49299999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435633,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1187188,"kb_used_data":153708,"kb_used_omap":4,"kb_used_meta":1033467,"kb_avail":92557964,"statfs":{"total":95995035648,"available":94779355136,"internally_reserved":0,"allocated":157396992,"data_stored":154567103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1058270317},"hb_peers":[0,1,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":14.129,"5min":7.8550000000000004,"15min":4.8159999999999998},"min":{"1min":0.27100000000000002,"5min":0.191,"15min":0.186},"max":{"1min":219.80000000000001,"5min":219.80000000000001,"15min":219.80000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":13.737,"5min":8.3599999999999994,"15min":5.0659999999999998},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":221.36000000000001,"5min":221.36000000000001,"15min":221.36000000000001},"last":0.35299999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6520000000000001,"5min":2.9769999999999999,"15min":4.2160000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20300000000000001},"max":{"1min":9.0830000000000002,"5min":92.036000000000001,"15min":183.87},"last":0.373},{"interface":"front","average":{"1min":3.145,"5min":2.972,"15min":4.4349999999999996},"min":{"1min":0.251,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":19.513000000000002,"5min":92.210999999999999,"15min":185.84899999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":7.6070000000000002,"15min":5.4000000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.7549999999999999,"5min":128.369,"15min":163.83500000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":3.8420000000000001,"5min":7.4020000000000001,"15min":5.2409999999999997},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":15.286,"5min":126.497,"15min":163.92099999999999},"last":0.52200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.996,"15min":1.9179999999999999},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.28999999999999998},"max":{"1min":8.8350000000000009,"5min":54.046999999999997,"15min":58.472999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.988,"5min":2.6819999999999999,"15min":2.4470000000000001},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":5.4770000000000003,"5min":54.311,"15min":58.408000000000001},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":2.9159999999999999,"15min":3.4089999999999998},"min":{"1min":0.39400000000000002,"5min":0.32000000000000001,"15min":0.30299999999999999},"max":{"1min":7.5179999999999998,"5min":54.139000000000003,"15min":200.53200000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.359,"5min":2.573,"15min":3.4140000000000001},"min":{"1min":0.34899999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.5030000000000001,"5min":56.457999999999998,"15min":200.846},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.593,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":1.5329999999999999,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.3280000000000001,"15min":2.286},"min":{"1min":0.42699999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.2040000000000002,"5min":8.2629999999999999,"15min":82.233000000000004},"last":4.1509999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.706,"15min":2.5529999999999999},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.3940000000000001,"5min":17.986000000000001,"15min":139.465},"last":0.443},{"interface":"front","average":{"1min":1.4219999999999999,"5min":1.6299999999999999,"15min":2.5249999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.32800000000000001},"max":{"1min":8.9019999999999992,"5min":9.2289999999999992,"15min":141.56200000000001},"last":0.48199999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019395,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":597800,"kb_used_data":5920,"kb_used_omap":0,"kb_used_meta":591872,"kb_avail":93147352,"statfs":{"total":95995035648,"available":95382888448,"internally_reserved":0,"allocated":6062080,"data_stored":4473431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":606076928},"hb_peers":[0,1,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":4.5789999999999997,"5min":10.012,"15min":4.46},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.219},"max":{"1min":52.881,"5min":232.89500000000001,"15min":232.89500000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":4.5730000000000004,"5min":10.061,"15min":4.4669999999999996},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":52.890000000000001,"5min":232.851,"15min":232.851},"last":0.54000000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":2.6200000000000001,"15min":3.1440000000000001},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":6.9800000000000004,"5min":114.05200000000001,"15min":138.47999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.889,"5min":2.5350000000000001,"15min":3.1890000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.217},"max":{"1min":12.058,"5min":114.02800000000001,"15min":138.429},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":15.880000000000001,"5min":9.6950000000000003,"15min":6.7859999999999996},"min":{"1min":0.29899999999999999,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":204.345,"5min":204.345,"15min":204.345},"last":0.436},{"interface":"front","average":{"1min":15.916,"5min":9.6020000000000003,"15min":6.6429999999999998},"min":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":204.072,"5min":204.072,"15min":204.072},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":5.585,"5min":8.0199999999999996,"15min":5.7599999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":88.105999999999995,"5min":156.72300000000001,"15min":184.37100000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":5.7809999999999997,"5min":7.9210000000000003,"15min":5.8289999999999997},"min":{"1min":0.33200000000000002,"5min":0.32900000000000001,"15min":0.309},"max":{"1min":87.093999999999994,"5min":155.84800000000001,"15min":184.22300000000001},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.9379999999999999,"15min":1.6459999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":4.3970000000000002,"5min":114.788,"15min":176.81899999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":2.0099999999999998,"15min":1.6879999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.5590000000000002,"5min":113.822,"15min":176.172},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:17:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.0390000000000001,"15min":1.026},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":2.5110000000000001,"5min":114.232,"15min":114.232},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.3540000000000001,"15min":1.2130000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.247},"max":{"1min":4.085,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.772,"15min":1.4350000000000001},"min":{"1min":0.33100000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67600000000000005,"5min":113.861,"15min":113.861},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.7410000000000001,"15min":1.4850000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.222},"max":{"1min":1.6379999999999999,"5min":114.32299999999999,"15min":114.32299999999999},"last":0.49399999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757759,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665568,"kb_used_data":5848,"kb_used_omap":0,"kb_used_meta":659712,"kb_avail":93079584,"statfs":{"total":95995035648,"available":95313494016,"internally_reserved":0,"allocated":5988352,"data_stored":4404144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":675545088},"hb_peers":[0,1,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:42 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":9.9619999999999997,"15min":4.5990000000000002},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":12.819000000000001,"5min":166.57900000000001,"15min":166.57900000000001},"last":0.499},{"interface":"front","average":{"1min":14.805,"5min":11.692,"15min":5.2869999999999999},"min":{"1min":0.22,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":256.52199999999999,"5min":256.52199999999999,"15min":256.52199999999999},"last":0.56499999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.6279999999999999,"15min":3.4359999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.22},"max":{"1min":8.5289999999999999,"5min":17.010999999999999,"15min":150.16999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.607,"5min":1.659,"15min":3.411},"min":{"1min":0.27300000000000002,"5min":0.27200000000000002,"15min":0.21199999999999999},"max":{"1min":8.3420000000000005,"5min":18.879000000000001,"15min":150.12700000000001},"last":0.59799999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:20:21 2023","interfaces":[{"interface":"back","average":{"1min":17.131,"5min":6.5279999999999996,"15min":5.3019999999999996},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":208.495,"5min":208.495,"15min":240.57300000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":17.167999999999999,"5min":6.5869999999999997,"15min":5.3659999999999997},"min":{"1min":0.33500000000000002,"5min":0.33100000000000002,"15min":0.30299999999999999},"max":{"1min":208.58199999999999,"5min":208.58199999999999,"15min":240.464},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":5.7240000000000002,"5min":8.7750000000000004,"15min":6.0579999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":54.5,"5min":196.97800000000001,"15min":196.97800000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":5.008,"5min":8.7650000000000006,"15min":5.9420000000000002},"min":{"1min":0.35199999999999998,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":54.533000000000001,"5min":196.875,"15min":196.875},"last":0.57899999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.72899999999999998,"15min":0.77700000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.0169999999999999,"5min":4.8159999999999998,"15min":9.7240000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.60999999999999999,"15min":0.60899999999999999},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":2.6890000000000001,"5min":3.371,"15min":8.7360000000000007},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2490000000000001,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":4.04,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.73299999999999998,"15min":0.60999999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":1.609,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":1.492}]},{"osd":7,"last update":"Sat Sep 23 20:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.90600000000000003,"15min":0.96999999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.0469999999999997,"5min":6.8620000000000001,"15min":11.725},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.76700000000000002,"15min":0.91500000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0880000000000001,"5min":5.8499999999999996,"15min":11.041},"last":0.52900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":20,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":20,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:21:37.294 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:21:37.294 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 61 2023-09-23T20:21:37.294 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:37.326 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:37.331 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:21:37.331 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:21:37.331 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:21:37.331 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:21:37.331 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:21:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:37 smithi144 ceph-mon[194833]: from='client.36816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:37.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:37.538 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:21:37.563 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:37 smithi135 ceph-mon[221113]: from='client.36816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:37.563 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:37.563 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:37.563 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:37 smithi135 ceph-mon[213871]: from='client.36816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:37.564 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:37.564 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:37.913 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:21:37.913 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:21:37.913 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:21:37.913 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:21:37.914 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:21:37.927 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:21:37.927 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:21:37.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:21:37.928 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T20:21:37.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:21:37.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:21:37.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:21:37.928 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:21:37.928 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:21:37.930 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:37.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:37.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:38.034 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:38.061 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:38.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:38 smithi135 ceph-mon[213871]: pgmap v1529: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 905 KiB/s rd, 451 KiB/s wr, 13 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:38.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2395299403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:38 smithi135 ceph-mon[221113]: pgmap v1529: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 905 KiB/s rd, 451 KiB/s wr, 13 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2395299403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:38.548 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:21:38.607 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:38.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:38.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:38 smithi144 ceph-mon[194833]: pgmap v1529: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 905 KiB/s rd, 451 KiB/s wr, 13 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2395299403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:38.711 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:38.725 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:21:38.726 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:21:38.743 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:38.792 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:38.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:38.815 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:38.868 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:38.899 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:38.925 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:39.031 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:39.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:39.135 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:39.162 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:39.252 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:39.561 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:39.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:39.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:39.665 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:39.693 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:40.290 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:40.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:40.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:40.394 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:40.424 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:40 smithi144 ceph-mon[194833]: pgmap v1530: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:40.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:40 smithi135 ceph-mon[221113]: pgmap v1530: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:40.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:40 smithi135 ceph-mon[213871]: pgmap v1530: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s; 16/117 objects degraded (13.675%) 2023-09-23T20:21:40.721 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:40.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:40.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:40.727 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:21:40.824 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:40.857 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:40.905 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:40.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:41.013 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:41.041 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:41.087 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:41.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:41.192 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:41.219 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:41.261 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:41.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:41.366 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:41.398 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:41.699 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:41.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:41.804 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:41.831 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:41.874 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:41.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:41.980 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:42.009 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:42.096 INFO:teuthology.orchestra.run.smithi135.stderr:Error EINVAL: specified pg_num 61 < pg_num_max 32 2023-09-23T20:21:42.209 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:42.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:42.314 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:42.340 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:42.382 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:42.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:42.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:42 smithi135 ceph-mon[221113]: pgmap v1531: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:42.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1994621783' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:21:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:42 smithi135 ceph-mon[213871]: pgmap v1531: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:42.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1994621783' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:21:42.487 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:42.515 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:42 smithi144 ceph-mon[194833]: pgmap v1531: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1994621783' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:21:42.639 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:21:42.639 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:21:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:21:43.261 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:21:43.269 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:43.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:43.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:43.288 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:43.368 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:21:43.373 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:43.403 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:43.454 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:43.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/412063674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:43 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/412063674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:43 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:43.560 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:43.590 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:43.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/412063674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:43.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:43 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:21:43.885 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:21:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:21:43] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.43.0" 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:44.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:44.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:44.193 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:44.219 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:44.414 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:44.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[213871]: pgmap v1532: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[213871]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:21:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[213871]: osdmap e537: 8 total, 6 up, 8 in 2023-09-23T20:21:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[213871]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[213871]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[221113]: pgmap v1532: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:44.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:44.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[221113]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:21:44.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[221113]: osdmap e537: 8 total, 6 up, 8 in 2023-09-23T20:21:44.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[221113]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:44 smithi135 ceph-mon[221113]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.518 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:44.544 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:44.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:44 smithi144 ceph-mon[194833]: pgmap v1532: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:44 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:44 smithi144 ceph-mon[194833]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:44 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:21:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:44 smithi144 ceph-mon[194833]: osdmap e537: 8 total, 6 up, 8 in 2023-09-23T20:21:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:44 smithi144 ceph-mon[194833]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:44 smithi144 ceph-mon[194833]: failed to encode map e537 with expected crc 2023-09-23T20:21:44.764 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:44.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:44.868 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:44.869 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:44.896 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:45.000 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:45.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:45.104 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:45.134 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:45.211 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:45.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:45.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:45.314 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:45.347 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:45.394 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:45 smithi135 ceph-mon[221113]: 1.1d scrub starts 2023-09-23T20:21:45.394 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:45 smithi135 ceph-mon[221113]: 1.1d scrub ok 2023-09-23T20:21:45.395 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:45 smithi135 ceph-mon[213871]: 1.1d scrub starts 2023-09-23T20:21:45.395 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:45 smithi135 ceph-mon[213871]: 1.1d scrub ok 2023-09-23T20:21:45.396 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:45.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:45.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:45.501 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:45.529 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:45.574 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:45.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:45.603 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:21:45.622 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:21:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:45 smithi144 ceph-mon[194833]: 1.1d scrub starts 2023-09-23T20:21:45.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:45 smithi144 ceph-mon[194833]: 1.1d scrub ok 2023-09-23T20:21:45.641 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:21:45.660 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:21:45.679 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:21:45.681 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:45.698 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:21:45.698 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:21:45.698 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:21:45.698 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T20:21:45.698 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:21:45.698 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:21:45.699 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:21:45.699 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:21:45.699 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:21:45.707 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:45.749 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:45.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:45.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:45.854 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:45.880 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:45.921 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:45.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:45.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:46.026 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:46.055 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:46.316 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:21:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:46 smithi144 ceph-mon[194833]: pgmap v1534: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4149794468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:46.652 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:46.653 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:46 smithi135 ceph-mon[221113]: pgmap v1534: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:46.653 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4149794468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:46.654 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:46 smithi135 ceph-mon[213871]: pgmap v1534: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:46.654 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4149794468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:46.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:46.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:46.750 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:21:46.750 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:21:46.756 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:46.786 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:46.833 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:46.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:46.852 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:46.939 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:46.971 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:47.166 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:47.168 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:21:47.168 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:21:47.168 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T20:21:47.168 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:21:47.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:47.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:47.271 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:47.299 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:47.946 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:47.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:47.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:48.050 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:48.081 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:48.131 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:48.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:48.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:48.234 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6, 2] live_osds: [0, 1, 3, 4, 5, 7] 2023-09-23T20:21:48.235 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:21:48.235 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:21:48.235 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:21:48.239 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:48.305 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:48.403 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:48.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:48.425 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:21:48.507 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:48.539 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:48.588 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:48.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:48.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:48 smithi144 ceph-mon[194833]: pgmap v1535: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:48.668 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:48.695 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:48.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:48 smithi135 ceph-mon[221113]: pgmap v1535: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:48.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:48 smithi135 ceph-mon[213871]: pgmap v1535: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:48.723 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:48.752 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:21:48.769 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:48.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:48.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:48.875 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:48.907 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:48.955 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:48.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:49.063 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:49.092 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:49.136 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:49.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:49.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:49.241 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:49.272 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:49.276 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:49.319 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:49.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:49.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:49.378 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:21:49.424 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:49.437 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:49.452 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:49.497 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:49.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:49.600 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:49.628 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:49 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1190120381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:49.631 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:49.678 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:49.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:49.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:49 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1190120381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:49.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:49 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1190120381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:21:49.785 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:49.813 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:49.857 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:49.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:49.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:49.963 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:49.989 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:50.032 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:50.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:50.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:50.139 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:50.165 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:50.475 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:50.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:50.578 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:50.607 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:50 smithi144 ceph-mon[194833]: pgmap v1536: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:50 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:21:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:50 smithi144 ceph-mon[194833]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:50 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1190120381' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:50 smithi144 ceph-mon[194833]: osdmap e538: 8 total, 6 up, 8 in 2023-09-23T20:21:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:50 smithi144 ceph-mon[194833]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:50 smithi144 ceph-mon[194833]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[221113]: pgmap v1536: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:50.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:21:50.641 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[221113]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.642 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1190120381' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:50.642 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[221113]: osdmap e538: 8 total, 6 up, 8 in 2023-09-23T20:21:50.642 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[221113]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.642 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[221113]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.642 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[213871]: pgmap v1536: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:50.642 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:21:50.642 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[213871]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.642 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1190120381' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:21:50.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[213871]: osdmap e538: 8 total, 6 up, 8 in 2023-09-23T20:21:50.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[213871]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.643 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:50 smithi135 ceph-mon[213871]: failed to encode map e538 with expected crc 2023-09-23T20:21:50.653 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:50.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:50.757 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:50.783 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:50.825 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:50.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:50.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:50.932 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:50.958 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:51.000 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:51.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:51.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:51.107 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:51.133 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:51.175 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:51.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:51.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:51.283 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:51.309 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:51.351 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:51.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:51.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:51.459 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:51.484 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:51.526 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:51.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:51.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:51.629 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:51.660 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:52.257 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:52.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:52.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:52.362 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:52.394 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:52.443 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:52.444 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:52 smithi135 ceph-mon[213871]: pgmap v1538: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:52.444 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:52.444 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:52.444 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:52 smithi135 ceph-mon[221113]: pgmap v1538: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:52.444 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:52.444 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:52.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:52.549 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:52.563 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:21:52.565 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:21:52.578 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:52.624 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:52.627 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:52 smithi144 ceph-mon[194833]: pgmap v1538: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:52.627 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:21:52.628 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:52.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:52.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:52.693 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:21:52.693 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:21:52.693 INFO:teuthology.orchestra.run.smithi135.stdout: "complaint_time": 30, 2023-09-23T20:21:52.693 INFO:teuthology.orchestra.run.smithi135.stdout: "num_blocked_ops": 0 2023-09-23T20:21:52.693 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:21:52.729 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:52.755 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:52.779 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:21:52.797 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:52.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:52.904 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:52.933 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:53.428 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:21:53.447 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:21:53.466 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:21:53.485 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:21:53.508 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:21:53.527 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:21:53.528 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:21:53.528 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:21:53.528 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T20:21:53.528 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:21:53.528 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:21:53.528 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:21:53.528 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:21:53.528 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:21:53.603 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:53.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:53.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:53.691 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1538,"stamp":"2023-09-23T20:21:51.271383+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4899442,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":6068,"num_read_kb":21498,"num_write":956,"num_write_kb":16853,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":265,"num_bytes_recovered":8508882,"num_keys_recovered":10,"num_objects_omap":25,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6108,"ondisk_log_size":6108,"up":350,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":407,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6366256,"kb_used_data":567216,"kb_used_omap":38,"kb_used_meta":5798937,"kb_avail":743594960,"statfs":{"total":767960285184,"available":761441239040,"internally_reserved":0,"allocated":580829184,"data_stored":564191936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39803,"internal_metadata":5938111621},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002787"},"pg_stats":[{"pgid":"11.12","version":"533'1","reported_seq":12,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.355836+0000","last_change":"2023-09-23T20:21:26.429725+0000","last_active":"2023-09-23T20:21:34.355836+0000","last_peered":"2023-09-23T20:21:34.355836+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116655+0000","last_became_peered":"2023-09-23T20:21:26.116655+0000","last_unstale":"2023-09-23T20:21:34.355836+0000","last_undegraded":"2023-09-23T20:21:26.429489+0000","last_fullsized":"2023-09-23T20:21:26.094804+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:16.512855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":914,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.338298+0000","last_change":"2023-09-23T20:21:19.419918+0000","last_active":"2023-09-23T20:21:34.338298+0000","last_peered":"2023-09-23T20:21:34.338298+0000","last_clean":"2023-09-23T20:21:16.629752+0000","last_became_active":"2023-09-23T20:21:19.419918+0000","last_became_peered":"2023-09-23T20:21:19.419918+0000","last_unstale":"2023-09-23T20:21:34.338298+0000","last_undegraded":"2023-09-23T20:21:19.415497+0000","last_fullsized":"2023-09-23T20:21:19.415369+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:54.928103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:54.928103+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:34.843753+0000","scrub_duration":0.011135947,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":65,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863527+0000","last_change":"2023-09-23T20:21:19.425270+0000","last_active":"2023-09-23T20:21:34.863527+0000","last_peered":"2023-09-23T20:21:34.863527+0000","last_clean":"2023-09-23T20:21:16.602225+0000","last_became_active":"2023-09-23T20:21:19.425270+0000","last_became_peered":"2023-09-23T20:21:19.425270+0000","last_unstale":"2023-09-23T20:21:34.863527+0000","last_undegraded":"2023-09-23T20:21:34.863527+0000","last_fullsized":"2023-09-23T20:21:19.420753+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:59.536065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863532+0000","last_change":"2023-09-23T20:17:47.270409+0000","last_active":"2023-09-23T20:21:34.863532+0000","last_peered":"2023-09-23T20:21:34.863532+0000","last_clean":"2023-09-23T20:21:34.863532+0000","last_became_active":"2023-09-23T20:17:47.270086+0000","last_became_peered":"2023-09-23T20:17:47.270086+0000","last_unstale":"2023-09-23T20:21:34.863532+0000","last_undegraded":"2023-09-23T20:21:34.863532+0000","last_fullsized":"2023-09-23T20:21:34.863532+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.885372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.11","version":"533'3","reported_seq":16,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.338370+0000","last_change":"2023-09-23T20:21:26.435100+0000","last_active":"2023-09-23T20:21:34.338370+0000","last_peered":"2023-09-23T20:21:34.338370+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.098114+0000","last_became_peered":"2023-09-23T20:21:26.098114+0000","last_unstale":"2023-09-23T20:21:34.338370+0000","last_undegraded":"2023-09-23T20:21:26.434876+0000","last_fullsized":"2023-09-23T20:21:26.092627+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:46.204168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":613,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863326+0000","last_change":"2023-09-23T20:21:21.987636+0000","last_active":"2023-09-23T20:21:34.863326+0000","last_peered":"2023-09-23T20:21:34.863326+0000","last_clean":"2023-09-23T20:21:34.863326+0000","last_became_active":"2023-09-23T20:19:05.090104+0000","last_became_peered":"2023-09-23T20:19:05.090104+0000","last_unstale":"2023-09-23T20:21:34.863326+0000","last_undegraded":"2023-09-23T20:21:34.863326+0000","last_fullsized":"2023-09-23T20:21:34.863326+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:21.987455+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:21:21.987455+0000","last_clean_scrub_stamp":"2023-09-23T20:21:21.987455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:43.513652+0000","scrub_duration":0.0017008080000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863112+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:21:34.863112+0000","last_peered":"2023-09-23T20:21:34.863112+0000","last_clean":"2023-09-23T20:21:34.863112+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:21:34.863112+0000","last_undegraded":"2023-09-23T20:21:34.863112+0000","last_fullsized":"2023-09-23T20:21:34.863112+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":245,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356804+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:21:34.356804+0000","last_peered":"2023-09-23T20:21:34.356804+0000","last_clean":"2023-09-23T20:21:34.356804+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:21:34.356804+0000","last_undegraded":"2023-09-23T20:21:34.356804+0000","last_fullsized":"2023-09-23T20:21:34.356804+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.10","version":"0'0","reported_seq":10,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338184+0000","last_change":"2023-09-23T20:21:26.116757+0000","last_active":"2023-09-23T20:21:34.338184+0000","last_peered":"2023-09-23T20:21:34.338184+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116757+0000","last_became_peered":"2023-09-23T20:21:26.116757+0000","last_unstale":"2023-09-23T20:21:34.338184+0000","last_undegraded":"2023-09-23T20:21:34.338184+0000","last_fullsized":"2023-09-23T20:21:26.098487+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:15.997588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":922,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338310+0000","last_change":"2023-09-23T20:21:19.421805+0000","last_active":"2023-09-23T20:21:34.338310+0000","last_peered":"2023-09-23T20:21:34.338310+0000","last_clean":"2023-09-23T20:21:16.629910+0000","last_became_active":"2023-09-23T20:21:19.421805+0000","last_became_peered":"2023-09-23T20:21:19.421805+0000","last_unstale":"2023-09-23T20:21:34.338310+0000","last_undegraded":"2023-09-23T20:21:34.338310+0000","last_fullsized":"2023-09-23T20:21:19.415881+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:14.314819+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T20:21:14.314819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:21.358850+0000","scrub_duration":0.001607596,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863097+0000","last_change":"2023-09-23T20:17:47.279385+0000","last_active":"2023-09-23T20:21:34.863097+0000","last_peered":"2023-09-23T20:21:34.863097+0000","last_clean":"2023-09-23T20:21:34.863097+0000","last_became_active":"2023-09-23T20:17:47.279065+0000","last_became_peered":"2023-09-23T20:17:47.279065+0000","last_unstale":"2023-09-23T20:21:34.863097+0000","last_undegraded":"2023-09-23T20:21:34.863097+0000","last_fullsized":"2023-09-23T20:21:34.863097+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:19.638157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":502,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.339243+0000","last_change":"2023-09-23T20:17:47.270621+0000","last_active":"2023-09-23T20:21:34.339243+0000","last_peered":"2023-09-23T20:21:34.339243+0000","last_clean":"2023-09-23T20:21:34.339243+0000","last_became_active":"2023-09-23T20:17:47.270375+0000","last_became_peered":"2023-09-23T20:17:47.270375+0000","last_unstale":"2023-09-23T20:21:34.339243+0000","last_undegraded":"2023-09-23T20:21:34.339243+0000","last_fullsized":"2023-09-23T20:21:34.339243+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:16.961271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1f","version":"0'0","reported_seq":10,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.356535+0000","last_change":"2023-09-23T20:21:26.101606+0000","last_active":"2023-09-23T20:21:34.356535+0000","last_peered":"2023-09-23T20:21:34.356535+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.101606+0000","last_became_peered":"2023-09-23T20:21:26.101606+0000","last_unstale":"2023-09-23T20:21:34.356535+0000","last_undegraded":"2023-09-23T20:21:34.356535+0000","last_fullsized":"2023-09-23T20:21:26.096639+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:40.701088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":497,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338808+0000","last_change":"2023-09-23T20:21:19.421049+0000","last_active":"2023-09-23T20:21:34.338808+0000","last_peered":"2023-09-23T20:21:34.338808+0000","last_clean":"2023-09-23T19:54:37.307730+0000","last_became_active":"2023-09-23T20:21:19.421049+0000","last_became_peered":"2023-09-23T20:21:19.421049+0000","last_unstale":"2023-09-23T20:21:34.338808+0000","last_undegraded":"2023-09-23T20:21:34.338808+0000","last_fullsized":"2023-09-23T20:21:19.416316+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:13.927403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:13.927403+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":463,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.339147+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:21:34.339147+0000","last_peered":"2023-09-23T20:21:34.339147+0000","last_clean":"2023-09-23T20:21:34.339147+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:21:34.339147+0000","last_undegraded":"2023-09-23T20:21:34.339147+0000","last_fullsized":"2023-09-23T20:21:34.339147+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":402,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.337940+0000","last_change":"2023-09-23T20:17:47.265452+0000","last_active":"2023-09-23T20:21:34.337940+0000","last_peered":"2023-09-23T20:21:34.337940+0000","last_clean":"2023-09-23T20:21:34.337940+0000","last_became_active":"2023-09-23T20:17:47.265115+0000","last_became_peered":"2023-09-23T20:17:47.265115+0000","last_unstale":"2023-09-23T20:21:34.337940+0000","last_undegraded":"2023-09-23T20:21:34.337940+0000","last_fullsized":"2023-09-23T20:21:34.337940+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:04.898742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1e","version":"533'7","reported_seq":18,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.356705+0000","last_change":"2023-09-23T20:21:26.323190+0000","last_active":"2023-09-23T20:21:34.356705+0000","last_peered":"2023-09-23T20:21:34.356705+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.101718+0000","last_became_peered":"2023-09-23T20:21:26.101718+0000","last_unstale":"2023-09-23T20:21:34.356705+0000","last_undegraded":"2023-09-23T20:21:26.322494+0000","last_fullsized":"2023-09-23T20:21:26.096430+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:20.608027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":619,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.356489+0000","last_change":"2023-09-23T20:21:19.426242+0000","last_active":"2023-09-23T20:21:34.356489+0000","last_peered":"2023-09-23T20:21:34.356489+0000","last_clean":"2023-09-23T20:07:24.057948+0000","last_became_active":"2023-09-23T20:21:19.426242+0000","last_became_peered":"2023-09-23T20:21:19.426242+0000","last_unstale":"2023-09-23T20:21:34.356489+0000","last_undegraded":"2023-09-23T20:21:34.356489+0000","last_fullsized":"2023-09-23T20:21:19.422563+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:25.106528+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:25.106528+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":556,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338860+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:21:34.338860+0000","last_peered":"2023-09-23T20:21:34.338860+0000","last_clean":"2023-09-23T20:21:34.338860+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:21:34.338860+0000","last_undegraded":"2023-09-23T20:21:34.338860+0000","last_fullsized":"2023-09-23T20:21:34.338860+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":403,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338302+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:21:34.338302+0000","last_peered":"2023-09-23T20:21:34.338302+0000","last_clean":"2023-09-23T20:21:34.338302+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:21:34.338302+0000","last_undegraded":"2023-09-23T20:21:34.338302+0000","last_fullsized":"2023-09-23T20:21:34.338302+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1d","version":"0'0","reported_seq":13,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356621+0000","last_change":"2023-09-23T20:21:26.116969+0000","last_active":"2023-09-23T20:21:34.356621+0000","last_peered":"2023-09-23T20:21:34.356621+0000","last_clean":"2023-09-23T20:21:34.356621+0000","last_became_active":"2023-09-23T20:21:26.116655+0000","last_became_peered":"2023-09-23T20:21:26.116655+0000","last_unstale":"2023-09-23T20:21:34.356621+0000","last_undegraded":"2023-09-23T20:21:34.356621+0000","last_fullsized":"2023-09-23T20:21:34.356621+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:22.160495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":842,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338827+0000","last_change":"2023-09-23T20:19:42.182813+0000","last_active":"2023-09-23T20:21:34.338827+0000","last_peered":"2023-09-23T20:21:34.338827+0000","last_clean":"2023-09-23T20:21:34.338827+0000","last_became_active":"2023-09-23T20:19:05.072575+0000","last_became_peered":"2023-09-23T20:19:05.072575+0000","last_unstale":"2023-09-23T20:21:34.338827+0000","last_undegraded":"2023-09-23T20:21:34.338827+0000","last_fullsized":"2023-09-23T20:21:34.338827+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:42.182718+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:42.182718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:00.054669+0000","scrub_duration":0.0014414230000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":230,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356717+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:21:34.356717+0000","last_peered":"2023-09-23T20:21:34.356717+0000","last_clean":"2023-09-23T20:21:34.356717+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:21:34.356717+0000","last_undegraded":"2023-09-23T20:21:34.356717+0000","last_fullsized":"2023-09-23T20:21:34.356717+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863362+0000","last_change":"2023-09-23T20:21:19.424648+0000","last_active":"2023-09-23T20:21:34.863362+0000","last_peered":"2023-09-23T20:21:34.863362+0000","last_clean":"2023-09-23T20:21:16.602032+0000","last_became_active":"2023-09-23T20:21:19.424648+0000","last_became_peered":"2023-09-23T20:21:19.424648+0000","last_unstale":"2023-09-23T20:21:34.863362+0000","last_undegraded":"2023-09-23T20:21:34.863362+0000","last_fullsized":"2023-09-23T20:21:19.420444+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:31.660089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1c","version":"0'0","reported_seq":10,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.355948+0000","last_change":"2023-09-23T20:21:26.099470+0000","last_active":"2023-09-23T20:21:34.355948+0000","last_peered":"2023-09-23T20:21:34.355948+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.099470+0000","last_became_peered":"2023-09-23T20:21:26.099470+0000","last_unstale":"2023-09-23T20:21:34.355948+0000","last_undegraded":"2023-09-23T20:21:34.355948+0000","last_fullsized":"2023-09-23T20:21:26.095194+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:45.463006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":746,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:21:43.447838+0000","last_change":"2023-09-23T20:21:23.507116+0000","last_active":"2023-09-23T20:21:43.447838+0000","last_peered":"2023-09-23T20:21:43.447838+0000","last_clean":"2023-09-23T20:21:43.447838+0000","last_became_active":"2023-09-23T20:19:05.092425+0000","last_became_peered":"2023-09-23T20:19:05.092425+0000","last_unstale":"2023-09-23T20:21:43.447838+0000","last_undegraded":"2023-09-23T20:21:43.447838+0000","last_fullsized":"2023-09-23T20:21:43.447838+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:23.507023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:23.507023+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:31.827575+0000","scrub_duration":0.0014042600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":490,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.356215+0000","last_change":"2023-09-23T20:21:19.426310+0000","last_active":"2023-09-23T20:21:34.356215+0000","last_peered":"2023-09-23T20:21:34.356215+0000","last_clean":"2023-09-23T20:21:16.394690+0000","last_became_active":"2023-09-23T20:21:19.426310+0000","last_became_peered":"2023-09-23T20:21:19.426310+0000","last_unstale":"2023-09-23T20:21:34.356215+0000","last_undegraded":"2023-09-23T20:21:19.423095+0000","last_fullsized":"2023-09-23T20:21:19.422996+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:01.978193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":57,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863778+0000","last_change":"2023-09-23T20:18:38.012263+0000","last_active":"2023-09-23T20:21:34.863778+0000","last_peered":"2023-09-23T20:21:34.863778+0000","last_clean":"2023-09-23T20:17:54.133085+0000","last_became_active":"2023-09-23T20:18:38.012263+0000","last_became_peered":"2023-09-23T20:18:38.012263+0000","last_unstale":"2023-09-23T20:21:34.863778+0000","last_undegraded":"2023-09-23T20:21:34.863778+0000","last_fullsized":"2023-09-23T20:18:38.003813+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:47.449647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1b","version":"0'0","reported_seq":10,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.356632+0000","last_change":"2023-09-23T20:21:26.109483+0000","last_active":"2023-09-23T20:21:34.356632+0000","last_peered":"2023-09-23T20:21:34.356632+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.109483+0000","last_became_peered":"2023-09-23T20:21:26.109483+0000","last_unstale":"2023-09-23T20:21:34.356632+0000","last_undegraded":"2023-09-23T20:21:34.356632+0000","last_fullsized":"2023-09-23T20:21:26.096197+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:21.370916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":896,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338252+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:21:34.338252+0000","last_peered":"2023-09-23T20:21:34.338252+0000","last_clean":"2023-09-23T20:21:34.338252+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:21:34.338252+0000","last_undegraded":"2023-09-23T20:21:34.338252+0000","last_fullsized":"2023-09-23T20:21:34.338252+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":237,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.356812+0000","last_change":"2023-09-23T20:21:19.426288+0000","last_active":"2023-09-23T20:21:34.356812+0000","last_peered":"2023-09-23T20:21:34.356812+0000","last_clean":"2023-09-23T20:21:16.393528+0000","last_became_active":"2023-09-23T20:21:19.426288+0000","last_became_peered":"2023-09-23T20:21:19.426288+0000","last_unstale":"2023-09-23T20:21:34.356812+0000","last_undegraded":"2023-09-23T20:21:19.421126+0000","last_fullsized":"2023-09-23T20:21:19.421001+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:48.307728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863941+0000","last_change":"2023-09-23T20:17:47.266963+0000","last_active":"2023-09-23T20:21:34.863941+0000","last_peered":"2023-09-23T20:21:34.863941+0000","last_clean":"2023-09-23T20:21:34.863941+0000","last_became_active":"2023-09-23T20:17:47.266645+0000","last_became_peered":"2023-09-23T20:17:47.266645+0000","last_unstale":"2023-09-23T20:21:34.863941+0000","last_undegraded":"2023-09-23T20:21:34.863941+0000","last_fullsized":"2023-09-23T20:21:34.863941+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:05.052559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1a","version":"0'0","reported_seq":10,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338382+0000","last_change":"2023-09-23T20:21:26.101521+0000","last_active":"2023-09-23T20:21:34.338382+0000","last_peered":"2023-09-23T20:21:34.338382+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.101521+0000","last_became_peered":"2023-09-23T20:21:26.101521+0000","last_unstale":"2023-09-23T20:21:34.338382+0000","last_undegraded":"2023-09-23T20:21:34.338382+0000","last_fullsized":"2023-09-23T20:21:26.098176+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:23.406198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":585,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338655+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:21:34.338655+0000","last_peered":"2023-09-23T20:21:34.338655+0000","last_clean":"2023-09-23T20:21:34.338655+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:21:34.338655+0000","last_undegraded":"2023-09-23T20:21:34.338655+0000","last_fullsized":"2023-09-23T20:21:34.338655+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":245,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356001+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:21:34.356001+0000","last_peered":"2023-09-23T20:21:34.356001+0000","last_clean":"2023-09-23T20:21:34.356001+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:21:34.356001+0000","last_undegraded":"2023-09-23T20:21:34.356001+0000","last_fullsized":"2023-09-23T20:21:34.356001+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":535,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338920+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:21:34.338920+0000","last_peered":"2023-09-23T20:21:34.338920+0000","last_clean":"2023-09-23T20:21:34.338920+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:21:34.338920+0000","last_undegraded":"2023-09-23T20:21:34.338920+0000","last_fullsized":"2023-09-23T20:21:34.338920+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"0'0","reported_seq":10,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.356239+0000","last_change":"2023-09-23T20:21:26.116675+0000","last_active":"2023-09-23T20:21:34.356239+0000","last_peered":"2023-09-23T20:21:34.356239+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116675+0000","last_became_peered":"2023-09-23T20:21:26.116675+0000","last_unstale":"2023-09-23T20:21:34.356239+0000","last_undegraded":"2023-09-23T20:21:34.356239+0000","last_fullsized":"2023-09-23T20:21:26.095370+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:11.977313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":608,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.862939+0000","last_change":"2023-09-23T20:21:11.930313+0000","last_active":"2023-09-23T20:21:34.862939+0000","last_peered":"2023-09-23T20:21:34.862939+0000","last_clean":"2023-09-23T20:21:34.862939+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:21:34.862939+0000","last_undegraded":"2023-09-23T20:21:34.862939+0000","last_fullsized":"2023-09-23T20:21:34.862939+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:11.930203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.930203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:00.534581+0000","scrub_duration":0.0016982569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":241,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.355580+0000","last_change":"2023-09-23T20:18:38.011856+0000","last_active":"2023-09-23T20:21:34.355580+0000","last_peered":"2023-09-23T20:21:34.355580+0000","last_clean":"2023-09-23T20:18:35.260354+0000","last_became_active":"2023-09-23T20:18:38.011856+0000","last_became_peered":"2023-09-23T20:18:38.011856+0000","last_unstale":"2023-09-23T20:21:34.355580+0000","last_undegraded":"2023-09-23T20:21:34.355580+0000","last_fullsized":"2023-09-23T20:18:37.999747+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:58.093947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":527,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338817+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:21:34.338817+0000","last_peered":"2023-09-23T20:21:34.338817+0000","last_clean":"2023-09-23T20:21:34.338817+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:21:34.338817+0000","last_undegraded":"2023-09-23T20:21:34.338817+0000","last_fullsized":"2023-09-23T20:21:34.338817+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"0'0","reported_seq":10,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338351+0000","last_change":"2023-09-23T20:21:26.110226+0000","last_active":"2023-09-23T20:21:34.338351+0000","last_peered":"2023-09-23T20:21:34.338351+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.110226+0000","last_became_peered":"2023-09-23T20:21:26.110226+0000","last_unstale":"2023-09-23T20:21:34.338351+0000","last_undegraded":"2023-09-23T20:21:34.338351+0000","last_fullsized":"2023-09-23T20:21:26.098141+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:22:02.166653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":694,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338278+0000","last_change":"2023-09-23T20:17:50.332129+0000","last_active":"2023-09-23T20:21:34.338278+0000","last_peered":"2023-09-23T20:21:34.338278+0000","last_clean":"2023-09-23T20:21:34.338278+0000","last_became_active":"2023-09-23T20:17:50.331669+0000","last_became_peered":"2023-09-23T20:17:50.331669+0000","last_unstale":"2023-09-23T20:21:34.338278+0000","last_undegraded":"2023-09-23T20:21:34.338278+0000","last_fullsized":"2023-09-23T20:21:34.338278+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:23.893417+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":180,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338474+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:21:34.338474+0000","last_peered":"2023-09-23T20:21:34.338474+0000","last_clean":"2023-09-23T20:21:34.338474+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:21:34.338474+0000","last_undegraded":"2023-09-23T20:21:34.338474+0000","last_fullsized":"2023-09-23T20:21:34.338474+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":189,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338473+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:21:34.338473+0000","last_peered":"2023-09-23T20:21:34.338473+0000","last_clean":"2023-09-23T20:21:34.338473+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:21:34.338473+0000","last_undegraded":"2023-09-23T20:21:34.338473+0000","last_fullsized":"2023-09-23T20:21:34.338473+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"533'23","reported_seq":57,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338735+0000","last_change":"2023-09-23T20:21:26.108368+0000","last_active":"2023-09-23T20:21:34.338735+0000","last_peered":"2023-09-23T20:21:34.338735+0000","last_clean":"2023-09-23T20:21:34.338735+0000","last_became_active":"2023-09-23T20:21:26.108071+0000","last_became_peered":"2023-09-23T20:21:26.108071+0000","last_unstale":"2023-09-23T20:21:34.338735+0000","last_undegraded":"2023-09-23T20:21:34.338735+0000","last_fullsized":"2023-09-23T20:21:34.338735+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:18.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":40,"num_read_kb":39,"num_write":17,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":846,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356038+0000","last_change":"2023-09-23T20:21:22.116340+0000","last_active":"2023-09-23T20:21:34.356038+0000","last_peered":"2023-09-23T20:21:34.356038+0000","last_clean":"2023-09-23T20:21:34.356038+0000","last_became_active":"2023-09-23T20:19:05.065283+0000","last_became_peered":"2023-09-23T20:19:05.065283+0000","last_unstale":"2023-09-23T20:21:34.356038+0000","last_undegraded":"2023-09-23T20:21:34.356038+0000","last_fullsized":"2023-09-23T20:21:34.356038+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:22.116270+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:21:22.116270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:27.825796+0000","scrub_duration":0.001763922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":228,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.357019+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:21:34.357019+0000","last_peered":"2023-09-23T20:21:34.357019+0000","last_clean":"2023-09-23T20:21:34.357019+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:21:34.357019+0000","last_undegraded":"2023-09-23T20:21:34.357019+0000","last_fullsized":"2023-09-23T20:21:34.357019+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":545,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338513+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:21:34.338513+0000","last_peered":"2023-09-23T20:21:34.338513+0000","last_clean":"2023-09-23T20:21:34.338513+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:21:34.338513+0000","last_undegraded":"2023-09-23T20:21:34.338513+0000","last_fullsized":"2023-09-23T20:21:34.338513+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"533'1","reported_seq":13,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.863683+0000","last_change":"2023-09-23T20:21:26.430890+0000","last_active":"2023-09-23T20:21:34.863683+0000","last_peered":"2023-09-23T20:21:34.863683+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116526+0000","last_became_peered":"2023-09-23T20:21:26.116526+0000","last_unstale":"2023-09-23T20:21:34.863683+0000","last_undegraded":"2023-09-23T20:21:26.430638+0000","last_fullsized":"2023-09-23T20:21:26.096111+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:20.265662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":915,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338305+0000","last_change":"2023-09-23T20:21:11.597505+0000","last_active":"2023-09-23T20:21:34.338305+0000","last_peered":"2023-09-23T20:21:34.338305+0000","last_clean":"2023-09-23T20:21:34.338305+0000","last_became_active":"2023-09-23T20:19:05.085764+0000","last_became_peered":"2023-09-23T20:19:05.085764+0000","last_unstale":"2023-09-23T20:21:34.338305+0000","last_undegraded":"2023-09-23T20:21:34.338305+0000","last_fullsized":"2023-09-23T20:21:34.338305+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:21:11.597355+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.597355+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:24.991370+0000","scrub_duration":0.0021599240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":257,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356874+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:21:34.356874+0000","last_peered":"2023-09-23T20:21:34.356874+0000","last_clean":"2023-09-23T20:21:34.356874+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:21:34.356874+0000","last_undegraded":"2023-09-23T20:21:34.356874+0000","last_fullsized":"2023-09-23T20:21:34.356874+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863174+0000","last_change":"2023-09-23T20:17:47.264776+0000","last_active":"2023-09-23T20:21:34.863174+0000","last_peered":"2023-09-23T20:21:34.863174+0000","last_clean":"2023-09-23T20:21:34.863174+0000","last_became_active":"2023-09-23T20:17:47.264473+0000","last_became_peered":"2023-09-23T20:17:47.264473+0000","last_unstale":"2023-09-23T20:21:34.863174+0000","last_undegraded":"2023-09-23T20:21:34.863174+0000","last_fullsized":"2023-09-23T20:21:34.863174+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:40.906303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":13,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356414+0000","last_change":"2023-09-23T20:21:26.102464+0000","last_active":"2023-09-23T20:21:34.356414+0000","last_peered":"2023-09-23T20:21:34.356414+0000","last_clean":"2023-09-23T20:21:34.356414+0000","last_became_active":"2023-09-23T20:21:26.102159+0000","last_became_peered":"2023-09-23T20:21:26.102159+0000","last_unstale":"2023-09-23T20:21:34.356414+0000","last_undegraded":"2023-09-23T20:21:34.356414+0000","last_fullsized":"2023-09-23T20:21:34.356414+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:39.141820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":609,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338958+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:21:34.338958+0000","last_peered":"2023-09-23T20:21:34.338958+0000","last_clean":"2023-09-23T20:21:34.338958+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:21:34.338958+0000","last_undegraded":"2023-09-23T20:21:34.338958+0000","last_fullsized":"2023-09-23T20:21:34.338958+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":249,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.356580+0000","last_change":"2023-09-23T20:18:38.001952+0000","last_active":"2023-09-23T20:21:34.356580+0000","last_peered":"2023-09-23T20:21:34.356580+0000","last_clean":"2023-09-23T20:18:35.257306+0000","last_became_active":"2023-09-23T20:18:38.001952+0000","last_became_peered":"2023-09-23T20:18:38.001952+0000","last_unstale":"2023-09-23T20:21:34.356580+0000","last_undegraded":"2023-09-23T20:18:38.000196+0000","last_fullsized":"2023-09-23T20:18:38.000044+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:17.507792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863386+0000","last_change":"2023-09-23T20:17:47.266977+0000","last_active":"2023-09-23T20:21:34.863386+0000","last_peered":"2023-09-23T20:21:34.863386+0000","last_clean":"2023-09-23T20:21:34.863386+0000","last_became_active":"2023-09-23T20:17:47.266657+0000","last_became_peered":"2023-09-23T20:17:47.266657+0000","last_unstale":"2023-09-23T20:21:34.863386+0000","last_undegraded":"2023-09-23T20:21:34.863386+0000","last_fullsized":"2023-09-23T20:21:34.863386+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:33.774981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":9,"reported_epoch":536,"state":"undersized+peered","last_fresh":"2023-09-23T20:21:34.355910+0000","last_change":"2023-09-23T20:21:26.096160+0000","last_active":"2023-09-23T20:21:25.081362+0000","last_peered":"2023-09-23T20:21:34.355910+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T20:21:26.096160+0000","last_unstale":"2023-09-23T20:21:34.355910+0000","last_undegraded":"2023-09-23T20:21:34.355910+0000","last_fullsized":"2023-09-23T20:21:26.095059+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:05.996546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":769,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356472+0000","last_change":"2023-09-23T20:21:24.112801+0000","last_active":"2023-09-23T20:21:34.356472+0000","last_peered":"2023-09-23T20:21:34.356472+0000","last_clean":"2023-09-23T20:21:34.356472+0000","last_became_active":"2023-09-23T20:19:05.071700+0000","last_became_peered":"2023-09-23T20:19:05.071700+0000","last_unstale":"2023-09-23T20:21:34.356472+0000","last_undegraded":"2023-09-23T20:21:34.356472+0000","last_fullsized":"2023-09-23T20:21:34.356472+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:24.112725+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:21:24.112725+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:07.289445+0000","scrub_duration":0.0014886809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":595,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338529+0000","last_change":"2023-09-23T20:17:47.267657+0000","last_active":"2023-09-23T20:21:34.338529+0000","last_peered":"2023-09-23T20:21:34.338529+0000","last_clean":"2023-09-23T20:21:34.338529+0000","last_became_active":"2023-09-23T20:17:47.267288+0000","last_became_peered":"2023-09-23T20:17:47.267288+0000","last_unstale":"2023-09-23T20:21:34.338529+0000","last_undegraded":"2023-09-23T20:21:34.338529+0000","last_fullsized":"2023-09-23T20:21:34.338529+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.185253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863714+0000","last_change":"2023-09-23T20:17:47.280675+0000","last_active":"2023-09-23T20:21:34.863714+0000","last_peered":"2023-09-23T20:21:34.863714+0000","last_clean":"2023-09-23T20:21:34.863714+0000","last_became_active":"2023-09-23T20:17:47.280175+0000","last_became_peered":"2023-09-23T20:17:47.280175+0000","last_unstale":"2023-09-23T20:21:34.863714+0000","last_undegraded":"2023-09-23T20:21:34.863714+0000","last_fullsized":"2023-09-23T20:21:34.863714+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:08.499949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":13,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.355941+0000","last_change":"2023-09-23T20:21:26.101105+0000","last_active":"2023-09-23T20:21:34.355941+0000","last_peered":"2023-09-23T20:21:34.355941+0000","last_clean":"2023-09-23T20:21:34.355941+0000","last_became_active":"2023-09-23T20:21:26.100814+0000","last_became_peered":"2023-09-23T20:21:26.100814+0000","last_unstale":"2023-09-23T20:21:34.355941+0000","last_undegraded":"2023-09-23T20:21:34.355941+0000","last_fullsized":"2023-09-23T20:21:34.355941+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:46.990251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":946,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.356237+0000","last_change":"2023-09-23T20:21:19.424707+0000","last_active":"2023-09-23T20:21:34.356237+0000","last_peered":"2023-09-23T20:21:34.356237+0000","last_clean":"2023-09-23T20:21:16.394694+0000","last_became_active":"2023-09-23T20:21:19.424707+0000","last_became_peered":"2023-09-23T20:21:19.424707+0000","last_unstale":"2023-09-23T20:21:34.356237+0000","last_undegraded":"2023-09-23T20:21:34.356237+0000","last_fullsized":"2023-09-23T20:21:19.420966+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:47.181912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:47.181912+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.799364+0000","scrub_duration":0.0074047540000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":285,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338422+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:21:34.338422+0000","last_peered":"2023-09-23T20:21:34.338422+0000","last_clean":"2023-09-23T20:21:34.338422+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:21:34.338422+0000","last_undegraded":"2023-09-23T20:21:34.338422+0000","last_fullsized":"2023-09-23T20:21:34.338422+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.864023+0000","last_change":"2023-09-23T20:17:47.270450+0000","last_active":"2023-09-23T20:21:34.864023+0000","last_peered":"2023-09-23T20:21:34.864023+0000","last_clean":"2023-09-23T20:21:34.864023+0000","last_became_active":"2023-09-23T20:17:47.270040+0000","last_became_peered":"2023-09-23T20:17:47.270040+0000","last_unstale":"2023-09-23T20:21:34.864023+0000","last_undegraded":"2023-09-23T20:21:34.864023+0000","last_fullsized":"2023-09-23T20:21:34.864023+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:51.572016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.2","version":"533'1","reported_seq":13,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.338084+0000","last_change":"2023-09-23T20:21:26.120563+0000","last_active":"2023-09-23T20:21:34.338084+0000","last_peered":"2023-09-23T20:21:34.338084+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116639+0000","last_became_peered":"2023-09-23T20:21:26.116639+0000","last_unstale":"2023-09-23T20:21:34.338084+0000","last_undegraded":"2023-09-23T20:21:26.120189+0000","last_fullsized":"2023-09-23T20:21:26.091682+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:16.130580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":917,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338596+0000","last_change":"2023-09-23T20:21:19.423194+0000","last_active":"2023-09-23T20:21:34.338596+0000","last_peered":"2023-09-23T20:21:34.338596+0000","last_clean":"2023-09-23T20:21:16.629850+0000","last_became_active":"2023-09-23T20:21:19.423194+0000","last_became_peered":"2023-09-23T20:21:19.423194+0000","last_unstale":"2023-09-23T20:21:34.338596+0000","last_undegraded":"2023-09-23T20:21:34.338596+0000","last_fullsized":"2023-09-23T20:21:19.417294+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:43.954204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:43.954204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:15.279707+0000","scrub_duration":0.0019293979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863285+0000","last_change":"2023-09-23T20:17:47.266513+0000","last_active":"2023-09-23T20:21:34.863285+0000","last_peered":"2023-09-23T20:21:34.863285+0000","last_clean":"2023-09-23T20:21:34.863285+0000","last_became_active":"2023-09-23T20:17:47.266163+0000","last_became_peered":"2023-09-23T20:17:47.266163+0000","last_unstale":"2023-09-23T20:21:34.863285+0000","last_undegraded":"2023-09-23T20:21:34.863285+0000","last_fullsized":"2023-09-23T20:21:34.863285+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.645381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":406,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863911+0000","last_change":"2023-09-23T20:21:19.425194+0000","last_active":"2023-09-23T20:21:34.863911+0000","last_peered":"2023-09-23T20:21:34.863911+0000","last_clean":"2023-09-23T20:17:42.212478+0000","last_became_active":"2023-09-23T20:21:19.425194+0000","last_became_peered":"2023-09-23T20:21:19.425194+0000","last_unstale":"2023-09-23T20:21:34.863911+0000","last_undegraded":"2023-09-23T20:21:34.863911+0000","last_fullsized":"2023-09-23T20:21:19.419364+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:01.812348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.1","version":"533'15","reported_seq":26,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.337694+0000","last_change":"2023-09-23T20:21:26.422339+0000","last_active":"2023-09-23T20:21:34.337694+0000","last_peered":"2023-09-23T20:21:34.337694+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.116852+0000","last_became_peered":"2023-09-23T20:21:26.116852+0000","last_unstale":"2023-09-23T20:21:34.337694+0000","last_undegraded":"2023-09-23T20:21:34.337694+0000","last_fullsized":"2023-09-23T20:21:26.091379+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:33.974639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":613,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863545+0000","last_change":"2023-09-23T20:21:20.952512+0000","last_active":"2023-09-23T20:21:34.863545+0000","last_peered":"2023-09-23T20:21:34.863545+0000","last_clean":"2023-09-23T20:21:34.863545+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:21:34.863545+0000","last_undegraded":"2023-09-23T20:21:34.863545+0000","last_fullsized":"2023-09-23T20:21:34.863545+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:20.952352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:20.952352+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:28.266826+0000","scrub_duration":0.00171708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":322,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.356042+0000","last_change":"2023-09-23T20:21:19.426288+0000","last_active":"2023-09-23T20:21:34.356042+0000","last_peered":"2023-09-23T20:21:34.356042+0000","last_clean":"2023-09-23T20:07:24.062735+0000","last_became_active":"2023-09-23T20:21:19.426288+0000","last_became_peered":"2023-09-23T20:21:19.426288+0000","last_unstale":"2023-09-23T20:21:34.356042+0000","last_undegraded":"2023-09-23T20:21:34.356042+0000","last_fullsized":"2023-09-23T20:21:19.421386+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:00.204744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863967+0000","last_change":"2023-09-23T20:17:47.268811+0000","last_active":"2023-09-23T20:21:34.863967+0000","last_peered":"2023-09-23T20:21:34.863967+0000","last_clean":"2023-09-23T20:21:34.863967+0000","last_became_active":"2023-09-23T20:17:47.268391+0000","last_became_peered":"2023-09-23T20:17:47.268391+0000","last_unstale":"2023-09-23T20:21:34.863967+0000","last_undegraded":"2023-09-23T20:21:34.863967+0000","last_fullsized":"2023-09-23T20:21:34.863967+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:32.388705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":14,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863183+0000","last_change":"2023-09-23T20:21:26.101824+0000","last_active":"2023-09-23T20:21:34.863183+0000","last_peered":"2023-09-23T20:21:34.863183+0000","last_clean":"2023-09-23T20:21:34.863183+0000","last_became_active":"2023-09-23T20:21:26.101337+0000","last_became_peered":"2023-09-23T20:21:26.101337+0000","last_unstale":"2023-09-23T20:21:34.863183+0000","last_undegraded":"2023-09-23T20:21:34.863183+0000","last_fullsized":"2023-09-23T20:21:34.863183+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:24.734547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":929,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338817+0000","last_change":"2023-09-23T20:21:19.421445+0000","last_active":"2023-09-23T20:21:34.338817+0000","last_peered":"2023-09-23T20:21:34.338817+0000","last_clean":"2023-09-23T20:21:16.631116+0000","last_became_active":"2023-09-23T20:21:19.421445+0000","last_became_peered":"2023-09-23T20:21:19.421445+0000","last_unstale":"2023-09-23T20:21:34.338817+0000","last_undegraded":"2023-09-23T20:21:34.338817+0000","last_fullsized":"2023-09-23T20:21:19.416634+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:12.314623+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T20:21:12.314623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:35.040895+0000","scrub_duration":0.001743334,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":189,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.337699+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:21:34.337699+0000","last_peered":"2023-09-23T20:21:34.337699+0000","last_clean":"2023-09-23T20:21:34.337699+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:21:34.337699+0000","last_undegraded":"2023-09-23T20:21:34.337699+0000","last_fullsized":"2023-09-23T20:21:34.337699+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863717+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:21:34.863717+0000","last_peered":"2023-09-23T20:21:34.863717+0000","last_clean":"2023-09-23T20:21:34.863717+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:21:34.863717+0000","last_undegraded":"2023-09-23T20:21:34.863717+0000","last_fullsized":"2023-09-23T20:21:34.863717+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":9,"reported_epoch":536,"state":"undersized+peered","last_fresh":"2023-09-23T20:21:35.501548+0000","last_change":"2023-09-23T20:21:26.107994+0000","last_active":"2023-09-23T20:21:25.081362+0000","last_peered":"2023-09-23T20:21:35.501548+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T20:21:26.107994+0000","last_unstale":"2023-09-23T20:21:35.501548+0000","last_undegraded":"2023-09-23T20:21:35.501548+0000","last_fullsized":"2023-09-23T20:21:26.106806+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:34.085552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":842,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338566+0000","last_change":"2023-09-23T20:19:40.219448+0000","last_active":"2023-09-23T20:21:34.338566+0000","last_peered":"2023-09-23T20:21:34.338566+0000","last_clean":"2023-09-23T20:21:34.338566+0000","last_became_active":"2023-09-23T20:19:05.072187+0000","last_became_peered":"2023-09-23T20:19:05.072187+0000","last_unstale":"2023-09-23T20:21:34.338566+0000","last_undegraded":"2023-09-23T20:21:34.338566+0000","last_fullsized":"2023-09-23T20:21:34.338566+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:40.219310+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:40.219310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:29.003073+0000","scrub_duration":0.011178762,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'127","reported_seq":488,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.862925+0000","last_change":"2023-09-23T20:17:50.311646+0000","last_active":"2023-09-23T20:21:34.862925+0000","last_peered":"2023-09-23T20:21:34.862925+0000","last_clean":"2023-09-23T20:21:34.862925+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:21:34.862925+0000","last_undegraded":"2023-09-23T20:21:34.862925+0000","last_fullsized":"2023-09-23T20:21:34.862925+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029399349999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":564,"num_read_kb":1486,"num_write":149,"num_write_kb":5182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":544,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338785+0000","last_change":"2023-09-23T20:21:19.422507+0000","last_active":"2023-09-23T20:21:34.338785+0000","last_peered":"2023-09-23T20:21:34.338785+0000","last_clean":"2023-09-23T20:21:16.630306+0000","last_became_active":"2023-09-23T20:21:19.422507+0000","last_became_peered":"2023-09-23T20:21:19.422507+0000","last_unstale":"2023-09-23T20:21:34.338785+0000","last_undegraded":"2023-09-23T20:21:34.338785+0000","last_fullsized":"2023-09-23T20:21:19.416760+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:35.522169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863232+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:21:34.863232+0000","last_peered":"2023-09-23T20:21:34.863232+0000","last_clean":"2023-09-23T20:21:34.863232+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:21:34.863232+0000","last_undegraded":"2023-09-23T20:21:34.863232+0000","last_fullsized":"2023-09-23T20:21:34.863232+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":14,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338255+0000","last_change":"2023-09-23T20:21:26.117398+0000","last_active":"2023-09-23T20:21:34.338255+0000","last_peered":"2023-09-23T20:21:34.338255+0000","last_clean":"2023-09-23T20:21:34.338255+0000","last_became_active":"2023-09-23T20:21:26.117088+0000","last_became_peered":"2023-09-23T20:21:26.117088+0000","last_unstale":"2023-09-23T20:21:34.338255+0000","last_undegraded":"2023-09-23T20:21:34.338255+0000","last_fullsized":"2023-09-23T20:21:34.338255+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:42.026338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":887,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338285+0000","last_change":"2023-09-23T20:21:19.422275+0000","last_active":"2023-09-23T20:21:34.338285+0000","last_peered":"2023-09-23T20:21:34.338285+0000","last_clean":"2023-09-23T20:21:16.630920+0000","last_became_active":"2023-09-23T20:21:19.422275+0000","last_became_peered":"2023-09-23T20:21:19.422275+0000","last_unstale":"2023-09-23T20:21:34.338285+0000","last_undegraded":"2023-09-23T20:21:34.338285+0000","last_fullsized":"2023-09-23T20:21:19.415540+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:29.088569+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'72","reported_seq":1217,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338428+0000","last_change":"2023-09-23T20:17:47.305979+0000","last_active":"2023-09-23T20:21:34.338428+0000","last_peered":"2023-09-23T20:21:34.338428+0000","last_clean":"2023-09-23T20:21:34.338428+0000","last_became_active":"2023-09-23T20:17:47.274623+0000","last_became_peered":"2023-09-23T20:17:47.274623+0000","last_unstale":"2023-09-23T20:21:34.338428+0000","last_undegraded":"2023-09-23T20:21:34.338428+0000","last_fullsized":"2023-09-23T20:21:34.338428+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:27.293931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068506800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":794,"num_read_kb":2099,"num_write":43,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863361+0000","last_change":"2023-09-23T20:17:47.280498+0000","last_active":"2023-09-23T20:21:34.863361+0000","last_peered":"2023-09-23T20:21:34.863361+0000","last_clean":"2023-09-23T20:21:34.863361+0000","last_became_active":"2023-09-23T20:17:47.279716+0000","last_became_peered":"2023-09-23T20:17:47.279716+0000","last_unstale":"2023-09-23T20:21:34.863361+0000","last_undegraded":"2023-09-23T20:21:34.863361+0000","last_fullsized":"2023-09-23T20:21:34.863361+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:18.625303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":65,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863950+0000","last_change":"2023-09-23T20:21:19.421772+0000","last_active":"2023-09-23T20:21:34.863950+0000","last_peered":"2023-09-23T20:21:34.863950+0000","last_clean":"2023-09-23T20:21:16.602164+0000","last_became_active":"2023-09-23T20:21:19.421772+0000","last_became_peered":"2023-09-23T20:21:19.421772+0000","last_unstale":"2023-09-23T20:21:34.863950+0000","last_undegraded":"2023-09-23T20:21:34.863950+0000","last_fullsized":"2023-09-23T20:21:19.418648+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:32.788234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"533'12","reported_seq":30,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338169+0000","last_change":"2023-09-23T20:21:26.117274+0000","last_active":"2023-09-23T20:21:34.338169+0000","last_peered":"2023-09-23T20:21:34.338169+0000","last_clean":"2023-09-23T20:21:34.338169+0000","last_became_active":"2023-09-23T20:21:26.116768+0000","last_became_peered":"2023-09-23T20:21:26.116768+0000","last_unstale":"2023-09-23T20:21:34.338169+0000","last_undegraded":"2023-09-23T20:21:34.338169+0000","last_fullsized":"2023-09-23T20:21:34.338169+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:45.952246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":18,"num_read_kb":18,"num_write":9,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":775,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356001+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:21:34.356001+0000","last_peered":"2023-09-23T20:21:34.356001+0000","last_clean":"2023-09-23T20:21:34.356001+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:21:34.356001+0000","last_undegraded":"2023-09-23T20:21:34.356001+0000","last_fullsized":"2023-09-23T20:21:34.356001+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'113","reported_seq":878,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338093+0000","last_change":"2023-09-23T20:21:19.422746+0000","last_active":"2023-09-23T20:21:34.338093+0000","last_peered":"2023-09-23T20:21:34.338093+0000","last_clean":"2023-09-23T20:03:53.495528+0000","last_became_active":"2023-09-23T20:21:19.422746+0000","last_became_peered":"2023-09-23T20:21:19.422746+0000","last_unstale":"2023-09-23T20:21:34.338093+0000","last_undegraded":"2023-09-23T20:21:34.338093+0000","last_fullsized":"2023-09-23T20:21:19.415128+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:15.902841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018463139999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":889,"num_read_kb":3126,"num_write":88,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":66,"reported_epoch":537,"state":"active+undersized","last_fresh":"2023-09-23T20:21:43.700135+0000","last_change":"2023-09-23T20:21:19.422296+0000","last_active":"2023-09-23T20:21:43.700135+0000","last_peered":"2023-09-23T20:21:43.700135+0000","last_clean":"2023-09-23T20:21:16.601818+0000","last_became_active":"2023-09-23T20:21:19.422296+0000","last_became_peered":"2023-09-23T20:21:19.422296+0000","last_unstale":"2023-09-23T20:21:43.700135+0000","last_undegraded":"2023-09-23T20:21:43.700135+0000","last_fullsized":"2023-09-23T20:21:19.419475+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:26.118095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863719+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:21:34.863719+0000","last_peered":"2023-09-23T20:21:34.863719+0000","last_clean":"2023-09-23T20:21:34.863719+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:21:34.863719+0000","last_undegraded":"2023-09-23T20:21:34.863719+0000","last_fullsized":"2023-09-23T20:21:34.863719+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":13,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338529+0000","last_change":"2023-09-23T20:21:26.101788+0000","last_active":"2023-09-23T20:21:34.338529+0000","last_peered":"2023-09-23T20:21:34.338529+0000","last_clean":"2023-09-23T20:21:34.338529+0000","last_became_active":"2023-09-23T20:21:26.101484+0000","last_became_peered":"2023-09-23T20:21:26.101484+0000","last_unstale":"2023-09-23T20:21:34.338529+0000","last_undegraded":"2023-09-23T20:21:34.338529+0000","last_fullsized":"2023-09-23T20:21:34.338529+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:55.459652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'245","reported_seq":1452,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.338979+0000","last_change":"2023-09-23T20:21:19.424613+0000","last_active":"2023-09-23T20:21:34.338979+0000","last_peered":"2023-09-23T20:21:34.338979+0000","last_clean":"2023-09-23T20:21:16.630598+0000","last_became_active":"2023-09-23T20:21:19.424613+0000","last_became_peered":"2023-09-23T20:21:19.424613+0000","last_unstale":"2023-09-23T20:21:34.338979+0000","last_undegraded":"2023-09-23T20:21:19.416004+0000","last_fullsized":"2023-09-23T20:21:19.415888+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":245,"log_dups_size":0,"ondisk_log_size":245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:37.709441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045716499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1304,"num_read_kb":1726,"num_write":201,"num_write_kb":392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":752,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.355829+0000","last_change":"2023-09-23T20:17:50.313197+0000","last_active":"2023-09-23T20:21:34.355829+0000","last_peered":"2023-09-23T20:21:34.355829+0000","last_clean":"2023-09-23T20:21:34.355829+0000","last_became_active":"2023-09-23T20:17:50.312838+0000","last_became_peered":"2023-09-23T20:17:50.312838+0000","last_unstale":"2023-09-23T20:21:34.355829+0000","last_undegraded":"2023-09-23T20:21:34.355829+0000","last_fullsized":"2023-09-23T20:21:34.355829+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:31.530167+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":402,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338641+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:21:34.338641+0000","last_peered":"2023-09-23T20:21:34.338641+0000","last_clean":"2023-09-23T20:21:34.338641+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:21:34.338641+0000","last_undegraded":"2023-09-23T20:21:34.338641+0000","last_fullsized":"2023-09-23T20:21:34.338641+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":334,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356150+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:21:34.356150+0000","last_peered":"2023-09-23T20:21:34.356150+0000","last_clean":"2023-09-23T20:21:34.356150+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:21:34.356150+0000","last_undegraded":"2023-09-23T20:21:34.356150+0000","last_fullsized":"2023-09-23T20:21:34.356150+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":14,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.337843+0000","last_change":"2023-09-23T20:21:26.117225+0000","last_active":"2023-09-23T20:21:34.337843+0000","last_peered":"2023-09-23T20:21:34.337843+0000","last_clean":"2023-09-23T20:21:34.337843+0000","last_became_active":"2023-09-23T20:21:26.116915+0000","last_became_peered":"2023-09-23T20:21:26.116915+0000","last_unstale":"2023-09-23T20:21:34.337843+0000","last_undegraded":"2023-09-23T20:21:34.337843+0000","last_fullsized":"2023-09-23T20:21:34.337843+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:18.770524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":560,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.862928+0000","last_change":"2023-09-23T20:21:18.875984+0000","last_active":"2023-09-23T20:21:34.862928+0000","last_peered":"2023-09-23T20:21:34.862928+0000","last_clean":"2023-09-23T20:21:34.862928+0000","last_became_active":"2023-09-23T20:17:50.329940+0000","last_became_peered":"2023-09-23T20:17:50.329940+0000","last_unstale":"2023-09-23T20:21:34.862928+0000","last_undegraded":"2023-09-23T20:21:34.862928+0000","last_fullsized":"2023-09-23T20:21:34.862928+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:18.875863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:18.875863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:17.358094+0000","scrub_duration":0.0015811600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'161","reported_seq":1061,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.355656+0000","last_change":"2023-09-23T20:17:50.313157+0000","last_active":"2023-09-23T20:21:34.355656+0000","last_peered":"2023-09-23T20:21:34.355656+0000","last_clean":"2023-09-23T20:21:34.355656+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:21:34.355656+0000","last_undegraded":"2023-09-23T20:21:34.355656+0000","last_fullsized":"2023-09-23T20:21:34.355656+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000417403,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1383,"num_read_kb":1492,"num_write":121,"num_write_kb":199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":485,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.337934+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:21:34.337934+0000","last_peered":"2023-09-23T20:21:34.337934+0000","last_clean":"2023-09-23T20:21:34.337934+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:21:34.337934+0000","last_undegraded":"2023-09-23T20:21:34.337934+0000","last_fullsized":"2023-09-23T20:21:34.337934+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":65,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.864050+0000","last_change":"2023-09-23T20:21:19.422717+0000","last_active":"2023-09-23T20:21:34.864050+0000","last_peered":"2023-09-23T20:21:34.864050+0000","last_clean":"2023-09-23T20:21:16.602848+0000","last_became_active":"2023-09-23T20:21:19.422717+0000","last_became_peered":"2023-09-23T20:21:19.422717+0000","last_unstale":"2023-09-23T20:21:34.864050+0000","last_undegraded":"2023-09-23T20:21:34.864050+0000","last_fullsized":"2023-09-23T20:21:19.420290+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:44.277154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":11,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863784+0000","last_change":"2023-09-23T20:21:26.109485+0000","last_active":"2023-09-23T20:21:34.863784+0000","last_peered":"2023-09-23T20:21:34.863784+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.109485+0000","last_became_peered":"2023-09-23T20:21:26.109485+0000","last_unstale":"2023-09-23T20:21:34.863784+0000","last_undegraded":"2023-09-23T20:21:34.863784+0000","last_fullsized":"2023-09-23T20:21:26.095138+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:15.508535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":746,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:21:43.447995+0000","last_change":"2023-09-23T20:21:19.413132+0000","last_active":"2023-09-23T20:21:43.447995+0000","last_peered":"2023-09-23T20:21:43.447995+0000","last_clean":"2023-09-23T20:21:43.447995+0000","last_became_active":"2023-09-23T20:19:05.092070+0000","last_became_peered":"2023-09-23T20:19:05.092070+0000","last_unstale":"2023-09-23T20:21:43.447995+0000","last_undegraded":"2023-09-23T20:21:43.447995+0000","last_fullsized":"2023-09-23T20:21:43.447995+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:19.413043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:19.413043+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:11.511055+0000","scrub_duration":0.0014842639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'45","reported_seq":237,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356509+0000","last_change":"2023-09-23T20:07:50.840962+0000","last_active":"2023-09-23T20:21:34.356509+0000","last_peered":"2023-09-23T20:21:34.356509+0000","last_clean":"2023-09-23T20:21:34.356509+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:21:34.356509+0000","last_undegraded":"2023-09-23T20:21:34.356509+0000","last_fullsized":"2023-09-23T20:21:34.356509+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031785599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":125,"num_read_kb":81,"num_write":49,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863763+0000","last_change":"2023-09-23T20:21:19.422083+0000","last_active":"2023-09-23T20:21:34.863763+0000","last_peered":"2023-09-23T20:21:34.863763+0000","last_clean":"2023-09-23T20:21:16.602031+0000","last_became_active":"2023-09-23T20:21:19.422083+0000","last_became_peered":"2023-09-23T20:21:19.422083+0000","last_unstale":"2023-09-23T20:21:34.863763+0000","last_undegraded":"2023-09-23T20:21:34.863763+0000","last_fullsized":"2023-09-23T20:21:19.418680+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:45.222502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":363,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338892+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:21:34.338892+0000","last_peered":"2023-09-23T20:21:34.338892+0000","last_clean":"2023-09-23T20:21:34.338892+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:21:34.338892+0000","last_undegraded":"2023-09-23T20:21:34.338892+0000","last_fullsized":"2023-09-23T20:21:34.338892+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"533'7","reported_seq":20,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.356684+0000","last_change":"2023-09-23T20:21:27.159782+0000","last_active":"2023-09-23T20:21:34.356684+0000","last_peered":"2023-09-23T20:21:34.356684+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.099528+0000","last_became_peered":"2023-09-23T20:21:26.099528+0000","last_unstale":"2023-09-23T20:21:34.356684+0000","last_undegraded":"2023-09-23T20:21:34.356684+0000","last_fullsized":"2023-09-23T20:21:26.096941+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:38.049209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":6,"num_read_kb":8192,"num_write":4,"num_write_kb":4096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":497,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.339033+0000","last_change":"2023-09-23T20:21:19.421286+0000","last_active":"2023-09-23T20:21:34.339033+0000","last_peered":"2023-09-23T20:21:34.339033+0000","last_clean":"2023-09-23T19:54:37.307730+0000","last_became_active":"2023-09-23T20:21:19.421286+0000","last_became_peered":"2023-09-23T20:21:19.421286+0000","last_unstale":"2023-09-23T20:21:34.339033+0000","last_undegraded":"2023-09-23T20:21:34.339033+0000","last_fullsized":"2023-09-23T20:21:19.417211+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:11.941739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.941739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002908912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":379,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356697+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:21:34.356697+0000","last_peered":"2023-09-23T20:21:34.356697+0000","last_clean":"2023-09-23T20:21:34.356697+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:21:34.356697+0000","last_undegraded":"2023-09-23T20:21:34.356697+0000","last_fullsized":"2023-09-23T20:21:34.356697+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'140","reported_seq":616,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863966+0000","last_change":"2023-09-23T20:17:50.311535+0000","last_active":"2023-09-23T20:21:34.863966+0000","last_peered":"2023-09-23T20:21:34.863966+0000","last_clean":"2023-09-23T20:21:34.863966+0000","last_became_active":"2023-09-23T20:17:50.310876+0000","last_became_peered":"2023-09-23T20:17:50.310876+0000","last_unstale":"2023-09-23T20:21:34.863966+0000","last_undegraded":"2023-09-23T20:21:34.863966+0000","last_fullsized":"2023-09-23T20:21:34.863966+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.794641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019416430000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":465,"num_read_kb":2791,"num_write":126,"num_write_kb":2322,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":172,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.337596+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:21:34.337596+0000","last_peered":"2023-09-23T20:21:34.337596+0000","last_clean":"2023-09-23T20:21:34.337596+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:21:34.337596+0000","last_undegraded":"2023-09-23T20:21:34.337596+0000","last_fullsized":"2023-09-23T20:21:34.337596+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"533'1","reported_seq":16,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863731+0000","last_change":"2023-09-23T20:21:26.108186+0000","last_active":"2023-09-23T20:21:34.863731+0000","last_peered":"2023-09-23T20:21:34.863731+0000","last_clean":"2023-09-23T20:21:34.863731+0000","last_became_active":"2023-09-23T20:21:26.107787+0000","last_became_peered":"2023-09-23T20:21:26.107787+0000","last_unstale":"2023-09-23T20:21:34.863731+0000","last_undegraded":"2023-09-23T20:21:34.863731+0000","last_fullsized":"2023-09-23T20:21:34.863731+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:13.846369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":865,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338734+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:21:34.338734+0000","last_peered":"2023-09-23T20:21:34.338734+0000","last_clean":"2023-09-23T20:21:34.338734+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:21:34.338734+0000","last_undegraded":"2023-09-23T20:21:34.338734+0000","last_fullsized":"2023-09-23T20:21:34.338734+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'73","reported_seq":250,"reported_epoch":536,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:21:34.356921+0000","last_change":"2023-09-23T20:21:19.424950+0000","last_active":"2023-09-23T20:21:34.356921+0000","last_peered":"2023-09-23T20:21:34.356921+0000","last_clean":"2023-09-23T20:21:16.394241+0000","last_became_active":"2023-09-23T20:21:19.424950+0000","last_became_peered":"2023-09-23T20:21:19.424950+0000","last_unstale":"2023-09-23T20:21:34.356921+0000","last_undegraded":"2023-09-23T20:21:34.356921+0000","last_fullsized":"2023-09-23T20:21:19.422321+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:28.720349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028323399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":456,"num_read_kb":406,"num_write":69,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":63,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863820+0000","last_change":"2023-09-23T20:17:47.284676+0000","last_active":"2023-09-23T20:21:34.863820+0000","last_peered":"2023-09-23T20:21:34.863820+0000","last_clean":"2023-09-23T20:21:34.863820+0000","last_became_active":"2023-09-23T20:17:47.277321+0000","last_became_peered":"2023-09-23T20:17:47.277321+0000","last_unstale":"2023-09-23T20:21:34.863820+0000","last_undegraded":"2023-09-23T20:21:34.863820+0000","last_fullsized":"2023-09-23T20:21:34.863820+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.082445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":337,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338107+0000","last_change":"2023-09-23T20:18:38.011785+0000","last_active":"2023-09-23T20:21:34.338107+0000","last_peered":"2023-09-23T20:21:34.338107+0000","last_clean":"2023-09-23T20:18:35.261768+0000","last_became_active":"2023-09-23T20:18:38.011785+0000","last_became_peered":"2023-09-23T20:18:38.011785+0000","last_unstale":"2023-09-23T20:21:34.338107+0000","last_undegraded":"2023-09-23T20:21:34.338107+0000","last_fullsized":"2023-09-23T20:18:38.001274+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:00.398059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"0'0","reported_seq":13,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338856+0000","last_change":"2023-09-23T20:21:26.117122+0000","last_active":"2023-09-23T20:21:34.338856+0000","last_peered":"2023-09-23T20:21:34.338856+0000","last_clean":"2023-09-23T20:21:34.338856+0000","last_became_active":"2023-09-23T20:21:26.116794+0000","last_became_peered":"2023-09-23T20:21:26.116794+0000","last_unstale":"2023-09-23T20:21:34.338856+0000","last_undegraded":"2023-09-23T20:21:34.338856+0000","last_fullsized":"2023-09-23T20:21:34.338856+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:56.978661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":945,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.355772+0000","last_change":"2023-09-23T20:21:19.425177+0000","last_active":"2023-09-23T20:21:34.355772+0000","last_peered":"2023-09-23T20:21:34.355772+0000","last_clean":"2023-09-23T20:21:16.393190+0000","last_became_active":"2023-09-23T20:21:19.425177+0000","last_became_peered":"2023-09-23T20:21:19.425177+0000","last_unstale":"2023-09-23T20:21:34.355772+0000","last_undegraded":"2023-09-23T20:21:34.355772+0000","last_fullsized":"2023-09-23T20:21:19.421689+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_clean_scrub_stamp":"2023-09-23T20:19:56.180203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:52.985095+0000","scrub_duration":0.012799305,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":103,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.356058+0000","last_change":"2023-09-23T20:21:19.425051+0000","last_active":"2023-09-23T20:21:34.356058+0000","last_peered":"2023-09-23T20:21:34.356058+0000","last_clean":"2023-09-23T20:07:24.087089+0000","last_became_active":"2023-09-23T20:21:19.425051+0000","last_became_peered":"2023-09-23T20:21:19.425051+0000","last_unstale":"2023-09-23T20:21:34.356058+0000","last_undegraded":"2023-09-23T20:21:34.356058+0000","last_fullsized":"2023-09-23T20:21:19.421743+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:23.902489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":282,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863616+0000","last_change":"2023-09-23T20:21:19.422404+0000","last_active":"2023-09-23T20:21:34.863616+0000","last_peered":"2023-09-23T20:21:34.863616+0000","last_clean":"2023-09-23T20:17:42.211994+0000","last_became_active":"2023-09-23T20:21:19.422404+0000","last_became_peered":"2023-09-23T20:21:19.422404+0000","last_unstale":"2023-09-23T20:21:34.863616+0000","last_undegraded":"2023-09-23T20:21:34.863616+0000","last_fullsized":"2023-09-23T20:21:19.418975+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:54.318949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.14","version":"0'0","reported_seq":11,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863342+0000","last_change":"2023-09-23T20:21:26.107890+0000","last_active":"2023-09-23T20:21:34.863342+0000","last_peered":"2023-09-23T20:21:34.863342+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.107890+0000","last_became_peered":"2023-09-23T20:21:26.107890+0000","last_unstale":"2023-09-23T20:21:34.863342+0000","last_undegraded":"2023-09-23T20:21:34.863342+0000","last_fullsized":"2023-09-23T20:21:26.095766+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:47.598260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":765,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356481+0000","last_change":"2023-09-23T20:21:29.073767+0000","last_active":"2023-09-23T20:21:34.356481+0000","last_peered":"2023-09-23T20:21:34.356481+0000","last_clean":"2023-09-23T20:21:34.356481+0000","last_became_active":"2023-09-23T20:19:05.071787+0000","last_became_peered":"2023-09-23T20:19:05.071787+0000","last_unstale":"2023-09-23T20:21:34.356481+0000","last_undegraded":"2023-09-23T20:21:34.356481+0000","last_fullsized":"2023-09-23T20:21:34.356481+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:29.073665+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:21:29.073665+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:22.713699+0000","scrub_duration":0.00139634,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":568,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338407+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:21:34.338407+0000","last_peered":"2023-09-23T20:21:34.338407+0000","last_clean":"2023-09-23T20:21:34.338407+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:21:34.338407+0000","last_undegraded":"2023-09-23T20:21:34.338407+0000","last_fullsized":"2023-09-23T20:21:34.338407+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":568,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338521+0000","last_change":"2023-09-23T20:17:47.266215+0000","last_active":"2023-09-23T20:21:34.338521+0000","last_peered":"2023-09-23T20:21:34.338521+0000","last_clean":"2023-09-23T20:21:34.338521+0000","last_became_active":"2023-09-23T20:17:47.265890+0000","last_became_peered":"2023-09-23T20:17:47.265890+0000","last_unstale":"2023-09-23T20:21:34.338521+0000","last_undegraded":"2023-09-23T20:21:34.338521+0000","last_fullsized":"2023-09-23T20:21:34.338521+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:59.854975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"0'0","reported_seq":11,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863812+0000","last_change":"2023-09-23T20:21:26.107729+0000","last_active":"2023-09-23T20:21:34.863812+0000","last_peered":"2023-09-23T20:21:34.863812+0000","last_clean":"2023-09-23T20:21:25.081362+0000","last_became_active":"2023-09-23T20:21:26.107729+0000","last_became_peered":"2023-09-23T20:21:26.107729+0000","last_unstale":"2023-09-23T20:21:34.863812+0000","last_undegraded":"2023-09-23T20:21:34.863812+0000","last_fullsized":"2023-09-23T20:21:26.095358+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:35.064604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":558,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338671+0000","last_change":"2023-09-23T20:17:47.288753+0000","last_active":"2023-09-23T20:21:34.338671+0000","last_peered":"2023-09-23T20:21:34.338671+0000","last_clean":"2023-09-23T20:21:34.338671+0000","last_became_active":"2023-09-23T20:17:47.270488+0000","last_became_peered":"2023-09-23T20:17:47.270488+0000","last_unstale":"2023-09-23T20:21:34.338671+0000","last_undegraded":"2023-09-23T20:21:34.338671+0000","last_fullsized":"2023-09-23T20:21:34.338671+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:35.837184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":533,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:35.501078+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:21:35.501078+0000","last_peered":"2023-09-23T20:21:35.501078+0000","last_clean":"2023-09-23T20:21:35.501078+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:21:35.501078+0000","last_undegraded":"2023-09-23T20:21:35.501078+0000","last_fullsized":"2023-09-23T20:21:35.501078+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"533'11","reported_seq":29,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.338753+0000","last_change":"2023-09-23T20:21:26.101586+0000","last_active":"2023-09-23T20:21:34.338753+0000","last_peered":"2023-09-23T20:21:34.338753+0000","last_clean":"2023-09-23T20:21:34.338753+0000","last_became_active":"2023-09-23T20:21:26.101294+0000","last_became_peered":"2023-09-23T20:21:26.101294+0000","last_unstale":"2023-09-23T20:21:34.338753+0000","last_undegraded":"2023-09-23T20:21:34.338753+0000","last_fullsized":"2023-09-23T20:21:34.338753+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:53.208791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":29,"num_write":7,"num_write_kb":10,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":913,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:37.139948+0000","last_change":"2023-09-23T20:21:17.621262+0000","last_active":"2023-09-23T20:21:37.139948+0000","last_peered":"2023-09-23T20:21:37.139948+0000","last_clean":"2023-09-23T20:21:37.139948+0000","last_became_active":"2023-09-23T20:19:05.086254+0000","last_became_peered":"2023-09-23T20:19:05.086254+0000","last_unstale":"2023-09-23T20:21:37.139948+0000","last_undegraded":"2023-09-23T20:21:37.139948+0000","last_fullsized":"2023-09-23T20:21:37.139948+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:21:17.621169+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:21:17.621169+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:20.261133+0000","scrub_duration":0.0018371310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":217,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.863584+0000","last_change":"2023-09-23T20:21:19.422426+0000","last_active":"2023-09-23T20:21:34.863584+0000","last_peered":"2023-09-23T20:21:34.863584+0000","last_clean":"2023-09-23T20:17:42.212603+0000","last_became_active":"2023-09-23T20:21:19.422426+0000","last_became_peered":"2023-09-23T20:21:19.422426+0000","last_unstale":"2023-09-23T20:21:34.863584+0000","last_undegraded":"2023-09-23T20:21:34.863584+0000","last_fullsized":"2023-09-23T20:21:19.419497+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:47.408072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":195,"reported_epoch":536,"state":"active+undersized","last_fresh":"2023-09-23T20:21:34.338103+0000","last_change":"2023-09-23T20:21:19.422350+0000","last_active":"2023-09-23T20:21:34.338103+0000","last_peered":"2023-09-23T20:21:34.338103+0000","last_clean":"2023-09-23T19:49:51.368969+0000","last_became_active":"2023-09-23T20:21:19.422350+0000","last_became_peered":"2023-09-23T20:21:19.422350+0000","last_unstale":"2023-09-23T20:21:34.338103+0000","last_undegraded":"2023-09-23T20:21:34.338103+0000","last_fullsized":"2023-09-23T20:21:19.415245+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:28.648556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"0'0","reported_seq":13,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.356462+0000","last_change":"2023-09-23T20:21:26.103532+0000","last_active":"2023-09-23T20:21:34.356462+0000","last_peered":"2023-09-23T20:21:34.356462+0000","last_clean":"2023-09-23T20:21:34.356462+0000","last_became_active":"2023-09-23T20:21:26.103284+0000","last_became_peered":"2023-09-23T20:21:26.103284+0000","last_unstale":"2023-09-23T20:21:34.356462+0000","last_undegraded":"2023-09-23T20:21:34.356462+0000","last_fullsized":"2023-09-23T20:21:34.356462+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:21:25.081362+0000","last_clean_scrub_stamp":"2023-09-23T20:21:25.081362+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:07.217311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":842,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:21:45.082151+0000","last_change":"2023-09-23T20:21:45.082109+0000","last_active":"2023-09-23T20:21:45.082151+0000","last_peered":"2023-09-23T20:21:45.082151+0000","last_clean":"2023-09-23T20:21:45.082151+0000","last_became_active":"2023-09-23T20:19:05.067188+0000","last_became_peered":"2023-09-23T20:19:05.067188+0000","last_unstale":"2023-09-23T20:21:45.082151+0000","last_undegraded":"2023-09-23T20:21:45.082151+0000","last_fullsized":"2023-09-23T20:21:45.082151+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:45.082027+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:21:45.082027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:56.216190+0000","scrub_duration":0.0017658070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":389,"reported_epoch":536,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:21:34.339119+0000","last_change":"2023-09-23T20:21:19.422811+0000","last_active":"2023-09-23T20:21:34.339119+0000","last_peered":"2023-09-23T20:21:34.339119+0000","last_clean":"2023-09-23T20:21:16.633070+0000","last_became_active":"2023-09-23T20:21:19.422811+0000","last_became_peered":"2023-09-23T20:21:19.422811+0000","last_unstale":"2023-09-23T20:21:34.339119+0000","last_undegraded":"2023-09-23T20:21:19.416170+0000","last_fullsized":"2023-09-23T20:21:19.416064+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:41.667749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:21:34.863249+0000","last_change":"2023-09-23T20:17:47.267673+0000","last_active":"2023-09-23T20:21:34.863249+0000","last_peered":"2023-09-23T20:21:34.863249+0000","last_clean":"2023-09-23T20:21:34.863249+0000","last_became_active":"2023-09-23T20:17:47.267216+0000","last_became_peered":"2023-09-23T20:17:47.267216+0000","last_unstale":"2023-09-23T20:21:34.863249+0000","last_undegraded":"2023-09-23T20:21:34.863249+0000","last_fullsized":"2023-09-23T20:21:34.863249+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:37.855890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":32,"stat_sum":{"num_bytes":20,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":88,"num_read_kb":8291,"num_write":49,"num_write_kb":4128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":9,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":40,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":75,"acting":75,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":10952,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33632,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":84,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":699595,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":83,"acting":83,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":8388703,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234488,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":734528,"kb_used_data":150776,"kb_used_omap":0,"kb_used_meta":583744,"kb_avail":93010624,"statfs":{"total":95995035648,"available":95242878976,"internally_reserved":0,"allocated":154394624,"data_stored":152797683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":597753856},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":17.827999999999999,"5min":7.8849999999999998,"15min":4.6340000000000003},"min":{"1min":0.22700000000000001,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":230.952,"5min":230.952,"15min":230.952},"last":0.52800000000000002},{"interface":"front","average":{"1min":18.509,"5min":8.2889999999999997,"15min":4.8810000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.222},"max":{"1min":231.08699999999999,"5min":231.08699999999999,"15min":231.08699999999999},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":2.0550000000000002,"15min":2.4510000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.21299999999999999},"max":{"1min":13.337,"5min":61.113999999999997,"15min":91.335999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.4219999999999999,"5min":2.234,"15min":2.6909999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.223},"max":{"1min":6.3769999999999998,"5min":61.082999999999998,"15min":91.516999999999996},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:21:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":5.585,"15min":4.6130000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":18.73,"5min":177.852,"15min":177.852},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.6629999999999998,"5min":5.7220000000000004,"15min":4.827},"min":{"1min":0.27800000000000002,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":19.018000000000001,"5min":176.97900000000001,"15min":176.97900000000001},"last":0.38700000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":4.04,"15min":4.1109999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":19.620000000000001,"5min":112.291,"15min":185.17099999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.5179999999999998,"15min":4.2649999999999997},"min":{"1min":0.33700000000000002,"5min":0.32000000000000001,"15min":0.28999999999999998},"max":{"1min":19.855,"5min":112.887,"15min":184.99100000000001},"last":0.56100000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.466,"15min":0.98899999999999999},"min":{"1min":0.26900000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.55800000000000005,"5min":83.811999999999998,"15min":83.811999999999998},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.532,"15min":1.095},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":2.851,"5min":84.620000000000005,"15min":84.620000000000005},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.99399999999999999,"15min":1.179},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.6279999999999999,"5min":16.327000000000002,"15min":24.876000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":1.0609999999999999,"15min":1.073},"min":{"1min":0.314,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.57499999999999996,"5min":16.716000000000001,"15min":21.623000000000001},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.3779999999999999,"15min":1.0569999999999999},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.3420000000000001,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.163,"15min":0.82199999999999995},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1400000000000001,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.436}]}]},{"osd":6,"up_from":251,"seq":1078036791490,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273876,"kb_used_data":233868,"kb_used_omap":6,"kb_used_meta":39993,"kb_avail":93471276,"statfs":{"total":95995035648,"available":95714586624,"internally_reserved":0,"allocated":239480832,"data_stored":238014233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":40953471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":8.9100000000000001,"5min":3.8820000000000001,"15min":2.6890000000000001},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":53.780000000000001,"5min":74.787000000000006,"15min":126.64},"last":0.27000000000000002},{"interface":"front","average":{"1min":9.0090000000000003,"5min":4.0289999999999999,"15min":2.7570000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":53.768000000000001,"5min":74.751999999999995,"15min":126.658},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":4.3120000000000003,"15min":4.0830000000000002},"min":{"1min":0.26500000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":87.864999999999995,"5min":103.545,"15min":156.70699999999999},"last":5.4710000000000001},{"interface":"front","average":{"1min":5.1260000000000003,"5min":4.0279999999999996,"15min":4.0590000000000002},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.23899999999999999},"max":{"1min":87.772000000000006,"5min":100.018,"15min":156.68299999999999},"last":2.8690000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":6.891,"5min":2.4329999999999998,"15min":3.1179999999999999},"min":{"1min":0.315,"5min":0.254,"15min":0.23100000000000001},"max":{"1min":53.911999999999999,"5min":53.911999999999999,"15min":122.149},"last":0.47499999999999998},{"interface":"front","average":{"1min":7.3259999999999996,"5min":2.4129999999999998,"15min":3.0600000000000001},"min":{"1min":0.32400000000000001,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":54.125999999999998,"5min":54.125999999999998,"15min":122.23999999999999},"last":1.7789999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":5.181,"15min":3.8580000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":51.512999999999998,"5min":128.279,"15min":160.27699999999999},"last":4.8179999999999996},{"interface":"front","average":{"1min":3.4249999999999998,"5min":4.4969999999999999,"15min":3.8180000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":51.601999999999997,"5min":127.911,"15min":160.42699999999999},"last":2.9830000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.298,"15min":1.5780000000000001},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.248},"max":{"1min":2.641,"5min":34.534999999999997,"15min":145.29300000000001},"last":2.9140000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.077,"15min":1.538},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.3999999999999999,"5min":34.426000000000002,"15min":145.256},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":4.5880000000000001,"15min":2.972},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.25800000000000001},"max":{"1min":2.5329999999999999,"5min":218.58199999999999,"15min":218.58199999999999},"last":2.9500000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.343,"15min":2.8969999999999998},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.28499999999999998},"max":{"1min":2.4380000000000002,"5min":217.95599999999999,"15min":217.95599999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.1269999999999998,"15min":1.73},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.315},"max":{"1min":4.7380000000000004,"5min":137.63900000000001,"15min":137.63900000000001},"last":1},{"interface":"front","average":{"1min":0.751,"5min":2.8199999999999998,"15min":1.629},"min":{"1min":0.39700000000000002,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":2.3620000000000001,"5min":136.738,"15min":136.738},"last":0.48799999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762687,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856140,"kb_used_data":10756,"kb_used_omap":7,"kb_used_meta":845368,"kb_avail":92889012,"statfs":{"total":95995035648,"available":95118348288,"internally_reserved":0,"allocated":11014144,"data_stored":8472708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":865657658},"hb_peers":[0,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:38 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":5.4489999999999998,"15min":3.3039999999999998},"min":{"1min":0.221,"5min":0.19600000000000001,"15min":0.185},"max":{"1min":1.9690000000000001,"5min":145.81299999999999,"15min":145.81299999999999},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.476,"5min":6.0679999999999996,"15min":3.6459999999999999},"min":{"1min":0.25600000000000001,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":6.3700000000000001,"5min":146.83799999999999,"15min":146.83799999999999},"last":0.31900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:20:32 2023","interfaces":[{"interface":"back","average":{"1min":13.513,"5min":6.0099999999999998,"15min":4.9539999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":113.941,"5min":152.62,"15min":152.62},"last":0.33800000000000002},{"interface":"front","average":{"1min":14.683,"5min":6.234,"15min":4.702},"min":{"1min":0.318,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":114.34699999999999,"5min":154.84299999999999,"15min":154.84299999999999},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:21:38 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":3.0419999999999998,"15min":2.907},"min":{"1min":0.25,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.899,"5min":22.256,"15min":66.527000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.9039999999999999,"15min":2.2509999999999999},"min":{"1min":0.28100000000000003,"5min":0.219,"15min":0.219},"max":{"1min":6.1630000000000003,"5min":22.068000000000001,"15min":61.854999999999997},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:21:38 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":1.8919999999999999,"15min":1.6879999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.649,"5min":29.791,"15min":29.791},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.145,"5min":1.6890000000000001,"15min":1.6679999999999999},"min":{"1min":0.38900000000000001,"5min":0.34399999999999997,"15min":0.28100000000000003},"max":{"1min":5.9269999999999996,"5min":27.143999999999998,"15min":27.143999999999998},"last":0.34699999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:21:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.8220000000000001,"15min":2.0950000000000002},"min":{"1min":0.374,"5min":0.313,"15min":0.29299999999999998},"max":{"1min":5.8840000000000003,"5min":29.280000000000001,"15min":83.510999999999996},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.0419999999999998,"15min":2.2749999999999999},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.28699999999999998},"max":{"1min":6.5449999999999999,"5min":32.692,"15min":83.353999999999999},"last":1.306}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.2390000000000001,"15min":1.665},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":4.5830000000000002,"5min":9.2279999999999998,"15min":18.117999999999999},"last":5.6760000000000002},{"interface":"front","average":{"1min":0.75,"5min":1.325,"15min":1.843},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.0310000000000001,"5min":9.8659999999999997,"15min":20.734999999999999},"last":3.7330000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:21:38 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.879,"15min":2.52},"min":{"1min":0.36499999999999999,"5min":0.315,"15min":0.315},"max":{"1min":6.5140000000000002,"5min":12.599,"15min":127.03100000000001},"last":1.278},{"interface":"front","average":{"1min":0.70899999999999996,"5min":1.5860000000000001,"15min":2.1779999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.3999999999999999,"5min":10.340999999999999,"15min":125.095},"last":0.37}]}]},{"osd":0,"up_from":94,"seq":403726926225,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1104564,"kb_used_data":57068,"kb_used_omap":8,"kb_used_meta":1047479,"kb_avail":92640588,"statfs":{"total":95995035648,"available":94863962112,"internally_reserved":0,"allocated":58437632,"data_stored":55925588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1072619105},"hb_peers":[1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:21:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3740000000000001,"5min":2.1400000000000001,"15min":2.3159999999999998},"min":{"1min":0.26700000000000002,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":9.3529999999999998,"5min":17.425000000000001,"15min":72.534000000000006},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.774,"5min":2.9289999999999998,"15min":2.754},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":7.7830000000000004,"5min":21.949000000000002,"15min":72.667000000000002},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:20 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":2.8069999999999999,"15min":3.4100000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":63.710999999999999,"5min":63.710999999999999,"15min":160.542},"last":0.33100000000000002},{"interface":"front","average":{"1min":6.3049999999999997,"5min":3.5009999999999999,"15min":3.7730000000000001},"min":{"1min":0.255,"5min":0.23599999999999999,"15min":0.22600000000000001},"max":{"1min":60.518999999999998,"5min":60.518999999999998,"15min":160.75999999999999},"last":0.36599999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:21:15 2023","interfaces":[{"interface":"back","average":{"1min":12.856,"5min":6.2640000000000002,"15min":4.0860000000000003},"min":{"1min":0.29299999999999998,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":201.655,"5min":201.655,"15min":201.655},"last":0.42799999999999999},{"interface":"front","average":{"1min":13.087,"5min":6.0709999999999997,"15min":4.1840000000000002},"min":{"1min":0.33000000000000002,"5min":0.27800000000000002,"15min":0.27400000000000002},"max":{"1min":201.69399999999999,"5min":201.69399999999999,"15min":201.69399999999999},"last":0.371}]},{"osd":4,"last update":"Sat Sep 23 20:21:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":1.956,"15min":1.7490000000000001},"min":{"1min":0.33100000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.3680000000000003,"5min":16.593,"15min":16.593},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.69099999999999995,"5min":2.028,"15min":1.907},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.0470000000000002,"5min":21.899000000000001,"15min":105.282},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:21:41 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":2.8279999999999998,"15min":2.0299999999999998},"min":{"1min":0.379,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.4020000000000001,"5min":145.14500000000001,"15min":145.14500000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.3560000000000001,"5min":3.415,"15min":2.532},"min":{"1min":0.371,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":5.3120000000000003,"5min":145.04499999999999,"15min":145.04499999999999},"last":0.625}]},{"osd":6,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6339999999999999,"15min":1.6419999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":7.6500000000000004,"5min":7.6500000000000004,"15min":15.744999999999999},"last":4.1239999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.234,"15min":1.6020000000000001},"min":{"1min":0.44,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":9.8219999999999992,"15min":13.766999999999999},"last":0.82199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:21:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":2.5739999999999998,"15min":1.823},"min":{"1min":0.38900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.4320000000000004,"5min":99.486000000000004,"15min":99.486000000000004},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.52,"5min":3.4510000000000001,"15min":2.4449999999999998},"min":{"1min":0.39100000000000001,"5min":0.308,"15min":0.308},"max":{"1min":5.3440000000000003,"5min":99.668999999999997,"15min":99.668999999999997},"last":0.66400000000000003}]}]},{"osd":2,"up_from":104,"seq":446676599157,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004136,"kb_used_data":6816,"kb_used_omap":12,"kb_used_meta":997299,"kb_avail":92741016,"statfs":{"total":95995035648,"available":94966800384,"internally_reserved":0,"allocated":6979584,"data_stored":4470086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12290,"internal_metadata":1021235198},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9009999999999998,"5min":8.0299999999999994,"15min":4.2039999999999997},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":15.4,"5min":274.185,"15min":274.185},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.4729999999999999,"5min":5.8079999999999998,"15min":3.601},"min":{"1min":0.23200000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.8209999999999997,"5min":106.477,"15min":106.477},"last":0.26400000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":4.335,"5min":4.5,"15min":3.5249999999999999},"min":{"1min":0.39600000000000002,"5min":0.20399999999999999,"15min":0.19600000000000001},"max":{"1min":11.939,"5min":106.357,"15min":106.357},"last":0.27800000000000002},{"interface":"front","average":{"1min":4.1619999999999999,"5min":4.2969999999999997,"15min":3.1579999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":15.255000000000001,"5min":104.905,"15min":104.905},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 20:20:50 2023","interfaces":[{"interface":"back","average":{"1min":5.899,"5min":6.0259999999999998,"15min":4.1989999999999998},"min":{"1min":0.52000000000000002,"5min":0.21299999999999999,"15min":0.20799999999999999},"max":{"1min":28.178000000000001,"5min":126.345,"15min":132.82499999999999},"last":0.224},{"interface":"front","average":{"1min":4.5709999999999997,"5min":5.2510000000000003,"15min":3.5419999999999998},"min":{"1min":0.57199999999999995,"5min":0.23000000000000001,"15min":0.22600000000000001},"max":{"1min":15.302,"5min":126.40300000000001,"15min":130.97300000000001},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8450000000000002,"5min":2.2469999999999999,"15min":2.113},"min":{"1min":0.313,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.355,"5min":43.040999999999997,"15min":75.608000000000004},"last":0.377},{"interface":"front","average":{"1min":3.2549999999999999,"5min":2.8700000000000001,"15min":2.5470000000000002},"min":{"1min":0.36599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.864000000000001,"5min":48.951000000000001,"15min":76.161000000000001},"last":0.38400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":2.4049999999999998,"15min":2.6349999999999998},"min":{"1min":0.38500000000000001,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":15.378,"5min":43.716000000000001,"15min":135.202},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.585,"5min":2.5990000000000002,"15min":2.4710000000000001},"min":{"1min":0.41499999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.782,"5min":43.643999999999998,"15min":63.792000000000002},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.329,"15min":1.542},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5510000000000002,"5min":11.843,"15min":14.664},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.425,"15min":1.744},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.077,"5min":12.025,"15min":14.279999999999999},"last":4.9660000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.4889999999999999,"5min":2.6360000000000001,"15min":2.1560000000000001},"min":{"1min":0.46999999999999997,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":11.91,"5min":18.495999999999999,"15min":18.495999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.9990000000000001,"5min":2.077,"15min":1.7609999999999999},"min":{"1min":0.41299999999999998,"5min":0.39900000000000002,"15min":0.35299999999999998},"max":{"1min":15.327999999999999,"5min":15.327999999999999,"15min":15.327999999999999},"last":0.49299999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435636,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129612,"kb_used_data":96132,"kb_used_omap":4,"kb_used_meta":1033467,"kb_avail":92615540,"statfs":{"total":95995035648,"available":94838312960,"internally_reserved":0,"allocated":98439168,"data_stored":95601559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1058270317},"hb_peers":[0,1,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":14.129,"5min":7.8550000000000004,"15min":4.8159999999999998},"min":{"1min":0.27100000000000002,"5min":0.191,"15min":0.186},"max":{"1min":219.80000000000001,"5min":219.80000000000001,"15min":219.80000000000001},"last":0.372},{"interface":"front","average":{"1min":13.737,"5min":8.3599999999999994,"15min":5.0659999999999998},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":221.36000000000001,"5min":221.36000000000001,"15min":221.36000000000001},"last":0.29099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6520000000000001,"5min":2.9769999999999999,"15min":4.2160000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20300000000000001},"max":{"1min":9.0830000000000002,"5min":92.036000000000001,"15min":183.87},"last":0.27200000000000002},{"interface":"front","average":{"1min":3.145,"5min":2.972,"15min":4.4349999999999996},"min":{"1min":0.251,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":19.513000000000002,"5min":92.210999999999999,"15min":185.84899999999999},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":7.6070000000000002,"15min":5.4000000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.7549999999999999,"5min":128.369,"15min":163.83500000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":3.8420000000000001,"5min":7.4020000000000001,"15min":5.2409999999999997},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":15.286,"5min":126.497,"15min":163.92099999999999},"last":0.52200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.996,"15min":1.9179999999999999},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.28999999999999998},"max":{"1min":8.8350000000000009,"5min":54.046999999999997,"15min":58.472999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.988,"5min":2.6819999999999999,"15min":2.4470000000000001},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":5.4770000000000003,"5min":54.311,"15min":58.408000000000001},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":2.9159999999999999,"15min":3.4089999999999998},"min":{"1min":0.39400000000000002,"5min":0.32000000000000001,"15min":0.30299999999999999},"max":{"1min":7.5179999999999998,"5min":54.139000000000003,"15min":200.53200000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.359,"5min":2.573,"15min":3.4140000000000001},"min":{"1min":0.34899999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.5030000000000001,"5min":56.457999999999998,"15min":200.846},"last":0.62}]},{"osd":6,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.593,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":1.5329999999999999,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.3280000000000001,"15min":2.286},"min":{"1min":0.42699999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.2040000000000002,"5min":8.2629999999999999,"15min":82.233000000000004},"last":4.1509999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.706,"15min":2.5529999999999999},"min":{"1min":0.379,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.3940000000000001,"5min":17.986000000000001,"15min":139.465},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.4219999999999999,"5min":1.6299999999999999,"15min":2.5249999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.32800000000000001},"max":{"1min":8.9019999999999992,"5min":9.2289999999999992,"15min":141.56200000000001},"last":0.76600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019398,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":597816,"kb_used_data":5936,"kb_used_omap":0,"kb_used_meta":591872,"kb_avail":93147336,"statfs":{"total":95995035648,"available":95382872064,"internally_reserved":0,"allocated":6078464,"data_stored":4489683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":606076928},"hb_peers":[0,1,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:42 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":9.8670000000000009,"15min":4.2910000000000004},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":10.503,"5min":232.89500000000001,"15min":232.89500000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":9.859,"15min":4.2880000000000003},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":3.242,"5min":232.851,"15min":232.851},"last":0.435}]},{"osd":1,"last update":"Sat Sep 23 20:21:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9430000000000001,"5min":2.5049999999999999,"15min":3.0699999999999998},"min":{"1min":0.28199999999999997,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":26.420000000000002,"5min":114.05200000000001,"15min":138.47999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.6479999999999999,"5min":2.4289999999999998,"15min":3.1139999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.217},"max":{"1min":26.388999999999999,"5min":114.02800000000001,"15min":138.429},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":15.880000000000001,"5min":9.6950000000000003,"15min":6.7859999999999996},"min":{"1min":0.29899999999999999,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":204.345,"5min":204.345,"15min":204.345},"last":0.436},{"interface":"front","average":{"1min":15.916,"5min":9.6020000000000003,"15min":6.6429999999999998},"min":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":204.072,"5min":204.072,"15min":204.072},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:21:42 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":7.7670000000000003,"15min":5.3090000000000002},"min":{"1min":0.29899999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.3500000000000001,"5min":156.72300000000001,"15min":184.37100000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.51000000000000001,"5min":7.6760000000000002,"15min":5.3630000000000004},"min":{"1min":0.34999999999999998,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":0.77200000000000002,"5min":155.84800000000001,"15min":184.22300000000001},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:21:42 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":1.6719999999999999,"15min":1.649},"min":{"1min":0.29299999999999998,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":0.69099999999999995,"5min":114.788,"15min":176.81899999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.7450000000000001,"15min":1.6899999999999999},"min":{"1min":0.316,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.85199999999999998,"5min":113.822,"15min":176.172},"last":0.32000000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:17:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.0390000000000001,"15min":1.026},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":2.5110000000000001,"5min":114.232,"15min":114.232},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.3540000000000001,"15min":1.2130000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.247},"max":{"1min":4.085,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.772,"15min":1.4350000000000001},"min":{"1min":0.33100000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67600000000000005,"5min":113.861,"15min":113.861},"last":0.372},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.7410000000000001,"15min":1.4850000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.222},"max":{"1min":1.6379999999999999,"5min":114.32299999999999,"15min":114.32299999999999},"last":0.34999999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757762,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665584,"kb_used_data":5864,"kb_used_omap":0,"kb_used_meta":659712,"kb_avail":93079568,"statfs":{"total":95995035648,"available":95313477632,"internally_reserved":0,"allocated":6004736,"data_stored":4420396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":675545088},"hb_peers":[0,1,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":8.8770000000000007,"15min":4.1859999999999999},"min":{"1min":0.253,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":25.452000000000002,"5min":166.57900000000001,"15min":166.57900000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.81699999999999995,"5min":10.395,"15min":4.7300000000000004},"min":{"1min":0.32900000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.754,"5min":256.52199999999999,"15min":256.52199999999999},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.6279999999999999,"15min":3.4359999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.22},"max":{"1min":8.5289999999999999,"5min":17.010999999999999,"15min":150.16999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.607,"5min":1.659,"15min":3.411},"min":{"1min":0.27300000000000002,"5min":0.27200000000000002,"15min":0.21199999999999999},"max":{"1min":8.3420000000000005,"5min":18.879000000000001,"15min":150.12700000000001},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:20:21 2023","interfaces":[{"interface":"back","average":{"1min":17.131,"5min":6.5279999999999996,"15min":5.3019999999999996},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":208.495,"5min":208.495,"15min":240.57300000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":17.167999999999999,"5min":6.5869999999999997,"15min":5.3659999999999997},"min":{"1min":0.33500000000000002,"5min":0.33100000000000002,"15min":0.30299999999999999},"max":{"1min":208.58199999999999,"5min":208.58199999999999,"15min":240.464},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:21:01 2023","interfaces":[{"interface":"back","average":{"1min":5.7240000000000002,"5min":8.7750000000000004,"15min":6.0579999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":54.5,"5min":196.97800000000001,"15min":196.97800000000001},"last":0.439},{"interface":"front","average":{"1min":5.008,"5min":8.7650000000000006,"15min":5.9420000000000002},"min":{"1min":0.35199999999999998,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":54.533000000000001,"5min":196.875,"15min":196.875},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:21:42 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.63900000000000001,"15min":0.80400000000000005},"min":{"1min":0.28999999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.7279999999999998,"5min":5.7279999999999998,"15min":9.7240000000000002},"last":0.31},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.54400000000000004,"15min":0.60399999999999998},"min":{"1min":0.28299999999999997,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":1.048,"5min":3.016,"15min":8.7360000000000007},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2490000000000001,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":4.04,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.73299999999999998,"15min":0.60999999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":1.609,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":1.492}]},{"osd":7,"last update":"Sat Sep 23 20:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.90600000000000003,"15min":0.96999999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.0469999999999997,"5min":6.8620000000000001,"15min":11.725},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.76700000000000002,"15min":0.91500000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0880000000000001,"5min":5.8499999999999996,"15min":11.041},"last":0.39100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7544,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6529,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":20,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":20,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:21:53.700 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:21:53.700 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 71 2023-09-23T20:21:53.707 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:53.767 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:21:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:21:53] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.43.0" 2023-09-23T20:21:53.770 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:54.029 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:54.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:54.133 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:54.165 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:54.214 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:54.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:54.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:54.289 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:54 smithi135 ceph-mon[221113]: from='client.36852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:54.289 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:54 smithi135 ceph-mon[221113]: pgmap v1539: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:54.289 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:54 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/606295047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:54.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:54 smithi135 ceph-mon[213871]: from='client.36852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:54.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:54 smithi135 ceph-mon[213871]: pgmap v1539: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:54.289 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:54 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/606295047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:54.290 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:21:54.321 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:54.347 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:54 smithi144 ceph-mon[194833]: from='client.36852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:21:54.348 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:54 smithi144 ceph-mon[194833]: pgmap v1539: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:54.348 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:54 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/606295047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:21:54.350 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:54.395 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:54.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:54.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:54.502 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:54.529 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:54.545 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:21:54.546 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:21:54.572 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:54.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:54.675 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:54.676 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:54.699 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:54.701 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:54.728 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:54.742 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:54.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:54.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:54.848 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:54.874 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:54.915 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:54.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:55.021 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:55.048 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:55.142 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:55.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:55.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:55.246 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:55.273 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:55.316 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:55.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:55.421 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:55.449 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:55.695 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:55.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:55.799 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:55.828 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:55.877 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:55.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:55.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:55.982 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:56.010 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:56.055 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:56.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:56.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:56.161 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:56.189 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:56.231 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:56.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:56.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:56.337 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:56.363 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:56.404 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:56.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:56.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:56.511 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:56.537 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:56.547 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:21:56.548 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:56 smithi135 ceph-mon[221113]: pgmap v1540: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:56.548 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:56 smithi135 ceph-mon[213871]: pgmap v1540: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:56.583 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:56.587 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:56 smithi144 ceph-mon[194833]: pgmap v1540: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:56.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:56.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:56.689 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:56.715 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:56.761 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:56.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:56.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:56.865 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:56.892 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:56.969 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:56.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:56.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:57.073 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:57.100 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:57.143 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:57.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:57.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:57.247 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:21:57.275 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:57.371 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:21:57.374 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:57 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:57.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:21:57.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:57 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:57.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:57 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:57.476 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:21:57.505 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:57.556 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:21:57.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:57.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:21:57.590 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:57.661 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:57.662 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:21:57.662 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:21:57.690 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:57.834 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:57.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:57.940 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:57.966 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:58.011 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:58.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:58.116 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:58.142 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:58.190 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:21:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:58.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:21:58.294 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:21:58.323 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:58.344 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:58 smithi144 ceph-mon[194833]: pgmap v1541: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:58.345 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1950359053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:58.345 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:58 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:58.346 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:21:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:58 smithi135 ceph-mon[221113]: pgmap v1541: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1950359053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:58 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:58 smithi135 ceph-mon[213871]: pgmap v1541: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:21:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1950359053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:58.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:58 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:21:58.937 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:21:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:58.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:21:59.041 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:21:59.072 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:59.422 INFO:teuthology.orchestra.run.smithi135.stderr:Error EINVAL: specified pg_num 71 < pg_num_max 32 2023-09-23T20:21:59.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:59 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:59 smithi135 ceph-mon[221113]: failed to encode map e539 with expected crc 2023-09-23T20:21:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:59 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:21:59.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:21:59 smithi135 ceph-mon[221113]: osdmap e539: 8 total, 6 up, 8 in 2023-09-23T20:21:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:59 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:59 smithi135 ceph-mon[213871]: failed to encode map e539 with expected crc 2023-09-23T20:21:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:59 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:21:59.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:21:59 smithi135 ceph-mon[213871]: osdmap e539: 8 total, 6 up, 8 in 2023-09-23T20:21:59.525 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:21:59.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:59 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:21:59.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:59 smithi144 ceph-mon[194833]: failed to encode map e539 with expected crc 2023-09-23T20:21:59.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:59 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:21:59.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:21:59 smithi144 ceph-mon[194833]: osdmap e539: 8 total, 6 up, 8 in 2023-09-23T20:21:59.669 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:21:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:59.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:21:59.775 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:21:59.802 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:21:59.845 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:21:59.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:21:59.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:21:59.951 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:21:59.975 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:00.020 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:00.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:00.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:00.129 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:00.155 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:00.199 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:00.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:00.306 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:00.333 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:00.353 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[213871]: pgmap v1543: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:00.353 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2446572451' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:22:00.353 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[213871]: failed to encode map e539 with expected crc 2023-09-23T20:22:00.353 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[213871]: failed to encode map e539 with expected crc 2023-09-23T20:22:00.353 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:00.354 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:22:00.430 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:00.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:00.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:00.533 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:00.560 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:00.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:00 smithi144 ceph-mon[194833]: pgmap v1543: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:00 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2446572451' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:22:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:00 smithi144 ceph-mon[194833]: failed to encode map e539 with expected crc 2023-09-23T20:22:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:00 smithi144 ceph-mon[194833]: failed to encode map e539 with expected crc 2023-09-23T20:22:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:00 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:00.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:00 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:00.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[221113]: pgmap v1543: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2446572451' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:22:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[221113]: failed to encode map e539 with expected crc 2023-09-23T20:22:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[221113]: failed to encode map e539 with expected crc 2023-09-23T20:22:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:00.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:00 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:00.764 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:00.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:00.865 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:00.868 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:00.900 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:00.949 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:00.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:00.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:01.055 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:01.083 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:01.245 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:01.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:01.348 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:01.382 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:01.430 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:01.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:01.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:01.442 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:22:01.463 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:22:01.484 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:22:01.503 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:22:01.523 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:22:01.537 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:01.542 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:22:01.542 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:22:01.542 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:22:01.543 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T20:22:01.543 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:22:01.543 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:22:01.543 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:22:01.543 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:22:01.543 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:22:01.566 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:01.610 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:01.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:01.715 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:01.744 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:02.121 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:22:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:02 smithi135 ceph-mon[213871]: pgmap v1544: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:02.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3637522817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:02 smithi135 ceph-mon[221113]: pgmap v1544: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:02.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3637522817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:02.494 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:02.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:02.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:02.598 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:02.629 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:02.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:02 smithi144 ceph-mon[194833]: pgmap v1544: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3637522817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:02.723 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:22:02.724 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:22:02.774 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:02.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:02.879 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:02.912 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:02.960 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:02.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:03.026 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:03.067 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:03.095 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:03.139 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:03.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:03.244 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:03.269 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:22:03.269 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [], 2023-09-23T20:22:03.269 INFO:teuthology.orchestra.run.smithi135.stdout: "num_ops": 0 2023-09-23T20:22:03.269 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:22:03.271 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:03.313 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:03.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:03.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:03.419 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:03.449 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:03.946 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:03.951 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:22:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:22:03] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.43.0" 2023-09-23T20:22:03.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:03.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:04.052 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:04.080 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:04.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:04 smithi135 ceph-mon[221113]: pgmap v1545: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:04.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:04 smithi135 ceph-mon[213871]: pgmap v1545: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:04.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:04 smithi144 ceph-mon[194833]: pgmap v1545: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:04.435 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:22:04.725 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:22:04.742 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:04.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:04.860 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:04.893 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:05.161 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:05.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:05.266 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:05.299 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:05.335 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:05.352 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:05.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:05.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:05.367 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6, 2] live_osds: [0, 1, 3, 4, 5, 7] 2023-09-23T20:22:05.367 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:22:05.367 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-09-23T20:22:05.367 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-09-23T20:22:05.368 DEBUG:teuthology.orchestra.run.smithi135:osd.2> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T20:22:05.370 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl start ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T20:22:05.414 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:05.454 INFO:journalctl@ceph.osd.2.smithi135.stdout:-- Logs begin at Sat 2023-09-23 19:19:07 UTC. -- 2023-09-23T20:22:05.461 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:05.490 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:05.535 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:05.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:05.641 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:05.668 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:05.710 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:05.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:05.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:05.815 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:05.846 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:06.095 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:06.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:06.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:06.207 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:06.233 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:06.275 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:06.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:06.335 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[221113]: pgmap v1546: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:06.335 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2775386750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:06.336 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:06.336 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:06.336 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:06.336 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[213871]: pgmap v1546: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:06.336 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2775386750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:06.336 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:06.336 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:06.336 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:06 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:06.337 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:22:06.383 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:06.409 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:06.451 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:06.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:06.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:06.556 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:06.580 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:06 smithi144 ceph-mon[194833]: pgmap v1546: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:06.580 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:06 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2775386750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:06.580 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:06 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:06.580 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:06 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:06.580 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:06 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:06.582 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:06.623 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:06.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:06.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:06.728 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:06.759 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:07.413 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:07.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:07.520 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:07.549 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:07.597 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:07.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:07 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:07.600 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:07 smithi144 ceph-mon[194833]: failed to encode map e540 with expected crc 2023-09-23T20:22:07.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:07 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:07.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:07 smithi144 ceph-mon[194833]: osdmap e540: 8 total, 6 up, 8 in 2023-09-23T20:22:07.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:07 smithi144 ceph-mon[194833]: failed to encode map e540 with expected crc 2023-09-23T20:22:07.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:07.601 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:07.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:07.702 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:07.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[221113]: failed to encode map e540 with expected crc 2023-09-23T20:22:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[221113]: osdmap e540: 8 total, 6 up, 8 in 2023-09-23T20:22:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[221113]: failed to encode map e540 with expected crc 2023-09-23T20:22:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:07.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:07.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:07.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[213871]: failed to encode map e540 with expected crc 2023-09-23T20:22:07.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:07.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[213871]: osdmap e540: 8 total, 6 up, 8 in 2023-09-23T20:22:07.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[213871]: failed to encode map e540 with expected crc 2023-09-23T20:22:07.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:07.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:07.729 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:07.772 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:07.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:07.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:07.877 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:07.911 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:08.202 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:08.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:08.306 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:08.353 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:08.446 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:08 smithi135 ceph-mon[221113]: pgmap v1548: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:08.447 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:08 smithi135 ceph-mon[221113]: failed to encode map e540 with expected crc 2023-09-23T20:22:08.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:08 smithi135 ceph-mon[213871]: pgmap v1548: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:08.447 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:08 smithi135 ceph-mon[213871]: failed to encode map e540 with expected crc 2023-09-23T20:22:08.451 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:08.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:08.555 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:08.591 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:08 smithi144 ceph-mon[194833]: pgmap v1548: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:08.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:08 smithi144 ceph-mon[194833]: failed to encode map e540 with expected crc 2023-09-23T20:22:08.663 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:22:08.689 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:22:08.711 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:22:08.720 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:08 smithi135 bash[671489]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:22:08.720 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:08 smithi135 bash[671489]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T20:22:08.721 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:08 smithi135 bash[671489]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T20:22:08.721 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:08 smithi135 bash[671489]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T20:22:08.721 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:08 smithi135 bash[671489]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T20:22:08.721 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:08 smithi135 bash[671489]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T20:22:08.721 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:08 smithi135 bash[671489]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T20:22:08.721 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:08 smithi135 bash[671489]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T20:22:08.735 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:22:08.753 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:22:08.772 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:22:08.772 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:22:08.773 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:22:08.773 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T20:22:08.773 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:22:08.773 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:22:08.773 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:22:08.773 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:22:08.773 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:22:08.989 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:08.990 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:09.046 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:09.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:09.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:09.151 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:09.179 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:09.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/372324603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:09.646 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/372324603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:09.646 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/372324603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:09.908 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:22:09.978 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:09.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:10.081 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:10.108 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:10.156 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:10.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:10.234 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:22:10.260 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:10.292 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:10.341 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:10.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:10.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:10.449 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:10.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:10 smithi135 ceph-mon[213871]: pgmap v1549: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:10.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:10 smithi135 ceph-mon[221113]: pgmap v1549: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:10.478 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:10.525 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:10.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:10.631 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:10 smithi144 ceph-mon[194833]: pgmap v1549: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:10.659 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:10.702 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:10.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:10.732 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:22:10.732 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:22:10.811 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:10.837 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:10.881 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:10.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:10.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:10.986 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:11.016 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:11.175 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:22:11.268 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:11.319 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:11.322 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:11.326 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:11 smithi135 ceph-osd[673243]: -- 172.21.15.135:0/3139084185 <== mon.2 v2:172.21.15.135:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5578f2a484e0 con 0x5578f2a1ac00 2023-09-23T20:22:11.326 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:11 smithi135 bash[673077]: 4236ae85039f7fdfbabc1c46d6400d9f5852ab80f86fb86b309d59df93e8a0f0 2023-09-23T20:22:11.326 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:11 smithi135 systemd[1]: Started Ceph osd.2 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T20:22:11.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:11.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:11.428 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:11.456 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:11.498 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:11.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:11.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:11.607 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:11.632 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:11.676 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:11.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:11.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:11.781 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:11.811 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:11.970 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:11 smithi135 conmon[673209]: 2023-09-23T20:22:11.681+0000 7ff136cd4640 -1 Falling back to public interface 2023-09-23T20:22:12.611 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:12 smithi135 ceph-mon[213871]: pgmap v1550: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:12.611 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:12 smithi135 ceph-mon[221113]: pgmap v1550: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:12.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:12 smithi144 ceph-mon[194833]: pgmap v1550: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:12.734 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:22:13.458 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:13.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:13.562 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:13.595 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:13.644 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:13.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:13.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:13.721 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:22:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:22:13] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-09-23T20:22:13.752 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:13.788 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:13.851 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:13.933 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:14.021 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:14.021 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:14.074 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:14.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:14.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:14.179 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:14.211 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:14.260 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:14.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:14.354 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:14 smithi135 ceph-mon[221113]: pgmap v1551: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:14.355 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:14 smithi135 ceph-mon[213871]: pgmap v1551: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:14.367 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:14.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:14 smithi144 ceph-mon[194833]: pgmap v1551: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:14.396 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:14.441 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:14.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:14.546 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:14.582 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:16.462 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:16.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:16.568 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:16.597 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:16 smithi135 ceph-mon[221113]: pgmap v1552: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:16.597 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:16 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3192615596' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:22:16.598 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:16 smithi135 ceph-mon[213871]: pgmap v1552: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:16.598 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:16 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3192615596' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:22:16.602 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:16.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:16 smithi144 ceph-mon[194833]: pgmap v1552: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:16.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:16 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3192615596' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:22:16.970 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:22:17.096 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:22:17.211 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:17.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:17.316 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:17.328 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:17.335 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:17.335 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.335 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.345 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:17.439 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:17.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:17.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3192615596' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: osdmap e541: 8 total, 6 up, 8 in 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:17.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3192615596' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: osdmap e541: 8 total, 6 up, 8 in 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:17.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:17 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:17.542 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:17.572 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3192615596' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:22:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: osdmap e541: 8 total, 6 up, 8 in 2023-09-23T20:22:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: failed to encode map e541 with expected crc 2023-09-23T20:22:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:17.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:17.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:17 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:18.278 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:22:18.291 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:18.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:18.314 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:22:18.340 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:22:18.356 INFO:teuthology.orchestra.run.smithi135.stderr:no valid command found; 10 closest matches: 2023-09-23T20:22:18.356 INFO:teuthology.orchestra.run.smithi135.stderr:0 2023-09-23T20:22:18.356 INFO:teuthology.orchestra.run.smithi135.stderr:1 2023-09-23T20:22:18.356 INFO:teuthology.orchestra.run.smithi135.stderr:2 2023-09-23T20:22:18.356 INFO:teuthology.orchestra.run.smithi135.stderr:abort 2023-09-23T20:22:18.357 INFO:teuthology.orchestra.run.smithi135.stderr:assert 2023-09-23T20:22:18.357 INFO:teuthology.orchestra.run.smithi135.stderr:bluefs debug_inject_read_zeros 2023-09-23T20:22:18.357 INFO:teuthology.orchestra.run.smithi135.stderr:bluefs files list 2023-09-23T20:22:18.357 INFO:teuthology.orchestra.run.smithi135.stderr:bluefs stats 2023-09-23T20:22:18.357 INFO:teuthology.orchestra.run.smithi135.stderr:bluestore allocator dump block 2023-09-23T20:22:18.357 INFO:teuthology.orchestra.run.smithi135.stderr:bluestore allocator fragmentation block 2023-09-23T20:22:18.357 INFO:teuthology.orchestra.run.smithi135.stderr:admin_socket: invalid command 2023-09-23T20:22:18.360 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:22:18.379 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:22:18.396 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:22:18.399 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:22:18.430 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:18.551 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:18 smithi144 ceph-mon[194833]: pgmap v1554: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:18.551 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1529971281' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:18.678 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:18 smithi135 ceph-mon[221113]: pgmap v1554: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:18.678 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1529971281' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:18.678 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:18 smithi135 ceph-mon[213871]: pgmap v1554: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:18.678 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1529971281' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:18.745 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:22:18.745 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:22:18.745 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:22:18.745 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:22:19.342 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:22:19.536 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:19.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:19.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:22:19.549 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-2, ['dump_ops_in_flight'] 2023-09-23T20:22:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:22:19.597 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:22:19.642 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:19.671 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:19.690 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:19.717 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:19.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:19.821 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:19.852 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:19.896 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:19.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:19.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:19.970 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:19 smithi135 conmon[673209]: 2023-09-23T20:22:19.485+0000 7ff136cd4640 -1 osd.2 529 log_to_monitors true 2023-09-23T20:22:20.000 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:20.026 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:20.120 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:20.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:20.130 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:22:20.224 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:20.251 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:20.293 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:20.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:20.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:20 smithi144 ceph-mon[194833]: pgmap v1555: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:20.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:20 smithi144 ceph-mon[194833]: from='osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:22:20.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:20 smithi144 ceph-mon[194833]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:22:20.397 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:20.421 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:22:20 smithi135 conmon[673209]: 2023-09-23T20:22:20.372+0000 7ff12713f700 -1 osd.2 529 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:22:20.422 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:20 smithi135 ceph-mon[221113]: pgmap v1555: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:20.422 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:20 smithi135 ceph-mon[221113]: from='osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:22:20.422 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:20 smithi135 ceph-mon[221113]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:22:20.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:20 smithi135 ceph-mon[213871]: pgmap v1555: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:20.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:20 smithi135 ceph-mon[213871]: from='osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:22:20.422 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:20 smithi135 ceph-mon[213871]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:22:20.424 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:20.481 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:20.520 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:20.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:20.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:20.625 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:20.652 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:20.698 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:20.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:20.803 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:20.830 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:21.026 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:21.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:21.112 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:22:21.112 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:22:21.112 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:22:21.112 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:22:21.112 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:22:21.130 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:21.158 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:21.354 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:21.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:21.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:21.458 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:21.484 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:21.598 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:22:21.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[221113]: failed to encode map e542 with expected crc 2023-09-23T20:22:21.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[221113]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:22:21.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[221113]: osdmap e542: 8 total, 6 up, 8 in 2023-09-23T20:22:21.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[221113]: from='osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T20:22:21.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[221113]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T20:22:21.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[221113]: failed to encode map e542 with expected crc 2023-09-23T20:22:21.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[213871]: failed to encode map e542 with expected crc 2023-09-23T20:22:21.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[213871]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:22:21.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[213871]: osdmap e542: 8 total, 6 up, 8 in 2023-09-23T20:22:21.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[213871]: from='osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T20:22:21.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[213871]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T20:22:21.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:21 smithi135 ceph-mon[213871]: failed to encode map e542 with expected crc 2023-09-23T20:22:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:21 smithi144 ceph-mon[194833]: failed to encode map e542 with expected crc 2023-09-23T20:22:21.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:21 smithi144 ceph-mon[194833]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:22:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:21 smithi144 ceph-mon[194833]: osdmap e542: 8 total, 6 up, 8 in 2023-09-23T20:22:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:21 smithi144 ceph-mon[194833]: from='osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T20:22:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:21 smithi144 ceph-mon[194833]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-09-23T20:22:21.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:21 smithi144 ceph-mon[194833]: failed to encode map e542 with expected crc 2023-09-23T20:22:21.681 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:22:21.683 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:21.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:21.788 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:21.813 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:21.857 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:21.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:21.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:21.962 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:21.993 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:22.029 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:22.046 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:22.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:22.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:22.066 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:22.151 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:22.177 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:22.272 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:22.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:22.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:22.376 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:22.390 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:22:22.393 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.get_size (55959 ms) 2023-09-23T20:22:22.393 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T20:22:22.399 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-09-23T20:22:22.399 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T20:22:22.410 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:22.418 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.protection_status (18 ms) 2023-09-23T20:22:22.418 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T20:22:22.425 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2023-09-23T20:22:22.425 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T20:22:22.436 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: pgmap v1557: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:22.436 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 16/117 objects degraded (13.675%), 11 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: failed to encode map e543 with expected crc 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: failed to encode map e543 with expected crc 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916] boot 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: osdmap e543: 8 total, 7 up, 8 in 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/438172333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:22.437 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:22 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:22.455 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:22.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:22.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:22.469 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.parents_v1 (43 ms) 2023-09-23T20:22:22.469 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T20:22:22.505 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.parents_v2 (37 ms) 2023-09-23T20:22:22.505 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T20:22:22.529 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.snapshots (23 ms) 2023-09-23T20:22:22.529 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T20:22:22.535 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.snapid_race (7 ms) 2023-09-23T20:22:22.535 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T20:22:22.544 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2023-09-23T20:22:22.544 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T20:22:22.547 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2023-09-23T20:22:22.547 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T20:22:22.554 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.object_map_resize (6 ms) 2023-09-23T20:22:22.554 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T20:22:22.558 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:22.561 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.object_map_update (8 ms) 2023-09-23T20:22:22.561 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T20:22:22.561 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T20:22:22.561 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T20:22:22.568 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2023-09-23T20:22:22.568 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T20:22:22.573 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2023-09-23T20:22:22.574 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T20:22:22.578 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-09-23T20:22:22.579 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T20:22:22.585 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:22.604 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.metadata (25 ms) 2023-09-23T20:22:22.604 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T20:22:22.613 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.set_features (9 ms) 2023-09-23T20:22:22.613 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T20:22:22.627 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: pgmap v1557: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:22.627 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 16/117 objects degraded (13.675%), 11 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: failed to encode map e543 with expected crc 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: failed to encode map e543 with expected crc 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916] boot 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: osdmap e543: 8 total, 7 up, 8 in 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/438172333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:22.628 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: pgmap v1557: 135 pgs: 1 active+undersized+remapped, 11 active+undersized+degraded, 2 undersized+peered, 38 active+undersized, 83 active+clean; 4.7 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 16/117 objects degraded (13.675%) 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 16/117 objects degraded (13.675%), 11 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: failed to encode map e543 with expected crc 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: failed to encode map e543 with expected crc 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: osd.2 [v2:172.21.15.135:6818/2108478916,v1:172.21.15.135:6819/2108478916] boot 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: osdmap e543: 8 total, 7 up, 8 in 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/438172333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:22.629 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:22 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:22.632 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:22.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:22.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:22.650 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.mirror (38 ms) 2023-09-23T20:22:22.650 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T20:22:22.666 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2023-09-23T20:22:22.666 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T20:22:22.736 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:22.762 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:22.802 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:22.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:22.906 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:22.931 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:23.155 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:23.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:23.260 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:23.287 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:23 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:22:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:23 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:23.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:23 smithi144 ceph-mon[194833]: osdmap e544: 8 total, 7 up, 8 in 2023-09-23T20:22:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:23 smithi144 ceph-mon[194833]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:23 smithi144 ceph-mon[194833]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:23 smithi144 ceph-mon[194833]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:23 smithi144 ceph-mon[194833]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:23 smithi144 ceph-mon[194833]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.382 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:23.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:23.486 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:23.518 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:23.562 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:23.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:23.605 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:22:23.605 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:22:23.605 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:22:23.605 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:22:23.605 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199768 9.4 9:22c43d57:::benchmark_data_smithi135_549536_object95913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.270504+0000", 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 346.33267018700002, 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.027306939, 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:23.606 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199768 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270504+0000", 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270504+0000", 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.607 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270502+0000", 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270509+0000", 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.6399999999999997e-06 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.608 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270525+0000", 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5775999999999999e-05 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270532+0000", 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9090000000000003e-06 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270605+0000", 2023-09-23T20:22:23.609 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2836000000000005e-05 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.270615+0000", 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0507e-05 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.290715+0000", 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0200998189999999 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.610 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.290845+0000", 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013020100000000001 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.291164+0000", 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000318645 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.611 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.296206+0000", 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0050415349999999998 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297663+0000", 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014569559999999999 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297758+0000", 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5564999999999997e-05 2023-09-23T20:22:23.612 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297786+0000", 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8314e-05 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297811+0000", 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4227e-05 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.613 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199783 9.4 9:2547b74f:::benchmark_data_smithi135_549536_object95928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.292081+0000", 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 346.311092675, 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0085980670000001, 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:23.614 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199783 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292081+0000", 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292081+0000", 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.615 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292080+0000", 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292086+0000", 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3169999999999998e-06 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.616 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292102+0000", 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5699999999999999e-05 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292114+0000", 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1664e-05 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292216+0000", 2023-09-23T20:22:23.617 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102756 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292228+0000", 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1745e-05 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.293068+0000", 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0008400669999999 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.618 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.293169+0000", 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000100556 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.293381+0000", 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000212189 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.619 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297718+0000", 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043367750000000002 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297841+0000", 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000122688 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300613+0000", 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027729080000000001 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.620 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300644+0000", 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0125e-05 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300679+0000", 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5787000000000001e-05 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199786 9.4 9:22231ef2:::benchmark_data_smithi135_549536_object95931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.292128+0000", 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 346.31104557700002, 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.020906723, 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:23.622 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199786 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292128+0000", 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292128+0000", 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.623 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292128+0000", 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292131+0000", 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.371e-06 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.624 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292138+0000", 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9979999999999997e-06 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292143+0000", 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4989999999999996e-06 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292248+0000", 2023-09-23T20:22:23.625 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000105475 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292255+0000", 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8399999999999997e-06 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.294615+0000", 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.002359612 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.626 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.294690+0000", 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5012000000000006e-05 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.294967+0000", 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00027756500000000002 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.627 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297846+0000", 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028783379999999998 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300715+0000", 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028695330000000001 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.312950+0000", 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012234910999999999 2023-09-23T20:22:23.628 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.312997+0000", 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6884e-05 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313035+0000", 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8204000000000002e-05 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.629 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199792 9.4 9:251d5e24:::benchmark_data_smithi135_549536_object95937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.292150+0000", 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 346.31102378100002, 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0209780289999999, 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:23.630 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199792 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292150+0000", 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292150+0000", 2023-09-23T20:22:23.631 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292150+0000", 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292153+0000", 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9979999999999999e-06 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.632 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292159+0000", 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6529999999999997e-06 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292164+0000", 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1389999999999997e-06 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292275+0000", 2023-09-23T20:22:23.633 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011100299999999999 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.292281+0000", 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6549999999999998e-06 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.296248+0000", 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0039670300000001 2023-09-23T20:22:23.634 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.296347+0000", 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.8867000000000001e-05 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.296539+0000", 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019160800000000001 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.635 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300566+0000", 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0040269700000000004 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300753+0000", 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018712399999999999 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313076+0000", 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012322783 2023-09-23T20:22:23.636 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313105+0000", 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.9830999999999999e-05 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313128+0000", 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2787000000000001e-05 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.637 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199801 9.4 9:24182093:::benchmark_data_smithi135_549536_object95946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.309108+0000", 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 346.29406601099998, 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0041302670000001, 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:23.638 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199801 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309108+0000", 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309108+0000", 2023-09-23T20:22:23.639 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309106+0000", 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309114+0000", 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2629999999999997e-06 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.640 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309130+0000", 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.6359999999999999e-05 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309137+0000", 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.939e-06 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.641 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309194+0000", 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6867999999999998e-05 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.309207+0000", 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2972e-05 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297877+0000", 2023-09-23T20:22:23.642 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98867036799999997 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.297942+0000", 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4992000000000004e-05 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.298141+0000", 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019871600000000001 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.643 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300720+0000", 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025793499999999998 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.303457+0000", 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0027370979999999999 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.644 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313162+0000", 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0097042410000000006 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313204+0000", 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2326000000000001e-05 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.313238+0000", 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4251999999999997e-05 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.645 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199833 9.4 9:23b4d44b:::benchmark_data_smithi135_549536_object95978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.536850+0000", 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 346.06632390300001, 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.78208931299999995, 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.646 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199833 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536850+0000", 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536850+0000", 2023-09-23T20:22:23.647 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536848+0000", 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536857+0000", 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2799999999999992e-06 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.648 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536888+0000", 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0354000000000001e-05 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536896+0000", 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.4409999999999993e-06 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.649 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536953+0000", 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6715000000000002e-05 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.536960+0000", 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5830000000000001e-06 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.299194+0000", 2023-09-23T20:22:23.650 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.76223389399999997 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.299260+0000", 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6210999999999993e-05 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.299488+0000", 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00022723599999999999 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.651 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.302080+0000", 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0025918429999999999 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.306190+0000", 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0041102559999999996 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.318857+0000", 2023-09-23T20:22:23.652 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012667223999999999 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.318905+0000", 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8121000000000002e-05 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.318939+0000", 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4177000000000002e-05 2023-09-23T20:22:23.653 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199843 9.4 9:202c07b4:::benchmark_data_smithi135_549536_object95988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.537102+0000", 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 346.066071722, 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.78193986900000001, 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.654 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199843 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537102+0000", 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.655 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537102+0000", 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537101+0000", 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537107+0000", 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0020000000000003e-06 2023-09-23T20:22:23.656 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537123+0000", 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5865000000000001e-05 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.537130+0000", 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6340000000000003e-06 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.657 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.540971+0000", 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038409939999999999 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.540981+0000", 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0468e-05 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300776+0000", 2023-09-23T20:22:23.658 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.759794788 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.300844+0000", 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7973999999999997e-05 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.301042+0000", 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00019837699999999999 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.659 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.306138+0000", 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0050955109999999996 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.306230+0000", 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.2410000000000005e-05 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.318982+0000", 2023-09-23T20:22:23.660 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.012751302000000001 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319017+0000", 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5030000000000002e-05 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319042+0000", 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5355999999999999e-05 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.661 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:199857 9.4 9:240ce8b0:::benchmark_data_smithi135_549536_object96002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:16:37.562071+0000", 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 346.04110250999997, 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.75710239400000001, 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.662 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 199857 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562071+0000", 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562071+0000", 2023-09-23T20:22:23.663 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562070+0000", 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562078+0000", 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7649999999999993e-06 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.664 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562097+0000", 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8938000000000001e-05 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562104+0000", 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5000000000000002e-06 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.665 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562164+0000", 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9426999999999999e-05 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:37.562175+0000", 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1391e-05 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.302116+0000", 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.73994101000000001 2023-09-23T20:22:23.666 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.302228+0000", 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000112235 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.302434+0000", 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000205183 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.667 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.306235+0000", 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038017849999999998 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.311586+0000", 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0053509880000000001 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.668 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319090+0000", 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.007503984 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319136+0000", 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5797999999999998e-05 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:16:38.319174+0000", 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7664999999999999e-05 2023-09-23T20:22:23.669 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:39763 10.2 10:4a79b0ff:::benchmark_data_smithi135_602286_object39762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:17:26.864094+0000", 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 296.73907986099999, 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21568243400000001, 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.670 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 39763 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864094+0000", 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.671 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864094+0000", 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864093+0000", 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864135+0000", 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1718999999999998e-05 2023-09-23T20:22:23.672 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864154+0000", 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9562999999999999e-05 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864162+0000", 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8159999999999994e-06 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.673 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864212+0000", 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9691000000000002e-05 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864351+0000", 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000139431 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.864500+0000", 2023-09-23T20:22:23.674 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014922500000000001 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.866289+0000", 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0017887249999999999 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.866811+0000", 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00052236999999999997 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.675 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.079697+0000", 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.21288605899999999 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.079744+0000", 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6805000000000003e-05 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.079777+0000", 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.2262999999999999e-05 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:39765 10.2 10:41a8f286:::benchmark_data_smithi135_602286_object39764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:17:26.868355+0000", 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 296.73481864500002, 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.214626228, 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.677 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 39765 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868355+0000", 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.678 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868355+0000", 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868354+0000", 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.679 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868446+0000", 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.1996999999999993e-05 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868460+0000", 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4025e-05 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868467+0000", 2023-09-23T20:22:23.680 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8680000000000001e-06 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868510+0000", 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3621999999999997e-05 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868642+0000", 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013202299999999999 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.681 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.868782+0000", 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000139572 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.870475+0000", 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001693022 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.682 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:26.873987+0000", 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0035118979999999998 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.082915+0000", 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.208928475 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.082954+0000", 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8390999999999999e-05 2023-09-23T20:22:23.683 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.082982+0000", 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7738000000000001e-05 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:40985 10.2 10:4d9f6e00:::benchmark_data_smithi135_602286_object40984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:17:27.832431+0000", 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 295.77074327999998, 2023-09-23T20:22:23.684 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28747203500000001, 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 40985 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832431+0000", 2023-09-23T20:22:23.685 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832431+0000", 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832430+0000", 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:22:23.686 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832488+0000", 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8402000000000001e-05 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832502+0000", 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4431e-05 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.687 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832509+0000", 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6499999999999999e-06 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832560+0000", 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0796e-05 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832723+0000", 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016322900000000001 2023-09-23T20:22:23.688 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.832857+0000", 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013411900000000001 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.834290+0000", 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0014332259999999999 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.689 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:27.834849+0000", 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00055816899999999996 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:28.119790+0000", 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.28494164500000002 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.690 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:28.119844+0000", 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3257000000000002e-05 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:17:28.119903+0000", 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.9191000000000003e-05 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.691 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:123189 10.f e514/503 10:f5618735:::benchmark_data_smithi135_602286_object30963:head v 514'7613, mlcod=499'5274)", 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:28.403820+0000", 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 175.199353562, 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15784634, 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403820+0000", 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.692 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403820+0000", 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403820+0000", 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403827+0000", 2023-09-23T20:22:23.693 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.5320000000000001e-06 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403835+0000", 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2520000000000002e-06 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403839+0000", 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.9790000000000004e-06 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.694 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403870+0000", 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1056999999999998e-05 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.403889+0000", 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.9298999999999999e-05 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.695 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.561622+0000", 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.157733082 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:28.561667+0000", 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4728000000000003e-05 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.696 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:124933 10.c e514/479 10:3ab62de7:::benchmark_data_smithi135_602286_object32707:head v 514'7830, mlcod=514'7828)", 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.284926+0000", 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 172.31824776799999, 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17721135599999999, 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284926+0000", 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.697 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284926+0000", 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284926+0000", 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284932+0000", 2023-09-23T20:22:23.698 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3659999999999997e-06 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284940+0000", 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.2740000000000007e-06 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284945+0000", 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8779999999999997e-06 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.699 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.284990+0000", 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4820000000000001e-05 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285008+0000", 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.8425999999999999e-05 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462103+0000", 2023-09-23T20:22:23.700 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17709439900000001 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462138+0000", 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4789000000000002e-05 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.701 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:124930 10.d e514/479 10:b5f42205:::benchmark_data_smithi135_602286_object32704:head v 514'7780, mlcod=514'7778)", 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.285891+0000", 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 172.31728305600001, 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.179623752, 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285891+0000", 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.702 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285891+0000", 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285890+0000", 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.703 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285896+0000", 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2319999999999998e-06 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285903+0000", 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285907+0000", 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5389999999999999e-06 2023-09-23T20:22:23.704 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285944+0000", 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6776999999999997e-05 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285957+0000", 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2959000000000001e-05 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.705 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.465467+0000", 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17951017699999999 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.465515+0000", 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7811e-05 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.706 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "MOSDPGPush(10.f 514/503 [PushOp(10:fca9ccf4:::benchmark_data_smithi135_602286_object89690:head, version: 500'5562, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:fca9ccf4:::benchmark_data_smithi135_602286_object89690:head@500'5562, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.285917+0000", 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 172.31725657300001, 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17891938700000001, 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "started", 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285917+0000", 2023-09-23T20:22:23.707 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285917+0000", 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285917+0000", 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967296 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.708 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285948+0000", 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0762999999999997e-05 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285957+0000", 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9120000000000004e-06 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.709 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285963+0000", 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8860000000000001e-06 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462080+0000", 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17611687100000001 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462095+0000", 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5579000000000001e-05 2023-09-23T20:22:23.710 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.464837+0000", 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002741556 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:124931 10.b e514/490 10:da9f67d3:::benchmark_data_smithi135_602286_object32705:head v 514'7623, mlcod=0'0)", 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.285983+0000", 2023-09-23T20:22:23.711 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 172.317190719, 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.179452687, 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285983+0000", 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.712 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285983+0000", 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285983+0000", 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285986+0000", 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3730000000000001e-06 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.713 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285990+0000", 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0940000000000001e-06 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.285993+0000", 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.4899999999999999e-06 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.714 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462116+0000", 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.17612323599999999 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462144+0000", 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8013000000000001e-05 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.465401+0000", 2023-09-23T20:22:23.715 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003256802 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.465436+0000", 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4978000000000003e-05 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:124932 10.4 e514/479 10:2bf1ccf2:::benchmark_data_smithi135_602286_object32706:head v 514'7828, mlcod=514'7826)", 2023-09-23T20:22:23.716 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:31.286005+0000", 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 172.317169272, 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.18492788800000001, 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.286005+0000", 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.717 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.286005+0000", 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.286004+0000", 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967296 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.286007+0000", 2023-09-23T20:22:23.718 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8289999999999999e-06 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462166+0000", 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.1761588 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462173+0000", 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5400000000000001e-06 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.719 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462440+0000", 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000267125 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.462460+0000", 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0367999999999999e-05 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.470898+0000", 2023-09-23T20:22:23.720 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0084382220000000004 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:31.470933+0000", 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4193000000000003e-05 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:125856 10.0 e514/511 10:00559810:::benchmark_data_smithi135_602286_object33630:head v 514'7809, mlcod=514'7806)", 2023-09-23T20:22:23.721 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:32.871097+0000", 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 170.732076869, 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.166058025, 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871097+0000", 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.722 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871097+0000", 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871096+0000", 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871106+0000", 2023-09-23T20:22:23.723 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9869999999999994e-06 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871116+0000", 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4970000000000007e-06 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871120+0000", 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1439999999999996e-06 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.724 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871151+0000", 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0666000000000001e-05 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.871163+0000", 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2278e-05 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037105+0000", 2023-09-23T20:22:23.725 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.16594193300000001 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037155+0000", 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0197000000000001e-05 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:125865 10.1 e514/479 10:822727c4:::benchmark_data_smithi135_602286_object33639:head v 514'7863, mlcod=514'7860)", 2023-09-23T20:22:23.726 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:32.872211+0000", 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 170.730963301, 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.165180407, 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872211+0000", 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.727 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872211+0000", 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872210+0000", 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872215+0000", 2023-09-23T20:22:23.728 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3140000000000003e-06 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.030919+0000", 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15870314899999999 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.030942+0000", 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3493e-05 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.729 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.031058+0000", 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115739 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.031111+0000", 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3075999999999998e-05 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037357+0000", 2023-09-23T20:22:23.730 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0062462519999999999 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037391+0000", 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3937999999999999e-05 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_repop(client.46418.0:125864 10.6 e514/479 10:6a1ce244:::benchmark_data_smithi135_602286_object33638:head v 514'7883, mlcod=514'7881)", 2023-09-23T20:22:23.731 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:32.872248+0000", 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 170.73092585500001, 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.165563234, 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872248+0000", 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.732 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872248+0000", 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872248+0000", 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:32.872252+0000", 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9130000000000002e-06 2023-09-23T20:22:23.733 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.030954+0000", 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.15870204299999999 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.030976+0000", 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1614e-05 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.734 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.031087+0000", 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011085400000000001 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.031143+0000", 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5578999999999998e-05 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:23.735 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037776+0000", 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0066330169999999997 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:33.037811+0000", 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5790999999999997e-05 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:23.736 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:22:23.737 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:23.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:22:23.737 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[221113]: osdmap e544: 8 total, 7 up, 8 in 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[221113]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[221113]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[221113]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[221113]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[221113]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:23.738 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[213871]: osdmap e544: 8 total, 7 up, 8 in 2023-09-23T20:22:23.739 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[213871]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.739 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[213871]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.739 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[213871]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.739 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[213871]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.739 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:23 smithi135 ceph-mon[213871]: failed to encode map e544 with expected crc 2023-09-23T20:22:23.791 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:23.836 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:23.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:23.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:23.940 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:23.969 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:22:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:22:23] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-09-23T20:22:23.972 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:24.080 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:24.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:24.185 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:24.214 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:24.226 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:22:24.257 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:24.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:24.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:24.363 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:24.399 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:24.507 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:24.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:24.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:24.549 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:22:24.574 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:22:24.581 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[221113]: 1.5 deep-scrub starts 2023-09-23T20:22:24.582 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[221113]: 1.5 deep-scrub ok 2023-09-23T20:22:24.582 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[221113]: pgmap v1560: 135 pgs: 1 remapped+peering, 1 peering, 7 active+undersized+degraded, 1 undersized+peered, 32 active+undersized, 93 active+clean; 4.7 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s; 8/126 objects degraded (6.349%); 3 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T20:22:24.582 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[221113]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.582 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[221113]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.582 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[221113]: osdmap e545: 8 total, 7 up, 8 in 2023-09-23T20:22:24.582 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[221113]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.583 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[213871]: 1.5 deep-scrub starts 2023-09-23T20:22:24.583 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[213871]: 1.5 deep-scrub ok 2023-09-23T20:22:24.583 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[213871]: pgmap v1560: 135 pgs: 1 remapped+peering, 1 peering, 7 active+undersized+degraded, 1 undersized+peered, 32 active+undersized, 93 active+clean; 4.7 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s; 8/126 objects degraded (6.349%); 3 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T20:22:24.583 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[213871]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.583 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[213871]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.584 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[213871]: osdmap e545: 8 total, 7 up, 8 in 2023-09-23T20:22:24.584 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:24 smithi135 ceph-mon[213871]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.597 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:24.602 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:22:24.611 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:24.629 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:22:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:24 smithi144 ceph-mon[194833]: 1.5 deep-scrub starts 2023-09-23T20:22:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:24 smithi144 ceph-mon[194833]: 1.5 deep-scrub ok 2023-09-23T20:22:24.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:24 smithi144 ceph-mon[194833]: pgmap v1560: 135 pgs: 1 remapped+peering, 1 peering, 7 active+undersized+degraded, 1 undersized+peered, 32 active+undersized, 93 active+clean; 4.7 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s; 8/126 objects degraded (6.349%); 3 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T20:22:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:24 smithi144 ceph-mon[194833]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:24 smithi144 ceph-mon[194833]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:24 smithi144 ceph-mon[194833]: osdmap e545: 8 total, 7 up, 8 in 2023-09-23T20:22:24.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:24 smithi144 ceph-mon[194833]: failed to encode map e545 with expected crc 2023-09-23T20:22:24.640 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:24.666 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:22:24.690 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:22:24.710 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:24.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:24.714 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:22:24.740 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:22:24.740 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:22:24.740 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:22:24.740 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:22:24.740 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:22:24.740 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:22:24.741 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:22:24.741 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:22:24.741 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:22:24.813 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:24.848 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:24.954 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:24.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:25.059 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:25.085 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:25.350 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:22:25.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/931319006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:25.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/931319006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:25.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/931319006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:25.790 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:25.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:25.894 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:25.902 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:22:25.902 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:22:25.932 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:25.980 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:25.980 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:25.992 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:26.002 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:26.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:26.016 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:26.109 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:26.140 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:26.295 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:26.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:26.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:26.307 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.mirror_image_status (3640 ms) 2023-09-23T20:22:26.307 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T20:22:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:26 smithi144 ceph-mon[194833]: 1.14 scrub starts 2023-09-23T20:22:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:26 smithi144 ceph-mon[194833]: 1.14 scrub ok 2023-09-23T20:22:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:26 smithi144 ceph-mon[194833]: pgmap v1562: 135 pgs: 1 remapped+peering, 1 peering, 4 active+undersized+degraded, 1 undersized+peered, 18 active+undersized, 110 active+clean; 4.7 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 13 KiB/s wr, 37 op/s; 4/147 objects degraded (2.721%); 4 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T20:22:26.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:26 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:22:26.397 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.mirror_image_map (90 ms) 2023-09-23T20:22:26.397 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T20:22:26.399 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:26.405 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-09-23T20:22:26.405 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T20:22:26.417 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.mirror_snapshot (13 ms) 2023-09-23T20:22:26.417 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T20:22:26.420 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-09-23T20:22:26.420 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T20:22:26.422 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-09-23T20:22:26.422 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T20:22:26.425 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T20:22:26.425 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T20:22:26.429 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:26 smithi135 ceph-mon[221113]: 1.14 scrub starts 2023-09-23T20:22:26.429 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:26 smithi135 ceph-mon[221113]: 1.14 scrub ok 2023-09-23T20:22:26.430 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:26 smithi135 ceph-mon[221113]: pgmap v1562: 135 pgs: 1 remapped+peering, 1 peering, 4 active+undersized+degraded, 1 undersized+peered, 18 active+undersized, 110 active+clean; 4.7 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 13 KiB/s wr, 37 op/s; 4/147 objects degraded (2.721%); 4 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T20:22:26.430 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:26 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:22:26.430 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:26 smithi135 ceph-mon[213871]: 1.14 scrub starts 2023-09-23T20:22:26.430 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:26 smithi135 ceph-mon[213871]: 1.14 scrub ok 2023-09-23T20:22:26.430 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:26 smithi135 ceph-mon[213871]: pgmap v1562: 135 pgs: 1 remapped+peering, 1 peering, 4 active+undersized+degraded, 1 undersized+peered, 18 active+undersized, 110 active+clean; 4.7 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 13 KiB/s wr, 37 op/s; 4/147 objects degraded (2.721%); 4 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T20:22:26.430 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:26 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:22:26.431 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-09-23T20:22:26.431 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T20:22:26.432 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:26.435 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-09-23T20:22:26.435 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T20:22:26.439 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2023-09-23T20:22:26.439 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T20:22:26.441 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-09-23T20:22:26.441 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T20:22:26.444 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T20:22:26.445 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T20:22:26.448 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-09-23T20:22:26.448 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T20:22:26.452 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-09-23T20:22:26.452 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T20:22:26.455 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-09-23T20:22:26.455 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T20:22:26.458 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-09-23T20:22:26.458 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T20:22:26.460 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-23T20:22:26.461 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T20:22:26.462 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-23T20:22:26.462 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T20:22:26.464 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T20:22:26.464 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T20:22:26.467 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2023-09-23T20:22:26.467 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T20:22:26.470 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-09-23T20:22:26.470 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T20:22:26.472 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-09-23T20:22:26.472 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T20:22:26.475 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-23T20:22:26.475 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T20:22:26.491 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (15 ms) 2023-09-23T20:22:26.491 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T20:22:26.568 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:26.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:26.622 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:22:26.672 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:26.699 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:26.706 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (215 ms) 2023-09-23T20:22:26.706 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T20:22:26.710 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T20:22:26.710 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T20:22:26.712 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T20:22:26.712 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T20:22:26.720 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.trash_methods (9 ms) 2023-09-23T20:22:26.720 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T20:22:26.729 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-09-23T20:22:26.729 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T20:22:26.752 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.clone_parent (23 ms) 2023-09-23T20:22:26.752 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T20:22:26.754 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:26.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:26.761 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-09-23T20:22:26.761 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T20:22:26.773 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2023-09-23T20:22:26.773 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T20:22:26.779 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2023-09-23T20:22:26.779 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T20:22:26.789 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-09-23T20:22:26.789 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T20:22:26.800 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2023-09-23T20:22:26.800 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T20:22:26.859 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:26.887 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:26.991 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:27.035 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:27.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:27.139 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:27.181 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:27.225 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:27.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:27.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:27.329 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:27.358 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:27.407 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:27.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:27.511 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:27.537 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:27.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:27 smithi144 ceph-mon[194833]: 1.15 scrub starts 2023-09-23T20:22:27.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:27 smithi144 ceph-mon[194833]: 1.15 scrub ok 2023-09-23T20:22:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:27 smithi135 ceph-mon[221113]: 1.15 scrub starts 2023-09-23T20:22:27.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:27 smithi135 ceph-mon[221113]: 1.15 scrub ok 2023-09-23T20:22:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:27 smithi135 ceph-mon[213871]: 1.15 scrub starts 2023-09-23T20:22:27.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:27 smithi135 ceph-mon[213871]: 1.15 scrub ok 2023-09-23T20:22:27.903 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:22:27.951 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:27.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:28.056 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:28.086 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:28.173 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:28.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:28.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:28.277 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:28.305 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:28.353 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:28.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:28.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:28.386 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1586 ms) 2023-09-23T20:22:28.386 INFO:tasks.workunit.client.0.smithi135.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T20:22:28.427 INFO:tasks.workunit.client.0.smithi135.stdout:[ OK ] TestClsRbd.sparsify (41 ms) 2023-09-23T20:22:28.457 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:28.460 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:28.461 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:28.488 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:28.532 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:28.535 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:28 smithi144 ceph-mon[194833]: pgmap v1563: 135 pgs: 1 remapped+peering, 2 active+undersized+degraded, 4 active+undersized, 128 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 867 KiB/s rd, 453 KiB/s wr, 1.20k op/s; 5/198 objects degraded (2.525%); 0 B/s, 4 keys/s, 0 objects/s recovering 2023-09-23T20:22:28.535 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:28 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 5/198 objects degraded (2.525%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:28.535 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:28 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:22:28.535 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:28 smithi144 ceph-mon[194833]: failed to encode map e546 with expected crc 2023-09-23T20:22:28.535 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:28 smithi144 ceph-mon[194833]: osdmap e546: 8 total, 7 up, 8 in 2023-09-23T20:22:28.535 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:28 smithi144 ceph-mon[194833]: failed to encode map e546 with expected crc 2023-09-23T20:22:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:28.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:28.587 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[221113]: pgmap v1563: 135 pgs: 1 remapped+peering, 2 active+undersized+degraded, 4 active+undersized, 128 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 867 KiB/s rd, 453 KiB/s wr, 1.20k op/s; 5/198 objects degraded (2.525%); 0 B/s, 4 keys/s, 0 objects/s recovering 2023-09-23T20:22:28.587 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 5/198 objects degraded (2.525%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:28.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:22:28.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[221113]: failed to encode map e546 with expected crc 2023-09-23T20:22:28.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[221113]: osdmap e546: 8 total, 7 up, 8 in 2023-09-23T20:22:28.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[221113]: failed to encode map e546 with expected crc 2023-09-23T20:22:28.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[213871]: pgmap v1563: 135 pgs: 1 remapped+peering, 2 active+undersized+degraded, 4 active+undersized, 128 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 867 KiB/s rd, 453 KiB/s wr, 1.20k op/s; 5/198 objects degraded (2.525%); 0 B/s, 4 keys/s, 0 objects/s recovering 2023-09-23T20:22:28.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 5/198 objects degraded (2.525%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:28.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:22:28.589 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[213871]: failed to encode map e546 with expected crc 2023-09-23T20:22:28.589 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[213871]: osdmap e546: 8 total, 7 up, 8 in 2023-09-23T20:22:28.589 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:28 smithi135 ceph-mon[213871]: failed to encode map e546 with expected crc 2023-09-23T20:22:28.637 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:28.663 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:28.701 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:28.757 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:28.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:28.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:28.802 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:28.802 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:28.862 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:28.888 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:29.060 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:29.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:29.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:29.113 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:22:29.164 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:29.192 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:29.234 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:29.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:29.340 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:29.367 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:29.391 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[213871]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.391 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[213871]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[213871]: osdmap e547: 8 total, 7 up, 8 in 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[213871]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/31193453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[221113]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[221113]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[221113]: osdmap e547: 8 total, 7 up, 8 in 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[221113]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.392 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/31193453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:29.393 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:29 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:29.400 INFO:tasks.workunit.client.0.smithi135.stdout:[----------] 70 tests from TestClsRbd (62333 ms total) 2023-09-23T20:22:29.400 INFO:tasks.workunit.client.0.smithi135.stdout: 2023-09-23T20:22:29.400 INFO:tasks.workunit.client.0.smithi135.stdout:[----------] Global test environment tear-down 2023-09-23T20:22:29.400 INFO:tasks.workunit.client.0.smithi135.stdout:[==========] 70 tests from 1 test suite ran. (65101 ms total) 2023-09-23T20:22:29.400 INFO:tasks.workunit.client.0.smithi135.stdout:[ PASSED ] 70 tests. 2023-09-23T20:22:29.401 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:22:29.402 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T20:22:29.402 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T20:22:29.465 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:29.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:29.472 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:29.473 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T20:22:29.474 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:22:29.569 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:29.596 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:29.666 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:29 smithi144 ceph-mon[194833]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.666 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:29 smithi144 ceph-mon[194833]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.666 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:29 smithi144 ceph-mon[194833]: osdmap e547: 8 total, 7 up, 8 in 2023-09-23T20:22:29.666 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:29 smithi144 ceph-mon[194833]: failed to encode map e547 with expected crc 2023-09-23T20:22:29.666 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/31193453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:29.666 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:29 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:29.735 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:29.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:29.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:29.752 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:29.752 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:22:29.822 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:22:29.822 DEBUG:teuthology.orchestra.run.smithi135:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:22:29.841 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:29.888 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:22:29.888 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:22:29.888 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T20:22:29.888 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:22:29.889 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T20:22:29.889 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:22:29.889 DEBUG:teuthology.orchestra.run.smithi135:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:22:29.899 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:29.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:22:29.916 INFO:teuthology.orchestra.run.smithi135.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:22:29.916 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:22:29.943 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:29.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:29.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:29.958 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:22:29.959 DEBUG:teuthology.orchestra.run.smithi135:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:22:30.019 INFO:tasks.workunit:timeout=3h 2023-09-23T20:22:30.019 INFO:tasks.workunit:cleanup=True 2023-09-23T20:22:30.020 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:22:30.046 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:30.077 INFO:tasks.workunit.client.0.smithi135.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T20:22:30.087 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:30.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[213871]: pgmap v1566: 135 pgs: 1 active+clean+remapped, 2 active+undersized+degraded, 4 active+undersized, 128 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 905 KiB/s rd, 467 KiB/s wr, 1.26k op/s; 5/243 objects degraded (2.058%); 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T20:22:30.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:30.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:22:30.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[213871]: osdmap e548: 8 total, 7 up, 8 in 2023-09-23T20:22:30.389 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[213871]: failed to encode map e548 with expected crc 2023-09-23T20:22:30.582 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:30.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:30.650 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:30 smithi144 ceph-mon[194833]: pgmap v1566: 135 pgs: 1 active+clean+remapped, 2 active+undersized+degraded, 4 active+undersized, 128 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 905 KiB/s rd, 467 KiB/s wr, 1.26k op/s; 5/243 objects degraded (2.058%); 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T20:22:30.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:30 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:30.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:30 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:22:30.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:30 smithi144 ceph-mon[194833]: osdmap e548: 8 total, 7 up, 8 in 2023-09-23T20:22:30.651 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:30 smithi144 ceph-mon[194833]: failed to encode map e548 with expected crc 2023-09-23T20:22:30.685 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:30.713 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:30.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[221113]: pgmap v1566: 135 pgs: 1 active+clean+remapped, 2 active+undersized+degraded, 4 active+undersized, 128 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 905 KiB/s rd, 467 KiB/s wr, 1.26k op/s; 5/243 objects degraded (2.058%); 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T20:22:30.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:30.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:22:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[221113]: osdmap e548: 8 total, 7 up, 8 in 2023-09-23T20:22:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:30 smithi135 ceph-mon[221113]: failed to encode map e548 with expected crc 2023-09-23T20:22:30.809 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:30.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:30.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:30.866 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:30.866 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:30.913 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:30.940 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:31.033 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:31.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:31.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:31.137 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:31.162 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:31.205 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:31.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:31.308 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:31.335 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:31.484 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:31.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:31 smithi144 ceph-mon[194833]: failed to encode map e548 with expected crc 2023-09-23T20:22:31.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:31 smithi144 ceph-mon[194833]: failed to encode map e548 with expected crc 2023-09-23T20:22:31.486 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:31 smithi144 ceph-mon[194833]: pgmap v1568: 103 pgs: 1 active+clean+remapped, 1 active+undersized+degraded, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:31.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:31.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:31.546 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:22:31.584 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:22:31.588 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:31.602 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:22:31.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:31 smithi135 ceph-mon[221113]: failed to encode map e548 with expected crc 2023-09-23T20:22:31.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:31 smithi135 ceph-mon[221113]: failed to encode map e548 with expected crc 2023-09-23T20:22:31.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:31 smithi135 ceph-mon[221113]: pgmap v1568: 103 pgs: 1 active+clean+remapped, 1 active+undersized+degraded, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:31.615 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:31 smithi135 ceph-mon[213871]: failed to encode map e548 with expected crc 2023-09-23T20:22:31.615 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:31 smithi135 ceph-mon[213871]: failed to encode map e548 with expected crc 2023-09-23T20:22:31.615 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:31 smithi135 ceph-mon[213871]: pgmap v1568: 103 pgs: 1 active+clean+remapped, 1 active+undersized+degraded, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:31.618 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:31.623 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:22:31.650 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:22:31.666 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:31.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:31.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:31.671 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:22:31.690 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:22:31.709 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:22:31.709 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:22:31.709 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:22:31.709 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:22:31.710 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:22:31.710 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:22:31.710 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:22:31.710 INFO:teuthology.orchestra.run.smithi135.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:22:31.710 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:22:31.771 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:31.800 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:31.845 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:31.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:31.895 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:31.952 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:31.978 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:32.019 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:32.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:32.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:32.123 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:32.149 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:32.191 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:32.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:32.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:32.295 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:32.323 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:32.419 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:32.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:32.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:32.483 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:22:32.483 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:22:32.523 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:32.550 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:32.596 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:32.598 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1166794007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:32.599 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1166794007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:32.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:32.700 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:32.727 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:32.773 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:32.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:32.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:32.825 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1166794007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:32.881 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:32.911 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:32.938 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:22:33.006 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:33.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:33.110 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:33.139 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:33.182 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:33.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:33.286 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:33.298 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:22:33.298 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:22:33.298 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:22:33.298 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:22:33.299 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:33.312 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:33.456 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:33.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:33.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:33.560 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:33.588 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:33.632 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:33.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:33.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:33.683 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:33 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:33.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:33 smithi135 ceph-mon[221113]: pgmap v1569: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:33 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:33.684 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:33 smithi135 ceph-mon[213871]: pgmap v1569: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:33.735 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:33.762 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:33.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:33 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:33.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:33 smithi144 ceph-mon[194833]: pgmap v1569: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:33.904 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:22:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:22:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:22:33] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-09-23T20:22:34.157 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:34.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:34.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:34.246 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:34.262 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:34.289 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:34.369 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:34.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:34.473 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:34.484 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:22:34.500 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:34.543 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:34.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:34.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:34.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:34.675 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:34.765 DEBUG:teuthology.orchestra.run.smithi135:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T20:22:34.822 INFO:tasks.workunit.client.0.smithi135.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T20:22:34.824 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:22:34.824 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T20:22:34.833 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 3, 4, 5, 7, 2] 2023-09-23T20:22:34.834 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:22:34.834 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-09-23T20:22:34.834 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-09-23T20:22:34.834 DEBUG:teuthology.orchestra.run.smithi144:osd.6> sudo journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T20:22:34.837 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl start ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T20:22:34.865 INFO:journalctl@ceph.osd.6.smithi144.stdout:-- Logs begin at Sat 2023-09-23 19:18:57 UTC. -- 2023-09-23T20:22:34.887 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T20:22:35.110 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T20:22:35.131 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:34 smithi144 systemd[1]: Starting Ceph osd.6 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:22:35.168 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:22:35.169 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:22:35.169 INFO:tasks.workunit.client.0.smithi135.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T20:22:35.191 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:35.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:35.222 INFO:tasks.workunit.client.0.smithi135.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:22:35.225 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-09-23T20:22:35.225 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T20:22:35.283 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T20:22:35.284 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T20:22:35.284 DEBUG:teuthology.orchestra.run.smithi135:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T20:22:35.296 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:35.354 INFO:tasks.workunit.client.0.smithi135.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T20:22:35.354 INFO:tasks.workunit.client.0.smithi135.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T20:22:35.354 INFO:tasks.workunit.client.0.smithi135.stderr:+ TMPDIR=/tmp/rbd_import_export_682894 2023-09-23T20:22:35.354 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm -rf /tmp/rbd_import_export_682894 2023-09-23T20:22:35.355 INFO:tasks.workunit.client.0.smithi135.stderr:+ mkdir /tmp/rbd_import_export_682894 2023-09-23T20:22:35.355 INFO:tasks.workunit.client.0.smithi135.stderr:+ trap 'rm -rf /tmp/rbd_import_export_682894' INT TERM EXIT 2023-09-23T20:22:35.356 INFO:tasks.workunit.client.0.smithi135.stderr:+ mkdir foo.682894 2023-09-23T20:22:35.356 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:35.356 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import foo.682894 foo.dir 2023-09-23T20:22:35.385 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:35.411 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:35.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:35.428 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: cannot import a directory 2023-09-23T20:22:35.428 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 0% complete...failed. 2023-09-23T20:22:35.428 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: import failed: (21) Is a directory 2023-09-23T20:22:35.432 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T20:22:35.432 INFO:tasks.workunit.client.0.smithi135.stderr:+ rmdir foo.682894 2023-09-23T20:22:35.432 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_682894/img bs=1k count=1 seek=10 2023-09-23T20:22:35.433 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T20:22:35.433 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T20:22:35.433 INFO:tasks.workunit.client.0.smithi135.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.9694e-05 s, 20.6 MB/s 2023-09-23T20:22:35.433 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_682894/img bs=1k count=10 seek=100 2023-09-23T20:22:35.434 INFO:tasks.workunit.client.0.smithi135.stderr:10+0 records in 2023-09-23T20:22:35.435 INFO:tasks.workunit.client.0.smithi135.stderr:10+0 records out 2023-09-23T20:22:35.435 INFO:tasks.workunit.client.0.smithi135.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000436987 s, 23.4 MB/s 2023-09-23T20:22:35.435 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_682894/img bs=1k count=100 seek=1000 2023-09-23T20:22:35.435 INFO:tasks.workunit.client.0.smithi135.stderr:70+1 records in 2023-09-23T20:22:35.435 INFO:tasks.workunit.client.0.smithi135.stderr:70+1 records out 2023-09-23T20:22:35.435 INFO:tasks.workunit.client.0.smithi135.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000177994 s, 405 MB/s 2023-09-23T20:22:35.436 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_682894/img bs=1k seek=10000 2023-09-23T20:22:35.436 INFO:tasks.workunit.client.0.smithi135.stderr:139+1 records in 2023-09-23T20:22:35.437 INFO:tasks.workunit.client.0.smithi135.stderr:139+1 records out 2023-09-23T20:22:35.437 INFO:tasks.workunit.client.0.smithi135.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.00033281 s, 430 MB/s 2023-09-23T20:22:35.437 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_682894/img bs=1k seek=100000 2023-09-23T20:22:35.437 INFO:tasks.workunit.client.0.smithi135.stderr:70+1 records in 2023-09-23T20:22:35.437 INFO:tasks.workunit.client.0.smithi135.stderr:70+1 records out 2023-09-23T20:22:35.437 INFO:tasks.workunit.client.0.smithi135.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.000180934 s, 399 MB/s 2023-09-23T20:22:35.438 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_682894/img bs=1k seek=1000000 2023-09-23T20:22:35.439 INFO:tasks.workunit.client.0.smithi135.stderr:189+1 records in 2023-09-23T20:22:35.439 INFO:tasks.workunit.client.0.smithi135.stderr:189+1 records out 2023-09-23T20:22:35.440 INFO:tasks.workunit.client.0.smithi135.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000459453 s, 422 MB/s 2023-09-23T20:22:35.440 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm testimg 2023-09-23T20:22:35.477 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 0% complete...failed. 2023-09-23T20:22:35.477 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:22:35.480 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T20:22:35.480 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format /tmp/rbd_import_export_682894/img testimg 2023-09-23T20:22:35.518 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:35.555 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:35.607 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:35.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:35.711 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:35.736 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:22:35.736 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:22:35.736 INFO:teuthology.orchestra.run.smithi144.stdout: "complaint_time": 30, 2023-09-23T20:22:35.736 INFO:teuthology.orchestra.run.smithi144.stdout: "num_blocked_ops": 0 2023-09-23T20:22:35.736 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:22:35.746 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:35.901 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:35.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:36.006 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:36.034 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:36.078 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:36.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:36.115 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T20:22:36.117 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:22:36.119 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg /tmp/rbd_import_export_682894/img2 2023-09-23T20:22:36.182 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:36.209 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:36.353 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is set 2023-09-23T20:22:36.390 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:36.393 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg - 2023-09-23T20:22:36.446 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:36.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:36.513 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:22:36.550 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:36.585 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:36.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:36 smithi144 ceph-mon[194833]: pgmap v1570: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/153272771' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:36 smithi144 bash[399799]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:22:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:36 smithi144 bash[399799]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T20:22:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:36 smithi144 bash[399799]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2023-09-23T20:22:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:36 smithi144 bash[399799]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2023-09-23T20:22:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:36 smithi144 bash[399799]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2023-09-23T20:22:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:36 smithi144 bash[399799]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T20:22:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:36 smithi144 bash[399799]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T20:22:36.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:36 smithi144 bash[399799]: --> ceph-volume lvm activate successful for osd ID: 6 2023-09-23T20:22:36.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:36 smithi135 ceph-mon[213871]: pgmap v1570: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:36.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/153272771' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:36.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:36 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:36 smithi135 ceph-mon[221113]: pgmap v1570: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/153272771' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:36.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:36.785 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:36.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:36.888 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:36.918 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:37.182 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:37.186 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm testimg 2023-09-23T20:22:37.274 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T20:22:37.277 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp /tmp/rbd_import_export_682894/img /tmp/rbd_import_export_682894/img2 2023-09-23T20:22:37.309 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:37.621 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:37.622 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[221113]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[221113]: osdmap e549: 8 total, 7 up, 8 in 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[221113]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[221113]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[221113]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[213871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:37.623 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[213871]: osdmap e549: 8 total, 7 up, 8 in 2023-09-23T20:22:37.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[213871]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[213871]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[213871]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.624 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:37.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:37.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:37 smithi144 ceph-mon[194833]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:37 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:37.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:37 smithi144 ceph-mon[194833]: osdmap e549: 8 total, 7 up, 8 in 2023-09-23T20:22:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:37 smithi144 ceph-mon[194833]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:37 smithi144 ceph-mon[194833]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:37 smithi144 ceph-mon[194833]: failed to encode map e549 with expected crc 2023-09-23T20:22:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:37.726 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:37.764 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:37.990 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp /tmp/rbd_import_export_682894/img /tmp/rbd_import_export_682894/img3 2023-09-23T20:22:38.262 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:38.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:38.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:38.365 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:38.394 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:38.427 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:22:38.436 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img2 /tmp/rbd_import_export_682894/img3 2023-09-23T20:22:38.443 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:38.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:38.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:38.551 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:38.602 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:38 smithi135 ceph-mon[221113]: pgmap v1572: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:38.602 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:38 smithi135 ceph-mon[213871]: pgmap v1572: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:38.605 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:38.616 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:38 smithi144 ceph-osd[400912]: -- 172.21.15.144:0/554993266 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5589a3d86b60 con 0x5589a3d4cc00 2023-09-23T20:22:38.617 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:38 smithi144 bash[400734]: 052ceac7488dfdb2a0c0266b3062a18a827392fdaed4811ffeab55915425bd38 2023-09-23T20:22:38.617 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:38 smithi144 systemd[1]: Started Ceph osd.6 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T20:22:38.617 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:38 smithi144 ceph-mon[194833]: pgmap v1572: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:22:38.647 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:38.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:38.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:38.750 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:38.781 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:38.830 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:38.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:38.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:38.846 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:22:38.846 INFO:teuthology.orchestra.run.smithi144.stdout: "size": 20, 2023-09-23T20:22:38.846 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 600, 2023-09-23T20:22:38.846 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [ 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71190 9.13 9:cf33cd0d:::benchmark_data_smithi135_549536_object71189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.171993+0000", 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.670855362, 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.143627628, 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.847 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71190 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.171993+0000", 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.171993+0000", 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.171993+0000", 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172033+0000", 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0908999999999997e-05 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.849 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172041+0000", 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2049999999999996e-06 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172045+0000", 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4460000000000003e-06 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.850 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172091+0000", 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6124000000000001e-05 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172336+0000", 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000244808 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.172485+0000", 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000149238 2023-09-23T20:22:38.851 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.176940+0000", 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0044543389999999999 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.182374+0000", 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0054341659999999998 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.852 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.182383+0000", 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9039999999999998e-06 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238107+0000", 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0557239190000001 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238118+0000", 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1071e-05 2023-09-23T20:22:38.853 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.308271+0000", 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.070152966 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.308949+0000", 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00067792799999999995 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.854 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.315503+0000", 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0065539700000000001 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.315621+0000", 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000118136 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.855 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71200 9.13 9:c81b3d8c:::benchmark_data_smithi135_549536_object71199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.177390+0000", 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.66545849199997, 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1402376740000002, 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71200 2023-09-23T20:22:38.856 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177390+0000", 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177390+0000", 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.857 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177389+0000", 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177431+0000", 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.2224000000000001e-05 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.858 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177455+0000", 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3859e-05 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177462+0000", 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5350000000000001e-06 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177606+0000", 2023-09-23T20:22:38.859 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014430399999999999 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177723+0000", 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000116592 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.177854+0000", 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000131643 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.860 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.180105+0000", 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002250307 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.182419+0000", 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023146220000000001 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.861 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.182426+0000", 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7460000000000003e-06 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238592+0000", 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.056166167 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238600+0000", 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3780000000000001e-06 2023-09-23T20:22:38.862 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.309843+0000", 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0712429160000001 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.310503+0000", 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000659814 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.863 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317571+0000", 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070675709999999999 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317628+0000", 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6878e-05 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.864 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71205 9.13 9:cb3316ae:::benchmark_data_smithi135_549536_object71204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.186043+0000", 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.65680502999999, 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.132378675, 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71205 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.865 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186043+0000", 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186043+0000", 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.866 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186042+0000", 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186073+0000", 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.0552999999999997e-05 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.867 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186082+0000", 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.7750000000000004e-06 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186089+0000", 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.2949999999999999e-06 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186219+0000", 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013041600000000001 2023-09-23T20:22:38.868 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.186245+0000", 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5329000000000001e-05 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238625+0000", 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0523801159999999 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.869 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238629+0000", 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.8300000000000003e-06 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.310545+0000", 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.071915256 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.310712+0000", 2023-09-23T20:22:38.870 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016684600000000001 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.310846+0000", 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013424800000000001 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.315448+0000", 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0046022349999999997 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.871 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317667+0000", 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0022188849999999999 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317698+0000", 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.1201999999999997e-05 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.872 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.317726+0000", 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.8175e-05 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.318422+0000", 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000695793 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.873 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71209 9.13 9:cba57e8e:::benchmark_data_smithi135_549536_object71208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.189594+0000", 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.65325402899998, 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1435122980000001, 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71209 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.874 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189594+0000", 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189594+0000", 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.875 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189593+0000", 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189628+0000", 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4752000000000002e-05 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189642+0000", 2023-09-23T20:22:38.876 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4295e-05 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189649+0000", 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.4209999999999998e-06 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189751+0000", 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000101164 2023-09-23T20:22:38.877 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.189760+0000", 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0389999999999993e-06 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238657+0000", 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.048896922 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.878 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238663+0000", 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6460000000000004e-06 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.318462+0000", 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0797985910000001 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.879 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.318644+0000", 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018246500000000001 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.318779+0000", 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013493000000000001 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.327867+0000", 2023-09-23T20:22:38.880 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0090877849999999993 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331013+0000", 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.003146542 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331155+0000", 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000141643 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.881 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331220+0000", 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4449000000000006e-05 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.333107+0000", 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018871409999999999 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.882 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71233 9.13 9:c9dee9fd:::benchmark_data_smithi135_549536_object71232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.215209+0000", 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.62763959999995, 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1197944849999999, 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.883 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71233 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215209+0000", 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215209+0000", 2023-09-23T20:22:38.884 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215208+0000", 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215307+0000", 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.9681999999999999e-05 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.885 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215319+0000", 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1343e-05 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215325+0000", 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.3090000000000001e-06 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.886 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215380+0000", 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4784000000000003e-05 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.215389+0000", 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.0459999999999994e-06 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238685+0000", 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.023296427 2023-09-23T20:22:38.887 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238689+0000", 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.5209999999999998e-06 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.320194+0000", 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.081505452 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.888 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.320322+0000", 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000127627 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.320464+0000", 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000142464 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.889 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.327873+0000", 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074090789999999998 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331119+0000", 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032455510000000002 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.333192+0000", 2023-09-23T20:22:38.890 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020725729999999999 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.333244+0000", 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.2225999999999998e-05 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.335003+0000", 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0017595390000000001 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.891 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71281 9.13 9:c9e738f5:::benchmark_data_smithi135_549536_object71280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.259961+0000", 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.58288759599998, 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0788080020000002, 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.892 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71281 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.259961+0000", 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.893 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.259961+0000", 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.259960+0000", 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260004+0000", 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4518000000000003e-05 2023-09-23T20:22:38.894 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260025+0000", 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1234999999999999e-05 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260032+0000", 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7209999999999997e-06 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.895 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260115+0000", 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.3078000000000007e-05 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.260124+0000", 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9150000000000002e-06 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238714+0000", 2023-09-23T20:22:38.896 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97859040399999997 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238721+0000", 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.392e-06 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.321605+0000", 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0828845419999999 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.897 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.321795+0000", 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000189614 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.321900+0000", 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00010535900000000001 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.898 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.327879+0000", 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0059790650000000004 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.333156+0000", 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0052770200000000003 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.336887+0000", 2023-09-23T20:22:38.899 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0037310339999999998 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.336943+0000", 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.5621000000000002e-05 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.338769+0000", 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001825824 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.900 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71322 9.13 9:cf6bc6c4:::benchmark_data_smithi135_549536_object71321:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.300630+0000", 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.54221829699998, 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0399141589999998, 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.901 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71322 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300630+0000", 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300630+0000", 2023-09-23T20:22:38.902 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300630+0000", 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300666+0000", 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.6192000000000002e-05 2023-09-23T20:22:38.903 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300672+0000", 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6390000000000003e-06 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300676+0000", 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.7720000000000001e-06 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.904 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300721+0000", 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4458e-05 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.300729+0000", 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6170000000000007e-06 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.905 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238744+0000", 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.93801429700000005 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238747+0000", 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.591e-06 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.323227+0000", 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0844800809999999 2023-09-23T20:22:38.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.323358+0000", 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013022399999999999 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.323539+0000", 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00018142700000000001 2023-09-23T20:22:38.907 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.330959+0000", 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0074204730000000003 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.336842+0000", 2023-09-23T20:22:38.909 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005882473 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.338832+0000", 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0019901390000000001 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.338896+0000", 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4029999999999995e-05 2023-09-23T20:22:38.910 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.340544+0000", 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001648229 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71336 9.13 9:c90857ba:::benchmark_data_smithi135_549536_object71335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:22:38.911 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.315133+0000", 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.52771511000003, 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0271613479999999, 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71336 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.912 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315133+0000", 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315133+0000", 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315132+0000", 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:22:38.913 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315249+0000", 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000117068 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315265+0000", 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5707000000000001e-05 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315271+0000", 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.285e-06 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315331+0000", 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0112999999999999e-05 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.915 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.315341+0000", 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5699999999999999e-06 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238773+0000", 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.92343184199999995 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238779+0000", 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.4250000000000003e-06 2023-09-23T20:22:38.916 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.324804+0000", 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.0860250680000001 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.324955+0000", 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000150808 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.917 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.325109+0000", 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015406099999999999 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.330968+0000", 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005858963 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.918 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.340608+0000", 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0096397800000000006 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.340660+0000", 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1718000000000002e-05 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.340713+0000", 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.3627e-05 2023-09-23T20:22:38.919 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.342295+0000", 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0015814010000000001 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71362 9.13 9:cf72cd0f:::benchmark_data_smithi135_549536_object71361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:22:38.920 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.342398+0000", 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.50045078300002, 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.0014112530000001, 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71362 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.921 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342398+0000", 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342398+0000", 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.922 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342397+0000", 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342441+0000", 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.3605e-05 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342466+0000", 2023-09-23T20:22:38.923 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.5137999999999999e-05 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342471+0000", 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0289999999999996e-06 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342530+0000", 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8743000000000003e-05 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.924 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.342539+0000", 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3300000000000005e-06 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238819+0000", 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.89628013900000003 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238827+0000", 2023-09-23T20:22:38.925 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0199999999999994e-06 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.326346+0000", 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.087519015 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.326480+0000", 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013393199999999999 2023-09-23T20:22:38.926 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.326631+0000", 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015099699999999999 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.331025+0000", 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0043942 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.927 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.342335+0000", 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011309703000000001 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.342363+0000", 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7868000000000002e-05 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.342413+0000", 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0590999999999997e-05 2023-09-23T20:22:38.928 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.343809+0000", 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0013954779999999999 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:71382 9.13 9:c87487c7:::benchmark_data_smithi135_549536_object71381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:05.363816+0000", 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 513.47903254400001, 2023-09-23T20:22:38.929 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.981403325, 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 71382 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.930 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363816+0000", 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363816+0000", 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363815+0000", 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:22:38.931 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363902+0000", 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.6953999999999995e-05 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363915+0000", 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2771e-05 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.932 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363921+0000", 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.127e-06 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.363988+0000", 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7515999999999995e-05 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.933 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:05.364001+0000", 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.257e-05 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238860+0000", 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.87485905600000002 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:06.238876+0000", 2023-09-23T20:22:38.934 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.628e-05 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.327905+0000", 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.089029228 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.328028+0000", 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012292999999999999 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.935 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.328183+0000", 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000154401 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.336849+0000", 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0086667700000000007 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.936 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.343845+0000", 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0069960309999999998 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.343880+0000", 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4644000000000002e-05 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.343925+0000", 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4440000000000001e-05 2023-09-23T20:22:38.937 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:07.345219+0000", 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0012947220000000001 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97522 9.e 9:703747eb:::benchmark_data_smithi135_549536_object97521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.826971+0000", 2023-09-23T20:22:38.938 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 485.01587749999999, 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3623128769999999, 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97522 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.939 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.826971+0000", 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.826971+0000", 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.826969+0000", 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:22:38.940 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827040+0000", 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0655999999999996e-05 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827061+0000", 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.1370999999999999e-05 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.941 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827071+0000", 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4169999999999998e-06 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827522+0000", 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00045141400000000003 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827645+0000", 2023-09-23T20:22:38.942 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000122705 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.828027+0000", 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000382698 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.831892+0000", 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0038651029999999999 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.943 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.841958+0000", 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.010065960000000001 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861337+0000", 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.01937854 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.944 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861344+0000", 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.3819999999999998e-06 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854480+0000", 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99313559799999995 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854495+0000", 2023-09-23T20:22:38.945 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4635e-05 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.189169+0000", 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3346740459999999 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.189284+0000", 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000115198 2023-09-23T20:22:38.946 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97523 9.1e 9:7df3908a:::benchmark_data_smithi135_549536_object97522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.827083+0000", 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 485.01576540999997, 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2868424049999998, 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:22:38.947 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97523 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827083+0000", 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.948 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827083+0000", 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827082+0000", 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827143+0000", 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0285000000000003e-05 2023-09-23T20:22:38.949 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827152+0000", 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.7220000000000005e-06 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.827156+0000", 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.6059999999999996e-06 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.950 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.830286+0000", 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0031297999999999999 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.830437+0000", 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00015097799999999999 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.830731+0000", 2023-09-23T20:22:38.951 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000293646 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.833156+0000", 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0024251110000000002 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.842278+0000", 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0091223970000000008 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.952 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861255+0000", 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.018977355000000001 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861264+0000", 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.9400000000000008e-06 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.953 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854571+0000", 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99330715300000005 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854579+0000", 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.0979999999999996e-06 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.113800+0000", 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2592216139999999 2023-09-23T20:22:38.954 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.113925+0000", 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000125239 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97525 9.1e 9:795065da:::benchmark_data_smithi135_549536_object97524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T20:22:38.955 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.840530+0000", 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 485.002318946, 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3525597380000001, 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "reached pg", 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97525 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.956 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840530+0000", 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840530+0000", 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840528+0000", 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.957 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840584+0000", 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.6082000000000003e-05 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840600+0000", 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.5248e-05 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840608+0000", 2023-09-23T20:22:38.958 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.1680000000000004e-06 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840681+0000", 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.2731999999999994e-05 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.840817+0000", 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00013614500000000001 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.959 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.841107+0000", 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00029028600000000001 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861290+0000", 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.020182736 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.960 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.861296+0000", 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8610000000000003e-06 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854618+0000", 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.99332210300000001 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854625+0000", 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1439999999999997e-06 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.961 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.113961+0000", 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2593362510000001 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for degraded object", 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.115314+0000", 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001353103 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.193009+0000", 2023-09-23T20:22:38.962 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.077694632 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.193089+0000", 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.0479e-05 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97537 9.e 9:724684c9:::benchmark_data_smithi135_549536_object97536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:22:38.963 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.873780+0000", 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 484.96906894099999, 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3318544399999999, 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97537 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.964 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873780+0000", 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873780+0000", 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.965 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873778+0000", 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999981 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873818+0000", 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0425e-05 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873836+0000", 2023-09-23T20:22:38.966 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.7958999999999999e-05 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873844+0000", 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.8939999999999993e-06 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873912+0000", 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.7990000000000005e-05 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.967 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.873925+0000", 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2796000000000001e-05 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854523+0000", 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.98059804800000006 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.968 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854534+0000", 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1277000000000001e-05 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.191453+0000", 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3369192000000001 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.191601+0000", 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014799100000000001 2023-09-23T20:22:38.969 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.192021+0000", 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00041995500000000001 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.201308+0000", 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0092868169999999993 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.970 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.201346+0000", 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8073e-05 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.203702+0000", 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.002355518 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.203759+0000", 2023-09-23T20:22:38.971 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.7172000000000001e-05 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205634+0000", 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0018750539999999999 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97541 9.d 9:b6092fac:::benchmark_data_smithi135_549536_object97540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:22:38.972 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.879991+0000", 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 484.96285756600003, 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.2314555679999999, 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97541 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.973 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.879991+0000", 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.879991+0000", 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.879990+0000", 2023-09-23T20:22:38.974 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880016+0000", 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.618e-05 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880026+0000", 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.6879999999999994e-06 2023-09-23T20:22:38.975 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880031+0000", 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.0880000000000002e-06 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880083+0000", 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.1801000000000001e-05 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.976 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.880094+0000", 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1e-05 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.857379+0000", 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.97728484299999996 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.977 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.857388+0000", 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.4949999999999997e-06 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.073732+0000", 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.2163436000000001 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.073897+0000", 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00016534899999999999 2023-09-23T20:22:38.978 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.074331+0000", 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00043391399999999998 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.085762+0000", 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.011431139 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.979 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.092805+0000", 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0070426810000000003 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.109301+0000", 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.016496459000000002 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.109356+0000", 2023-09-23T20:22:38.980 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4648000000000003e-05 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.111446+0000", 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020903089999999998 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.36186.0:97546 9.e 9:74898ffe:::benchmark_data_smithi135_549536_object97545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T20:22:38.981 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:14:33.888839+0000", 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 484.95400992200001, 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.3184715890000001, 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.36186", 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/3734991771", 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 97546 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.982 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888839+0000", 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888839+0000", 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888838+0000", 2023-09-23T20:22:38.983 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888899+0000", 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0520000000000003e-05 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888906+0000", 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.729e-06 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.984 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888910+0000", 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.3100000000000001e-06 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888979+0000", 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9623999999999996e-05 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.985 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:33.888994+0000", 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4786999999999999e-05 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854637+0000", 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.96564282000000001 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for peered", 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:34.854649+0000", 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1688999999999999e-05 2023-09-23T20:22:38.986 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.193660+0000", 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.3390118229999999 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.193809+0000", 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00014898999999999999 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.987 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.194207+0000", 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00039802099999999998 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.201380+0000", 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0071722790000000002 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205652+0000", 2023-09-23T20:22:38.988 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0042719180000000004 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205692+0000", 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.0253999999999998e-05 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.205746+0000", 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4187999999999997e-05 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.989 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:14:36.207310+0000", 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.001564027 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:96745 10.1 10:8024395c:::benchmark_data_smithi135_602286_object4519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:18:45.899523+0000", 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 232.94332556000001, 2023-09-23T20:22:38.990 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27761313500000001, 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 96745 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899523+0000", 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.991 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899523+0000", 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899522+0000", 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.992 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899526+0000", 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.515e-06 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899536+0000", 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.3130000000000004e-06 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899541+0000", 2023-09-23T20:22:38.993 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9470000000000002e-06 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899576+0000", 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.4932999999999999e-05 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899656+0000", 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.9961000000000002e-05 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.994 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.899722+0000", 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.6236999999999996e-05 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.902981+0000", 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0032588629999999999 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.995 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:45.903184+0000", 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00020359299999999999 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:46.177032+0000", 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.27384766399999999 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:46.177090+0000", 2023-09-23T20:22:38.996 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.8505999999999999e-05 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:18:46.177136+0000", 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.5599999999999997e-05 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.997 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:112760 10.c 10:3b7ab588:::benchmark_data_smithi135_602286_object20534:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:19:11.524164+0000", 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 207.31868477699999, 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.23737845599999999, 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 112760 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:38.998 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524164+0000", 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524164+0000", 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:38.999 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524162+0000", 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999986 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524169+0000", 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.9450000000000004e-06 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524184+0000", 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.4538e-05 2023-09-23T20:22:39.000 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524192+0000", 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.7170000000000008e-06 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524241+0000", 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.9531999999999999e-05 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.001 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524345+0000", 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000104026 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.524440+0000", 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 9.5217000000000007e-05 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:39.002 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.527267+0000", 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0028268780000000001 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.529349+0000", 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0020817639999999998 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.761446+0000", 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.232097266 2023-09-23T20:22:39.003 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.761500+0000", 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 5.4014000000000002e-05 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:19:11.761542+0000", 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1779000000000001e-05 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:39.004 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:182401 10.c 10:38d6aa55:::benchmark_data_smithi135_602286_object90175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:20:58.815612+0000", 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 100.02723609100001, 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.236735467, 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:22:39.005 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182401 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815612+0000", 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815612+0000", 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:39.006 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815611+0000", 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.999999 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815620+0000", 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 8.8349999999999993e-06 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.007 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815631+0000", 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 1.1090000000000001e-05 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815637+0000", 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.0129999999999999e-06 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:39.008 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.817964+0000", 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0023266939999999998 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.818090+0000", 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00012637 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.818203+0000", 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.00011270800000000001 2023-09-23T20:22:39.009 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.823688+0000", 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.005484999 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.823754+0000", 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.5816999999999997e-05 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.010 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:59.052197+0000", 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.22844339899999999 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:59.052309+0000", 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000111882 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:59.052348+0000", 2023-09-23T20:22:39.011 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 3.8686999999999997e-05 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: "description": "osd_op(client.46418.0:182402 10.c 10:31c538bf:::benchmark_data_smithi135_602286_object90176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: "initiated_at": "2023-09-23T20:20:58.815643+0000", 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: "age": 100.027205112, 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.236856286, 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: "type_data": { 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:22:39.012 INFO:teuthology.orchestra.run.smithi144.stdout: "client_info": { 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: "client": "client.46418", 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: "client_addr": "172.21.15.135:0/1588303245", 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: "tid": 182402 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: "events": [ 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "initiated", 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815643+0000", 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.013 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled", 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815643+0000", 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "header_read", 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815643+0000", 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4294967295.9999995 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "all_read", 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815646+0000", 2023-09-23T20:22:39.014 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 2.7199999999999998e-06 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "dispatched", 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815652+0000", 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 6.015e-06 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "queued_for_pg", 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.815656+0000", 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.1620000000000001e-06 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.015 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "reached_pg", 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.820813+0000", 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0051569160000000001 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "started", 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.820916+0000", 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000103257 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:22:39.016 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.821019+0000", 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.000102908 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.827352+0000", 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.0063328400000000002 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "op_commit", 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:58.827359+0000", 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 7.1289999999999998e-06 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.017 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:59.052407+0000", 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 0.22504843199999999 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "commit_sent", 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:59.052455+0000", 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.7917000000000002e-05 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: { 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "done", 2023-09-23T20:22:39.018 INFO:teuthology.orchestra.run.smithi144.stdout: "time": "2023-09-23T20:20:59.052500+0000", 2023-09-23T20:22:39.019 INFO:teuthology.orchestra.run.smithi144.stdout: "duration": 4.4487000000000003e-05 2023-09-23T20:22:39.019 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:39.019 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:39.019 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:39.019 INFO:teuthology.orchestra.run.smithi144.stdout: } 2023-09-23T20:22:39.019 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2023-09-23T20:22:39.019 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:22:39.019 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:39.020 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:39.047 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:39.131 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:38 smithi144 conmon[400857]: 2023-09-23T20:22:38.891+0000 7f1a04f77540 -1 Falling back to public interface 2023-09-23T20:22:39.191 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:39.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:39.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:39.295 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:39.328 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:39.377 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:39.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:39.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:39.480 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:39.511 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:39.560 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:39.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:39.661 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:22:39.663 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:39.692 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:39.739 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:39.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:39.843 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:39.872 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:40.067 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:40.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:40.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:40.171 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:40.198 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:40.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:40 smithi135 ceph-mon[213871]: pgmap v1573: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 77 KiB/s rd, 31 KiB/s wr, 20 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:40.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:40 smithi135 ceph-mon[221113]: pgmap v1573: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 77 KiB/s rd, 31 KiB/s wr, 20 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:40.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:40 smithi144 ceph-mon[194833]: pgmap v1573: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 77 KiB/s rd, 31 KiB/s wr, 20 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:40.931 INFO:teuthology.orchestra.run.smithi144.stderr:no valid command found; 10 closest matches: 2023-09-23T20:22:40.931 INFO:teuthology.orchestra.run.smithi144.stderr:0 2023-09-23T20:22:40.931 INFO:teuthology.orchestra.run.smithi144.stderr:1 2023-09-23T20:22:40.931 INFO:teuthology.orchestra.run.smithi144.stderr:2 2023-09-23T20:22:40.931 INFO:teuthology.orchestra.run.smithi144.stderr:abort 2023-09-23T20:22:40.931 INFO:teuthology.orchestra.run.smithi144.stderr:assert 2023-09-23T20:22:40.931 INFO:teuthology.orchestra.run.smithi144.stderr:config diff 2023-09-23T20:22:40.932 INFO:teuthology.orchestra.run.smithi144.stderr:config diff get 2023-09-23T20:22:40.932 INFO:teuthology.orchestra.run.smithi144.stderr:config get 2023-09-23T20:22:40.932 INFO:teuthology.orchestra.run.smithi144.stderr:config help [] 2023-09-23T20:22:40.932 INFO:teuthology.orchestra.run.smithi144.stderr:config set ... 2023-09-23T20:22:40.932 INFO:teuthology.orchestra.run.smithi144.stderr:admin_socket: invalid command 2023-09-23T20:22:41.520 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:22:41.520 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-09-23T20:22:42.354 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:41 smithi144 conmon[400857]: 2023-09-23T20:22:41.961+0000 7f1a04f77540 -1 osd.6 501 log_to_monitors true 2023-09-23T20:22:42.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:22:42 smithi144 conmon[400857]: 2023-09-23T20:22:42.392+0000 7f19f4554700 -1 osd.6 550 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:22:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:42 smithi144 ceph-mon[194833]: pgmap v1574: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:42 smithi144 ceph-mon[194833]: from='osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:22:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:42 smithi144 ceph-mon[194833]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:22:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:42 smithi135 ceph-mon[213871]: pgmap v1574: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:42 smithi135 ceph-mon[213871]: from='osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:22:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:42 smithi135 ceph-mon[213871]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:22:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:42 smithi135 ceph-mon[221113]: pgmap v1574: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:42 smithi135 ceph-mon[221113]: from='osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:22:42.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:42 smithi135 ceph-mon[221113]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:22:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:43 smithi144 ceph-mon[194833]: failed to encode map e550 with expected crc 2023-09-23T20:22:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:43 smithi144 ceph-mon[194833]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:22:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:43 smithi144 ceph-mon[194833]: osdmap e550: 8 total, 7 up, 8 in 2023-09-23T20:22:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:43 smithi144 ceph-mon[194833]: from='osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:22:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:43 smithi144 ceph-mon[194833]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:22:43.685 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[213871]: failed to encode map e550 with expected crc 2023-09-23T20:22:43.685 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[213871]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:22:43.685 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[213871]: osdmap e550: 8 total, 7 up, 8 in 2023-09-23T20:22:43.685 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[213871]: from='osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:22:43.685 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[213871]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:22:43.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[221113]: failed to encode map e550 with expected crc 2023-09-23T20:22:43.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[221113]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:22:43.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[221113]: osdmap e550: 8 total, 7 up, 8 in 2023-09-23T20:22:43.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[221113]: from='osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:22:43.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:43 smithi135 ceph-mon[221113]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2023-09-23T20:22:43.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:22:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:22:43] "GET /metrics HTTP/1.1" 200 34119 "" "Prometheus/2.43.0" 2023-09-23T20:22:44.436 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format - testimg 2023-09-23T20:22:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: failed to encode map e550 with expected crc 2023-09-23T20:22:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: pgmap v1576: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 149 KiB/s rd, 64 KiB/s wr, 38 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: failed to encode map e550 with expected crc 2023-09-23T20:22:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:22:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340] boot 2023-09-23T20:22:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: osdmap e551: 8 total, 8 up, 8 in 2023-09-23T20:22:44.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:22:44.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:44 smithi144 ceph-mon[194833]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: failed to encode map e550 with expected crc 2023-09-23T20:22:44.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: pgmap v1576: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 149 KiB/s rd, 64 KiB/s wr, 38 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:44.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: failed to encode map e550 with expected crc 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340] boot 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: osdmap e551: 8 total, 8 up, 8 in 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[213871]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: failed to encode map e550 with expected crc 2023-09-23T20:22:44.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: pgmap v1576: 103 pgs: 1 active+undersized+degraded, 1 active+clean+remapped, 3 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 149 KiB/s rd, 64 KiB/s wr, 38 op/s; 1/84 objects degraded (1.190%) 2023-09-23T20:22:44.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: failed to encode map e550 with expected crc 2023-09-23T20:22:44.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:22:44.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: osd.6 [v2:172.21.15.144:6816/1753202340,v1:172.21.15.144:6817/1753202340] boot 2023-09-23T20:22:44.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: osdmap e551: 8 total, 8 up, 8 in 2023-09-23T20:22:44.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:22:44.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:44 smithi135 ceph-mon[221113]: failed to encode map e551 with expected crc 2023-09-23T20:22:44.932 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:44.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:45.037 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:45.066 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:45.163 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:45.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:45.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:45.268 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:45.298 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:45.315 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:45.399 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:45.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:45.503 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:45.537 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:45.585 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:45.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:45.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:45.677 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:22:45.692 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e502 with expected crc 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e542 with expected crc 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e551 with expected crc 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e551 with expected crc 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.693 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[213871]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.694 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e502 with expected crc 2023-09-23T20:22:45.694 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e542 with expected crc 2023-09-23T20:22:45.694 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e551 with expected crc 2023-09-23T20:22:45.694 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e551 with expected crc 2023-09-23T20:22:45.694 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.694 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.694 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T20:22:45.694 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.695 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.695 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:45 smithi135 ceph-mon[221113]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.701 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:22:45.727 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:22:45.736 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:45.750 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:22:45.776 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:22:45.794 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:45.798 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:22:45.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:45.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:45.819 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:22:45.829 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:22:45.829 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:22:45.832 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg /tmp/rbd_import_export_682894/img2 2023-09-23T20:22:45.841 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:22:45.842 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:22:45.842 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:22:45.842 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:22:45.842 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:22:45.842 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:22:45.843 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:22:45.843 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:22:45.843 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:22:45.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e502 with expected crc 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e542 with expected crc 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e551 with expected crc 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e551 with expected crc 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:45 smithi144 ceph-mon[194833]: failed to encode map e552 with expected crc 2023-09-23T20:22:45.901 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:45.934 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:46.125 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:46.129 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg - 2023-09-23T20:22:46.521 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:22:46.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[221113]: pgmap v1579: 103 pgs: 1 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 2 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s rd, 34 KiB/s wr, 17 op/s; 1/87 objects degraded (1.149%) 2023-09-23T20:22:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[221113]: Health check update: Degraded data redundancy: 1/87 objects degraded (1.149%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[221113]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[221113]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[221113]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[221113]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T20:22:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/728400680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:46.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[213871]: pgmap v1579: 103 pgs: 1 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 2 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s rd, 34 KiB/s wr, 17 op/s; 1/87 objects degraded (1.149%) 2023-09-23T20:22:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[213871]: Health check update: Degraded data redundancy: 1/87 objects degraded (1.149%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[213871]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[213871]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[213871]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[213871]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T20:22:46.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/728400680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:46.864 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:46.864 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:22:46.867 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm testimg 2023-09-23T20:22:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:46 smithi144 ceph-mon[194833]: pgmap v1579: 103 pgs: 1 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 2 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s rd, 34 KiB/s wr, 17 op/s; 1/87 objects degraded (1.149%) 2023-09-23T20:22:46.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:46 smithi144 ceph-mon[194833]: Health check update: Degraded data redundancy: 1/87 objects degraded (1.149%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:22:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:46 smithi144 ceph-mon[194833]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:46 smithi144 ceph-mon[194833]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:46 smithi144 ceph-mon[194833]: failed to encode map e553 with expected crc 2023-09-23T20:22:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:46 smithi144 ceph-mon[194833]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T20:22:46.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/728400680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:46.951 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T20:22:46.954 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp /tmp/rbd_import_export_682894/img /tmp/rbd_import_export_682894/img2 2023-09-23T20:22:47.169 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:22:47.590 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp /tmp/rbd_import_export_682894/img /tmp/rbd_import_export_682894/img3 2023-09-23T20:22:47.653 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:47.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:47.757 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:47.785 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:48.053 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img /tmp/rbd_import_export_682894/img2 /tmp/rbd_import_export_682894/img3 2023-09-23T20:22:48.471 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:48.472 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:48.628 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:48 smithi144 ceph-mon[194833]: pgmap v1581: 103 pgs: 1 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 2 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 831 B/s rd, 623 B/s wr, 2 op/s; 1/90 objects degraded (1.111%) 2023-09-23T20:22:48.714 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd help export 2023-09-23T20:22:48.714 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -q export-format 2023-09-23T20:22:48.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:48 smithi135 ceph-mon[221113]: pgmap v1581: 103 pgs: 1 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 2 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 831 B/s rd, 623 B/s wr, 2 op/s; 1/90 objects degraded (1.111%) 2023-09-23T20:22:48.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:48 smithi135 ceph-mon[213871]: pgmap v1581: 103 pgs: 1 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 2 active+undersized, 98 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 831 B/s rd, 623 B/s wr, 2 op/s; 1/90 objects degraded (1.111%) 2023-09-23T20:22:48.734 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_682894/img bs=1k count=10 seek=100 2023-09-23T20:22:48.735 INFO:tasks.workunit.client.0.smithi135.stderr:10+0 records in 2023-09-23T20:22:48.735 INFO:tasks.workunit.client.0.smithi135.stderr:10+0 records out 2023-09-23T20:22:48.736 INFO:tasks.workunit.client.0.smithi135.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.3884e-05 s, 160 MB/s 2023-09-23T20:22:48.736 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format /tmp/rbd_import_export_682894/img testimg 2023-09-23T20:22:48.801 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:22:48.801 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:22:48.803 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap create testimg@snap 2023-09-23T20:22:49.412 INFO:tasks.workunit.client.0.smithi135.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:22:49.418 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T20:22:49.466 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd image-meta list testimg 2023-09-23T20:22:49.503 INFO:tasks.workunit.client.0.smithi135.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T20:22:49.503 INFO:tasks.workunit.client.0.smithi135.stderr: 2023-09-23T20:22:49.504 INFO:tasks.workunit.client.0.smithi135.stderr:Key Value 2023-09-23T20:22:49.504 INFO:tasks.workunit.client.0.smithi135.stderr:key1 value1' 2023-09-23T20:22:49.504 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_682894/img_v2 2023-09-23T20:22:49.504 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:49.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:49.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:49.544 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:49.547 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_682894/img_v2 testimg_import 2023-09-23T20:22:49.609 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:49.628 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:49.640 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:49.682 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:49.686 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:49 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T20:22:49.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:49.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:49.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:49 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T20:22:49.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:49 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T20:22:49.787 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:49.816 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.034 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:50.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:50.070 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:50.071 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:50.138 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:50.168 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.213 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:22:50.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:50.320 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:50.347 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.390 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:50.422 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:22:50.425 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import 2023-09-23T20:22:50.458 INFO:tasks.workunit.client.0.smithi135.stdout:rbd image 'testimg_import': 2023-09-23T20:22:50.458 INFO:tasks.workunit.client.0.smithi135.stdout: size 110 KiB in 1 objects 2023-09-23T20:22:50.458 INFO:tasks.workunit.client.0.smithi135.stdout: order 22 (4 MiB objects) 2023-09-23T20:22:50.458 INFO:tasks.workunit.client.0.smithi135.stdout: snapshot_count: 1 2023-09-23T20:22:50.458 INFO:tasks.workunit.client.0.smithi135.stdout: id: ddd72151a44 2023-09-23T20:22:50.459 INFO:tasks.workunit.client.0.smithi135.stdout: block_name_prefix: rbd_data.ddd72151a44 2023-09-23T20:22:50.459 INFO:tasks.workunit.client.0.smithi135.stdout: format: 2 2023-09-23T20:22:50.459 INFO:tasks.workunit.client.0.smithi135.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T20:22:50.459 INFO:tasks.workunit.client.0.smithi135.stdout: op_features: 2023-09-23T20:22:50.459 INFO:tasks.workunit.client.0.smithi135.stdout: flags: 2023-09-23T20:22:50.459 INFO:tasks.workunit.client.0.smithi135.stdout: create_timestamp: Sat Sep 23 20:22:49 2023 2023-09-23T20:22:50.459 INFO:tasks.workunit.client.0.smithi135.stdout: access_timestamp: Sat Sep 23 20:22:49 2023 2023-09-23T20:22:50.459 INFO:tasks.workunit.client.0.smithi135.stdout: modify_timestamp: Sat Sep 23 20:22:49 2023 2023-09-23T20:22:50.461 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import@snap 2023-09-23T20:22:50.493 INFO:tasks.workunit.client.0.smithi135.stdout:rbd image 'testimg_import': 2023-09-23T20:22:50.493 INFO:tasks.workunit.client.0.smithi135.stdout: size 110 KiB in 1 objects 2023-09-23T20:22:50.493 INFO:tasks.workunit.client.0.smithi135.stdout: order 22 (4 MiB objects) 2023-09-23T20:22:50.493 INFO:tasks.workunit.client.0.smithi135.stdout: snapshot_count: 1 2023-09-23T20:22:50.493 INFO:tasks.workunit.client.0.smithi135.stdout: id: ddd72151a44 2023-09-23T20:22:50.493 INFO:tasks.workunit.client.0.smithi135.stdout: block_name_prefix: rbd_data.ddd72151a44 2023-09-23T20:22:50.494 INFO:tasks.workunit.client.0.smithi135.stdout: format: 2 2023-09-23T20:22:50.494 INFO:tasks.workunit.client.0.smithi135.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T20:22:50.494 INFO:tasks.workunit.client.0.smithi135.stdout: op_features: 2023-09-23T20:22:50.494 INFO:tasks.workunit.client.0.smithi135.stdout: flags: 2023-09-23T20:22:50.494 INFO:tasks.workunit.client.0.smithi135.stdout: create_timestamp: Sat Sep 23 20:22:49 2023 2023-09-23T20:22:50.494 INFO:tasks.workunit.client.0.smithi135.stdout: access_timestamp: Sat Sep 23 20:22:49 2023 2023-09-23T20:22:50.494 INFO:tasks.workunit.client.0.smithi135.stdout: modify_timestamp: Sat Sep 23 20:22:49 2023 2023-09-23T20:22:50.494 INFO:tasks.workunit.client.0.smithi135.stdout: protected: False 2023-09-23T20:22:50.496 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd image-meta list testimg_import 2023-09-23T20:22:50.497 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:50.521 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: pgmap v1582: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 154 KiB/s rd, 74 KiB/s wr, 27 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T20:22:50.521 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/90 objects degraded (1.111%), 1 pg degraded, 3 pgs undersized) 2023-09-23T20:22:50.521 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.521 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.521 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.522 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T20:22:50.522 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T20:22:50.522 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.522 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:50 smithi144 ceph-mon[194833]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.523 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr: 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:Key Value 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:key1 value1' 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr: 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:Key Value 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr: 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:Key Value 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:key1 value1' ']' 2023-09-23T20:22:50.535 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg_import /tmp/rbd_import_export_682894/img_import 2023-09-23T20:22:50.566 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:22:50.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:50.630 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete...done. 2023-09-23T20:22:50.633 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_682894/img /tmp/rbd_import_export_682894/img_import 2023-09-23T20:22:50.633 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_682894/img /tmp/rbd_import_export_682894/img_import 2023-09-23T20:22:50.634 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/img --format %b 2023-09-23T20:22:50.635 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=24 2023-09-23T20:22:50.635 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/img_import --format %b 2023-09-23T20:22:50.636 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=224 2023-09-23T20:22:50.636 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=200 2023-09-23T20:22:50.636 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=200 2023-09-23T20:22:50.636 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T20:22:50.636 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_682894/img_snap 2023-09-23T20:22:50.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: pgmap v1582: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 154 KiB/s rd, 74 KiB/s wr, 27 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T20:22:50.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/90 objects degraded (1.111%), 1 pg degraded, 3 pgs undersized) 2023-09-23T20:22:50.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.657 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T20:22:50.658 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T20:22:50.658 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.658 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[221113]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: pgmap v1582: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 154 KiB/s rd, 74 KiB/s wr, 27 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T20:22:50.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/90 objects degraded (1.111%), 1 pg degraded, 3 pgs undersized) 2023-09-23T20:22:50.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.658 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.659 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T20:22:50.659 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T20:22:50.659 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.659 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:50 smithi135 ceph-mon[213871]: failed to encode map e554 with expected crc 2023-09-23T20:22:50.671 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete...done. 2023-09-23T20:22:50.672 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:50.675 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_682894/img_snap_import 2023-09-23T20:22:50.701 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.710 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete...done. 2023-09-23T20:22:50.714 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_682894/img_snap /tmp/rbd_import_export_682894/img_snap_import 2023-09-23T20:22:50.714 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_682894/img_snap /tmp/rbd_import_export_682894/img_snap_import 2023-09-23T20:22:50.714 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/img_snap --format %b 2023-09-23T20:22:50.715 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=224 2023-09-23T20:22:50.715 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/img_snap_import --format %b 2023-09-23T20:22:50.717 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=224 2023-09-23T20:22:50.717 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:22:50.717 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:22:50.717 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:22:50.717 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img_v2 2023-09-23T20:22:50.717 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img_import 2023-09-23T20:22:50.718 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img_snap 2023-09-23T20:22:50.719 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img_snap_import 2023-09-23T20:22:50.719 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T20:22:51.260 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:22:51.419 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:22:51.425 INFO:tasks.workunit.client.0.smithi135.stderr: Removing snap: 100% complete...done. 2023-09-23T20:22:51.432 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T20:22:51.450 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:22:51.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:51.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:51.540 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:22:51.543 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap rm testimg@snap 2023-09-23T20:22:51.554 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:51.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[213871]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.587 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[213871]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[213871]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[213871]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[213871]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[213871]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[213871]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[213871]: pgmap v1585: 103 pgs: 4 remapped+peering, 2 unknown, 97 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 231 KiB/s rd, 106 KiB/s wr, 91 op/s; 31 KiB/s, 1 objects/s recovering 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[221113]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[221113]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[221113]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.588 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[221113]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T20:22:51.589 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[221113]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.589 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[221113]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.589 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[221113]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.589 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:51 smithi135 ceph-mon[221113]: pgmap v1585: 103 pgs: 4 remapped+peering, 2 unknown, 97 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 231 KiB/s rd, 106 KiB/s wr, 91 op/s; 31 KiB/s, 1 objects/s recovering 2023-09-23T20:22:51.591 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:51.698 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:51.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:51.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:51.776 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:51.802 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:51.826 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:51 smithi144 ceph-mon[194833]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.827 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:51 smithi144 ceph-mon[194833]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.827 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:51 smithi144 ceph-mon[194833]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.827 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:51 smithi144 ceph-mon[194833]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T20:22:51.827 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:51 smithi144 ceph-mon[194833]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.827 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:51 smithi144 ceph-mon[194833]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.827 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:51 smithi144 ceph-mon[194833]: failed to encode map e555 with expected crc 2023-09-23T20:22:51.827 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:51 smithi144 ceph-mon[194833]: pgmap v1585: 103 pgs: 4 remapped+peering, 2 unknown, 97 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 231 KiB/s rd, 106 KiB/s wr, 91 op/s; 31 KiB/s, 1 objects/s recovering 2023-09-23T20:22:51.829 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:51.870 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:51.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:51.975 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:52.001 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:52.042 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:52.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:52.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:52.147 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:52.172 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:52.214 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:22:52.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:52.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:52.320 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:52.347 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:52.349 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:52.429 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:22:52.429 INFO:tasks.workunit.client.0.smithi135.stderr: Removing snap: 100% complete...done. 2023-09-23T20:22:52.435 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm testimg 2023-09-23T20:22:52.503 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:52.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:52.534 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:22:52.537 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --order 20 /tmp/rbd_import_export_682894/img testimg 2023-09-23T20:22:52.592 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:22:52.593 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:22:52.594 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_682894/img_v2 2023-09-23T20:22:52.608 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:52.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: osdmap e556: 8 total, 8 up, 8 in 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4173484466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:22:52.634 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:52.674 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:22:52.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:52.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:52.704 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:52.707 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_682894/img_v2 testimg_import 2023-09-23T20:22:52.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: osdmap e556: 8 total, 8 up, 8 in 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4173484466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: osdmap e556: 8 total, 8 up, 8 in 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: failed to encode map e556 with expected crc 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:52.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4173484466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:22:52.765 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:22:52.768 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import 2023-09-23T20:22:52.769 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep order 2023-09-23T20:22:52.769 INFO:tasks.workunit.client.0.smithi135.stderr:+ awk '{print $2}' 2023-09-23T20:22:52.769 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 20 2023-09-23T20:22:52.781 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:52.804 INFO:tasks.workunit.client.0.smithi135.stdout:20 2023-09-23T20:22:52.804 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img_v2 2023-09-23T20:22:52.804 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T20:22:52.807 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:52.850 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:52.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:52.890 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:22:52.893 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg 2023-09-23T20:22:52.956 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:52.978 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:22:52.981 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_682894/img testimg 2023-09-23T20:22:52.983 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.025 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:22:53.027 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:22:53.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:53.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:53.031 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd info testimg 2023-09-23T20:22:53.031 INFO:tasks.workunit.client.0.smithi135.stderr:++ grep features 2023-09-23T20:22:53.067 INFO:tasks.workunit.client.0.smithi135.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T20:22:53.067 INFO:tasks.workunit.client.0.smithi135.stderr: op_features: ' 2023-09-23T20:22:53.067 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_682894/img_v2 2023-09-23T20:22:53.103 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:53.106 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_682894/img_v2 testimg_import 2023-09-23T20:22:53.131 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:53.156 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.161 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:22:53.164 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd info testimg_import 2023-09-23T20:22:53.165 INFO:tasks.workunit.client.0.smithi135.stderr:++ grep features 2023-09-23T20:22:53.197 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:53.198 INFO:tasks.workunit.client.0.smithi135.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T20:22:53.198 INFO:tasks.workunit.client.0.smithi135.stderr: op_features: ' 2023-09-23T20:22:53.199 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' ' features: layering 2023-09-23T20:22:53.199 INFO:tasks.workunit.client.0.smithi135.stderr: op_features: ' '!=' ' features: layering 2023-09-23T20:22:53.199 INFO:tasks.workunit.client.0.smithi135.stderr: op_features: ' ']' 2023-09-23T20:22:53.199 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img_v2 2023-09-23T20:22:53.199 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T20:22:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:53.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:53.302 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:53.332 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.339 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:22:53.342 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg 2023-09-23T20:22:53.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[213871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:53.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[213871]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[213871]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4173484466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:22:53.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[213871]: osdmap e557: 8 total, 8 up, 8 in 2023-09-23T20:22:53.410 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[213871]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.411 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[213871]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.411 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[213871]: pgmap v1588: 103 pgs: 3 peering, 1 active+undersized+remapped, 1 remapped+peering, 2 unknown, 96 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 260 KiB/s rd, 65 KiB/s wr, 272 op/s; 46 KiB/s, 0 objects/s recovering 2023-09-23T20:22:53.412 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:22:53.415 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_682894/img testimg 2023-09-23T20:22:53.483 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:22:53.486 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_682894/img_v2 2023-09-23T20:22:53.523 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:53.526 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_682894/img_v2 testimg_import 2023-09-23T20:22:53.591 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:22:53.594 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import 2023-09-23T20:22:53.594 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 'stripe unit' 2023-09-23T20:22:53.594 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T20:22:53.633 INFO:tasks.workunit.client.0.smithi135.stdout: stripe unit: 4 KiB 2023-09-23T20:22:53.634 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import 2023-09-23T20:22:53.634 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 'stripe count' 2023-09-23T20:22:53.634 INFO:tasks.workunit.client.0.smithi135.stderr:+ awk '{print $3}' 2023-09-23T20:22:53.634 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 1000 2023-09-23T20:22:53.669 INFO:tasks.workunit.client.0.smithi135.stdout:1000 2023-09-23T20:22:53.669 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/img_v2 2023-09-23T20:22:53.670 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T20:22:53.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[221113]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:53.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[221113]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[221113]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4173484466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:22:53.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[221113]: osdmap e557: 8 total, 8 up, 8 in 2023-09-23T20:22:53.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[221113]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[221113]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:53 smithi135 ceph-mon[221113]: pgmap v1588: 103 pgs: 3 peering, 1 active+undersized+remapped, 1 remapped+peering, 2 unknown, 96 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 260 KiB/s rd, 65 KiB/s wr, 272 op/s; 46 KiB/s, 0 objects/s recovering 2023-09-23T20:22:53.764 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T20:22:53.767 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg 2023-09-23T20:22:53.776 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:53.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:53.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:53.866 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T20:22:53.869 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_682894/img testimg 2023-09-23T20:22:53.881 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:53 smithi144 ceph-mon[194833]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:53 smithi144 ceph-mon[194833]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:53 smithi144 ceph-mon[194833]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:53 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4173484466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:22:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:53 smithi144 ceph-mon[194833]: osdmap e557: 8 total, 8 up, 8 in 2023-09-23T20:22:53.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:53 smithi144 ceph-mon[194833]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:53 smithi144 ceph-mon[194833]: failed to encode map e557 with expected crc 2023-09-23T20:22:53.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:53 smithi144 ceph-mon[194833]: pgmap v1588: 103 pgs: 3 peering, 1 active+undersized+remapped, 1 remapped+peering, 2 unknown, 96 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 260 KiB/s rd, 65 KiB/s wr, 272 op/s; 46 KiB/s, 0 objects/s recovering 2023-09-23T20:22:53.912 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.927 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:22:53.930 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap create testimg@snap1 2023-09-23T20:22:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:22:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:22:53] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2023-09-23T20:22:54.011 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 7, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 7, 2, 6] 2023-09-23T20:22:54.011 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:22:54.011 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:22:54.012 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:22:54.319 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:54.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:54.327 INFO:teuthology.orchestra.run.smithi135.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:54.327 INFO:teuthology.orchestra.run.smithi135.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:54.423 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:54.445 INFO:tasks.workunit.client.0.smithi135.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:22:54.452 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap create testimg@snap2 2023-09-23T20:22:54.457 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: osdmap e558: 8 total, 8 up, 8 in 2023-09-23T20:22:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: 1.1a scrub starts 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[213871]: 1.1a scrub ok 2023-09-23T20:22:54.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: osdmap e558: 8 total, 8 up, 8 in 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: 1.1a scrub starts 2023-09-23T20:22:54.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:54 smithi135 ceph-mon[221113]: 1.1a scrub ok 2023-09-23T20:22:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: osdmap e558: 8 total, 8 up, 8 in 2023-09-23T20:22:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: failed to encode map e558 with expected crc 2023-09-23T20:22:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: 1.1a scrub starts 2023-09-23T20:22:54.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:54 smithi144 ceph-mon[194833]: 1.1a scrub ok 2023-09-23T20:22:55.449 INFO:tasks.workunit.client.0.smithi135.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:22:55.456 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap protect testimg@snap2 2023-09-23T20:22:55.503 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_682894/snap_protect 2023-09-23T20:22:55.543 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:22:55.546 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_682894/snap_protect testimg_import 2023-09-23T20:22:55.599 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:22:55.619 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:55.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:55.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:55.674 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[221113]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[221113]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[221113]: osdmap e559: 8 total, 8 up, 8 in 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[221113]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[221113]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[221113]: pgmap v1591: 103 pgs: 3 peering, 1 active+undersized+remapped, 1 remapped+peering, 98 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 358 KiB/s rd, 50 KiB/s wr, 456 op/s; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[213871]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[213871]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[213871]: osdmap e559: 8 total, 8 up, 8 in 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[213871]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[213871]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:55 smithi135 ceph-mon[213871]: pgmap v1591: 103 pgs: 3 peering, 1 active+undersized+remapped, 1 remapped+peering, 98 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 358 KiB/s rd, 50 KiB/s wr, 456 op/s; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:22:55.727 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:55.755 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:55 smithi144 ceph-mon[194833]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:55 smithi144 ceph-mon[194833]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:55 smithi144 ceph-mon[194833]: osdmap e559: 8 total, 8 up, 8 in 2023-09-23T20:22:55.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:55 smithi144 ceph-mon[194833]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:55 smithi144 ceph-mon[194833]: failed to encode map e559 with expected crc 2023-09-23T20:22:55.756 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:55 smithi144 ceph-mon[194833]: pgmap v1591: 103 pgs: 3 peering, 1 active+undersized+remapped, 1 remapped+peering, 98 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 358 KiB/s rd, 50 KiB/s wr, 456 op/s; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:22:55.759 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:55.779 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:22:55.809 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:55.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:55.916 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:55.945 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:55.988 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:55.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:55.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:56.093 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:56.126 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.250 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:22:56.279 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:22:56.305 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:22:56.322 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:22:56.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:56.330 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:22:56.352 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:22:56.371 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:22:56.390 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:22:56.410 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:22:56.411 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:22:56.411 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:22:56.411 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:22:56.412 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:22:56.412 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:22:56.412 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:22:56.412 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:22:56.412 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:22:56.426 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:56.456 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.499 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:56.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:56.605 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:56.631 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.672 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:56.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: 1.1b scrub starts 2023-09-23T20:22:56.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: 1.1b scrub ok 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: osdmap e560: 8 total, 8 up, 8 in 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2745924496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: 1.1b scrub starts 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: 1.1b scrub ok 2023-09-23T20:22:56.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: osdmap e560: 8 total, 8 up, 8 in 2023-09-23T20:22:56.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2745924496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:56.777 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:56.801 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: 1.1b scrub starts 2023-09-23T20:22:56.801 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: 1.1b scrub ok 2023-09-23T20:22:56.801 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.801 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.801 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: osdmap e560: 8 total, 8 up, 8 in 2023-09-23T20:22:56.802 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.802 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.802 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: failed to encode map e560 with expected crc 2023-09-23T20:22:56.802 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2745924496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:56.803 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.844 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:56.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:56.949 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:56.975 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:57.016 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:22:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:57.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:57.123 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:57.152 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:57.467 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:22:57.470 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import@snap1 2023-09-23T20:22:57.470 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 'protected: False' 2023-09-23T20:22:57.509 INFO:tasks.workunit.client.0.smithi135.stdout: protected: False 2023-09-23T20:22:57.509 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd info testimg_import@snap2 2023-09-23T20:22:57.509 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep 'protected: True' 2023-09-23T20:22:57.550 INFO:tasks.workunit.client.0.smithi135.stdout: protected: True 2023-09-23T20:22:57.550 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/snap_protect 2023-09-23T20:22:57.551 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T20:22:57.593 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T20:22:57.601 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:57.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:57.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:57.640 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap purge testimg 2023-09-23T20:22:57.644 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:57.705 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:57.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[221113]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[221113]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[221113]: osdmap e561: 8 total, 8 up, 8 in 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[221113]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[221113]: 1.1e scrub starts 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[221113]: 1.1e scrub ok 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[221113]: pgmap v1594: 103 pgs: 3 peering, 1 remapped+peering, 99 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 239 KiB/s rd, 39 KiB/s wr, 321 op/s; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[213871]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[213871]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[213871]: osdmap e561: 8 total, 8 up, 8 in 2023-09-23T20:22:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[213871]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[213871]: 1.1e scrub starts 2023-09-23T20:22:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[213871]: 1.1e scrub ok 2023-09-23T20:22:57.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:57 smithi135 ceph-mon[213871]: pgmap v1594: 103 pgs: 3 peering, 1 remapped+peering, 99 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 239 KiB/s rd, 39 KiB/s wr, 321 op/s; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:22:57.729 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:57 smithi144 ceph-mon[194833]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.729 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:57 smithi144 ceph-mon[194833]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.729 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:57 smithi144 ceph-mon[194833]: osdmap e561: 8 total, 8 up, 8 in 2023-09-23T20:22:57.730 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:57 smithi144 ceph-mon[194833]: failed to encode map e561 with expected crc 2023-09-23T20:22:57.730 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:57 smithi144 ceph-mon[194833]: 1.1e scrub starts 2023-09-23T20:22:57.730 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:57 smithi144 ceph-mon[194833]: 1.1e scrub ok 2023-09-23T20:22:57.730 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:57 smithi144 ceph-mon[194833]: pgmap v1594: 103 pgs: 3 peering, 1 remapped+peering, 99 active+clean; 4.8 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 239 KiB/s rd, 39 KiB/s wr, 321 op/s; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:22:57.732 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:57.748 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:57.773 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:57.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:57.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:57.880 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:57.905 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:57.946 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:57.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:57.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:58.054 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:58.085 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:58.184 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:58.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:58.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:58.288 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:58.314 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:58.320 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:22:58.322 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:22:58.355 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:58.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:58.444 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: 1.17 scrub starts 2023-09-23T20:22:58.460 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:58.506 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:58.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: 1.17 scrub starts 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: 1.17 scrub ok 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: osdmap e562: 8 total, 8 up, 8 in 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: 1.1d scrub starts 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: 1.1d scrub ok 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: 1.1c scrub starts 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: 1.1c scrub ok 2023-09-23T20:22:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:22:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: osdmap e563: 8 total, 8 up, 8 in 2023-09-23T20:22:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: failed to encode map e563 with expected crc 2023-09-23T20:22:58.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:58 smithi144 ceph-mon[194833]: from='client.47198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: 1.17 scrub starts 2023-09-23T20:22:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: 1.17 scrub ok 2023-09-23T20:22:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: osdmap e562: 8 total, 8 up, 8 in 2023-09-23T20:22:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: 1.1d scrub starts 2023-09-23T20:22:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: 1.1d scrub ok 2023-09-23T20:22:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: 1.1c scrub starts 2023-09-23T20:22:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: 1.1c scrub ok 2023-09-23T20:22:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:22:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: osdmap e563: 8 total, 8 up, 8 in 2023-09-23T20:22:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: failed to encode map e563 with expected crc 2023-09-23T20:22:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[221113]: from='client.47198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:58.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: 1.17 scrub ok 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: osdmap e562: 8 total, 8 up, 8 in 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: failed to encode map e562 with expected crc 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: 1.1d scrub starts 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: 1.1d scrub ok 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: 1.1c scrub starts 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: 1.1c scrub ok 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:22:58.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: osdmap e563: 8 total, 8 up, 8 in 2023-09-23T20:22:58.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: failed to encode map e563 with expected crc 2023-09-23T20:22:58.724 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:58 smithi135 ceph-mon[213871]: from='client.47198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:59.103 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:22:59.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:59.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:59.122 INFO:tasks.workunit.client.0.smithi135.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T20:22:59.130 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd snap purge testimg_import 2023-09-23T20:22:59.207 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:59.237 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:59.282 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:22:59.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:59.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:59.388 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:59.414 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:59.456 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:22:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:59.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:59.543 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1596,"stamp":"2023-09-23T20:22:58.105445+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4889209,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":8623,"num_read_kb":17467,"num_write":1303,"num_write_kb":14103,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":249,"num_bytes_recovered":8892327,"num_keys_recovered":10,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6469,"ondisk_log_size":6469,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":316,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5194076,"kb_used_data":277580,"kb_used_omap":46,"kb_used_meta":4916369,"kb_avail":744767140,"statfs":{"total":767960285184,"available":762641551360,"internally_reserved":0,"allocated":284241920,"data_stored":267114430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47576,"internal_metadata":5034362408},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-112605,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":622,"num_read_kb":494,"num_write":100,"num_write_kb":65,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.675864"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":969,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.431768+0000","last_change":"2023-09-23T20:22:53.431768+0000","last_active":"2023-09-23T20:22:53.431768+0000","last_peered":"2023-09-23T20:22:53.431768+0000","last_clean":"2023-09-23T20:22:53.431768+0000","last_became_active":"2023-09-23T20:22:53.431378+0000","last_became_peered":"2023-09-23T20:22:53.431378+0000","last_unstale":"2023-09-23T20:22:53.431768+0000","last_undegraded":"2023-09-23T20:22:53.431768+0000","last_fullsized":"2023-09-23T20:22:53.431768+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:54.928103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:54.928103+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:57.671137+0000","scrub_duration":0.011135947,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":96,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421370+0000","last_change":"2023-09-23T20:22:22.380877+0000","last_active":"2023-09-23T20:22:53.421370+0000","last_peered":"2023-09-23T20:22:53.421370+0000","last_clean":"2023-09-23T20:22:53.421370+0000","last_became_active":"2023-09-23T20:22:22.380224+0000","last_became_peered":"2023-09-23T20:22:22.380224+0000","last_unstale":"2023-09-23T20:22:53.421370+0000","last_undegraded":"2023-09-23T20:22:53.421370+0000","last_fullsized":"2023-09-23T20:22:53.421370+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:14.627193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.422388+0000","last_change":"2023-09-23T20:17:47.270409+0000","last_active":"2023-09-23T20:22:53.422388+0000","last_peered":"2023-09-23T20:22:53.422388+0000","last_clean":"2023-09-23T20:22:53.422388+0000","last_became_active":"2023-09-23T20:17:47.270086+0000","last_became_peered":"2023-09-23T20:17:47.270086+0000","last_unstale":"2023-09-23T20:22:53.422388+0000","last_undegraded":"2023-09-23T20:22:53.422388+0000","last_fullsized":"2023-09-23T20:22:53.422388+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.885372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":584,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428576+0000","last_change":"2023-09-23T20:22:53.428576+0000","last_active":"2023-09-23T20:22:53.428576+0000","last_peered":"2023-09-23T20:22:53.428576+0000","last_clean":"2023-09-23T20:22:53.428576+0000","last_became_active":"2023-09-23T20:22:53.428227+0000","last_became_peered":"2023-09-23T20:22:53.428227+0000","last_unstale":"2023-09-23T20:22:53.428576+0000","last_undegraded":"2023-09-23T20:22:53.428576+0000","last_fullsized":"2023-09-23T20:22:53.428576+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:21.987455+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:21:21.987455+0000","last_clean_scrub_stamp":"2023-09-23T20:21:21.987455+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0022394540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.420839+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:22:53.420839+0000","last_peered":"2023-09-23T20:22:53.420839+0000","last_clean":"2023-09-23T20:22:53.420839+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:22:53.420839+0000","last_undegraded":"2023-09-23T20:22:53.420839+0000","last_fullsized":"2023-09-23T20:22:53.420839+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":265,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.415935+0000","last_change":"2023-09-23T20:11:17.700009+0000","last_active":"2023-09-23T20:22:51.415935+0000","last_peered":"2023-09-23T20:22:51.415935+0000","last_clean":"2023-09-23T20:22:51.415935+0000","last_became_active":"2023-09-23T20:11:17.699778+0000","last_became_peered":"2023-09-23T20:11:17.699778+0000","last_unstale":"2023-09-23T20:22:51.415935+0000","last_undegraded":"2023-09-23T20:22:51.415935+0000","last_fullsized":"2023-09-23T20:22:51.415935+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:36.216474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":421,"reported_epoch":557,"state":"peering","last_fresh":"2023-09-23T20:22:52.420257+0000","last_change":"2023-09-23T20:22:52.420257+0000","last_active":"2023-09-23T19:52:54.308942+0000","last_peered":"2023-09-23T19:45:37.119152+0000","last_clean":"2023-09-23T19:45:37.119152+0000","last_became_active":"2023-09-23T19:40:07.159949+0000","last_became_peered":"2023-09-23T19:40:07.159949+0000","last_unstale":"2023-09-23T20:22:52.420257+0000","last_undegraded":"2023-09-23T20:22:52.420257+0000","last_fullsized":"2023-09-23T20:22:52.420257+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:14.314819+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T20:21:14.314819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.420876+0000","last_change":"2023-09-23T20:17:47.279385+0000","last_active":"2023-09-23T20:22:53.420876+0000","last_peered":"2023-09-23T20:22:53.420876+0000","last_clean":"2023-09-23T20:22:53.420876+0000","last_became_active":"2023-09-23T20:17:47.279065+0000","last_became_peered":"2023-09-23T20:17:47.279065+0000","last_unstale":"2023-09-23T20:22:53.420876+0000","last_undegraded":"2023-09-23T20:22:53.420876+0000","last_fullsized":"2023-09-23T20:22:53.420876+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:19.638157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":523,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.424930+0000","last_change":"2023-09-23T20:17:47.270621+0000","last_active":"2023-09-23T20:22:52.424930+0000","last_peered":"2023-09-23T20:22:52.424930+0000","last_clean":"2023-09-23T20:22:52.424930+0000","last_became_active":"2023-09-23T20:17:47.270375+0000","last_became_peered":"2023-09-23T20:17:47.270375+0000","last_unstale":"2023-09-23T20:22:52.424930+0000","last_undegraded":"2023-09-23T20:22:52.424930+0000","last_fullsized":"2023-09-23T20:22:52.424930+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:16.961271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":828,"reported_epoch":557,"state":"peering","last_fresh":"2023-09-23T20:22:52.419622+0000","last_change":"2023-09-23T20:22:52.419622+0000","last_active":"2023-09-23T20:22:50.407306+0000","last_peered":"2023-09-23T20:22:45.390055+0000","last_clean":"2023-09-23T20:22:45.390055+0000","last_became_active":"2023-09-23T20:22:22.382251+0000","last_became_peered":"2023-09-23T20:22:22.382251+0000","last_unstale":"2023-09-23T20:22:52.419622+0000","last_undegraded":"2023-09-23T20:22:52.419622+0000","last_fullsized":"2023-09-23T20:22:52.419622+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:25.555266+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:22:25.555266+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:42.004440+0000","scrub_duration":0.001735737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":484,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.424843+0000","last_change":"2023-09-23T20:01:04.468660+0000","last_active":"2023-09-23T20:22:52.424843+0000","last_peered":"2023-09-23T20:22:52.424843+0000","last_clean":"2023-09-23T20:22:52.424843+0000","last_became_active":"2023-09-23T20:01:04.468275+0000","last_became_peered":"2023-09-23T20:01:04.468275+0000","last_unstale":"2023-09-23T20:22:52.424843+0000","last_undegraded":"2023-09-23T20:22:52.424843+0000","last_fullsized":"2023-09-23T20:22:52.424843+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:45.923818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":427,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436128+0000","last_change":"2023-09-23T20:17:47.265452+0000","last_active":"2023-09-23T20:22:55.436128+0000","last_peered":"2023-09-23T20:22:55.436128+0000","last_clean":"2023-09-23T20:22:55.436128+0000","last_became_active":"2023-09-23T20:17:47.265115+0000","last_became_peered":"2023-09-23T20:17:47.265115+0000","last_unstale":"2023-09-23T20:22:55.436128+0000","last_undegraded":"2023-09-23T20:22:55.436128+0000","last_fullsized":"2023-09-23T20:22:55.436128+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:04.898742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":868,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428375+0000","last_change":"2023-09-23T20:22:24.576681+0000","last_active":"2023-09-23T20:22:53.428375+0000","last_peered":"2023-09-23T20:22:53.428375+0000","last_clean":"2023-09-23T20:22:53.428375+0000","last_became_active":"2023-09-23T20:22:22.381258+0000","last_became_peered":"2023-09-23T20:22:22.381258+0000","last_unstale":"2023-09-23T20:22:53.428375+0000","last_undegraded":"2023-09-23T20:22:53.428375+0000","last_fullsized":"2023-09-23T20:22:53.428375+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:22:24.576567+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:22:24.576567+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:58.491034+0000","scrub_duration":0.0019668609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":577,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.424560+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:22:52.424560+0000","last_peered":"2023-09-23T20:22:52.424560+0000","last_clean":"2023-09-23T20:22:52.424560+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:22:52.424560+0000","last_undegraded":"2023-09-23T20:22:52.424560+0000","last_fullsized":"2023-09-23T20:22:52.424560+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":428,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436758+0000","last_change":"2023-09-23T20:07:36.968330+0000","last_active":"2023-09-23T20:22:55.436758+0000","last_peered":"2023-09-23T20:22:55.436758+0000","last_clean":"2023-09-23T20:22:55.436758+0000","last_became_active":"2023-09-23T20:07:36.968008+0000","last_became_peered":"2023-09-23T20:07:36.968008+0000","last_unstale":"2023-09-23T20:22:55.436758+0000","last_undegraded":"2023-09-23T20:22:55.436758+0000","last_fullsized":"2023-09-23T20:22:55.436758+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:57.086136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":468,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.431561+0000","last_change":"2023-09-23T20:22:53.431561+0000","last_active":"2023-09-23T20:22:53.431561+0000","last_peered":"2023-09-23T20:22:53.431561+0000","last_clean":"2023-09-23T20:22:53.431561+0000","last_became_active":"2023-09-23T20:22:53.431238+0000","last_became_peered":"2023-09-23T20:22:53.431238+0000","last_unstale":"2023-09-23T20:22:53.431561+0000","last_undegraded":"2023-09-23T20:22:53.431561+0000","last_fullsized":"2023-09-23T20:22:53.431561+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:42.182718+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:42.182718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":250,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.416780+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:22:51.416780+0000","last_peered":"2023-09-23T20:22:51.416780+0000","last_clean":"2023-09-23T20:22:51.416780+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:22:51.416780+0000","last_undegraded":"2023-09-23T20:22:51.416780+0000","last_fullsized":"2023-09-23T20:22:51.416780+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":96,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421276+0000","last_change":"2023-09-23T20:22:22.381040+0000","last_active":"2023-09-23T20:22:53.421276+0000","last_peered":"2023-09-23T20:22:53.421276+0000","last_clean":"2023-09-23T20:22:53.421276+0000","last_became_active":"2023-09-23T20:22:22.380591+0000","last_became_peered":"2023-09-23T20:22:22.380591+0000","last_unstale":"2023-09-23T20:22:53.421276+0000","last_undegraded":"2023-09-23T20:22:53.421276+0000","last_fullsized":"2023-09-23T20:22:53.421276+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:18.489909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":774,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.424880+0000","last_change":"2023-09-23T20:22:53.424578+0000","last_active":"2023-09-23T20:22:53.424880+0000","last_peered":"2023-09-23T20:22:53.424880+0000","last_clean":"2023-09-23T20:22:53.424880+0000","last_became_active":"2023-09-23T20:22:53.424250+0000","last_became_peered":"2023-09-23T20:22:53.424250+0000","last_unstale":"2023-09-23T20:22:53.424880+0000","last_undegraded":"2023-09-23T20:22:53.424880+0000","last_fullsized":"2023-09-23T20:22:53.424880+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:23.507023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:23.507023+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:09.885578+0000","scrub_duration":0.0014042600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":520,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.416879+0000","last_change":"2023-09-23T20:22:22.385072+0000","last_active":"2023-09-23T20:22:51.416879+0000","last_peered":"2023-09-23T20:22:51.416879+0000","last_clean":"2023-09-23T20:22:51.416879+0000","last_became_active":"2023-09-23T20:22:22.384755+0000","last_became_peered":"2023-09-23T20:22:22.384755+0000","last_unstale":"2023-09-23T20:22:51.416879+0000","last_undegraded":"2023-09-23T20:22:51.416879+0000","last_fullsized":"2023-09-23T20:22:51.416879+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:07.014454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":33,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.439002+0000","last_change":"2023-09-23T20:22:44.517176+0000","last_active":"2023-09-23T20:22:55.439002+0000","last_peered":"2023-09-23T20:22:55.439002+0000","last_clean":"2023-09-23T20:22:55.439002+0000","last_became_active":"2023-09-23T20:22:44.516814+0000","last_became_peered":"2023-09-23T20:22:44.516814+0000","last_unstale":"2023-09-23T20:22:55.439002+0000","last_undegraded":"2023-09-23T20:22:55.439002+0000","last_fullsized":"2023-09-23T20:22:55.439002+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:05.669125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":917,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.422565+0000","last_change":"2023-09-23T20:13:27.219419+0000","last_active":"2023-09-23T20:22:52.422565+0000","last_peered":"2023-09-23T20:22:52.422565+0000","last_clean":"2023-09-23T20:22:52.422565+0000","last_became_active":"2023-09-23T20:12:57.744767+0000","last_became_peered":"2023-09-23T20:12:57.744767+0000","last_unstale":"2023-09-23T20:22:52.422565+0000","last_undegraded":"2023-09-23T20:22:52.422565+0000","last_fullsized":"2023-09-23T20:22:52.422565+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:27.219320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:27.219320+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:35.612568+0000","scrub_duration":0.0051160670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":267,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.414633+0000","last_change":"2023-09-23T20:22:22.384480+0000","last_active":"2023-09-23T20:22:51.414633+0000","last_peered":"2023-09-23T20:22:51.414633+0000","last_clean":"2023-09-23T20:22:51.414633+0000","last_became_active":"2023-09-23T20:22:22.384129+0000","last_became_peered":"2023-09-23T20:22:22.384129+0000","last_unstale":"2023-09-23T20:22:51.414633+0000","last_undegraded":"2023-09-23T20:22:51.414633+0000","last_fullsized":"2023-09-23T20:22:51.414633+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:15.885311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.422444+0000","last_change":"2023-09-23T20:17:47.266963+0000","last_active":"2023-09-23T20:22:53.422444+0000","last_peered":"2023-09-23T20:22:53.422444+0000","last_clean":"2023-09-23T20:22:53.422444+0000","last_became_active":"2023-09-23T20:17:47.266645+0000","last_became_peered":"2023-09-23T20:17:47.266645+0000","last_unstale":"2023-09-23T20:22:53.422444+0000","last_undegraded":"2023-09-23T20:22:53.422444+0000","last_fullsized":"2023-09-23T20:22:53.422444+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:05.052559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":606,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423229+0000","last_change":"2023-09-23T20:13:35.274566+0000","last_active":"2023-09-23T20:22:52.423229+0000","last_peered":"2023-09-23T20:22:52.423229+0000","last_clean":"2023-09-23T20:22:52.423229+0000","last_became_active":"2023-09-23T20:12:57.746429+0000","last_became_peered":"2023-09-23T20:12:57.746429+0000","last_unstale":"2023-09-23T20:22:52.423229+0000","last_undegraded":"2023-09-23T20:22:52.423229+0000","last_fullsized":"2023-09-23T20:22:52.423229+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:31.713951+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":265,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.414911+0000","last_change":"2023-09-23T20:11:17.701390+0000","last_active":"2023-09-23T20:22:51.414911+0000","last_peered":"2023-09-23T20:22:51.414911+0000","last_clean":"2023-09-23T20:22:51.414911+0000","last_became_active":"2023-09-23T20:11:17.701085+0000","last_became_peered":"2023-09-23T20:11:17.701085+0000","last_unstale":"2023-09-23T20:22:51.414911+0000","last_undegraded":"2023-09-23T20:22:51.414911+0000","last_fullsized":"2023-09-23T20:22:51.414911+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:36.311685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":556,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423984+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:22:52.423984+0000","last_peered":"2023-09-23T20:22:52.423984+0000","last_clean":"2023-09-23T20:22:52.423984+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:22:52.423984+0000","last_undegraded":"2023-09-23T20:22:52.423984+0000","last_fullsized":"2023-09-23T20:22:52.423984+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":630,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421506+0000","last_change":"2023-09-23T20:21:11.930313+0000","last_active":"2023-09-23T20:22:53.421506+0000","last_peered":"2023-09-23T20:22:53.421506+0000","last_clean":"2023-09-23T20:22:53.421506+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:22:53.421506+0000","last_undegraded":"2023-09-23T20:22:53.421506+0000","last_fullsized":"2023-09-23T20:22:53.421506+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:11.930203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.930203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:00.534581+0000","scrub_duration":0.0016982569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":271,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.415697+0000","last_change":"2023-09-23T20:22:44.517176+0000","last_active":"2023-09-23T20:22:51.415697+0000","last_peered":"2023-09-23T20:22:51.415697+0000","last_clean":"2023-09-23T20:22:51.415697+0000","last_became_active":"2023-09-23T20:22:44.516833+0000","last_became_peered":"2023-09-23T20:22:44.516833+0000","last_unstale":"2023-09-23T20:22:51.415697+0000","last_undegraded":"2023-09-23T20:22:51.415697+0000","last_fullsized":"2023-09-23T20:22:51.415697+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:43.417704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":552,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.437478+0000","last_change":"2023-09-23T20:07:36.966699+0000","last_active":"2023-09-23T20:22:55.437478+0000","last_peered":"2023-09-23T20:22:55.437478+0000","last_clean":"2023-09-23T20:22:55.437478+0000","last_became_active":"2023-09-23T20:07:36.966420+0000","last_became_peered":"2023-09-23T20:07:36.966420+0000","last_unstale":"2023-09-23T20:22:55.437478+0000","last_undegraded":"2023-09-23T20:22:55.437478+0000","last_fullsized":"2023-09-23T20:22:55.437478+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:36.701322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":715,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423324+0000","last_change":"2023-09-23T20:17:50.332129+0000","last_active":"2023-09-23T20:22:52.423324+0000","last_peered":"2023-09-23T20:22:52.423324+0000","last_clean":"2023-09-23T20:22:52.423324+0000","last_became_active":"2023-09-23T20:17:50.331669+0000","last_became_peered":"2023-09-23T20:17:50.331669+0000","last_unstale":"2023-09-23T20:22:52.423324+0000","last_undegraded":"2023-09-23T20:22:52.423324+0000","last_fullsized":"2023-09-23T20:22:52.423324+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:23.893417+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":201,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.422950+0000","last_change":"2023-09-23T20:12:36.046150+0000","last_active":"2023-09-23T20:22:52.422950+0000","last_peered":"2023-09-23T20:22:52.422950+0000","last_clean":"2023-09-23T20:22:52.422950+0000","last_became_active":"2023-09-23T20:11:10.067146+0000","last_became_peered":"2023-09-23T20:11:10.067146+0000","last_unstale":"2023-09-23T20:22:52.422950+0000","last_undegraded":"2023-09-23T20:22:52.422950+0000","last_fullsized":"2023-09-23T20:22:52.422950+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.790235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":213,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436806+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:22:55.436806+0000","last_peered":"2023-09-23T20:22:55.436806+0000","last_clean":"2023-09-23T20:22:55.436806+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:22:55.436806+0000","last_undegraded":"2023-09-23T20:22:55.436806+0000","last_fullsized":"2023-09-23T20:22:55.436806+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":866,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.416328+0000","last_change":"2023-09-23T20:21:22.116340+0000","last_active":"2023-09-23T20:22:51.416328+0000","last_peered":"2023-09-23T20:22:51.416328+0000","last_clean":"2023-09-23T20:22:51.416328+0000","last_became_active":"2023-09-23T20:19:05.065283+0000","last_became_peered":"2023-09-23T20:19:05.065283+0000","last_unstale":"2023-09-23T20:22:51.416328+0000","last_undegraded":"2023-09-23T20:22:51.416328+0000","last_fullsized":"2023-09-23T20:22:51.416328+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:22.116270+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:21:22.116270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:27.825796+0000","scrub_duration":0.001763922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":248,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.415342+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:22:51.415342+0000","last_peered":"2023-09-23T20:22:51.415342+0000","last_clean":"2023-09-23T20:22:51.415342+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:22:51.415342+0000","last_undegraded":"2023-09-23T20:22:51.415342+0000","last_fullsized":"2023-09-23T20:22:51.415342+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":566,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423030+0000","last_change":"2023-09-23T20:07:37.029345+0000","last_active":"2023-09-23T20:22:52.423030+0000","last_peered":"2023-09-23T20:22:52.423030+0000","last_clean":"2023-09-23T20:22:52.423030+0000","last_became_active":"2023-09-23T20:07:37.029022+0000","last_became_peered":"2023-09-23T20:07:37.029022+0000","last_unstale":"2023-09-23T20:22:52.423030+0000","last_undegraded":"2023-09-23T20:22:52.423030+0000","last_fullsized":"2023-09-23T20:22:52.423030+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:24.152600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":939,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436401+0000","last_change":"2023-09-23T20:21:11.597505+0000","last_active":"2023-09-23T20:22:55.436401+0000","last_peered":"2023-09-23T20:22:55.436401+0000","last_clean":"2023-09-23T20:22:55.436401+0000","last_became_active":"2023-09-23T20:19:05.085764+0000","last_became_peered":"2023-09-23T20:19:05.085764+0000","last_unstale":"2023-09-23T20:22:55.436401+0000","last_undegraded":"2023-09-23T20:22:55.436401+0000","last_fullsized":"2023-09-23T20:22:55.436401+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:21:11.597355+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.597355+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:24.991370+0000","scrub_duration":0.0021599240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":277,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.416029+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:22:51.416029+0000","last_peered":"2023-09-23T20:22:51.416029+0000","last_clean":"2023-09-23T20:22:51.416029+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:22:51.416029+0000","last_undegraded":"2023-09-23T20:22:51.416029+0000","last_fullsized":"2023-09-23T20:22:51.416029+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.420943+0000","last_change":"2023-09-23T20:17:47.264776+0000","last_active":"2023-09-23T20:22:53.420943+0000","last_peered":"2023-09-23T20:22:53.420943+0000","last_clean":"2023-09-23T20:22:53.420943+0000","last_became_active":"2023-09-23T20:17:47.264473+0000","last_became_peered":"2023-09-23T20:17:47.264473+0000","last_unstale":"2023-09-23T20:22:53.420943+0000","last_undegraded":"2023-09-23T20:22:53.420943+0000","last_fullsized":"2023-09-23T20:22:53.420943+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:40.906303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":630,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.424753+0000","last_change":"2023-09-23T20:13:33.256443+0000","last_active":"2023-09-23T20:22:52.424753+0000","last_peered":"2023-09-23T20:22:52.424753+0000","last_clean":"2023-09-23T20:22:52.424753+0000","last_became_active":"2023-09-23T20:12:57.744832+0000","last_became_peered":"2023-09-23T20:12:57.744832+0000","last_unstale":"2023-09-23T20:22:52.424753+0000","last_undegraded":"2023-09-23T20:22:52.424753+0000","last_fullsized":"2023-09-23T20:22:52.424753+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:33.256362+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:13:33.256362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:37.626005+0000","scrub_duration":0.0079882870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":279,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.416539+0000","last_change":"2023-09-23T20:22:44.388257+0000","last_active":"2023-09-23T20:22:51.416539+0000","last_peered":"2023-09-23T20:22:51.416539+0000","last_clean":"2023-09-23T20:22:51.416539+0000","last_became_active":"2023-09-23T20:22:44.387884+0000","last_became_peered":"2023-09-23T20:22:44.387884+0000","last_unstale":"2023-09-23T20:22:51.416539+0000","last_undegraded":"2023-09-23T20:22:51.416539+0000","last_fullsized":"2023-09-23T20:22:51.416539+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:26.931322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421282+0000","last_change":"2023-09-23T20:17:47.266977+0000","last_active":"2023-09-23T20:22:53.421282+0000","last_peered":"2023-09-23T20:22:53.421282+0000","last_clean":"2023-09-23T20:22:53.421282+0000","last_became_active":"2023-09-23T20:17:47.266657+0000","last_became_peered":"2023-09-23T20:17:47.266657+0000","last_unstale":"2023-09-23T20:22:53.421282+0000","last_undegraded":"2023-09-23T20:22:53.421282+0000","last_fullsized":"2023-09-23T20:22:53.421282+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:33.774981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":789,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.416903+0000","last_change":"2023-09-23T20:21:24.112801+0000","last_active":"2023-09-23T20:22:51.416903+0000","last_peered":"2023-09-23T20:22:51.416903+0000","last_clean":"2023-09-23T20:22:51.416903+0000","last_became_active":"2023-09-23T20:19:05.071700+0000","last_became_peered":"2023-09-23T20:19:05.071700+0000","last_unstale":"2023-09-23T20:22:51.416903+0000","last_undegraded":"2023-09-23T20:22:51.416903+0000","last_fullsized":"2023-09-23T20:22:51.416903+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:24.112725+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:21:24.112725+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:07.289445+0000","scrub_duration":0.0014886809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":616,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423858+0000","last_change":"2023-09-23T20:17:47.267657+0000","last_active":"2023-09-23T20:22:52.423858+0000","last_peered":"2023-09-23T20:22:52.423858+0000","last_clean":"2023-09-23T20:22:52.423858+0000","last_became_active":"2023-09-23T20:17:47.267288+0000","last_became_peered":"2023-09-23T20:17:47.267288+0000","last_unstale":"2023-09-23T20:22:52.423858+0000","last_undegraded":"2023-09-23T20:22:52.423858+0000","last_fullsized":"2023-09-23T20:22:52.423858+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:22.185253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421243+0000","last_change":"2023-09-23T20:17:47.280675+0000","last_active":"2023-09-23T20:22:53.421243+0000","last_peered":"2023-09-23T20:22:53.421243+0000","last_clean":"2023-09-23T20:22:53.421243+0000","last_became_active":"2023-09-23T20:17:47.280175+0000","last_became_peered":"2023-09-23T20:17:47.280175+0000","last_unstale":"2023-09-23T20:22:53.421243+0000","last_undegraded":"2023-09-23T20:22:53.421243+0000","last_fullsized":"2023-09-23T20:22:53.421243+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:08.499949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":976,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.415588+0000","last_change":"2023-09-23T20:22:22.386488+0000","last_active":"2023-09-23T20:22:51.415588+0000","last_peered":"2023-09-23T20:22:51.415588+0000","last_clean":"2023-09-23T20:22:51.415588+0000","last_became_active":"2023-09-23T20:22:22.386137+0000","last_became_peered":"2023-09-23T20:22:22.386137+0000","last_unstale":"2023-09-23T20:22:51.415588+0000","last_undegraded":"2023-09-23T20:22:51.415588+0000","last_fullsized":"2023-09-23T20:22:51.415588+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:47.181912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:47.181912+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:26.092523+0000","scrub_duration":0.0074047540000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":309,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436319+0000","last_change":"2023-09-23T20:13:57.748149+0000","last_active":"2023-09-23T20:22:55.436319+0000","last_peered":"2023-09-23T20:22:55.436319+0000","last_clean":"2023-09-23T20:22:55.436319+0000","last_became_active":"2023-09-23T20:13:57.747803+0000","last_became_peered":"2023-09-23T20:13:57.747803+0000","last_unstale":"2023-09-23T20:22:55.436319+0000","last_undegraded":"2023-09-23T20:22:55.436319+0000","last_fullsized":"2023-09-23T20:22:55.436319+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.144395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.422482+0000","last_change":"2023-09-23T20:17:47.270450+0000","last_active":"2023-09-23T20:22:53.422482+0000","last_peered":"2023-09-23T20:22:53.422482+0000","last_clean":"2023-09-23T20:22:53.422482+0000","last_became_active":"2023-09-23T20:17:47.270040+0000","last_became_peered":"2023-09-23T20:17:47.270040+0000","last_unstale":"2023-09-23T20:22:53.422482+0000","last_undegraded":"2023-09-23T20:22:53.422482+0000","last_fullsized":"2023-09-23T20:22:53.422482+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:51.572016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":948,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423116+0000","last_change":"2023-09-23T20:22:22.377942+0000","last_active":"2023-09-23T20:22:52.423116+0000","last_peered":"2023-09-23T20:22:52.423116+0000","last_clean":"2023-09-23T20:22:52.423116+0000","last_became_active":"2023-09-23T20:22:22.377619+0000","last_became_peered":"2023-09-23T20:22:22.377619+0000","last_unstale":"2023-09-23T20:22:52.423116+0000","last_undegraded":"2023-09-23T20:22:52.423116+0000","last_fullsized":"2023-09-23T20:22:52.423116+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:43.954204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:43.954204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:00.412768+0000","scrub_duration":0.0019293979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421179+0000","last_change":"2023-09-23T20:17:47.266513+0000","last_active":"2023-09-23T20:22:53.421179+0000","last_peered":"2023-09-23T20:22:53.421179+0000","last_clean":"2023-09-23T20:22:53.421179+0000","last_became_active":"2023-09-23T20:17:47.266163+0000","last_became_peered":"2023-09-23T20:17:47.266163+0000","last_unstale":"2023-09-23T20:22:53.421179+0000","last_undegraded":"2023-09-23T20:22:53.421179+0000","last_fullsized":"2023-09-23T20:22:53.421179+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.645381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":427,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428303+0000","last_change":"2023-09-23T20:22:22.384798+0000","last_active":"2023-09-23T20:22:53.428303+0000","last_peered":"2023-09-23T20:22:53.428303+0000","last_clean":"2023-09-23T20:22:53.428303+0000","last_became_active":"2023-09-23T20:22:22.384476+0000","last_became_peered":"2023-09-23T20:22:22.384476+0000","last_unstale":"2023-09-23T20:22:53.428303+0000","last_undegraded":"2023-09-23T20:22:53.428303+0000","last_fullsized":"2023-09-23T20:22:53.428303+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.000282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":635,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421403+0000","last_change":"2023-09-23T20:21:20.952512+0000","last_active":"2023-09-23T20:22:53.421403+0000","last_peered":"2023-09-23T20:22:53.421403+0000","last_clean":"2023-09-23T20:22:53.421403+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:22:53.421403+0000","last_undegraded":"2023-09-23T20:22:53.421403+0000","last_fullsized":"2023-09-23T20:22:53.421403+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:20.952352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:20.952352+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:28.266826+0000","scrub_duration":0.00171708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":428,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428530+0000","last_change":"2023-09-23T20:22:22.380486+0000","last_active":"2023-09-23T20:22:53.428530+0000","last_peered":"2023-09-23T20:22:53.428530+0000","last_clean":"2023-09-23T20:22:53.428530+0000","last_became_active":"2023-09-23T20:22:22.380085+0000","last_became_peered":"2023-09-23T20:22:22.380085+0000","last_unstale":"2023-09-23T20:22:53.428530+0000","last_undegraded":"2023-09-23T20:22:53.428530+0000","last_fullsized":"2023-09-23T20:22:53.428530+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:51.480406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.423047+0000","last_change":"2023-09-23T20:17:47.268811+0000","last_active":"2023-09-23T20:22:53.423047+0000","last_peered":"2023-09-23T20:22:53.423047+0000","last_clean":"2023-09-23T20:22:53.423047+0000","last_became_active":"2023-09-23T20:17:47.268391+0000","last_became_peered":"2023-09-23T20:17:47.268391+0000","last_unstale":"2023-09-23T20:22:53.423047+0000","last_undegraded":"2023-09-23T20:22:53.423047+0000","last_fullsized":"2023-09-23T20:22:53.423047+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:32.388705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":960,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.424602+0000","last_change":"2023-09-23T20:22:22.380878+0000","last_active":"2023-09-23T20:22:52.424602+0000","last_peered":"2023-09-23T20:22:52.424602+0000","last_clean":"2023-09-23T20:22:52.424602+0000","last_became_active":"2023-09-23T20:22:22.380365+0000","last_became_peered":"2023-09-23T20:22:22.380365+0000","last_unstale":"2023-09-23T20:22:52.424602+0000","last_undegraded":"2023-09-23T20:22:52.424602+0000","last_fullsized":"2023-09-23T20:22:52.424602+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:12.314623+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T20:21:12.314623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:01.955991+0000","scrub_duration":0.001743334,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":214,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.435849+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:22:55.435849+0000","last_peered":"2023-09-23T20:22:55.435849+0000","last_clean":"2023-09-23T20:22:55.435849+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:22:55.435849+0000","last_undegraded":"2023-09-23T20:22:55.435849+0000","last_fullsized":"2023-09-23T20:22:55.435849+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.422503+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:22:53.422503+0000","last_peered":"2023-09-23T20:22:53.422503+0000","last_clean":"2023-09-23T20:22:53.422503+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:22:53.422503+0000","last_undegraded":"2023-09-23T20:22:53.422503+0000","last_fullsized":"2023-09-23T20:22:53.422503+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":867,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436893+0000","last_change":"2023-09-23T20:19:40.219448+0000","last_active":"2023-09-23T20:22:55.436893+0000","last_peered":"2023-09-23T20:22:55.436893+0000","last_clean":"2023-09-23T20:22:55.436893+0000","last_became_active":"2023-09-23T20:19:05.072187+0000","last_became_peered":"2023-09-23T20:19:05.072187+0000","last_unstale":"2023-09-23T20:22:55.436893+0000","last_undegraded":"2023-09-23T20:22:55.436893+0000","last_fullsized":"2023-09-23T20:22:55.436893+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:40.219310+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:40.219310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:29.003073+0000","scrub_duration":0.011178762,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"558'195","reported_seq":799,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.906619+0000","last_change":"2023-09-23T20:22:52.425956+0000","last_active":"2023-09-23T20:22:53.906619+0000","last_peered":"2023-09-23T20:22:53.906619+0000","last_clean":"2023-09-23T20:22:53.906619+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:22:53.906619+0000","last_undegraded":"2023-09-23T20:22:53.906619+0000","last_fullsized":"2023-09-23T20:22:53.906619+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":195,"log_dups_size":0,"ondisk_log_size":195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057036900000000004,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1083,"num_read_kb":2102,"num_write":206,"num_write_kb":5287,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.5","version":"0'0","reported_seq":575,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423901+0000","last_change":"2023-09-23T20:22:22.379055+0000","last_active":"2023-09-23T20:22:52.423901+0000","last_peered":"2023-09-23T20:22:52.423901+0000","last_clean":"2023-09-23T20:22:52.423901+0000","last_became_active":"2023-09-23T20:22:22.378812+0000","last_became_peered":"2023-09-23T20:22:22.378812+0000","last_unstale":"2023-09-23T20:22:52.423901+0000","last_undegraded":"2023-09-23T20:22:52.423901+0000","last_fullsized":"2023-09-23T20:22:52.423901+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:26.700741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421195+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:22:53.421195+0000","last_peered":"2023-09-23T20:22:53.421195+0000","last_clean":"2023-09-23T20:22:53.421195+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:22:53.421195+0000","last_undegraded":"2023-09-23T20:22:53.421195+0000","last_fullsized":"2023-09-23T20:22:53.421195+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":918,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.422653+0000","last_change":"2023-09-23T20:22:22.379736+0000","last_active":"2023-09-23T20:22:52.422653+0000","last_peered":"2023-09-23T20:22:52.422653+0000","last_clean":"2023-09-23T20:22:52.422653+0000","last_became_active":"2023-09-23T20:22:22.379411+0000","last_became_peered":"2023-09-23T20:22:22.379411+0000","last_unstale":"2023-09-23T20:22:52.422653+0000","last_undegraded":"2023-09-23T20:22:52.422653+0000","last_fullsized":"2023-09-23T20:22:52.422653+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:21.231359+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"558'120","reported_seq":1551,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.591269+0000","last_change":"2023-09-23T20:22:52.424780+0000","last_active":"2023-09-23T20:22:53.591269+0000","last_peered":"2023-09-23T20:22:53.591269+0000","last_clean":"2023-09-23T20:22:53.591269+0000","last_became_active":"2023-09-23T20:17:47.274623+0000","last_became_peered":"2023-09-23T20:17:47.274623+0000","last_unstale":"2023-09-23T20:22:53.591269+0000","last_undegraded":"2023-09-23T20:22:53.591269+0000","last_fullsized":"2023-09-23T20:22:53.591269+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:27.293931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013222590000000001,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1264,"num_read_kb":2699,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421120+0000","last_change":"2023-09-23T20:17:47.280498+0000","last_active":"2023-09-23T20:22:53.421120+0000","last_peered":"2023-09-23T20:22:53.421120+0000","last_clean":"2023-09-23T20:22:53.421120+0000","last_became_active":"2023-09-23T20:17:47.279716+0000","last_became_peered":"2023-09-23T20:17:47.279716+0000","last_unstale":"2023-09-23T20:22:53.421120+0000","last_undegraded":"2023-09-23T20:22:53.421120+0000","last_fullsized":"2023-09-23T20:22:53.421120+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:18.625303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.422406+0000","last_change":"2023-09-23T20:22:22.382763+0000","last_active":"2023-09-23T20:22:53.422406+0000","last_peered":"2023-09-23T20:22:53.422406+0000","last_clean":"2023-09-23T20:22:53.422406+0000","last_became_active":"2023-09-23T20:22:22.381690+0000","last_became_peered":"2023-09-23T20:22:22.381690+0000","last_unstale":"2023-09-23T20:22:53.422406+0000","last_undegraded":"2023-09-23T20:22:53.422406+0000","last_fullsized":"2023-09-23T20:22:53.422406+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:51.102791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":795,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.414999+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:22:51.414999+0000","last_peered":"2023-09-23T20:22:51.414999+0000","last_clean":"2023-09-23T20:22:51.414999+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:22:51.414999+0000","last_undegraded":"2023-09-23T20:22:51.414999+0000","last_fullsized":"2023-09-23T20:22:51.414999+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"559'166","reported_seq":1354,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T20:22:54.497562+0000","last_change":"2023-09-23T20:22:52.423074+0000","last_active":"2023-09-23T20:22:54.497562+0000","last_peered":"2023-09-23T20:22:54.497562+0000","last_clean":"2023-09-23T20:22:54.497562+0000","last_became_active":"2023-09-23T20:22:22.387896+0000","last_became_peered":"2023-09-23T20:22:22.387896+0000","last_unstale":"2023-09-23T20:22:54.497562+0000","last_undegraded":"2023-09-23T20:22:54.497562+0000","last_fullsized":"2023-09-23T20:22:54.497562+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":166,"log_dups_size":0,"ondisk_log_size":166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:11.339762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030435999999999999,"stat_sum":{"num_bytes":16,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1125,"num_read_kb":3659,"num_write":139,"num_write_kb":2524,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.0","version":"0'0","reported_seq":96,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.420855+0000","last_change":"2023-09-23T20:22:22.381123+0000","last_active":"2023-09-23T20:22:53.420855+0000","last_peered":"2023-09-23T20:22:53.420855+0000","last_clean":"2023-09-23T20:22:53.420855+0000","last_became_active":"2023-09-23T20:22:22.379609+0000","last_became_peered":"2023-09-23T20:22:22.379609+0000","last_unstale":"2023-09-23T20:22:53.420855+0000","last_undegraded":"2023-09-23T20:22:53.420855+0000","last_fullsized":"2023-09-23T20:22:53.420855+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:55.146453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.422579+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:22:53.422579+0000","last_peered":"2023-09-23T20:22:53.422579+0000","last_clean":"2023-09-23T20:22:53.422579+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:22:53.422579+0000","last_undegraded":"2023-09-23T20:22:53.422579+0000","last_fullsized":"2023-09-23T20:22:53.422579+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"558'308","reported_seq":1649,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.426351+0000","last_change":"2023-09-23T20:22:52.425192+0000","last_active":"2023-09-23T20:22:53.426351+0000","last_peered":"2023-09-23T20:22:53.426351+0000","last_clean":"2023-09-23T20:22:53.426351+0000","last_became_active":"2023-09-23T20:22:22.379019+0000","last_became_peered":"2023-09-23T20:22:22.379019+0000","last_unstale":"2023-09-23T20:22:53.426351+0000","last_undegraded":"2023-09-23T20:22:53.426351+0000","last_fullsized":"2023-09-23T20:22:53.426351+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":308,"log_dups_size":0,"ondisk_log_size":308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:35.091435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045003900000000001,"stat_sum":{"num_bytes":4096,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1543,"num_read_kb":2592,"num_write":257,"num_write_kb":758,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"1.0","version":"90'161","reported_seq":772,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.416977+0000","last_change":"2023-09-23T20:17:50.313197+0000","last_active":"2023-09-23T20:22:51.416977+0000","last_peered":"2023-09-23T20:22:51.416977+0000","last_clean":"2023-09-23T20:22:51.416977+0000","last_became_active":"2023-09-23T20:17:50.312838+0000","last_became_peered":"2023-09-23T20:17:50.312838+0000","last_unstale":"2023-09-23T20:22:51.416977+0000","last_undegraded":"2023-09-23T20:22:51.416977+0000","last_fullsized":"2023-09-23T20:22:51.416977+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:31.530167+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":423,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.424450+0000","last_change":"2023-09-23T20:08:04.749523+0000","last_active":"2023-09-23T20:22:52.424450+0000","last_peered":"2023-09-23T20:22:52.424450+0000","last_clean":"2023-09-23T20:22:52.424450+0000","last_became_active":"2023-09-23T20:08:04.749181+0000","last_became_peered":"2023-09-23T20:08:04.749181+0000","last_unstale":"2023-09-23T20:22:52.424450+0000","last_undegraded":"2023-09-23T20:22:52.424450+0000","last_fullsized":"2023-09-23T20:22:52.424450+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:22.319977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":354,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.416410+0000","last_change":"2023-09-23T20:13:57.752460+0000","last_active":"2023-09-23T20:22:51.416410+0000","last_peered":"2023-09-23T20:22:51.416410+0000","last_clean":"2023-09-23T20:22:51.416410+0000","last_became_active":"2023-09-23T20:13:57.752186+0000","last_became_peered":"2023-09-23T20:13:57.752186+0000","last_unstale":"2023-09-23T20:22:51.416410+0000","last_undegraded":"2023-09-23T20:22:51.416410+0000","last_fullsized":"2023-09-23T20:22:51.416410+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:40.032897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":582,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421323+0000","last_change":"2023-09-23T20:21:18.875984+0000","last_active":"2023-09-23T20:22:53.421323+0000","last_peered":"2023-09-23T20:22:53.421323+0000","last_clean":"2023-09-23T20:22:53.421323+0000","last_became_active":"2023-09-23T20:17:50.329940+0000","last_became_peered":"2023-09-23T20:17:50.329940+0000","last_unstale":"2023-09-23T20:22:53.421323+0000","last_undegraded":"2023-09-23T20:22:53.421323+0000","last_fullsized":"2023-09-23T20:22:53.421323+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:18.875863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:18.875863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:17.358094+0000","scrub_duration":0.0015811600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"557'242","reported_seq":1394,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.973055+0000","last_change":"2023-09-23T20:22:52.419473+0000","last_active":"2023-09-23T20:22:52.973055+0000","last_peered":"2023-09-23T20:22:52.973055+0000","last_clean":"2023-09-23T20:22:52.973055+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:22:52.973055+0000","last_undegraded":"2023-09-23T20:22:52.973055+0000","last_fullsized":"2023-09-23T20:22:52.973055+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":242,"log_dups_size":0,"ondisk_log_size":242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001396221,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1962,"num_read_kb":2176,"num_write":187,"num_write_kb":327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.1","version":"0'0","reported_seq":510,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436398+0000","last_change":"2023-09-23T20:13:57.747973+0000","last_active":"2023-09-23T20:22:55.436398+0000","last_peered":"2023-09-23T20:22:55.436398+0000","last_clean":"2023-09-23T20:22:55.436398+0000","last_became_active":"2023-09-23T20:13:57.747623+0000","last_became_peered":"2023-09-23T20:13:57.747623+0000","last_unstale":"2023-09-23T20:22:55.436398+0000","last_undegraded":"2023-09-23T20:22:55.436398+0000","last_fullsized":"2023-09-23T20:22:55.436398+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:21.399684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":96,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.423118+0000","last_change":"2023-09-23T20:22:22.379579+0000","last_active":"2023-09-23T20:22:53.423118+0000","last_peered":"2023-09-23T20:22:53.423118+0000","last_clean":"2023-09-23T20:22:53.423118+0000","last_became_active":"2023-09-23T20:22:22.379052+0000","last_became_peered":"2023-09-23T20:22:22.379052+0000","last_unstale":"2023-09-23T20:22:53.423118+0000","last_undegraded":"2023-09-23T20:22:53.423118+0000","last_fullsized":"2023-09-23T20:22:53.423118+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:59.658895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":768,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T20:22:54.436558+0000","last_change":"2023-09-23T20:21:19.413132+0000","last_active":"2023-09-23T20:22:54.436558+0000","last_peered":"2023-09-23T20:22:54.436558+0000","last_clean":"2023-09-23T20:22:54.436558+0000","last_became_active":"2023-09-23T20:19:05.092070+0000","last_became_peered":"2023-09-23T20:19:05.092070+0000","last_unstale":"2023-09-23T20:22:54.436558+0000","last_undegraded":"2023-09-23T20:22:54.436558+0000","last_fullsized":"2023-09-23T20:22:54.436558+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:19.413043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:19.413043+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:11.511055+0000","scrub_duration":0.0014842639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"557'59","reported_seq":283,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.883147+0000","last_change":"2023-09-23T20:22:52.421267+0000","last_active":"2023-09-23T20:22:52.883147+0000","last_peered":"2023-09-23T20:22:52.883147+0000","last_clean":"2023-09-23T20:22:52.883147+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:22:52.883147+0000","last_undegraded":"2023-09-23T20:22:52.883147+0000","last_fullsized":"2023-09-23T20:22:52.883147+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":59,"log_dups_size":0,"ondisk_log_size":59,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011330999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":163,"num_read_kb":109,"num_write":66,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.4","version":"0'0","reported_seq":96,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.422327+0000","last_change":"2023-09-23T20:22:22.382963+0000","last_active":"2023-09-23T20:22:53.422327+0000","last_peered":"2023-09-23T20:22:53.422327+0000","last_clean":"2023-09-23T20:22:53.422327+0000","last_became_active":"2023-09-23T20:22:22.382376+0000","last_became_peered":"2023-09-23T20:22:22.382376+0000","last_unstale":"2023-09-23T20:22:53.422327+0000","last_undegraded":"2023-09-23T20:22:53.422327+0000","last_fullsized":"2023-09-23T20:22:53.422327+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:29.101975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":384,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.424639+0000","last_change":"2023-09-23T20:07:36.976883+0000","last_active":"2023-09-23T20:22:52.424639+0000","last_peered":"2023-09-23T20:22:52.424639+0000","last_clean":"2023-09-23T20:22:52.424639+0000","last_became_active":"2023-09-23T20:07:36.975206+0000","last_became_peered":"2023-09-23T20:07:36.975206+0000","last_unstale":"2023-09-23T20:22:52.424639+0000","last_undegraded":"2023-09-23T20:22:52.424639+0000","last_fullsized":"2023-09-23T20:22:52.424639+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:51.034811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":830,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428388+0000","last_change":"2023-09-23T20:22:22.526312+0000","last_active":"2023-09-23T20:22:53.428388+0000","last_peered":"2023-09-23T20:22:53.428388+0000","last_clean":"2023-09-23T20:22:53.428388+0000","last_became_active":"2023-09-23T20:22:22.381610+0000","last_became_peered":"2023-09-23T20:22:22.381610+0000","last_unstale":"2023-09-23T20:22:53.428388+0000","last_undegraded":"2023-09-23T20:22:53.428388+0000","last_fullsized":"2023-09-23T20:22:53.428388+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:22.526188+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:22:22.526188+0000","last_clean_scrub_stamp":"2023-09-23T20:22:22.526188+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:31.726686+0000","scrub_duration":0.0013404529999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":399,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.414911+0000","last_change":"2023-09-23T20:08:04.756053+0000","last_active":"2023-09-23T20:22:51.414911+0000","last_peered":"2023-09-23T20:22:51.414911+0000","last_clean":"2023-09-23T20:22:51.414911+0000","last_became_active":"2023-09-23T20:08:04.755733+0000","last_became_peered":"2023-09-23T20:08:04.755733+0000","last_unstale":"2023-09-23T20:22:51.414911+0000","last_undegraded":"2023-09-23T20:22:51.414911+0000","last_fullsized":"2023-09-23T20:22:51.414911+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:47.108242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"558'179","reported_seq":722,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.865277+0000","last_change":"2023-09-23T20:22:52.428074+0000","last_active":"2023-09-23T20:22:53.865277+0000","last_peered":"2023-09-23T20:22:53.865277+0000","last_clean":"2023-09-23T20:22:53.865277+0000","last_became_active":"2023-09-23T20:17:50.310876+0000","last_became_peered":"2023-09-23T20:17:50.310876+0000","last_unstale":"2023-09-23T20:22:53.865277+0000","last_undegraded":"2023-09-23T20:22:53.865277+0000","last_fullsized":"2023-09-23T20:22:53.865277+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":179,"log_dups_size":0,"ondisk_log_size":179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.794641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011362569999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":570,"num_read_kb":2844,"num_write":157,"num_write_kb":2337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":196,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.435843+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:22:55.435843+0000","last_peered":"2023-09-23T20:22:55.435843+0000","last_clean":"2023-09-23T20:22:55.435843+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:22:55.435843+0000","last_undegraded":"2023-09-23T20:22:55.435843+0000","last_fullsized":"2023-09-23T20:22:55.435843+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":890,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436638+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:22:55.436638+0000","last_peered":"2023-09-23T20:22:55.436638+0000","last_clean":"2023-09-23T20:22:55.436638+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:22:55.436638+0000","last_undegraded":"2023-09-23T20:22:55.436638+0000","last_fullsized":"2023-09-23T20:22:55.436638+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"561'150","reported_seq":196,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.446648+0000","last_change":"2023-09-23T20:22:52.422461+0000","last_active":"2023-09-23T20:22:56.446648+0000","last_peered":"2023-09-23T20:22:56.446648+0000","last_clean":"2023-09-23T20:22:56.446648+0000","last_became_active":"2023-09-23T20:22:45.391193+0000","last_became_peered":"2023-09-23T20:22:45.391193+0000","last_unstale":"2023-09-23T20:22:56.446648+0000","last_undegraded":"2023-09-23T20:22:56.446648+0000","last_fullsized":"2023-09-23T20:22:56.446648+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.102405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038057599999999998,"stat_sum":{"num_bytes":35,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":880,"num_read_kb":1223,"num_write":135,"num_write_kb":1326,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.6","version":"68'1","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.422823+0000","last_change":"2023-09-23T20:17:47.284676+0000","last_active":"2023-09-23T20:22:53.422823+0000","last_peered":"2023-09-23T20:22:53.422823+0000","last_clean":"2023-09-23T20:22:53.422823+0000","last_became_active":"2023-09-23T20:17:47.277321+0000","last_became_peered":"2023-09-23T20:17:47.277321+0000","last_unstale":"2023-09-23T20:22:53.422823+0000","last_undegraded":"2023-09-23T20:22:53.422823+0000","last_fullsized":"2023-09-23T20:22:53.422823+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.082445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":368,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.422787+0000","last_change":"2023-09-23T20:22:44.517201+0000","last_active":"2023-09-23T20:22:52.422787+0000","last_peered":"2023-09-23T20:22:52.422787+0000","last_clean":"2023-09-23T20:22:52.422787+0000","last_became_active":"2023-09-23T20:22:44.516833+0000","last_became_peered":"2023-09-23T20:22:44.516833+0000","last_unstale":"2023-09-23T20:22:52.422787+0000","last_undegraded":"2023-09-23T20:22:52.422787+0000","last_fullsized":"2023-09-23T20:22:52.422787+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:48.130916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":721,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.438936+0000","last_change":"2023-09-23T20:22:53.430626+0000","last_active":"2023-09-23T20:22:55.438936+0000","last_peered":"2023-09-23T20:22:55.438936+0000","last_clean":"2023-09-23T20:22:55.438936+0000","last_became_active":"2023-09-23T20:22:53.430079+0000","last_became_peered":"2023-09-23T20:22:53.430079+0000","last_unstale":"2023-09-23T20:22:55.438936+0000","last_undegraded":"2023-09-23T20:22:55.438936+0000","last_fullsized":"2023-09-23T20:22:55.438936+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_clean_scrub_stamp":"2023-09-23T20:19:56.180203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039477649999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":334,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428699+0000","last_change":"2023-09-23T20:22:22.385579+0000","last_active":"2023-09-23T20:22:53.428699+0000","last_peered":"2023-09-23T20:22:53.428699+0000","last_clean":"2023-09-23T20:22:53.428699+0000","last_became_active":"2023-09-23T20:22:22.385218+0000","last_became_peered":"2023-09-23T20:22:22.385218+0000","last_unstale":"2023-09-23T20:22:53.428699+0000","last_undegraded":"2023-09-23T20:22:53.428699+0000","last_fullsized":"2023-09-23T20:22:53.428699+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:40.582088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":303,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428286+0000","last_change":"2023-09-23T20:22:22.388204+0000","last_active":"2023-09-23T20:22:53.428286+0000","last_peered":"2023-09-23T20:22:53.428286+0000","last_clean":"2023-09-23T20:22:53.428286+0000","last_became_active":"2023-09-23T20:22:22.387640+0000","last_became_peered":"2023-09-23T20:22:22.387640+0000","last_unstale":"2023-09-23T20:22:53.428286+0000","last_undegraded":"2023-09-23T20:22:53.428286+0000","last_fullsized":"2023-09-23T20:22:53.428286+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:58.015528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":685,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.760347+0000","last_change":"2023-09-23T20:22:56.760430+0000","last_active":"2023-09-23T20:22:56.760347+0000","last_peered":"2023-09-23T20:22:56.760347+0000","last_clean":"2023-09-23T20:22:56.760347+0000","last_became_active":"2023-09-23T20:22:53.431327+0000","last_became_peered":"2023-09-23T20:22:53.431327+0000","last_unstale":"2023-09-23T20:22:56.760347+0000","last_undegraded":"2023-09-23T20:22:56.760347+0000","last_fullsized":"2023-09-23T20:22:56.760347+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:56.760329+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:22:56.760329+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:13.013667+0000","scrub_duration":0.001887615,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":589,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423759+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:22:52.423759+0000","last_peered":"2023-09-23T20:22:52.423759+0000","last_clean":"2023-09-23T20:22:52.423759+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:22:52.423759+0000","last_undegraded":"2023-09-23T20:22:52.423759+0000","last_fullsized":"2023-09-23T20:22:52.423759+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":592,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T20:22:55.436349+0000","last_change":"2023-09-23T20:17:47.266215+0000","last_active":"2023-09-23T20:22:55.436349+0000","last_peered":"2023-09-23T20:22:55.436349+0000","last_clean":"2023-09-23T20:22:55.436349+0000","last_became_active":"2023-09-23T20:17:47.265890+0000","last_became_peered":"2023-09-23T20:17:47.265890+0000","last_unstale":"2023-09-23T20:22:55.436349+0000","last_undegraded":"2023-09-23T20:22:55.436349+0000","last_fullsized":"2023-09-23T20:22:55.436349+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:59.854975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":579,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.423798+0000","last_change":"2023-09-23T20:17:47.288753+0000","last_active":"2023-09-23T20:22:52.423798+0000","last_peered":"2023-09-23T20:22:52.423798+0000","last_clean":"2023-09-23T20:22:52.423798+0000","last_became_active":"2023-09-23T20:17:47.270488+0000","last_became_peered":"2023-09-23T20:17:47.270488+0000","last_unstale":"2023-09-23T20:22:52.423798+0000","last_undegraded":"2023-09-23T20:22:52.423798+0000","last_fullsized":"2023-09-23T20:22:52.423798+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:35.837184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":553,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-09-23T20:22:51.421468+0000","last_change":"2023-09-23T19:59:09.264192+0000","last_active":"2023-09-23T20:22:51.421468+0000","last_peered":"2023-09-23T20:22:51.421468+0000","last_clean":"2023-09-23T20:22:51.421468+0000","last_became_active":"2023-09-23T19:59:09.263914+0000","last_became_peered":"2023-09-23T19:59:09.263914+0000","last_unstale":"2023-09-23T20:22:51.421468+0000","last_undegraded":"2023-09-23T20:22:51.421468+0000","last_fullsized":"2023-09-23T20:22:51.421468+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:57.517530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":725,"reported_epoch":557,"state":"peering","last_fresh":"2023-09-23T20:22:52.420665+0000","last_change":"2023-09-23T20:22:52.420665+0000","last_active":"2023-09-23T20:12:53.717116+0000","last_peered":"2023-09-23T20:12:51.474658+0000","last_clean":"2023-09-23T20:12:51.474658+0000","last_became_active":"2023-09-23T20:09:56.064411+0000","last_became_peered":"2023-09-23T20:09:56.064411+0000","last_unstale":"2023-09-23T20:22:52.420665+0000","last_undegraded":"2023-09-23T20:22:52.420665+0000","last_fullsized":"2023-09-23T20:22:52.420665+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:21:17.621169+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:21:17.621169+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0073627120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":238,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428616+0000","last_change":"2023-09-23T20:22:22.387830+0000","last_active":"2023-09-23T20:22:53.428616+0000","last_peered":"2023-09-23T20:22:53.428616+0000","last_clean":"2023-09-23T20:22:53.428616+0000","last_became_active":"2023-09-23T20:22:22.387480+0000","last_became_peered":"2023-09-23T20:22:22.387480+0000","last_unstale":"2023-09-23T20:22:53.428616+0000","last_undegraded":"2023-09-23T20:22:53.428616+0000","last_fullsized":"2023-09-23T20:22:53.428616+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:11.294140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":386,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.428046+0000","last_change":"2023-09-23T20:22:22.388889+0000","last_active":"2023-09-23T20:22:53.428046+0000","last_peered":"2023-09-23T20:22:53.428046+0000","last_clean":"2023-09-23T20:22:53.428046+0000","last_became_active":"2023-09-23T20:22:22.388445+0000","last_became_peered":"2023-09-23T20:22:22.388445+0000","last_unstale":"2023-09-23T20:22:53.428046+0000","last_undegraded":"2023-09-23T20:22:53.428046+0000","last_fullsized":"2023-09-23T20:22:53.428046+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:07.308257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":778,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.426413+0000","last_change":"2023-09-23T20:22:53.426093+0000","last_active":"2023-09-23T20:22:53.426413+0000","last_peered":"2023-09-23T20:22:53.426413+0000","last_clean":"2023-09-23T20:22:53.426413+0000","last_became_active":"2023-09-23T20:22:53.425726+0000","last_became_peered":"2023-09-23T20:22:53.425726+0000","last_unstale":"2023-09-23T20:22:53.426413+0000","last_undegraded":"2023-09-23T20:22:53.426413+0000","last_fullsized":"2023-09-23T20:22:53.426413+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:45.082027+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:21:45.082027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0020677999999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":420,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.424823+0000","last_change":"2023-09-23T20:22:22.382650+0000","last_active":"2023-09-23T20:22:52.424823+0000","last_peered":"2023-09-23T20:22:52.424823+0000","last_clean":"2023-09-23T20:22:52.424823+0000","last_became_active":"2023-09-23T20:22:22.382296+0000","last_became_peered":"2023-09-23T20:22:22.382296+0000","last_unstale":"2023-09-23T20:22:52.424823+0000","last_undegraded":"2023-09-23T20:22:52.424823+0000","last_fullsized":"2023-09-23T20:22:52.424823+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:56.429985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.421429+0000","last_change":"2023-09-23T20:17:47.267673+0000","last_active":"2023-09-23T20:22:53.421429+0000","last_peered":"2023-09-23T20:22:53.421429+0000","last_clean":"2023-09-23T20:22:53.421429+0000","last_became_active":"2023-09-23T20:17:47.267216+0000","last_became_peered":"2023-09-23T20:17:47.267216+0000","last_unstale":"2023-09-23T20:22:53.421429+0000","last_undegraded":"2023-09-23T20:22:53.421429+0000","last_fullsized":"2023-09-23T20:22:53.421429+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:37.855890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":10952,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41405,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":685235,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":17,"num_write_kb":165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":492672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4198470,"num_objects":9,"num_object_clones":0,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":8590,"num_read_kb":17404,"num_write":1225,"num_write_kb":13909,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":200,"num_bytes_recovered":8388703,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12701696,"data_stored":12638520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":1419,"ondisk_log_size":1419,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234501,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":615544,"kb_used_data":26408,"kb_used_omap":0,"kb_used_meta":589120,"kb_avail":93129608,"statfs":{"total":95995035648,"available":95364718592,"internally_reserved":0,"allocated":27041792,"data_stored":25435272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603258880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":6.3289999999999997,"15min":4.5750000000000002},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":0.72299999999999998,"5min":230.952,"15min":230.952},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.442,"5min":6.7119999999999997,"15min":4.8200000000000003},"min":{"1min":0.29899999999999999,"5min":0.224,"15min":0.222},"max":{"1min":0.624,"5min":231.08699999999999,"15min":231.08699999999999},"last":0.51300000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":1.8180000000000001,"15min":2.3730000000000002},"min":{"1min":0.313,"5min":0.253,"15min":0.21299999999999999},"max":{"1min":0.54800000000000004,"5min":61.113999999999997,"15min":91.335999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.438,"5min":1.952,"15min":2.6299999999999999},"min":{"1min":0.312,"5min":0.248,"15min":0.248},"max":{"1min":0.70599999999999996,"5min":61.082999999999998,"15min":91.516999999999996},"last":0.59199999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:21:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":5.585,"15min":4.6130000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":18.73,"5min":177.852,"15min":177.852},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.6629999999999998,"5min":5.7220000000000004,"15min":4.827},"min":{"1min":0.27800000000000002,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":19.018000000000001,"5min":176.97900000000001,"15min":176.97900000000001},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":3.121,"15min":3.831},"min":{"1min":0.35199999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69099999999999995,"5min":112.291,"15min":185.17099999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":3.5459999999999998,"15min":4.0039999999999996},"min":{"1min":0.35399999999999998,"5min":0.32000000000000001,"15min":0.28999999999999998},"max":{"1min":0.73899999999999999,"5min":112.887,"15min":184.99100000000001},"last":0.55400000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":1.232,"15min":0.96599999999999997},"min":{"1min":0.27500000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.57499999999999996,"5min":83.811999999999998,"15min":83.811999999999998},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.3,"15min":1.075},"min":{"1min":0.26400000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.66700000000000004,"5min":84.620000000000005,"15min":84.620000000000005},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.69899999999999995,"15min":1.0580000000000001},"min":{"1min":0.27800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.621,"5min":4.6959999999999997,"15min":16.327000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.65200000000000002,"15min":0.97099999999999997},"min":{"1min":0.29399999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60099999999999998,"5min":4.077,"15min":16.716000000000001},"last":0.443}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.3779999999999999,"15min":1.0569999999999999},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":2.3420000000000001,"5min":8.4120000000000008,"15min":41.902999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.163,"15min":0.82199999999999995},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1400000000000001,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.56899999999999995}]}]},{"osd":6,"up_from":551,"seq":2366526980100,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":227240,"kb_used_data":201952,"kb_used_omap":9,"kb_used_meta":25270,"kb_avail":93517912,"statfs":{"total":95995035648,"available":95762341888,"internally_reserved":0,"allocated":206798848,"data_stored":205215740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9991,"internal_metadata":25876729},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51000000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33700000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44500000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48699999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.502}]}]},{"osd":1,"up_from":99,"seq":425201762700,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":852292,"kb_used_data":11004,"kb_used_omap":7,"kb_used_meta":841272,"kb_avail":92892860,"statfs":{"total":95995035648,"available":95122288640,"internally_reserved":0,"allocated":11268096,"data_stored":8650216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":861463354},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.27700000000000002,"5min":2.9169999999999998,"15min":3.2610000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.185},"max":{"1min":0.373,"5min":62.710999999999999,"15min":145.81299999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.32800000000000001,"5min":3.5,"15min":3.5329999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.19800000000000001},"max":{"1min":0.76900000000000002,"5min":62.610999999999997,"15min":146.83799999999999},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:20:32 2023","interfaces":[{"interface":"back","average":{"1min":13.513,"5min":6.0099999999999998,"15min":4.9539999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":113.941,"5min":152.62,"15min":152.62},"last":0.48999999999999999},{"interface":"front","average":{"1min":14.683,"5min":6.234,"15min":4.702},"min":{"1min":0.318,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":114.34699999999999,"5min":154.84299999999999,"15min":154.84299999999999},"last":0.26900000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":2.8969999999999998,"15min":2.8159999999999998},"min":{"1min":0.23000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.91700000000000004,"5min":22.256,"15min":66.527000000000001},"last":0.31},{"interface":"front","average":{"1min":0.36299999999999999,"5min":1.859,"15min":2.1890000000000001},"min":{"1min":0.223,"5min":0.219,"15min":0.219},"max":{"1min":0.89000000000000001,"5min":22.068000000000001,"15min":61.854999999999997},"last":0.64600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":1.7749999999999999,"15min":1.6240000000000001},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.63400000000000001,"5min":29.791,"15min":29.791},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.6459999999999999,"15min":1.6319999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.98299999999999998,"5min":27.143999999999998,"15min":27.143999999999998},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":1.752,"15min":2.04},"min":{"1min":0.31,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":0.93700000000000006,"5min":29.280000000000001,"15min":83.510999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":2,"15min":2.2469999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.306,"5min":32.692,"15min":83.353999999999999},"last":0.35399999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.2390000000000001,"15min":1.665},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":4.5830000000000002,"5min":9.2279999999999998,"15min":18.117999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.75,"5min":1.325,"15min":1.843},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.0310000000000001,"5min":9.8659999999999997,"15min":20.734999999999999},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.7869999999999999,"15min":2.4950000000000001},"min":{"1min":0.33100000000000002,"5min":0.315,"15min":0.315},"max":{"1min":1.278,"5min":12.599,"15min":127.03100000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":1.5389999999999999,"15min":2.117},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.71199999999999997,"5min":10.340999999999999,"15min":125.095},"last":0.47499999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926238,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1054712,"kb_used_data":11312,"kb_used_omap":8,"kb_used_meta":1043383,"kb_avail":92690440,"statfs":{"total":95995035648,"available":94915010560,"internally_reserved":0,"allocated":11583488,"data_stored":8998737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1068424801},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":2.0459999999999998,"15min":2.2160000000000002},"min":{"1min":0.20899999999999999,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":1.1699999999999999,"5min":17.425000000000001,"15min":72.534000000000006},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.374,"5min":2.9049999999999998,"15min":2.6579999999999999},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":0.99199999999999999,"5min":21.949000000000002,"15min":72.667000000000002},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:20 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":2.8069999999999999,"15min":3.4100000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":63.710999999999999,"5min":63.710999999999999,"15min":160.542},"last":0.41799999999999998},{"interface":"front","average":{"1min":6.3049999999999997,"5min":3.5009999999999999,"15min":3.7730000000000001},"min":{"1min":0.255,"5min":0.23599999999999999,"15min":0.22600000000000001},"max":{"1min":60.518999999999998,"5min":60.518999999999998,"15min":160.75999999999999},"last":0.66900000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":5.702,"15min":3.9710000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.0249999999999999,"5min":201.655,"15min":201.655},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":5.5570000000000004,"15min":4.1040000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":4.7009999999999996,"5min":201.69399999999999,"15min":201.69399999999999},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.8600000000000001,"15min":1.7030000000000001},"min":{"1min":0.307,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.5249999999999999,"5min":16.593,"15min":16.593},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.51000000000000001,"5min":1.9610000000000001,"15min":1.883},"min":{"1min":0.34000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.77200000000000002,"5min":21.899000000000001,"15min":105.282},"last":0.64200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":2.7170000000000001,"15min":2.0070000000000001},"min":{"1min":0.32000000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.153,"5min":145.14500000000001,"15min":145.14500000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":3.2759999999999998,"15min":2.4430000000000001},"min":{"1min":0.35899999999999999,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":1.478,"5min":145.04499999999999,"15min":145.04499999999999},"last":0.63500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6339999999999999,"15min":1.6419999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":7.6500000000000004,"5min":7.6500000000000004,"15min":15.744999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.381,"5min":1.234,"15min":1.6020000000000001},"min":{"1min":0.44,"5min":0.35599999999999998,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":9.8219999999999992,"15min":13.766999999999999},"last":0.70299999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.2470000000000001,"15min":1.8009999999999999},"min":{"1min":0.34999999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.1259999999999999,"5min":8.0269999999999992,"15min":99.486000000000004},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.622,"5min":2.0779999999999998,"15min":2.3839999999999999},"min":{"1min":0.35599999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.5569999999999999,"5min":16.617000000000001,"15min":99.668999999999997},"last":0.61499999999999999}]}]},{"osd":2,"up_from":543,"seq":2332167241736,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":146256,"kb_used_data":7176,"kb_used_omap":16,"kb_used_meta":139055,"kb_avail":93598896,"statfs":{"total":95995035648,"available":95845269504,"internally_reserved":0,"allocated":7348224,"data_stored":4732112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16601,"internal_metadata":142393127},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.377}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435650,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1045172,"kb_used_data":7596,"kb_used_omap":4,"kb_used_meta":1037563,"kb_avail":92699980,"statfs":{"total":95995035648,"available":94924779520,"internally_reserved":0,"allocated":7778304,"data_stored":4852496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1062464621},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":7.218,"15min":4.6050000000000004},"min":{"1min":0.245,"5min":0.191,"15min":0.186},"max":{"1min":0.60199999999999998,"5min":219.80000000000001,"15min":219.80000000000001},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.34599999999999997,"5min":7.6529999999999996,"15min":4.8410000000000002},"min":{"1min":0.23100000000000001,"5min":0.20899999999999999,"15min":0.20100000000000001},"max":{"1min":0.69399999999999995,"5min":221.36000000000001,"15min":221.36000000000001},"last":0.26100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":2.9060000000000001,"15min":3.6880000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.58699999999999997,"5min":92.036000000000001,"15min":183.87},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.46999999999999997,"5min":2.8029999999999999,"15min":3.8540000000000001},"min":{"1min":0.26800000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.4300000000000002,"5min":92.210999999999999,"15min":185.84899999999999},"last":0.33200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":7.6070000000000002,"15min":5.4000000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.7549999999999999,"5min":128.369,"15min":163.83500000000001},"last":0.317},{"interface":"front","average":{"1min":3.8420000000000001,"5min":7.4020000000000001,"15min":5.2409999999999997},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":15.286,"5min":126.497,"15min":163.92099999999999},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":1.8759999999999999,"15min":1.829},"min":{"1min":0.35499999999999998,"5min":0.34799999999999998,"15min":0.28999999999999998},"max":{"1min":0.76600000000000001,"5min":54.046999999999997,"15min":58.472999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.505,"5min":2.3250000000000002,"15min":2.327},"min":{"1min":0.34699999999999998,"5min":0.32400000000000001,"15min":0.28299999999999997},"max":{"1min":0.83199999999999996,"5min":54.311,"15min":58.408000000000001},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":2.4430000000000001,"15min":3.29},"min":{"1min":0.33000000000000002,"5min":0.32000000000000001,"15min":0.30299999999999999},"max":{"1min":0.88200000000000001,"5min":54.139000000000003,"15min":200.53200000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":2.1339999999999999,"15min":3.3340000000000001},"min":{"1min":0.375,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.871,"5min":56.457999999999998,"15min":200.846},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:18:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.593,"15min":2.2200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":1.5329999999999999,"5min":8.9610000000000003,"15min":82.281999999999996},"last":0.441},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.3280000000000001,"15min":2.286},"min":{"1min":0.42699999999999999,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":2.2040000000000002,"5min":8.2629999999999999,"15min":82.233000000000004},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.542,"15min":2.4710000000000001},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.81899999999999995,"5min":17.986000000000001,"15min":139.465},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":1.329,"15min":2.423},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.32800000000000001},"max":{"1min":0.79200000000000004,"5min":9.2289999999999992,"15min":141.56200000000001},"last":0.5}]}]},{"osd":4,"up_from":30,"seq":128849019411,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":582372,"kb_used_data":6104,"kb_used_omap":0,"kb_used_meta":576256,"kb_avail":93162780,"statfs":{"total":95995035648,"available":95398686720,"internally_reserved":0,"allocated":6250496,"data_stored":4649580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":590086144},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":8.1210000000000004,"15min":4.0899999999999999},"min":{"1min":0.32800000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.5779999999999998,"5min":232.89500000000001,"15min":232.89500000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.45600000000000002,"5min":8.0589999999999993,"15min":4.0800000000000001},"min":{"1min":0.34799999999999998,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":0.63,"5min":232.851,"15min":232.851},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.29,"15min":3.0379999999999998},"min":{"1min":0.32800000000000001,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":3.5,"5min":26.420000000000002,"15min":138.47999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.232,"15min":3.0680000000000001},"min":{"1min":0.33500000000000002,"5min":0.25,"15min":0.217},"max":{"1min":0.626,"5min":26.388999999999999,"15min":138.429},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:20:22 2023","interfaces":[{"interface":"back","average":{"1min":15.880000000000001,"5min":9.6950000000000003,"15min":6.7859999999999996},"min":{"1min":0.29899999999999999,"5min":0.27500000000000002,"15min":0.245},"max":{"1min":204.345,"5min":204.345,"15min":204.345},"last":0.31900000000000001},{"interface":"front","average":{"1min":15.916,"5min":9.6020000000000003,"15min":6.6429999999999998},"min":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":204.072,"5min":204.072,"15min":204.072},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":5.9930000000000003,"15min":4.7759999999999998},"min":{"1min":0.34100000000000003,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.66400000000000003,"5min":156.72300000000001,"15min":184.37100000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.46400000000000002,"5min":5.8890000000000002,"15min":4.8300000000000001},"min":{"1min":0.34100000000000003,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":0.68899999999999995,"5min":155.84800000000001,"15min":184.22300000000001},"last":0.435}]},{"osd":5,"last update":"Sat Sep 23 20:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.51000000000000001,"15min":1.0780000000000001},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":0.65700000000000003,"5min":4.3970000000000002,"15min":114.788},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.53600000000000003,"15min":1.101},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.72099999999999997,"5min":4.0460000000000003,"15min":113.822},"last":0.37}]},{"osd":6,"last update":"Sat Sep 23 20:17:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.0390000000000001,"15min":1.026},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":2.5110000000000001,"5min":114.232,"15min":114.232},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.3540000000000001,"15min":1.2130000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.247},"max":{"1min":4.085,"5min":114.07299999999999,"15min":114.07299999999999},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.52500000000000002,"15min":1.405},"min":{"1min":0.32100000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.61699999999999999,"5min":4.9050000000000002,"15min":113.861},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.59799999999999998,"15min":1.4570000000000001},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.222},"max":{"1min":0.629,"5min":4.6710000000000003,"15min":114.32299999999999},"last":0.38200000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757775,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670488,"kb_used_data":6028,"kb_used_omap":0,"kb_used_meta":664448,"kb_avail":93074664,"statfs":{"total":95995035648,"available":95308455936,"internally_reserved":0,"allocated":6172672,"data_stored":4580277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":680394752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":6.8090000000000002,"15min":4.1459999999999999},"min":{"1min":0.26100000000000001,"5min":0.253,"15min":0.20699999999999999},"max":{"1min":0.59899999999999998,"5min":166.57900000000001,"15min":166.57900000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":8.5269999999999992,"15min":4.6680000000000001},"min":{"1min":0.29599999999999999,"5min":0.22,"15min":0.21099999999999999},"max":{"1min":0.626,"5min":256.52199999999999,"15min":256.52199999999999},"last":0.66700000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:22:02 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":1.2969999999999999,"15min":2.903},"min":{"1min":0.32000000000000001,"5min":0.24299999999999999,"15min":0.22},"max":{"1min":0.56299999999999994,"5min":17.010999999999999,"15min":150.16999999999999},"last":0.621},{"interface":"front","average":{"1min":0.495,"5min":1.5740000000000001,"15min":2.8450000000000002},"min":{"1min":0.29899999999999999,"5min":0.27300000000000002,"15min":0.21199999999999999},"max":{"1min":0.67800000000000005,"5min":18.879000000000001,"15min":150.12700000000001},"last":0.59699999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:20:21 2023","interfaces":[{"interface":"back","average":{"1min":17.131,"5min":6.5279999999999996,"15min":5.3019999999999996},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":208.495,"5min":208.495,"15min":240.57300000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":17.167999999999999,"5min":6.5869999999999997,"15min":5.3659999999999997},"min":{"1min":0.33500000000000002,"5min":0.33100000000000002,"15min":0.30299999999999999},"max":{"1min":208.58199999999999,"5min":208.58199999999999,"15min":240.464},"last":0.65500000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:22:02 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":5.8079999999999998,"15min":5.5490000000000004},"min":{"1min":0.32000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.55400000000000005,"5min":196.97800000000001,"15min":196.97800000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":5.734,"15min":5.4020000000000001},"min":{"1min":0.36599999999999999,"5min":0.313,"15min":0.29999999999999999},"max":{"1min":0.69899999999999995,"5min":196.875,"15min":196.875},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.56299999999999994,"15min":0.72099999999999997},"min":{"1min":0.26200000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.55500000000000005,"5min":5.7279999999999998,"15min":8},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.51400000000000001,"15min":0.55200000000000005},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":0.52000000000000002,"5min":2.6890000000000001,"15min":3.371},"last":0.433}]},{"osd":6,"last update":"Sat Sep 23 20:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2490000000000001,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":4.04,"5min":7.5250000000000004,"15min":9.9009999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.73299999999999998,"15min":0.60999999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":1.609,"5min":3.3180000000000001,"15min":5.0119999999999996},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.63700000000000001,"15min":0.94299999999999995},"min":{"1min":0.27400000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55700000000000005,"5min":5.0469999999999997,"15min":11.725},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.438,"5min":0.63700000000000001,"15min":0.89400000000000002},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.59699999999999998,"5min":5.8499999999999996,"15min":11.041},"last":0.63400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4235264,"data_stored":4214889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4214784,"data_stored":4204598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":51,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":10272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":16,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":4115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":35,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4206592,"data_stored":4204544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11855,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9991,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:22:59.549 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 71 2023-09-23T20:22:59.564 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:59.611 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:59 smithi144 ceph-mon[194833]: failed to encode map e563 with expected crc 2023-09-23T20:22:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:59 smithi144 ceph-mon[194833]: failed to encode map e563 with expected crc 2023-09-23T20:22:59.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:59 smithi144 ceph-mon[194833]: 1.19 scrub starts 2023-09-23T20:22:59.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:59 smithi144 ceph-mon[194833]: 1.19 scrub ok 2023-09-23T20:22:59.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:59 smithi144 ceph-mon[194833]: failed to encode map e564 with expected crc 2023-09-23T20:22:59.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:59 smithi144 ceph-mon[194833]: osdmap e564: 8 total, 8 up, 8 in 2023-09-23T20:22:59.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:22:59 smithi144 ceph-mon[194833]: pgmap v1598: 103 pgs: 103 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 193 KiB/s rd, 0 B/s wr, 263 op/s 2023-09-23T20:22:59.671 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:22:59.689 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[213871]: failed to encode map e563 with expected crc 2023-09-23T20:22:59.689 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[213871]: failed to encode map e563 with expected crc 2023-09-23T20:22:59.689 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[213871]: 1.19 scrub starts 2023-09-23T20:22:59.689 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[213871]: 1.19 scrub ok 2023-09-23T20:22:59.689 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[213871]: failed to encode map e564 with expected crc 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[213871]: osdmap e564: 8 total, 8 up, 8 in 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[213871]: pgmap v1598: 103 pgs: 103 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 193 KiB/s rd, 0 B/s wr, 263 op/s 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[221113]: failed to encode map e563 with expected crc 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[221113]: failed to encode map e563 with expected crc 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[221113]: 1.19 scrub starts 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[221113]: 1.19 scrub ok 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[221113]: failed to encode map e564 with expected crc 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[221113]: osdmap e564: 8 total, 8 up, 8 in 2023-09-23T20:22:59.690 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:22:59 smithi135 ceph-mon[221113]: pgmap v1598: 103 pgs: 103 active+clean; 4.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 193 KiB/s rd, 0 B/s wr, 263 op/s 2023-09-23T20:22:59.715 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:22:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:59.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:59.821 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:59.848 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:00.163 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:23:00.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:00.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:00.267 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:00.299 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:00.531 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:23:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:00.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:00.593 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:00.615 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:00.636 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:00.668 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:00.669 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: "size": 20, 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 600, 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: "ops": [ 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:138 11.d 11:b1b22a0f:::image17:head [call rbd.get_size in=11b] snapc 0=[] RETRY=2 ondisk+retry+read+known_if_redirected+supports_pool_eio e543)", 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:21.368515+0000", 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 39.298948813000003, 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0145893050000001, 2023-09-23T20:23:00.670 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 138 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:21.368515+0000", 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.671 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:21.368515+0000", 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:21.368513+0000", 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999976 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.672 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:21.368521+0000", 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4700000000000002e-06 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:21.368533+0000", 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2437e-05 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.673 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:21.368545+0000", 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1863e-05 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:21.373252+0000", 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0047067849999999998 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for peered", 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:21.373263+0000", 2023-09-23T20:23:00.674 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0519e-05 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.383001+0000", 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0097386669999999 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.383104+0000", 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000102713 2023-09-23T20:23:00.675 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:139 11.d 11:b1b22a0f:::image17:head [call rbd.create in=9b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:22.388026+0000", 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 38.279437534000003, 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026652070000000002, 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.676 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 139 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388026+0000", 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.677 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388026+0000", 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388024+0000", 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388032+0000", 2023-09-23T20:23:00.678 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.379e-06 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388041+0000", 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6020000000000006e-06 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388052+0000", 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1113e-05 2023-09-23T20:23:00.679 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388089+0000", 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6238000000000001e-05 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388208+0000", 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000119077 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.680 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4", 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.388452+0000", 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000244474 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.389780+0000", 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001327564 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.390565+0000", 2023-09-23T20:23:00.681 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00078549499999999999 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.390612+0000", 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6786000000000002e-05 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.390691+0000", 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.9033e-05 2023-09-23T20:23:00.682 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:343 11.10 11:0d2ddece:::image28:head [call rbd.snap_add in=11b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:22.530931+0000", 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 38.136532789, 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.003300435, 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.683 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 343 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.530931+0000", 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.684 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.530931+0000", 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.530930+0000", 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.530934+0000", 2023-09-23T20:23:00.685 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.267e-06 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.530944+0000", 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.4970000000000007e-06 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.530956+0000", 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2652e-05 2023-09-23T20:23:00.686 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.530981+0000", 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.425e-05 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.531032+0000", 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.1542000000000002e-05 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.687 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.533466+0000", 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0024335210000000001 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.533779+0000", 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000313753 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.688 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.533935+0000", 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015557599999999999 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.534153+0000", 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000217574 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.534198+0000", 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5120000000000002e-05 2023-09-23T20:23:00.689 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.534231+0000", 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3442999999999998e-05 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:375 11.d 11:b7102ad3:::image35:head [call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:23:00.690 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:22.563413+0000", 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 38.104050665999999, 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0041929649999999999, 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 375 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.691 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.563413+0000", 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.563413+0000", 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.692 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.563412+0000", 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.563416+0000", 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1760000000000003e-06 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.563426+0000", 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3340000000000008e-06 2023-09-23T20:23:00.693 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.563431+0000", 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0610000000000003e-06 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.563462+0000", 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1007000000000003e-05 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.694 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.563523+0000", 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1247000000000005e-05 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 4", 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.566526+0000", 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0030031609999999999 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.695 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.566817+0000", 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000290864 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.567470+0000", 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00065298000000000005 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.567518+0000", 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7865999999999999e-05 2023-09-23T20:23:00.696 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.567606+0000", 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.7934000000000002e-05 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:563 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_instance_get in=28b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:22.698277+0000", 2023-09-23T20:23:00.697 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.969186082, 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28631972999999999, 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 563 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.698 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.698277+0000", 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.698277+0000", 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.699 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.698277+0000", 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.698281+0000", 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.1849999999999997e-06 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.698287+0000", 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.6089999999999997e-06 2023-09-23T20:23:00.700 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.698292+0000", 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.0860000000000004e-06 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.698322+0000", 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0613e-05 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.701 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.698367+0000", 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4876000000000002e-05 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.984597+0000", 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.28623012399999997 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.702 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:575 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_status_get_summary in=34b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:22.992060+0000", 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.675403682000002, 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.031154758000000001, 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "started", 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 575 2023-09-23T20:23:00.703 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.992060+0000", 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.992060+0000", 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.704 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.992059+0000", 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:22.992064+0000", 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.7190000000000001e-06 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.705 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.023028+0000", 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.030964447999999999 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.023040+0000", 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1938e-05 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.706 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.023082+0000", 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.2039999999999997e-05 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.023127+0000", 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4527999999999998e-05 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.707 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.023215+0000", 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8001e-05 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:652 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_set in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:23.094908+0000", 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.572555373999997, 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026565870000000002, 2023-09-23T20:23:00.708 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 652 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.094908+0000", 2023-09-23T20:23:00.709 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.094908+0000", 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.094903+0000", 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999943 2023-09-23T20:23:00.710 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.094913+0000", 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0064999999999999e-05 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.094925+0000", 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2181000000000001e-05 2023-09-23T20:23:00.711 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.094930+0000", 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7010000000000001e-06 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.094976+0000", 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6020999999999999e-05 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.712 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.095065+0000", 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8436000000000005e-05 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.095300+0000", 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000234692 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.713 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.095683+0000", 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00038331900000000002 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.097479+0000", 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017956700000000001 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.714 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.097509+0000", 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0164000000000001e-05 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.097542+0000", 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3379e-05 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.715 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.097565+0000", 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.2527999999999998e-05 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:921 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_status_set in=26b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:23.359631+0000", 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.307832857000001, 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023561699999999999, 2023-09-23T20:23:00.716 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 921 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359631+0000", 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.717 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359631+0000", 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359630+0000", 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.718 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359634+0000", 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.6399999999999996e-06 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359645+0000", 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0045e-05 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.719 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359650+0000", 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0119999999999996e-06 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359700+0000", 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0713e-05 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359785+0000", 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4294000000000003e-05 2023-09-23T20:23:00.720 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.359985+0000", 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00020032599999999999 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.360947+0000", 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00096187400000000004 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.721 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.361861+0000", 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00091467899999999995 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.361898+0000", 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6573999999999999e-05 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.361949+0000", 2023-09-23T20:23:00.722 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0883000000000001e-05 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.361987+0000", 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7929000000000002e-05 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:922 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_set in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:23:00.723 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:23.362077+0000", 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.305386734000002, 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031646589999999998, 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 922 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.724 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362077+0000", 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362077+0000", 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.725 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362076+0000", 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362082+0000", 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1290000000000001e-06 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.726 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362096+0000", 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4392999999999999e-05 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362104+0000", 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.6799999999999993e-06 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362139+0000", 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.4977000000000001e-05 2023-09-23T20:23:00.727 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362188+0000", 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9610000000000001e-05 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362340+0000", 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00015194500000000001 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.728 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.362864+0000", 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00052327299999999999 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.364807+0000", 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0019435419999999999 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.729 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.365180+0000", 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037333199999999998 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.365216+0000", 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5493000000000001e-05 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.365241+0000", 2023-09-23T20:23:00.730 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.5517e-05 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:928 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_set in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:23.371880+0000", 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.295583762, 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002579745, 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.731 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 928 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.371880+0000", 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.732 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.371880+0000", 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.371879+0000", 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.733 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.371885+0000", 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7509999999999997e-06 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.371899+0000", 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.4119e-05 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.371916+0000", 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.768e-05 2023-09-23T20:23:00.734 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.371971+0000", 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4938000000000002e-05 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.372046+0000", 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4697000000000007e-05 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.735 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.372241+0000", 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00019464499999999999 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.373316+0000", 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0010749589999999999 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.374135+0000", 2023-09-23T20:23:00.736 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00081963800000000003 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.374372+0000", 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00023714299999999999 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.374422+0000", 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9119999999999997e-05 2023-09-23T20:23:00.737 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.374459+0000", 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7947000000000001e-05 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:1064 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_set in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:23.512727+0000", 2023-09-23T20:23:00.738 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 37.154736022999998, 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.14515451600000001, 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 1064 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.739 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512727+0000", 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512727+0000", 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.740 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512727+0000", 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512733+0000", 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.3840000000000002e-06 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512742+0000", 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.8710000000000003e-06 2023-09-23T20:23:00.741 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512747+0000", 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.2429999999999996e-06 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512787+0000", 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9394000000000003e-05 2023-09-23T20:23:00.742 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512841+0000", 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4014999999999997e-05 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.512977+0000", 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013639899999999999 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.743 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.513335+0000", 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000357995 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.513597+0000", 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00026199099999999999 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.744 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.657759+0000", 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.144162226 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.657828+0000", 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.8496000000000003e-05 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:23.657882+0000", 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.4061999999999998e-05 2023-09-23T20:23:00.745 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:1455 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_status_set in=26b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:24.095604+0000", 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.571859191000001, 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0040836320000000002, 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.746 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 1455 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095604+0000", 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095604+0000", 2023-09-23T20:23:00.747 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095604+0000", 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095608+0000", 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4789999999999998e-06 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.748 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095618+0000", 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0416e-05 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095625+0000", 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7000000000000002e-06 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.749 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095661+0000", 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5626000000000001e-05 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095722+0000", 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.1053e-05 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.095858+0000", 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013596999999999999 2023-09-23T20:23:00.750 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.096440+0000", 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00058213700000000004 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.096805+0000", 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000364722 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.751 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.099617+0000", 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0028120459999999999 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.099656+0000", 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.9428999999999998e-05 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.099688+0000", 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.1822999999999997e-05 2023-09-23T20:23:00.752 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:1567 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_status_set in=26b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:24.234252+0000", 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.433211722000003, 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033915289999999999, 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.753 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 1567 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234252+0000", 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.754 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234252+0000", 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234251+0000", 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.755 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234254+0000", 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.2789999999999999e-06 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234263+0000", 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5029999999999997e-06 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234269+0000", 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6969999999999998e-06 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.756 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234308+0000", 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8949000000000003e-05 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234392+0000", 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.4831000000000001e-05 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.757 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234530+0000", 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013745900000000001 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.234990+0000", 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000459837 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.235272+0000", 2023-09-23T20:23:00.758 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028268100000000001 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.237588+0000", 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002315915 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.237619+0000", 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.0720999999999999e-05 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.759 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.237643+0000", 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.4229000000000001e-05 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:1630 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_set in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:24.309980+0000", 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.357483178999999, 2023-09-23T20:23:00.760 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0040670660000000003, 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 1630 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.309980+0000", 2023-09-23T20:23:00.761 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.309980+0000", 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.309979+0000", 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.762 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.309984+0000", 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4649999999999996e-06 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.309994+0000", 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6439999999999995e-06 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.763 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.310001+0000", 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.0809999999999996e-06 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.310034+0000", 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.3393999999999999e-05 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.310101+0000", 2023-09-23T20:23:00.764 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.7199999999999994e-05 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.310311+0000", 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000209769 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.310946+0000", 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00063466499999999997 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.765 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.311174+0000", 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00022828799999999999 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.313955+0000", 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002781366 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.766 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.314009+0000", 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3544000000000001e-05 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.314047+0000", 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8454000000000001e-05 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.767 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:1664 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_set in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:24.360628+0000", 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.306835341000003, 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0033592330000000001, 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 1664 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.768 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360628+0000", 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360628+0000", 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.769 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360628+0000", 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360631+0000", 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.534e-06 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360640+0000", 2023-09-23T20:23:00.770 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.9439999999999994e-06 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360645+0000", 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5510000000000001e-06 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360681+0000", 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6569999999999997e-05 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.771 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360739+0000", 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.7748999999999999e-05 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.360875+0000", 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013656700000000001 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.772 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.362895+0000", 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.002019589 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.363152+0000", 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00025651299999999999 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.363913+0000", 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00076151699999999997 2023-09-23T20:23:00.773 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.363949+0000", 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.5408999999999999e-05 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.363987+0000", 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.8855e-05 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.774 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:1666 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_set in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:24.366292+0000", 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.301171592000003, 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00266395, 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.775 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 1666 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366292+0000", 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366292+0000", 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.776 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366291+0000", 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366296+0000", 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6670000000000001e-06 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.777 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366310+0000", 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3131e-05 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366317+0000", 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.413e-06 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.778 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366362+0000", 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5339000000000001e-05 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366440+0000", 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.7263999999999998e-05 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.366664+0000", 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00022482099999999999 2023-09-23T20:23:00.779 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.367488+0000", 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00082313199999999997 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.368832+0000", 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.001344402 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.780 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.368861+0000", 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.9054e-05 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.368899+0000", 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.7966000000000002e-05 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.368956+0000", 2023-09-23T20:23:00.781 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6836999999999998e-05 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46700.0:1698 11.6 11:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_set in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:24.411801+0000", 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 36.255662534999999, 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023551779999999999, 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.782 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46700", 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1890555918", 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 1698 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.411801+0000", 2023-09-23T20:23:00.783 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.411801+0000", 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.411800+0000", 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.999999 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.784 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.411805+0000", 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8199999999999996e-06 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.411813+0000", 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.5059999999999995e-06 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.785 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.411818+0000", 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.8189999999999999e-06 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.411855+0000", 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.6618000000000001e-05 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.411948+0000", 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.3156000000000001e-05 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.412121+0000", 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000173525 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.414008+0000", 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0018862939999999999 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.787 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.414053+0000", 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4861999999999998e-05 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.414063+0000", 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8989999999999996e-06 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.788 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.414113+0000", 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.0336000000000001e-05 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:24.414156+0000", 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.3273999999999999e-05 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.789 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.46883.0:1 2.1 2:968b5ac7:::testimg.rbd:head [stat] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:44.464129+0000", 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 16.203334366, 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0025829099999999999, 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "reached pg", 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.46883", 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/1396473456", 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 1 2023-09-23T20:23:00.790 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.464129+0000", 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.464129+0000", 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.791 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.464127+0000", 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.464135+0000", 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 7.4549999999999998e-06 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.792 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.464146+0000", 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1215e-05 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.464156+0000", 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0654e-05 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.793 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.464222+0000", 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.5517000000000003e-05 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.464232+0000", 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8889999999999997e-06 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.466578+0000", 2023-09-23T20:23:00.794 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0023458709999999998 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:44.466712+0000", 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00013430000000000001 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.795 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.47192.0:26 2.1 2:9bfbd4e6:::rbd_header.91716bc963a3:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e563)", 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:58.110318+0000", 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 2.5571459079999999, 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0052863379999999998, 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.47192", 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/3939304308", 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 26 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.796 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.110318+0000", 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.110318+0000", 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.797 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.110316+0000", 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.110326+0000", 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.8090000000000004e-06 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.798 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.110345+0000", 2023-09-23T20:23:00.799 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.9613e-05 2023-09-23T20:23:00.799 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.799 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.799 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.110357+0000", 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.2094e-05 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.110413+0000", 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.6119999999999998e-05 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.801 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.110424+0000", 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.0331e-05 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.113080+0000", 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0026559270000000002 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.802 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.113190+0000", 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.000110387 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.113565+0000", 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00037445499999999999 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.115325+0000", 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0017608739999999999 2023-09-23T20:23:00.803 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.115409+0000", 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 8.3687e-05 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.115510+0000", 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00010042199999999999 2023-09-23T20:23:00.804 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.115559+0000", 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.9234999999999999e-05 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:58.115604+0000", 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.5108999999999998e-05 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.805 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: "description": "osd_op(client.47192.0:41 2.1 2:9bfbd4e6:::rbd_header.91716bc963a3:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e564)", 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: "initiated_at": "2023-09-23T20:22:59.117541+0000", 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: "age": 1.5499219960000001, 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0031424949999999999, 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: "type_data": { 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: "client_info": { 2023-09-23T20:23:00.806 INFO:teuthology.orchestra.run.smithi135.stdout: "client": "client.47192", 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: "client_addr": "172.21.15.135:0/3939304308", 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: "tid": 41 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: "events": [ 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "initiated", 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.117541+0000", 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.807 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "throttled", 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.117541+0000", 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "header_read", 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.117539+0000", 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "all_read", 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.117551+0000", 2023-09-23T20:23:00.808 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.1963e-05 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "dispatched", 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.117573+0000", 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 2.213e-05 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "queued_for_pg", 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.117587+0000", 2023-09-23T20:23:00.809 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 1.3908e-05 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.117650+0000", 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 6.2210000000000002e-05 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op must wait for map", 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.117655+0000", 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.3730000000000001e-06 2023-09-23T20:23:00.810 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "reached_pg", 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.118941+0000", 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.0012863099999999999 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "started", 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.119038+0000", 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 9.6527999999999995e-05 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.811 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.119321+0000", 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00028307700000000001 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "op_commit", 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.120044+0000", 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00072275599999999996 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.812 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.120546+0000", 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 0.00050193000000000004 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.120584+0000", 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 3.854e-05 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "commit_sent", 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.120640+0000", 2023-09-23T20:23:00.813 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 5.5701000000000001e-05 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: }, 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: { 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: "event": "done", 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: "time": "2023-09-23T20:22:59.120684+0000", 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: "duration": 4.4152e-05 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: } 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-09-23T20:23:00.814 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-09-23T20:23:00.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[221113]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[221113]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[221113]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T20:23:00.815 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[221113]: osdmap e565: 8 total, 8 up, 8 in 2023-09-23T20:23:00.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[213871]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[213871]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[213871]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T20:23:00.816 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:00 smithi135 ceph-mon[213871]: osdmap e565: 8 total, 8 up, 8 in 2023-09-23T20:23:00.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:00 smithi144 ceph-mon[194833]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:00 smithi144 ceph-mon[194833]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:00 smithi144 ceph-mon[194833]: failed to encode map e564 with expected crc 2023-09-23T20:23:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:00 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T20:23:00.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:00 smithi144 ceph-mon[194833]: osdmap e565: 8 total, 8 up, 8 in 2023-09-23T20:23:01.112 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:23:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:01.131 INFO:tasks.workunit.client.0.smithi135.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T20:23:01.158 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg 2023-09-23T20:23:01.216 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:01.247 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:01.256 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:23:01.260 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd remove testimg_import 2023-09-23T20:23:01.294 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:23:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:01.347 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:23:01.350 INFO:tasks.workunit.client.0.smithi135.stderr:+ tiered=0 2023-09-23T20:23:01.350 INFO:tasks.workunit.client.0.smithi135.stderr:+ ceph osd dump 2023-09-23T20:23:01.350 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep '^pool' 2023-09-23T20:23:01.350 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep ''\''rbd'\''' 2023-09-23T20:23:01.351 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep tier 2023-09-23T20:23:01.399 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:01.426 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:01.528 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:23:01.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:01 smithi144 ceph-mon[194833]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:01 smithi144 ceph-mon[194833]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:01 smithi144 ceph-mon[194833]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:01 smithi144 ceph-mon[194833]: osdmap e566: 8 total, 8 up, 8 in 2023-09-23T20:23:01.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:01 smithi144 ceph-mon[194833]: pgmap v1601: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 205 KiB/s rd, 0 B/s wr, 284 op/s 2023-09-23T20:23:01.632 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:01.659 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:01.700 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:23:01.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:01.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[221113]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[221113]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[221113]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[221113]: osdmap e566: 8 total, 8 up, 8 in 2023-09-23T20:23:01.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[221113]: pgmap v1601: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 205 KiB/s rd, 0 B/s wr, 284 op/s 2023-09-23T20:23:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[213871]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[213871]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[213871]: failed to encode map e565 with expected crc 2023-09-23T20:23:01.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[213871]: osdmap e566: 8 total, 8 up, 8 in 2023-09-23T20:23:01.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:01 smithi135 ceph-mon[213871]: pgmap v1601: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 205 KiB/s rd, 0 B/s wr, 284 op/s 2023-09-23T20:23:01.806 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:01.814 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:23:01.832 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:01.875 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:23:01.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:01.984 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:02.010 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:02.054 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:23:02.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:02.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:02.092 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_682894/sparse1 2023-09-23T20:23:02.098 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T20:23:02.099 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T20:23:02.099 INFO:tasks.workunit.client.0.smithi135.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00558006 s, 188 MB/s 2023-09-23T20:23:02.099 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_682894/sparse2 2023-09-23T20:23:02.105 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T20:23:02.105 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T20:23:02.105 INFO:tasks.workunit.client.0.smithi135.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00504093 s, 208 MB/s 2023-09-23T20:23:02.105 INFO:tasks.workunit.client.0.smithi135.stderr:+ truncate /tmp/rbd_import_export_682894/sparse2 -s 2M 2023-09-23T20:23:02.105 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse1 2023-09-23T20:23:02.145 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 0% complete...failed. 2023-09-23T20:23:02.145 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:23:02.148 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T20:23:02.148 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_682894/sparse1 2023-09-23T20:23:02.161 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:02.187 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:02.188 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:02.213 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T20:23:02.213 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:23:02.213 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:23:02.216 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd ls -l 2023-09-23T20:23:02.216 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep sparse1 2023-09-23T20:23:02.216 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T20:23:02.235 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:23:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:02.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:02.251 INFO:tasks.workunit.client.0.smithi135.stdout:sparse1 2 MiB 2 2023-09-23T20:23:02.251 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse1 2023-09-23T20:23:02.251 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse1 2023-09-23T20:23:02.252 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse1 2023-09-23T20:23:02.252 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T20:23:02.252 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T20:23:02.287 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.b8886627829d 2023-09-23T20:23:02.288 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.b8886627829d 2023-09-23T20:23:02.288 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse1 2023-09-23T20:23:02.288 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.b8886627829d\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:23:02.288 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse1 2023-09-23T20:23:02.288 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T20:23:02.288 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse1 2023-09-23T20:23:02.288 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:23:02.288 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:23:02.323 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T20:23:02.323 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T20:23:02.323 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T20:23:02.323 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T20:23:02.323 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T20:23:02.340 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:02.357 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=1 2023-09-23T20:23:02.357 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 1 2023-09-23T20:23:02.358 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:23:02.358 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse1 /tmp/rbd_import_export_682894/sparse1.out 2023-09-23T20:23:02.371 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:02.402 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T20:23:02.406 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_682894/sparse1 /tmp/rbd_import_export_682894/sparse1.out 2023-09-23T20:23:02.406 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_682894/sparse1 /tmp/rbd_import_export_682894/sparse1.out 2023-09-23T20:23:02.408 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse1 --format %b 2023-09-23T20:23:02.409 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=2048 2023-09-23T20:23:02.409 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse1.out --format %b 2023-09-23T20:23:02.411 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=2048 2023-09-23T20:23:02.411 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:23:02.411 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:23:02.411 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:23:02.411 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/sparse1.out 2023-09-23T20:23:02.411 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse1 2023-09-23T20:23:02.457 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:23:02.510 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T20:23:02.512 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse2 2023-09-23T20:23:02.550 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 0% complete...failed. 2023-09-23T20:23:02.550 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:23:02.554 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T20:23:02.554 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_682894/sparse2 2023-09-23T20:23:02.622 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T20:23:02.623 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:23:02.623 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:23:02.625 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd ls -l 2023-09-23T20:23:02.626 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep sparse2 2023-09-23T20:23:02.626 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T20:23:02.661 INFO:tasks.workunit.client.0.smithi135.stdout:sparse2 2 MiB 2 2023-09-23T20:23:02.662 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse2 2023-09-23T20:23:02.662 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse2 2023-09-23T20:23:02.662 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse2 2023-09-23T20:23:02.663 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T20:23:02.663 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T20:23:02.700 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.92168acc4b98 2023-09-23T20:23:02.701 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.92168acc4b98 2023-09-23T20:23:02.701 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse2 2023-09-23T20:23:02.701 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.92168acc4b98\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:23:02.701 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse2 2023-09-23T20:23:02.702 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T20:23:02.702 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse2 2023-09-23T20:23:02.702 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:23:02.703 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:23:02.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[221113]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[221113]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[221113]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/964884391' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:02.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1494941835' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:23:02.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1494941835' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:23:02.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[213871]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[213871]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[213871]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/964884391' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:02.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1494941835' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:23:02.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1494941835' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:23:02.725 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:23:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:02.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:02.739 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T20:23:02.739 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T20:23:02.739 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T20:23:02.740 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T20:23:02.740 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T20:23:02.772 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=0 2023-09-23T20:23:02.772 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 0 2023-09-23T20:23:02.772 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T20:23:02.772 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse2 /tmp/rbd_import_export_682894/sparse2.out 2023-09-23T20:23:02.813 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T20:23:02.817 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_682894/sparse2 /tmp/rbd_import_export_682894/sparse2.out 2023-09-23T20:23:02.818 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_682894/sparse2 /tmp/rbd_import_export_682894/sparse2.out 2023-09-23T20:23:02.818 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:02 smithi144 ceph-mon[194833]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.818 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:02 smithi144 ceph-mon[194833]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.819 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:02 smithi144 ceph-mon[194833]: failed to encode map e566 with expected crc 2023-09-23T20:23:02.819 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/964884391' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:02.819 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1494941835' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:23:02.819 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1494941835' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:23:02.820 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse2 --format %b 2023-09-23T20:23:02.820 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=2048 2023-09-23T20:23:02.821 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse2.out --format %b 2023-09-23T20:23:02.821 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=2048 2023-09-23T20:23:02.822 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:23:02.822 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:23:02.822 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:23:02.822 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/sparse2.out 2023-09-23T20:23:02.823 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse2 2023-09-23T20:23:02.829 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:02.860 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:02.914 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T20:23:02.917 INFO:tasks.workunit.client.0.smithi135.stderr:+ truncate /tmp/rbd_import_export_682894/sparse1 -s 10M 2023-09-23T20:23:02.994 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T20:23:03.315 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:23:03.315 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:23:03.315 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:23:03.318 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd ls -l 2023-09-23T20:23:03.318 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep sparse1 2023-09-23T20:23:03.318 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T20:23:03.359 INFO:tasks.workunit.client.0.smithi135.stdout:sparse1 10 MiB 2 2023-09-23T20:23:03.359 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse1 2023-09-23T20:23:03.359 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse1 2023-09-23T20:23:03.360 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse1 2023-09-23T20:23:03.360 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T20:23:03.360 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T20:23:03.395 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.92437f7432c3 2023-09-23T20:23:03.397 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.92437f7432c3 2023-09-23T20:23:03.397 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.92437f7432c3\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:23:03.397 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse1 2023-09-23T20:23:03.397 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse1 2023-09-23T20:23:03.397 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T20:23:03.397 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse1 2023-09-23T20:23:03.397 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:23:03.397 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:23:03.433 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T20:23:03.433 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T20:23:03.433 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T20:23:03.433 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T20:23:03.433 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T20:23:03.465 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=1 2023-09-23T20:23:03.466 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 1 2023-09-23T20:23:03.466 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:23:03.466 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse1 /tmp/rbd_import_export_682894/sparse1.out 2023-09-23T20:23:03.503 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T20:23:03.506 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_682894/sparse1 /tmp/rbd_import_export_682894/sparse1.out 2023-09-23T20:23:03.507 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_682894/sparse1 /tmp/rbd_import_export_682894/sparse1.out 2023-09-23T20:23:03.514 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse1 --format %b 2023-09-23T20:23:03.515 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=2048 2023-09-23T20:23:03.515 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse1.out --format %b 2023-09-23T20:23:03.516 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=2048 2023-09-23T20:23:03.516 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:23:03.516 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:23:03.516 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:23:03.517 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/sparse1.out 2023-09-23T20:23:03.517 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse1 2023-09-23T20:23:03.580 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:23:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:03.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:03.604 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T20:23:03.607 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_682894/sparse2 oflag=append conv=notrunc 2023-09-23T20:23:03.618 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T20:23:03.618 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T20:23:03.618 INFO:tasks.workunit.client.0.smithi135.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00998503 s, 210 MB/s 2023-09-23T20:23:03.618 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T20:23:03.684 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:03.705 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:23:03.705 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:23:03.706 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:23:03.708 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd ls -l 2023-09-23T20:23:03.709 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep sparse2 2023-09-23T20:23:03.709 INFO:tasks.workunit.client.0.smithi135.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T20:23:03.711 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:23:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:23:03] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/964884391' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: osdmap e567: 8 total, 8 up, 8 in 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: failed to encode map e568 with expected crc 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: osdmap e568: 8 total, 8 up, 8 in 2023-09-23T20:23:03.712 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[221113]: pgmap v1604: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 56 KiB/s rd, 8.0 KiB/s wr, 81 op/s 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/964884391' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: osdmap e567: 8 total, 8 up, 8 in 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: failed to encode map e568 with expected crc 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: osdmap e568: 8 total, 8 up, 8 in 2023-09-23T20:23:03.713 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:03 smithi135 ceph-mon[213871]: pgmap v1604: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 56 KiB/s rd, 8.0 KiB/s wr, 81 op/s 2023-09-23T20:23:03.714 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:03.748 INFO:tasks.workunit.client.0.smithi135.stdout:sparse2 4 MiB 2 2023-09-23T20:23:03.749 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse2 2023-09-23T20:23:03.749 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse2 2023-09-23T20:23:03.749 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse2 2023-09-23T20:23:03.749 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T20:23:03.749 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T20:23:03.788 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.927038e9a9fc 2023-09-23T20:23:03.789 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.927038e9a9fc 2023-09-23T20:23:03.789 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse2 2023-09-23T20:23:03.790 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse2 2023-09-23T20:23:03.790 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.927038e9a9fc\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:23:03.790 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T20:23:03.792 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse2 2023-09-23T20:23:03.792 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:23:03.792 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:23:03.834 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T20:23:03.834 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T20:23:03.835 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T20:23:03.835 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T20:23:03.835 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T20:23:03.867 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:23:03.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:03.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:03.877 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects='0 2023-09-23T20:23:03.877 INFO:tasks.workunit.client.0.smithi135.stderr:2 2023-09-23T20:23:03.877 INFO:tasks.workunit.client.0.smithi135.stderr:3' 2023-09-23T20:23:03.878 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 0 2 3 2023-09-23T20:23:03.878 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T20:23:03.878 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse2 /tmp/rbd_import_export_682894/sparse2.out 2023-09-23T20:23:03.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:23:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: failed to encode map e567 with expected crc 2023-09-23T20:23:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/964884391' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: osdmap e567: 8 total, 8 up, 8 in 2023-09-23T20:23:03.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: failed to encode map e568 with expected crc 2023-09-23T20:23:03.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: osdmap e568: 8 total, 8 up, 8 in 2023-09-23T20:23:03.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:03 smithi144 ceph-mon[194833]: pgmap v1604: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 56 KiB/s rd, 8.0 KiB/s wr, 81 op/s 2023-09-23T20:23:03.916 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T20:23:03.919 INFO:tasks.workunit.client.0.smithi135.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_682894/sparse2 /tmp/rbd_import_export_682894/sparse2.out 2023-09-23T20:23:03.919 INFO:tasks.workunit.client.0.smithi135.stderr:+ cmp -l /tmp/rbd_import_export_682894/sparse2 /tmp/rbd_import_export_682894/sparse2.out 2023-09-23T20:23:03.922 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse2 --format %b 2023-09-23T20:23:03.923 INFO:tasks.workunit.client.0.smithi135.stderr:+ origsize=6144 2023-09-23T20:23:03.924 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse2.out --format %b 2023-09-23T20:23:03.925 INFO:tasks.workunit.client.0.smithi135.stderr:+ exportsize=6144 2023-09-23T20:23:03.925 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:23:03.925 INFO:tasks.workunit.client.0.smithi135.stderr:+ difference=0 2023-09-23T20:23:03.925 INFO:tasks.workunit.client.0.smithi135.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:23:03.925 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/sparse2.out 2023-09-23T20:23:03.926 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse2 2023-09-23T20:23:03.972 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:04.001 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:04.016 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T20:23:04.020 INFO:tasks.workunit.client.0.smithi135.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T20:23:04.020 INFO:tasks.workunit.client.0.smithi135.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T20:23:04.020 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_682894/sparse1 sparse 2023-09-23T20:23:04.051 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:23:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:04.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:04.092 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T20:23:04.093 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:23:04.093 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:23:04.097 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse 2023-09-23T20:23:04.097 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse 2023-09-23T20:23:04.097 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse 2023-09-23T20:23:04.098 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T20:23:04.098 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T20:23:04.135 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.b942d99847ea 2023-09-23T20:23:04.136 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.b942d99847ea 2023-09-23T20:23:04.136 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse 2023-09-23T20:23:04.136 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse 2023-09-23T20:23:04.136 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.b942d99847ea\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:23:04.136 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T20:23:04.138 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse 2023-09-23T20:23:04.138 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:23:04.138 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:23:04.152 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2023-09-23T20:23:04.152 INFO:teuthology.orchestra.run.smithi144.stdout: "ops": [], 2023-09-23T20:23:04.152 INFO:teuthology.orchestra.run.smithi144.stdout: "num_ops": 0 2023-09-23T20:23:04.152 INFO:teuthology.orchestra.run.smithi144.stdout:} 2023-09-23T20:23:04.155 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:04.177 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T20:23:04.177 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T20:23:04.178 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T20:23:04.178 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T20:23:04.178 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T20:23:04.187 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:04.219 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=1 2023-09-23T20:23:04.219 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 1 2023-09-23T20:23:04.219 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:23:04.220 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse 2023-09-23T20:23:04.312 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T20:23:04.315 INFO:tasks.workunit.client.0.smithi135.stdout:zeros import through stdin to sparse image 2023-09-23T20:23:04.316 INFO:tasks.workunit.client.0.smithi135.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T20:23:04.316 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T20:23:04.316 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd import --new-format - sparse 2023-09-23T20:23:04.369 INFO:tasks.workunit.client.0.smithi135.stderr:4+0 records in 2023-09-23T20:23:04.369 INFO:tasks.workunit.client.0.smithi135.stderr:4+0 records out 2023-09-23T20:23:04.369 INFO:tasks.workunit.client.0.smithi135.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0523484 s, 80.1 MB/s 2023-09-23T20:23:04.380 INFO:tasks.workunit.client.0.smithi135.stderr: Importing image: 100% complete...done. 2023-09-23T20:23:04.381 INFO:teuthology.orchestra.run.smithi135.stdout:osd.0: {} 2023-09-23T20:23:04.381 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:23:04.385 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse 2023-09-23T20:23:04.385 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse 2023-09-23T20:23:04.386 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse 2023-09-23T20:23:04.386 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T20:23:04.386 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T20:23:04.390 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:23:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:04.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:04.401 INFO:teuthology.orchestra.run.smithi135.stdout:osd.1: {} 2023-09-23T20:23:04.424 INFO:teuthology.orchestra.run.smithi135.stdout:osd.2: {} 2023-09-23T20:23:04.425 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.92bb6cf9bd93 2023-09-23T20:23:04.426 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.92bb6cf9bd93 2023-09-23T20:23:04.426 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse 2023-09-23T20:23:04.426 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse 2023-09-23T20:23:04.426 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.92bb6cf9bd93\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:23:04.426 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T20:23:04.427 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse 2023-09-23T20:23:04.427 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:23:04.427 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:23:04.445 INFO:teuthology.orchestra.run.smithi135.stdout:osd.3: {} 2023-09-23T20:23:04.471 INFO:teuthology.orchestra.run.smithi135.stdout:osd.4: {} 2023-09-23T20:23:04.474 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T20:23:04.474 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T20:23:04.474 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T20:23:04.474 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T20:23:04.474 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T20:23:04.490 INFO:teuthology.orchestra.run.smithi135.stdout:osd.5: {} 2023-09-23T20:23:04.496 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:04.507 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects= 2023-09-23T20:23:04.507 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 2023-09-23T20:23:04.507 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T20:23:04.507 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse 2023-09-23T20:23:04.525 INFO:teuthology.orchestra.run.smithi135.stdout:osd.6: {} 2023-09-23T20:23:04.526 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:04.546 INFO:teuthology.orchestra.run.smithi135.stdout:osd.7: {} 2023-09-23T20:23:04.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:23:04.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:23:04.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:23:04.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:23:04.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:23:04.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:23:04.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:23:04.548 INFO:teuthology.orchestra.run.smithi135.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:23:04.598 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:23:04.601 INFO:tasks.workunit.client.0.smithi135.stderr:+ echo 'zeros export to sparse file' 2023-09-23T20:23:04.601 INFO:tasks.workunit.client.0.smithi135.stdout:zeros export to sparse file 2023-09-23T20:23:04.602 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T20:23:04.645 INFO:tasks.workunit.client.0.smithi135.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:23:04.647 INFO:tasks.workunit.client.0.smithi135.stderr:++ rbd info sparse 2023-09-23T20:23:04.647 INFO:tasks.workunit.client.0.smithi135.stderr:++ grep block_name_prefix 2023-09-23T20:23:04.648 INFO:tasks.workunit.client.0.smithi135.stderr:++ awk '{print $NF;}' 2023-09-23T20:23:04.683 INFO:tasks.workunit.client.0.smithi135.stderr:+ prefix=rbd_data.92e8d04d1cf7 2023-09-23T20:23:04.683 INFO:tasks.workunit.client.0.smithi135.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T20:23:04.684 INFO:tasks.workunit.client.0.smithi135.stderr:++ get_image_data_pool sparse 2023-09-23T20:23:04.684 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse 2023-09-23T20:23:04.684 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse 2023-09-23T20:23:04.685 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep 'data_pool: ' 2023-09-23T20:23:04.685 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T20:23:04.698 INFO:teuthology.orchestra.run.smithi135.stderr:Error EINVAL: specified pgp_num 71 > pg_num 31 2023-09-23T20:23:04.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:04 smithi135 ceph-mon[221113]: failed to encode map e568 with expected crc 2023-09-23T20:23:04.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:04 smithi135 ceph-mon[221113]: failed to encode map e568 with expected crc 2023-09-23T20:23:04.714 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:04 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3779961042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:04.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:04 smithi135 ceph-mon[213871]: failed to encode map e568 with expected crc 2023-09-23T20:23:04.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:04 smithi135 ceph-mon[213871]: failed to encode map e568 with expected crc 2023-09-23T20:23:04.715 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:04 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3779961042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:04.723 INFO:tasks.workunit.client.0.smithi135.stderr:++ data_pool= 2023-09-23T20:23:04.723 INFO:tasks.workunit.client.0.smithi135.stderr:++ '[' -z ']' 2023-09-23T20:23:04.723 INFO:tasks.workunit.client.0.smithi135.stderr:++ data_pool=rbd 2023-09-23T20:23:04.723 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo rbd 2023-09-23T20:23:04.723 INFO:tasks.workunit.client.0.smithi135.stderr:+ rados -p rbd put rbd_data.92e8d04d1cf7.000000000000 - 2023-09-23T20:23:04.751 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:23:04.752 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records in 2023-09-23T20:23:04.752 INFO:tasks.workunit.client.0.smithi135.stderr:1+0 records out 2023-09-23T20:23:04.752 INFO:tasks.workunit.client.0.smithi135.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0680323 s, 61.7 MB/s 2023-09-23T20:23:04.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:04 smithi144 ceph-mon[194833]: failed to encode map e568 with expected crc 2023-09-23T20:23:04.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:04 smithi144 ceph-mon[194833]: failed to encode map e568 with expected crc 2023-09-23T20:23:04.753 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:04 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3779961042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:04.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:04.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:04.780 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects sparse 2023-09-23T20:23:04.780 INFO:tasks.workunit.client.0.smithi135.stderr:++ image=sparse 2023-09-23T20:23:04.780 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rbd info sparse 2023-09-23T20:23:04.780 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep block_name_prefix 2023-09-23T20:23:04.780 INFO:tasks.workunit.client.0.smithi135.stderr:+++ awk '{print $NF;}' 2023-09-23T20:23:04.807 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:23:04.820 INFO:tasks.workunit.client.0.smithi135.stderr:++ prefix=rbd_data.92e8d04d1cf7 2023-09-23T20:23:04.821 INFO:tasks.workunit.client.0.smithi135.stderr:+++ grep rbd_data.92e8d04d1cf7 2023-09-23T20:23:04.821 INFO:tasks.workunit.client.0.smithi135.stderr:++++ get_image_data_pool sparse 2023-09-23T20:23:04.821 INFO:tasks.workunit.client.0.smithi135.stderr:++++ image=sparse 2023-09-23T20:23:04.821 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sed -e 's/rbd_data.92e8d04d1cf7\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:23:04.821 INFO:tasks.workunit.client.0.smithi135.stderr:+++ sort -u 2023-09-23T20:23:04.821 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ rbd info sparse 2023-09-23T20:23:04.821 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:23:04.822 INFO:tasks.workunit.client.0.smithi135.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:23:04.855 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:04.857 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool= 2023-09-23T20:23:04.858 INFO:tasks.workunit.client.0.smithi135.stderr:++++ '[' -z ']' 2023-09-23T20:23:04.858 INFO:tasks.workunit.client.0.smithi135.stderr:++++ data_pool=rbd 2023-09-23T20:23:04.858 INFO:tasks.workunit.client.0.smithi135.stderr:++++ echo rbd 2023-09-23T20:23:04.858 INFO:tasks.workunit.client.0.smithi135.stderr:+++ rados ls -p rbd 2023-09-23T20:23:04.883 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:04.892 INFO:tasks.workunit.client.0.smithi135.stderr:++ objects=0 2023-09-23T20:23:04.893 INFO:tasks.workunit.client.0.smithi135.stderr:++ echo 0 2023-09-23T20:23:04.893 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T20:23:04.893 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/sparse 2023-09-23T20:23:04.893 INFO:tasks.workunit.client.0.smithi135.stderr:rm: cannot remove '/tmp/rbd_import_export_682894/sparse': No such file or directory 2023-09-23T20:23:04.894 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T20:23:04.894 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd export sparse /tmp/rbd_import_export_682894/sparse 2023-09-23T20:23:04.932 INFO:tasks.workunit.client.0.smithi135.stderr: Exporting image: 100% complete...done. 2023-09-23T20:23:04.936 INFO:tasks.workunit.client.0.smithi135.stderr:++ stat /tmp/rbd_import_export_682894/sparse --format=%b 2023-09-23T20:23:04.937 INFO:tasks.workunit.client.0.smithi135.stderr:+ '[' 0 = 0 ']' 2023-09-23T20:23:04.937 INFO:tasks.workunit.client.0.smithi135.stderr:+ rbd rm sparse 2023-09-23T20:23:05.020 INFO:tasks.workunit.client.0.smithi135.stderr: Removing image: 100% complete...done. 2023-09-23T20:23:05.023 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm /tmp/rbd_import_export_682894/sparse /tmp/rbd_import_export_682894/sparse1 /tmp/rbd_import_export_682894/sparse2 /tmp/rbd_import_export_682894/sparse3 2023-09-23T20:23:05.025 INFO:tasks.workunit.client.0.smithi135.stderr:rm: cannot remove '/tmp/rbd_import_export_682894/sparse3': No such file or directory 2023-09-23T20:23:05.025 INFO:tasks.workunit.client.0.smithi135.stderr:+ true 2023-09-23T20:23:05.025 INFO:tasks.workunit.client.0.smithi135.stderr:+ echo OK 2023-09-23T20:23:05.026 INFO:tasks.workunit.client.0.smithi135.stdout:OK 2023-09-23T20:23:05.027 INFO:tasks.workunit.client.0.smithi135.stderr:+ rm -rf /tmp/rbd_import_export_682894 2023-09-23T20:23:05.027 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T20:23:05.027 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T20:23:05.095 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T20:23:05.095 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:23:05.166 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:05.564 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:23:05.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:05.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2524889834' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T20:23:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:05 smithi144 ceph-mon[194833]: pgmap v1605: 103 pgs: 103 active+clean; 5.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 638 KiB/s rd, 611 KiB/s wr, 319 op/s 2023-09-23T20:23:05.668 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:05.695 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:05.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2524889834' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T20:23:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:05 smithi135 ceph-mon[221113]: pgmap v1605: 103 pgs: 103 active+clean; 5.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 638 KiB/s rd, 611 KiB/s wr, 319 op/s 2023-09-23T20:23:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2524889834' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T20:23:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:05 smithi135 ceph-mon[213871]: pgmap v1605: 103 pgs: 103 active+clean; 5.7 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 638 KiB/s rd, 611 KiB/s wr, 319 op/s 2023-09-23T20:23:05.738 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:23:05.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:05.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:05.841 DEBUG:teuthology.orchestra.run.smithi135:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:05.868 INFO:teuthology.orchestra.run.smithi135.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:05.871 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:23:06.133 DEBUG:teuthology.parallel:result is None 2023-09-23T20:23:06.133 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:23:06.164 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:23:06.165 DEBUG:teuthology.orchestra.run.smithi135:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:23:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:23:06.205 INFO:teuthology.orchestra.run.smithi135.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:23:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:06.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi135 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:06.221 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:23:06.221 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:23:06.221 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T20:23:06.221 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:23:06.221 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T20:23:06.221 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:23:06.222 DEBUG:teuthology.orchestra.run.smithi135:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:23:06.228 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:23:06.245 INFO:teuthology.orchestra.run.smithi135.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:23:06.245 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:23:06.302 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:23:06.302 DEBUG:teuthology.orchestra.run.smithi135:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:23:06.311 DEBUG:teuthology.orchestra.run.smithi144:> sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:06.341 INFO:teuthology.orchestra.run.smithi144.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:06.359 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T20:23:06.435 INFO:teuthology.orchestra.run.smithi144.stderr:Error: no container with name or ID ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:23:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:06.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi144 with status 125: 'sudo docker kill -s 1 ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:06.599 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:06.599 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:07.239 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:23:07.587 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:08 smithi144 ceph-mon[194833]: pgmap v1606: 103 pgs: 103 active+clean; 5.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 685 KiB/s rd, 840 KiB/s wr, 250 op/s 2023-09-23T20:23:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:08 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:08.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:08 smithi144 ceph-mon[194833]: osdmap e569: 8 total, 8 up, 8 in 2023-09-23T20:23:08.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[221113]: pgmap v1606: 103 pgs: 103 active+clean; 5.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 685 KiB/s rd, 840 KiB/s wr, 250 op/s 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[221113]: osdmap e569: 8 total, 8 up, 8 in 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[213871]: pgmap v1606: 103 pgs: 103 active+clean; 5.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 685 KiB/s rd, 840 KiB/s wr, 250 op/s 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:08.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:08 smithi135 ceph-mon[213871]: osdmap e569: 8 total, 8 up, 8 in 2023-09-23T20:23:09.045 INFO:teuthology.orchestra.run.smithi144.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:09.045 INFO:teuthology.orchestra.run.smithi144.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:09.196 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is set 2023-09-23T20:23:09.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:09 smithi144 ceph-mon[194833]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:09 smithi144 ceph-mon[194833]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:09 smithi144 ceph-mon[194833]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/71560621' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:09.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:09 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[221113]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[221113]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[221113]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/71560621' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:09.474 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:09.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[213871]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[213871]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[213871]: failed to encode map e569 with expected crc 2023-09-23T20:23:09.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/71560621' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:09.474 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:09 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:10.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[221113]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[221113]: failed to encode map e570 with expected crc 2023-09-23T20:23:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[221113]: failed to encode map e570 with expected crc 2023-09-23T20:23:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[221113]: osdmap e570: 8 total, 8 up, 8 in 2023-09-23T20:23:10.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[221113]: pgmap v1609: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 403 op/s 2023-09-23T20:23:10.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[213871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:10.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[213871]: failed to encode map e570 with expected crc 2023-09-23T20:23:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[213871]: failed to encode map e570 with expected crc 2023-09-23T20:23:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[213871]: osdmap e570: 8 total, 8 up, 8 in 2023-09-23T20:23:10.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:10 smithi135 ceph-mon[213871]: pgmap v1609: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 403 op/s 2023-09-23T20:23:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:10 smithi144 ceph-mon[194833]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:10 smithi144 ceph-mon[194833]: failed to encode map e570 with expected crc 2023-09-23T20:23:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:10 smithi144 ceph-mon[194833]: failed to encode map e570 with expected crc 2023-09-23T20:23:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:10 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:10 smithi144 ceph-mon[194833]: osdmap e570: 8 total, 8 up, 8 in 2023-09-23T20:23:10.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:10 smithi144 ceph-mon[194833]: pgmap v1609: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 403 op/s 2023-09-23T20:23:11.170 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T20:23:11.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:11 smithi135 ceph-mon[221113]: failed to encode map e570 with expected crc 2023-09-23T20:23:11.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:11 smithi135 ceph-mon[213871]: failed to encode map e570 with expected crc 2023-09-23T20:23:11.536 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:11.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:11 smithi144 ceph-mon[194833]: failed to encode map e570 with expected crc 2023-09-23T20:23:11.975 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:12 smithi135 ceph-mon[221113]: pgmap v1610: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 331 op/s 2023-09-23T20:23:12.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:12 smithi135 ceph-mon[213871]: pgmap v1610: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 331 op/s 2023-09-23T20:23:12.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:12 smithi144 ceph-mon[194833]: pgmap v1610: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 331 op/s 2023-09-23T20:23:12.771 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:13.173 INFO:teuthology.orchestra.run.smithi135.stdout:mon.a: {} 2023-09-23T20:23:13.183 INFO:teuthology.orchestra.run.smithi135.stdout:mon.b: {} 2023-09-23T20:23:13.192 INFO:teuthology.orchestra.run.smithi135.stdout:mon.c: {} 2023-09-23T20:23:13.194 INFO:teuthology.orchestra.run.smithi135.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:23:13.194 INFO:teuthology.orchestra.run.smithi135.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:23:13.194 INFO:teuthology.orchestra.run.smithi135.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:23:13.949 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:23:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:23:13] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2023-09-23T20:23:13.967 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:23:14.220 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[221113]: failed to encode map e571 with expected crc 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[221113]: osdmap e571: 8 total, 8 up, 8 in 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[221113]: failed to encode map e571 with expected crc 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/624642570' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[221113]: pgmap v1612: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 656 KiB/s rd, 1.0 MiB/s wr, 194 op/s 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[213871]: failed to encode map e571 with expected crc 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[213871]: osdmap e571: 8 total, 8 up, 8 in 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[213871]: failed to encode map e571 with expected crc 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/624642570' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:23:14.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:14 smithi135 ceph-mon[213871]: pgmap v1612: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 656 KiB/s rd, 1.0 MiB/s wr, 194 op/s 2023-09-23T20:23:14.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:14 smithi144 ceph-mon[194833]: failed to encode map e571 with expected crc 2023-09-23T20:23:14.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:14 smithi144 ceph-mon[194833]: osdmap e571: 8 total, 8 up, 8 in 2023-09-23T20:23:14.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:14 smithi144 ceph-mon[194833]: failed to encode map e571 with expected crc 2023-09-23T20:23:14.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:14 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/624642570' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:23:14.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:14 smithi144 ceph-mon[194833]: pgmap v1612: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 656 KiB/s rd, 1.0 MiB/s wr, 194 op/s 2023-09-23T20:23:14.719 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:15.386 INFO:teuthology.orchestra.run.smithi135.stderr:noscrub is unset 2023-09-23T20:23:15.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2843299307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:15.409 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:15 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:15.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2843299307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:15.410 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:15 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2843299307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:15.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:15 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:16.553 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:23:16.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[221113]: pgmap v1613: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 503 KiB/s rd, 289 KiB/s wr, 100 op/s 2023-09-23T20:23:16.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[221113]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-09-23T20:23:16.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:23:16.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[221113]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[221113]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[221113]: osdmap e572: 8 total, 8 up, 8 in 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[221113]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[213871]: pgmap v1613: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 503 KiB/s rd, 289 KiB/s wr, 100 op/s 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[213871]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[213871]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[213871]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:16.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[213871]: osdmap e572: 8 total, 8 up, 8 in 2023-09-23T20:23:16.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:16 smithi135 ceph-mon[213871]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:16 smithi144 ceph-mon[194833]: pgmap v1613: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 503 KiB/s rd, 289 KiB/s wr, 100 op/s 2023-09-23T20:23:16.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:16 smithi144 ceph-mon[194833]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-09-23T20:23:16.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:16 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:23:16.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:16 smithi144 ceph-mon[194833]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:16 smithi144 ceph-mon[194833]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:16 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:16.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:16 smithi144 ceph-mon[194833]: osdmap e572: 8 total, 8 up, 8 in 2023-09-23T20:23:16.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:16 smithi144 ceph-mon[194833]: failed to encode map e572 with expected crc 2023-09-23T20:23:16.968 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:17.401 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:23:17.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:17 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4157886479' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:17.428 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:17 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:23:17.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:17 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4157886479' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:17.428 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:17 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:23:17.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:17 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4157886479' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:17.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:17 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:23:18.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[221113]: pgmap v1615: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 383 B/s wr, 18 op/s 2023-09-23T20:23:18.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[221113]: failed to encode map e573 with expected crc 2023-09-23T20:23:18.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[221113]: failed to encode map e573 with expected crc 2023-09-23T20:23:18.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4157886479' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:18.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[221113]: osdmap e573: 8 total, 8 up, 8 in 2023-09-23T20:23:18.662 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[221113]: failed to encode map e573 with expected crc 2023-09-23T20:23:18.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[213871]: pgmap v1615: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 383 B/s wr, 18 op/s 2023-09-23T20:23:18.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[213871]: failed to encode map e573 with expected crc 2023-09-23T20:23:18.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[213871]: failed to encode map e573 with expected crc 2023-09-23T20:23:18.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4157886479' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:18.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[213871]: osdmap e573: 8 total, 8 up, 8 in 2023-09-23T20:23:18.663 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:18 smithi135 ceph-mon[213871]: failed to encode map e573 with expected crc 2023-09-23T20:23:18.664 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T20:23:18.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:18 smithi144 ceph-mon[194833]: pgmap v1615: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 383 B/s wr, 18 op/s 2023-09-23T20:23:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:18 smithi144 ceph-mon[194833]: failed to encode map e573 with expected crc 2023-09-23T20:23:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:18 smithi144 ceph-mon[194833]: failed to encode map e573 with expected crc 2023-09-23T20:23:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:18 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4157886479' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:18 smithi144 ceph-mon[194833]: osdmap e573: 8 total, 8 up, 8 in 2023-09-23T20:23:18.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:18 smithi144 ceph-mon[194833]: failed to encode map e573 with expected crc 2023-09-23T20:23:19.222 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:19.411 INFO:teuthology.orchestra.run.smithi135.stderr:nodeep-scrub is unset 2023-09-23T20:23:19.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:19 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1982361356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:19.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:19 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1982361356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:19.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:19 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1982361356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:20.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[221113]: pgmap v1617: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[221113]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[221113]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[221113]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1982361356' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:20.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[221113]: osdmap e574: 8 total, 8 up, 8 in 2023-09-23T20:23:20.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[213871]: pgmap v1617: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:20.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[213871]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[213871]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[213871]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1982361356' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:20.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:20 smithi135 ceph-mon[213871]: osdmap e574: 8 total, 8 up, 8 in 2023-09-23T20:23:20.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:20 smithi144 ceph-mon[194833]: pgmap v1617: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:20 smithi144 ceph-mon[194833]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:20 smithi144 ceph-mon[194833]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:20 smithi144 ceph-mon[194833]: failed to encode map e574 with expected crc 2023-09-23T20:23:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:20 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1982361356' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:20.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:20 smithi144 ceph-mon[194833]: osdmap e574: 8 total, 8 up, 8 in 2023-09-23T20:23:21.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:21.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:21 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:21.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:21 smithi135 ceph-mon[221113]: pgmap v1619: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:21.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:21.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:21 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:21.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:21 smithi135 ceph-mon[213871]: pgmap v1619: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:21.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:21.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:21 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:21.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:21 smithi144 ceph-mon[194833]: pgmap v1619: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:22.490 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T20:23:22.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:22 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2838376942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:22 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:22 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2838376942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:22 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:22.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:22 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2838376942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:22 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:22.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:23.317 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:23:23.678 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:23.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:23.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: osdmap e575: 8 total, 8 up, 8 in 2023-09-23T20:23:23.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[221113]: pgmap v1621: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.686 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:23.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: osdmap e575: 8 total, 8 up, 8 in 2023-09-23T20:23:23.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:23:23.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:23:23.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.687 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:23 smithi135 ceph-mon[213871]: pgmap v1621: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: osdmap e575: 8 total, 8 up, 8 in 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: failed to encode map e575 with expected crc 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:23:23.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:23:23.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:23.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:23 smithi144 ceph-mon[194833]: pgmap v1621: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:23.950 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:23:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:23:23] "GET /metrics HTTP/1.1" 200 34118 "" "Prometheus/2.43.0" 2023-09-23T20:23:24.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:24 smithi135 ceph-mon[221113]: failed to encode map e576 with expected crc 2023-09-23T20:23:24.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:24 smithi135 ceph-mon[221113]: osdmap e576: 8 total, 8 up, 8 in 2023-09-23T20:23:24.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:24 smithi135 ceph-mon[213871]: failed to encode map e576 with expected crc 2023-09-23T20:23:24.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:24 smithi135 ceph-mon[213871]: osdmap e576: 8 total, 8 up, 8 in 2023-09-23T20:23:24.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:24 smithi144 ceph-mon[194833]: failed to encode map e576 with expected crc 2023-09-23T20:23:24.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:24 smithi144 ceph-mon[194833]: osdmap e576: 8 total, 8 up, 8 in 2023-09-23T20:23:25.493 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:23:25.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:25 smithi144 ceph-mon[194833]: pgmap v1623: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:25.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:25 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1313718281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:23:25.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:25 smithi135 ceph-mon[221113]: pgmap v1623: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:25.971 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:25 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1313718281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:23:25.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:25 smithi135 ceph-mon[213871]: pgmap v1623: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:25.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:25 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1313718281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:23:26.259 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T20:23:26.610 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:26.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[221113]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[221113]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[221113]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1313718281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[221113]: osdmap e577: 8 total, 8 up, 8 in 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[213871]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[213871]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[213871]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1313718281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:23:26.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:26 smithi135 ceph-mon[213871]: osdmap e577: 8 total, 8 up, 8 in 2023-09-23T20:23:26.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:26 smithi144 ceph-mon[194833]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:26 smithi144 ceph-mon[194833]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:26 smithi144 ceph-mon[194833]: failed to encode map e577 with expected crc 2023-09-23T20:23:26.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:26 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1313718281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:23:26.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:26 smithi144 ceph-mon[194833]: osdmap e577: 8 total, 8 up, 8 in 2023-09-23T20:23:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:27 smithi144 ceph-mon[194833]: pgmap v1625: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T20:23:27.970 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:27 smithi135 ceph-mon[221113]: pgmap v1625: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T20:23:27.971 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:27 smithi135 ceph-mon[213871]: pgmap v1625: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T20:23:29.126 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T20:23:29.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:29 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1574537896' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:23:29.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:29 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1574537896' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:23:29.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:29 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1574537896' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:23:29.946 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:23:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[221113]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[221113]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1574537896' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:23:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[221113]: osdmap e578: 8 total, 8 up, 8 in 2023-09-23T20:23:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[221113]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[221113]: pgmap v1627: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T20:23:30.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[213871]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[213871]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1574537896' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:23:30.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[213871]: osdmap e578: 8 total, 8 up, 8 in 2023-09-23T20:23:30.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[213871]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:30 smithi135 ceph-mon[213871]: pgmap v1627: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T20:23:30.302 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:30 smithi144 ceph-mon[194833]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:30 smithi144 ceph-mon[194833]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:30 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1574537896' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:23:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:30 smithi144 ceph-mon[194833]: osdmap e578: 8 total, 8 up, 8 in 2023-09-23T20:23:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:30 smithi144 ceph-mon[194833]: failed to encode map e578 with expected crc 2023-09-23T20:23:30.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:30 smithi144 ceph-mon[194833]: pgmap v1627: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T20:23:31.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:31 smithi144 ceph-mon[194833]: failed to encode map e578 with expected crc 2023-09-23T20:23:31.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:31 smithi135 ceph-mon[221113]: failed to encode map e578 with expected crc 2023-09-23T20:23:31.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:31 smithi135 ceph-mon[213871]: failed to encode map e578 with expected crc 2023-09-23T20:23:32.135 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:23:32.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:32 smithi135 ceph-mon[213871]: pgmap v1628: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T20:23:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2908234670' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:32.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:32 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:32.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:32 smithi135 ceph-mon[221113]: pgmap v1628: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T20:23:32.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2908234670' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:32.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:32 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:32.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:32 smithi144 ceph-mon[194833]: pgmap v1628: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T20:23:32.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2908234670' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:32.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:32 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:32.808 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T20:23:33.135 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:33 smithi135 ceph-mon[221113]: failed to encode map e579 with expected crc 2023-09-23T20:23:33.135 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:33 smithi135 ceph-mon[221113]: failed to encode map e579 with expected crc 2023-09-23T20:23:33.136 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:33 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:23:33.136 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:33 smithi135 ceph-mon[221113]: osdmap e579: 8 total, 8 up, 8 in 2023-09-23T20:23:33.136 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:33 smithi135 ceph-mon[213871]: failed to encode map e579 with expected crc 2023-09-23T20:23:33.136 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:33 smithi135 ceph-mon[213871]: failed to encode map e579 with expected crc 2023-09-23T20:23:33.136 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:33 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:23:33.136 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:33 smithi135 ceph-mon[213871]: osdmap e579: 8 total, 8 up, 8 in 2023-09-23T20:23:33.163 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:33 smithi144 ceph-mon[194833]: failed to encode map e579 with expected crc 2023-09-23T20:23:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:33 smithi144 ceph-mon[194833]: failed to encode map e579 with expected crc 2023-09-23T20:23:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:33 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:23:33.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:33 smithi144 ceph-mon[194833]: osdmap e579: 8 total, 8 up, 8 in 2023-09-23T20:23:33.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:23:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:23:33] "GET /metrics HTTP/1.1" 200 34118 "" "Prometheus/2.43.0" 2023-09-23T20:23:34.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:34 smithi135 ceph-mon[221113]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:34 smithi135 ceph-mon[221113]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:34 smithi135 ceph-mon[221113]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:34 smithi135 ceph-mon[221113]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T20:23:34.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:34 smithi135 ceph-mon[213871]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:34 smithi135 ceph-mon[213871]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:34 smithi135 ceph-mon[213871]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:34 smithi135 ceph-mon[213871]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T20:23:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:34 smithi144 ceph-mon[194833]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:34 smithi144 ceph-mon[194833]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:34 smithi144 ceph-mon[194833]: failed to encode map e579 with expected crc 2023-09-23T20:23:34.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:34 smithi144 ceph-mon[194833]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T20:23:35.156 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T20:23:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:35 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2565163091' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:35.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:35 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:35 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2565163091' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:35 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:35 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2565163091' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:35.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:35 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:23:35.990 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:23:36.345 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:36.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[213871]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[213871]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[213871]: osdmap e580: 8 total, 8 up, 8 in 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[213871]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[213871]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[221113]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[221113]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:23:36.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[221113]: osdmap e580: 8 total, 8 up, 8 in 2023-09-23T20:23:36.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[221113]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:36 smithi135 ceph-mon[221113]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:36 smithi144 ceph-mon[194833]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:36 smithi144 ceph-mon[194833]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:36 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:23:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:36 smithi144 ceph-mon[194833]: osdmap e580: 8 total, 8 up, 8 in 2023-09-23T20:23:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:36 smithi144 ceph-mon[194833]: failed to encode map e580 with expected crc 2023-09-23T20:23:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:36 smithi144 ceph-mon[194833]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:37.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[213871]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[213871]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[213871]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[213871]: osdmap e581: 8 total, 8 up, 8 in 2023-09-23T20:23:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[213871]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[221113]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[221113]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[221113]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[221113]: osdmap e581: 8 total, 8 up, 8 in 2023-09-23T20:23:37.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:37 smithi135 ceph-mon[221113]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:37 smithi144 ceph-mon[194833]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:37 smithi144 ceph-mon[194833]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:37 smithi144 ceph-mon[194833]: failed to encode map e581 with expected crc 2023-09-23T20:23:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:37 smithi144 ceph-mon[194833]: osdmap e581: 8 total, 8 up, 8 in 2023-09-23T20:23:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:37 smithi144 ceph-mon[194833]: failed to encode map e581 with expected crc 2023-09-23T20:23:38.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:38 smithi135 ceph-mon[213871]: 1.19 scrub starts 2023-09-23T20:23:38.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:38 smithi135 ceph-mon[213871]: 1.19 scrub ok 2023-09-23T20:23:38.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:38 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:38.165 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:38 smithi135 ceph-mon[213871]: pgmap v1634: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:38.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:38 smithi135 ceph-mon[221113]: 1.19 scrub starts 2023-09-23T20:23:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:38 smithi135 ceph-mon[221113]: 1.19 scrub ok 2023-09-23T20:23:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:38 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:38.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:38 smithi135 ceph-mon[221113]: pgmap v1634: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:38 smithi144 ceph-mon[194833]: 1.19 scrub starts 2023-09-23T20:23:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:38 smithi144 ceph-mon[194833]: 1.19 scrub ok 2023-09-23T20:23:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:38 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:38 smithi144 ceph-mon[194833]: pgmap v1634: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:39.186 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:23:39.415 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: 1.11 scrub starts 2023-09-23T20:23:39.415 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: 1.11 scrub ok 2023-09-23T20:23:39.415 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4069319824' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:39.416 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:39.416 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]: dispatch 2023-09-23T20:23:39.416 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.d", "id": [7, 2]}]: dispatch 2023-09-23T20:23:39.416 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:23:39.416 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-09-23T20:23:39.416 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T20:23:39.416 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-09-23T20:23:39.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: 1.11 scrub starts 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: 1.11 scrub ok 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4069319824' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]: dispatch 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.d", "id": [7, 2]}]: dispatch 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T20:23:39.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:39 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-09-23T20:23:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: 1.11 scrub starts 2023-09-23T20:23:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: 1.11 scrub ok 2023-09-23T20:23:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4069319824' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]: dispatch 2023-09-23T20:23:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.d", "id": [7, 2]}]: dispatch 2023-09-23T20:23:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:23:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-09-23T20:23:39.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T20:23:39.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:39 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-09-23T20:23:39.945 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T20:23:40.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:23:40.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2023-09-23T20:23:40.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.d", "id": [7, 2]}]': finished 2023-09-23T20:23:40.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:23:40.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-09-23T20:23:40.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T20:23:40.183 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-09-23T20:23:40.184 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: osdmap e582: 8 total, 8 up, 8 in 2023-09-23T20:23:40.184 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.184 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.184 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.184 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.184 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[221113]: pgmap v1636: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:40.185 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.185 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:23:40.185 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2023-09-23T20:23:40.185 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.d", "id": [7, 2]}]': finished 2023-09-23T20:23:40.185 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:23:40.185 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-09-23T20:23:40.185 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T20:23:40.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-09-23T20:23:40.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: osdmap e582: 8 total, 8 up, 8 in 2023-09-23T20:23:40.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.186 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:40 smithi135 ceph-mon[213871]: pgmap v1636: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:40.315 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.d", "id": [7, 2]}]': finished 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: osdmap e582: 8 total, 8 up, 8 in 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: failed to encode map e582 with expected crc 2023-09-23T20:23:40.634 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:40 smithi144 ceph-mon[194833]: pgmap v1636: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:41.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[213871]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[213871]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[213871]: osdmap e583: 8 total, 8 up, 8 in 2023-09-23T20:23:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[213871]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[213871]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[213871]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[221113]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[221113]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[221113]: osdmap e583: 8 total, 8 up, 8 in 2023-09-23T20:23:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[221113]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[221113]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:41 smithi135 ceph-mon[221113]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:41 smithi144 ceph-mon[194833]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:41 smithi144 ceph-mon[194833]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:41 smithi144 ceph-mon[194833]: osdmap e583: 8 total, 8 up, 8 in 2023-09-23T20:23:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:41 smithi144 ceph-mon[194833]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:41 smithi144 ceph-mon[194833]: failed to encode map e583 with expected crc 2023-09-23T20:23:41.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:41 smithi144 ceph-mon[194833]: failed to encode map e583 with expected crc 2023-09-23T20:23:42.201 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T20:23:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[221113]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[221113]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[221113]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[221113]: osdmap e584: 8 total, 8 up, 8 in 2023-09-23T20:23:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[221113]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[221113]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[221113]: pgmap v1639: 103 pgs: 1 remapped+peering, 102 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:42.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2622262043' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[213871]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[213871]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[213871]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[213871]: osdmap e584: 8 total, 8 up, 8 in 2023-09-23T20:23:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[213871]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[213871]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[213871]: pgmap v1639: 103 pgs: 1 remapped+peering, 102 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:42.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2622262043' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:42 smithi144 ceph-mon[194833]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:42 smithi144 ceph-mon[194833]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:42 smithi144 ceph-mon[194833]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:42 smithi144 ceph-mon[194833]: osdmap e584: 8 total, 8 up, 8 in 2023-09-23T20:23:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:42 smithi144 ceph-mon[194833]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:42 smithi144 ceph-mon[194833]: failed to encode map e584 with expected crc 2023-09-23T20:23:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:42 smithi144 ceph-mon[194833]: pgmap v1639: 103 pgs: 1 remapped+peering, 102 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2622262043' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:23:43.014 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[221113]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[221113]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2622262043' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[221113]: osdmap e585: 8 total, 8 up, 8 in 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[221113]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[221113]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[213871]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[213871]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2622262043' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[213871]: osdmap e585: 8 total, 8 up, 8 in 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[213871]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:43 smithi135 ceph-mon[213871]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:43 smithi144 ceph-mon[194833]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:43 smithi144 ceph-mon[194833]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:43 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2622262043' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:23:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:43 smithi144 ceph-mon[194833]: osdmap e585: 8 total, 8 up, 8 in 2023-09-23T20:23:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:43 smithi144 ceph-mon[194833]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:43 smithi144 ceph-mon[194833]: failed to encode map e585 with expected crc 2023-09-23T20:23:43.956 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:23:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:23:43] "GET /metrics HTTP/1.1" 200 34130 "" "Prometheus/2.43.0" 2023-09-23T20:23:44.220 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[213871]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[213871]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[213871]: osdmap e586: 8 total, 8 up, 8 in 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[213871]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[213871]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[213871]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[213871]: pgmap v1642: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[213871]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[221113]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.221 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[221113]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[221113]: osdmap e586: 8 total, 8 up, 8 in 2023-09-23T20:23:44.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[221113]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[221113]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[221113]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[221113]: pgmap v1642: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:44.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:44 smithi135 ceph-mon[221113]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:44 smithi144 ceph-mon[194833]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:44 smithi144 ceph-mon[194833]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:44 smithi144 ceph-mon[194833]: osdmap e586: 8 total, 8 up, 8 in 2023-09-23T20:23:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:44 smithi144 ceph-mon[194833]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:44 smithi144 ceph-mon[194833]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:44 smithi144 ceph-mon[194833]: failed to encode map e586 with expected crc 2023-09-23T20:23:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:44 smithi144 ceph-mon[194833]: pgmap v1642: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:23:44.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:44 smithi144 ceph-mon[194833]: failed to encode map e586 with expected crc 2023-09-23T20:23:45.209 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:23:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[221113]: 1.d scrub starts 2023-09-23T20:23:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[221113]: 1.d scrub ok 2023-09-23T20:23:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[221113]: 1.1a deep-scrub starts 2023-09-23T20:23:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[221113]: 1.1a deep-scrub ok 2023-09-23T20:23:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-09-23T20:23:45.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2096634244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:23:45.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[213871]: 1.d scrub starts 2023-09-23T20:23:45.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[213871]: 1.d scrub ok 2023-09-23T20:23:45.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[213871]: 1.1a deep-scrub starts 2023-09-23T20:23:45.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[213871]: 1.1a deep-scrub ok 2023-09-23T20:23:45.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-09-23T20:23:45.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:45 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2096634244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:23:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:45 smithi144 ceph-mon[194833]: 1.d scrub starts 2023-09-23T20:23:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:45 smithi144 ceph-mon[194833]: 1.d scrub ok 2023-09-23T20:23:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:45 smithi144 ceph-mon[194833]: 1.1a deep-scrub starts 2023-09-23T20:23:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:45 smithi144 ceph-mon[194833]: 1.1a deep-scrub ok 2023-09-23T20:23:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:45 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-09-23T20:23:45.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:45 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2096634244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:23:46.047 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T20:23:46.395 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[221113]: 1.5 deep-scrub starts 2023-09-23T20:23:46.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[221113]: 1.5 deep-scrub ok 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[221113]: failed to encode map e587 with expected crc 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2096634244' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[221113]: osdmap e587: 8 total, 8 up, 8 in 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[221113]: failed to encode map e587 with expected crc 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[221113]: pgmap v1644: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.1 KiB/s, 0 objects/s recovering 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[213871]: 1.5 deep-scrub starts 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[213871]: 1.5 deep-scrub ok 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[213871]: failed to encode map e587 with expected crc 2023-09-23T20:23:46.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2096634244' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:23:46.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[213871]: osdmap e587: 8 total, 8 up, 8 in 2023-09-23T20:23:46.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[213871]: failed to encode map e587 with expected crc 2023-09-23T20:23:46.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:46 smithi135 ceph-mon[213871]: pgmap v1644: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.1 KiB/s, 0 objects/s recovering 2023-09-23T20:23:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:46 smithi144 ceph-mon[194833]: 1.5 deep-scrub starts 2023-09-23T20:23:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:46 smithi144 ceph-mon[194833]: 1.5 deep-scrub ok 2023-09-23T20:23:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:46 smithi144 ceph-mon[194833]: failed to encode map e587 with expected crc 2023-09-23T20:23:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:46 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2096634244' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:23:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:46 smithi144 ceph-mon[194833]: osdmap e587: 8 total, 8 up, 8 in 2023-09-23T20:23:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:46 smithi144 ceph-mon[194833]: failed to encode map e587 with expected crc 2023-09-23T20:23:46.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:46 smithi144 ceph-mon[194833]: pgmap v1644: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.1 KiB/s, 0 objects/s recovering 2023-09-23T20:23:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:47 smithi135 ceph-mon[221113]: 1.e deep-scrub starts 2023-09-23T20:23:47.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:47 smithi135 ceph-mon[221113]: 1.e deep-scrub ok 2023-09-23T20:23:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:47 smithi135 ceph-mon[213871]: 1.e deep-scrub starts 2023-09-23T20:23:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:47 smithi135 ceph-mon[213871]: 1.e deep-scrub ok 2023-09-23T20:23:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:47 smithi144 ceph-mon[194833]: 1.e deep-scrub starts 2023-09-23T20:23:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:47 smithi144 ceph-mon[194833]: 1.e deep-scrub ok 2023-09-23T20:23:48.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:48 smithi144 ceph-mon[194833]: pgmap v1645: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T20:23:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:48 smithi135 ceph-mon[213871]: pgmap v1645: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T20:23:48.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:48 smithi135 ceph-mon[221113]: pgmap v1645: 103 pgs: 5 activating, 98 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T20:23:49.125 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T20:23:49.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:49 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/295940710' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:23:49.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:49 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/295940710' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:23:49.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:49 smithi135 ceph-mon[221113]: osdmap e588: 8 total, 8 up, 8 in 2023-09-23T20:23:49.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:49 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/295940710' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:23:49.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:49 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/295940710' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:23:49.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:49 smithi135 ceph-mon[213871]: osdmap e588: 8 total, 8 up, 8 in 2023-09-23T20:23:49.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:49 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/295940710' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:23:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:49 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/295940710' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:23:49.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:49 smithi144 ceph-mon[194833]: osdmap e588: 8 total, 8 up, 8 in 2023-09-23T20:23:49.890 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:23:50.241 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:50.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[213871]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[213871]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[213871]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[213871]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.489 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[213871]: pgmap v1647: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 775 B/s, 0 objects/s recovering 2023-09-23T20:23:50.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[221113]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[221113]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.489 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[221113]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[221113]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.490 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:50 smithi135 ceph-mon[221113]: pgmap v1647: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 775 B/s, 0 objects/s recovering 2023-09-23T20:23:50.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:50 smithi144 ceph-mon[194833]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:50 smithi144 ceph-mon[194833]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:50 smithi144 ceph-mon[194833]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:50 smithi144 ceph-mon[194833]: failed to encode map e588 with expected crc 2023-09-23T20:23:50.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:50 smithi144 ceph-mon[194833]: pgmap v1647: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 775 B/s, 0 objects/s recovering 2023-09-23T20:23:51.540 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:51 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive) 2023-09-23T20:23:51.541 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:51 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:23:51.541 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:51 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive) 2023-09-23T20:23:51.541 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:51 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:23:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:51 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive) 2023-09-23T20:23:51.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:51 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:23:52.247 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:23:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:52 smithi144 ceph-mon[194833]: pgmap v1648: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:52 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3757289251' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:52 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:52.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:52 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:52 smithi135 ceph-mon[221113]: pgmap v1648: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:52 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3757289251' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:52 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:52.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:52 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:52 smithi135 ceph-mon[213871]: pgmap v1648: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:52 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3757289251' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:52.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:52 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:52.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:52 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:53.046 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T20:23:53.396 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:53.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:53 smithi135 ceph-mon[221113]: failed to encode map e589 with expected crc 2023-09-23T20:23:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:53 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:23:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:53 smithi135 ceph-mon[221113]: osdmap e589: 8 total, 8 up, 8 in 2023-09-23T20:23:53.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:53 smithi135 ceph-mon[221113]: failed to encode map e589 with expected crc 2023-09-23T20:23:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:53 smithi135 ceph-mon[213871]: failed to encode map e589 with expected crc 2023-09-23T20:23:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:53 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:23:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:53 smithi135 ceph-mon[213871]: osdmap e589: 8 total, 8 up, 8 in 2023-09-23T20:23:53.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:53 smithi135 ceph-mon[213871]: failed to encode map e589 with expected crc 2023-09-23T20:23:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:53 smithi144 ceph-mon[194833]: failed to encode map e589 with expected crc 2023-09-23T20:23:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:53 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:23:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:53 smithi144 ceph-mon[194833]: osdmap e589: 8 total, 8 up, 8 in 2023-09-23T20:23:53.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:53 smithi144 ceph-mon[194833]: failed to encode map e589 with expected crc 2023-09-23T20:23:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:23:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:23:53] "GET /metrics HTTP/1.1" 200 34118 "" "Prometheus/2.43.0" 2023-09-23T20:23:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:54 smithi144 ceph-mon[194833]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:54 smithi144 ceph-mon[194833]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:54 smithi144 ceph-mon[194833]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:54 smithi144 ceph-mon[194833]: pgmap v1650: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:54.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:54 smithi135 ceph-mon[221113]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:54 smithi135 ceph-mon[221113]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.707 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:54 smithi135 ceph-mon[221113]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.708 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:54 smithi135 ceph-mon[221113]: pgmap v1650: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:54.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:54 smithi135 ceph-mon[213871]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:54 smithi135 ceph-mon[213871]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:54 smithi135 ceph-mon[213871]: failed to encode map e589 with expected crc 2023-09-23T20:23:54.708 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:54 smithi135 ceph-mon[213871]: pgmap v1650: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:55.265 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T20:23:55.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:55 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3615742443' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:55.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:55 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3615742443' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:55.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:55 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3615742443' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:23:56.044 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:23:56.395 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:56.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:56 smithi135 ceph-mon[213871]: failed to encode map e590 with expected crc 2023-09-23T20:23:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:56 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3615742443' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:23:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:56 smithi135 ceph-mon[213871]: osdmap e590: 8 total, 8 up, 8 in 2023-09-23T20:23:56.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:56 smithi135 ceph-mon[213871]: pgmap v1652: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:56 smithi135 ceph-mon[221113]: failed to encode map e590 with expected crc 2023-09-23T20:23:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:56 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3615742443' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:23:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:56 smithi135 ceph-mon[221113]: osdmap e590: 8 total, 8 up, 8 in 2023-09-23T20:23:56.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:56 smithi135 ceph-mon[221113]: pgmap v1652: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:56 smithi144 ceph-mon[194833]: failed to encode map e590 with expected crc 2023-09-23T20:23:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:56 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3615742443' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:23:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:56 smithi144 ceph-mon[194833]: osdmap e590: 8 total, 8 up, 8 in 2023-09-23T20:23:56.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:56 smithi144 ceph-mon[194833]: pgmap v1652: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:58 smithi144 ceph-mon[194833]: pgmap v1653: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:58 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:58 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3378128083' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:23:58.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:23:58 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:23:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:58 smithi135 ceph-mon[213871]: pgmap v1653: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:58 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:58.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:58 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3378128083' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:23:58.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:23:58 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:23:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:58 smithi135 ceph-mon[221113]: pgmap v1653: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:23:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:58 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:23:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:58 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3378128083' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:23:58.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:23:58 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:23:59.131 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:24:00.010 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T20:24:00.365 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:00 smithi144 ceph-mon[194833]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.381 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:00 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:24:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:00 smithi144 ceph-mon[194833]: osdmap e591: 8 total, 8 up, 8 in 2023-09-23T20:24:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:00 smithi144 ceph-mon[194833]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:00 smithi144 ceph-mon[194833]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:00 smithi144 ceph-mon[194833]: pgmap v1655: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[213871]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.470 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[213871]: osdmap e591: 8 total, 8 up, 8 in 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[213871]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[213871]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[213871]: pgmap v1655: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[221113]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[221113]: osdmap e591: 8 total, 8 up, 8 in 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[221113]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[221113]: failed to encode map e591 with expected crc 2023-09-23T20:24:00.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:00 smithi135 ceph-mon[221113]: pgmap v1655: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:02.366 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T20:24:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:02 smithi144 ceph-mon[194833]: pgmap v1656: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:02.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:02 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1262517376' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:24:02.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:02 smithi135 ceph-mon[221113]: pgmap v1656: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:02.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:02 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1262517376' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:24:02.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:02 smithi135 ceph-mon[213871]: pgmap v1656: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:02.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:02 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1262517376' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:24:03.137 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:24:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1262517376' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:24:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[213871]: osdmap e592: 8 total, 8 up, 8 in 2023-09-23T20:24:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[213871]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[213871]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.369 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[213871]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.370 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[213871]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.490 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:03 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1262517376' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:24:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:03 smithi144 ceph-mon[194833]: osdmap e592: 8 total, 8 up, 8 in 2023-09-23T20:24:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:03 smithi144 ceph-mon[194833]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:03 smithi144 ceph-mon[194833]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:03 smithi144 ceph-mon[194833]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:03 smithi144 ceph-mon[194833]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1262517376' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:24:03.684 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[221113]: osdmap e592: 8 total, 8 up, 8 in 2023-09-23T20:24:03.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[221113]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[221113]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[221113]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.685 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:03 smithi135 ceph-mon[221113]: failed to encode map e592 with expected crc 2023-09-23T20:24:03.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:03 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:24:03] "GET /metrics HTTP/1.1" 200 34118 "" "Prometheus/2.43.0" 2023-09-23T20:24:04.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:04 smithi144 ceph-mon[194833]: pgmap v1658: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:04.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:04 smithi135 ceph-mon[221113]: pgmap v1658: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:04.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:04 smithi135 ceph-mon[213871]: pgmap v1658: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:05.393 INFO:teuthology.orchestra.run.smithi135.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:24:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:05 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2303546556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:05.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:05 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:05.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:05 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2303546556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:05.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:05 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:05 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2303546556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:05.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:05 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:06.261 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T20:24:06.654 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:24:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: osdmap e593: 8 total, 8 up, 8 in 2023-09-23T20:24:06.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[221113]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: osdmap e593: 8 total, 8 up, 8 in 2023-09-23T20:24:06.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:06 smithi135 ceph-mon[213871]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:24:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: osdmap e593: 8 total, 8 up, 8 in 2023-09-23T20:24:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: failed to encode map e593 with expected crc 2023-09-23T20:24:06.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:06 smithi144 ceph-mon[194833]: failed to encode map e593 with expected crc 2023-09-23T20:24:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: osdmap e594: 8 total, 8 up, 8 in 2023-09-23T20:24:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[213871]: osdmap e595: 8 total, 8 up, 8 in 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: osdmap e594: 8 total, 8 up, 8 in 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:07.473 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:07 smithi135 ceph-mon[221113]: osdmap e595: 8 total, 8 up, 8 in 2023-09-23T20:24:07.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: osdmap e594: 8 total, 8 up, 8 in 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: failed to encode map e594 with expected crc 2023-09-23T20:24:07.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:07.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:07 smithi144 ceph-mon[194833]: osdmap e595: 8 total, 8 up, 8 in 2023-09-23T20:24:08.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: pgmap v1662: 103 pgs: 3 remapped+peering, 100 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: osdmap e596: 8 total, 8 up, 8 in 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e596 with expected crc 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[213871]: failed to encode map e596 with expected crc 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: pgmap v1662: 103 pgs: 3 remapped+peering, 100 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.723 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: osdmap e596: 8 total, 8 up, 8 in 2023-09-23T20:24:08.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e596 with expected crc 2023-09-23T20:24:08.724 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:08 smithi135 ceph-mon[221113]: failed to encode map e596 with expected crc 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: pgmap v1662: 103 pgs: 3 remapped+peering, 100 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e595 with expected crc 2023-09-23T20:24:08.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: osdmap e596: 8 total, 8 up, 8 in 2023-09-23T20:24:08.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e596 with expected crc 2023-09-23T20:24:08.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:08 smithi144 ceph-mon[194833]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.133 INFO:teuthology.orchestra.run.smithi135.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T20:24:09.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2455098282' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2455098282' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: osdmap e597: 8 total, 8 up, 8 in 2023-09-23T20:24:09.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[221113]: pgmap v1666: 103 pgs: 8 peering, 3 remapped+peering, 92 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.2 KiB/s, 6 objects/s recovering 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2455098282' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:09.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2455098282' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:24:09.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: osdmap e597: 8 total, 8 up, 8 in 2023-09-23T20:24:09.473 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:09 smithi135 ceph-mon[213871]: pgmap v1666: 103 pgs: 8 peering, 3 remapped+peering, 92 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.2 KiB/s, 6 objects/s recovering 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: failed to encode map e596 with expected crc 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2455098282' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:09.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2455098282' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:24:09.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: osdmap e597: 8 total, 8 up, 8 in 2023-09-23T20:24:09.883 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:09 smithi144 ceph-mon[194833]: pgmap v1666: 103 pgs: 8 peering, 3 remapped+peering, 92 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.2 KiB/s, 6 objects/s recovering 2023-09-23T20:24:09.998 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T20:24:09.998 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T20:24:09.998 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T20:24:09.998 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T20:24:09.998 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:24:09.999 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:24:10.351 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:10.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[221113]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.722 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:10 smithi135 ceph-mon[213871]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:10 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:10 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:10 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:10 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:10 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:10 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:10 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:10.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:10 smithi144 ceph-mon[194833]: failed to encode map e597 with expected crc 2023-09-23T20:24:11.664 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:11 smithi135 ceph-mon[213871]: 1.f scrub starts 2023-09-23T20:24:11.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:11 smithi135 ceph-mon[213871]: 1.f scrub ok 2023-09-23T20:24:11.728 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:11 smithi135 ceph-mon[213871]: pgmap v1667: 103 pgs: 8 peering, 3 remapped+peering, 92 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 964 B/s, 8 objects/s recovering 2023-09-23T20:24:11.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:11 smithi135 ceph-mon[221113]: 1.f scrub starts 2023-09-23T20:24:11.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:11 smithi135 ceph-mon[221113]: 1.f scrub ok 2023-09-23T20:24:11.728 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:11 smithi135 ceph-mon[221113]: pgmap v1667: 103 pgs: 8 peering, 3 remapped+peering, 92 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 964 B/s, 8 objects/s recovering 2023-09-23T20:24:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:11 smithi144 ceph-mon[194833]: 1.f scrub starts 2023-09-23T20:24:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:11 smithi144 ceph-mon[194833]: 1.f scrub ok 2023-09-23T20:24:11.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:11 smithi144 ceph-mon[194833]: pgmap v1667: 103 pgs: 8 peering, 3 remapped+peering, 92 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 964 B/s, 8 objects/s recovering 2023-09-23T20:24:12.053 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:24:12.053 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":597,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T20:24:09.120330+0000","last_up_change":"2023-09-23T20:22:43.360320+0000","last_in_change":"2023-09-23T20:18:02.168483+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"554","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"508","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:40:39.075611+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"566","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":566,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:41:38.466408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:41:38.762420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":595,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6803","nonce":100269857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6805","nonce":100269857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6809","nonce":100269857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6807","nonce":100269857}]},"public_addr":"172.21.15.135:6803/100269857","cluster_addr":"172.21.15.135:6805/100269857","heartbeat_back_addr":"172.21.15.135:6809/100269857","heartbeat_front_addr":"172.21.15.135:6807/100269857","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":596,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6811","nonce":2160782760}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6813","nonce":2160782760}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6817","nonce":2160782760}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6815","nonce":2160782760}]},"public_addr":"172.21.15.135:6811/2160782760","cluster_addr":"172.21.15.135:6813/2160782760","heartbeat_back_addr":"172.21.15.135:6817/2160782760","heartbeat_front_addr":"172.21.15.135:6815/2160782760","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":104,"last_clean_end":529,"up_from":543,"up_thru":585,"down_at":530,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":2108478916},{"type":"v1","addr":"172.21.15.135:6819","nonce":2108478916}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":2108478916},{"type":"v1","addr":"172.21.15.135:6821","nonce":2108478916}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":2108478916},{"type":"v1","addr":"172.21.15.135:6825","nonce":2108478916}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":2108478916},{"type":"v1","addr":"172.21.15.135:6823","nonce":2108478916}]},"public_addr":"172.21.15.135:6819/2108478916","cluster_addr":"172.21.15.135:6821/2108478916","heartbeat_back_addr":"172.21.15.135:6825/2108478916","heartbeat_front_addr":"172.21.15.135:6823/2108478916","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":596,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6827","nonce":2232272678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6829","nonce":2232272678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6833","nonce":2232272678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6831","nonce":2232272678}]},"public_addr":"172.21.15.135:6827/2232272678","cluster_addr":"172.21.15.135:6829/2232272678","heartbeat_back_addr":"172.21.15.135:6833/2232272678","heartbeat_front_addr":"172.21.15.135:6831/2232272678","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":596,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":596,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":251,"last_clean_end":501,"up_from":551,"up_thru":596,"down_at":502,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1753202340},{"type":"v1","addr":"172.21.15.144:6817","nonce":1753202340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1753202340},{"type":"v1","addr":"172.21.15.144:6819","nonce":1753202340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1753202340},{"type":"v1","addr":"172.21.15.144:6823","nonce":1753202340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1753202340},{"type":"v1","addr":"172.21.15.144:6821","nonce":1753202340}]},"public_addr":"172.21.15.144:6817/1753202340","cluster_addr":"172.21.15.144:6819/1753202340","heartbeat_back_addr":"172.21.15.144:6823/1753202340","heartbeat_front_addr":"172.21.15.144:6821/1753202340","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":596,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":529},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":501},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.d","mappings":[{"from":7,"to":2}]},{"pgid":"3.1b","mappings":[{"from":4,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":3,"to":5}]},{"pgid":"4.1d","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:6801/1867076080":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/1867076080":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/3668294279":"2023-09-24T19:44:16.725201+0000","172.21.15.135:6801/1642850725":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/4238089144":"2023-09-24T19:44:16.725201+0000","172.21.15.135:6801/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1005337745":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.144:6834/1742442049":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/775791172":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/3361160768":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/173145882":"2023-09-24T19:40:37.220934+0000","172.21.15.144:0/3293023812":"2023-09-24T19:44:56.542176+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.144:6835/1742442049":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.144:0/3940572722":"2023-09-24T19:44:56.542176+0000","172.21.15.144:0/2226038972":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/456433686":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/1642850725":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/3693941440":"2023-09-24T19:44:16.725201+0000","172.21.15.144:0/3756303050":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/2749491827":"2023-09-24T19:45:36.991484+0000","172.21.15.144:0/1990744051":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.144:0/2993818522":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/3516425720":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/2123602186":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/4159692573":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/1017551752":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/1625057187":"2023-09-24T19:45:36.991484+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:24:12.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:12 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1072716729' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:24:12.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:12 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1072716729' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:24:12.842 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:24:12.843 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:24:12.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:12 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1072716729' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:24:13.197 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:13.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:13 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:24:13] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.43.0" 2023-09-23T20:24:14.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:14 smithi135 ceph-mon[213871]: pgmap v1668: 103 pgs: 7 peering, 96 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 54 KiB/s, 6 objects/s recovering 2023-09-23T20:24:14.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:14 smithi135 ceph-mon[221113]: pgmap v1668: 103 pgs: 7 peering, 96 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 54 KiB/s, 6 objects/s recovering 2023-09-23T20:24:14.633 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:14 smithi144 ceph-mon[194833]: pgmap v1668: 103 pgs: 7 peering, 96 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 54 KiB/s, 6 objects/s recovering 2023-09-23T20:24:14.926 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:24:14.926 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":597,"fsid":"2f95c858-5a48-11ee-8db4-212e2dc638e7","created":"2023-09-23T19:35:08.772740+0000","modified":"2023-09-23T20:24:09.120330+0000","last_up_change":"2023-09-23T20:22:43.360320+0000","last_in_change":"2023-09-23T20:18:02.168483+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T19:38:09.111581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"554","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"508","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T19:40:39.075611+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"566","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":566,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T19:41:38.466408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T19:41:38.762420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"369a8e37-43f6-4c41-a626-2b159656f305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":595,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6803","nonce":100269857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6805","nonce":100269857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6809","nonce":100269857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":100269857},{"type":"v1","addr":"172.21.15.135:6807","nonce":100269857}]},"public_addr":"172.21.15.135:6803/100269857","cluster_addr":"172.21.15.135:6805/100269857","heartbeat_back_addr":"172.21.15.135:6809/100269857","heartbeat_front_addr":"172.21.15.135:6807/100269857","state":["exists","up"]},{"osd":1,"uuid":"ccf1949a-3059-45d2-a46a-07dfbbc68fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":596,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6811","nonce":2160782760}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6813","nonce":2160782760}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6817","nonce":2160782760}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":2160782760},{"type":"v1","addr":"172.21.15.135:6815","nonce":2160782760}]},"public_addr":"172.21.15.135:6811/2160782760","cluster_addr":"172.21.15.135:6813/2160782760","heartbeat_back_addr":"172.21.15.135:6817/2160782760","heartbeat_front_addr":"172.21.15.135:6815/2160782760","state":["exists","up"]},{"osd":2,"uuid":"2a8e9267-6c97-4c1f-b56b-7ca1844a1334","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":104,"last_clean_end":529,"up_from":543,"up_thru":585,"down_at":530,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":2108478916},{"type":"v1","addr":"172.21.15.135:6819","nonce":2108478916}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":2108478916},{"type":"v1","addr":"172.21.15.135:6821","nonce":2108478916}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":2108478916},{"type":"v1","addr":"172.21.15.135:6825","nonce":2108478916}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":2108478916},{"type":"v1","addr":"172.21.15.135:6823","nonce":2108478916}]},"public_addr":"172.21.15.135:6819/2108478916","cluster_addr":"172.21.15.135:6821/2108478916","heartbeat_back_addr":"172.21.15.135:6825/2108478916","heartbeat_front_addr":"172.21.15.135:6823/2108478916","state":["exists","up"]},{"osd":3,"uuid":"21fd337a-25fa-42ee-9f4c-130d8882de6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":596,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6827","nonce":2232272678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6829","nonce":2232272678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6832","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6833","nonce":2232272678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":2232272678},{"type":"v1","addr":"172.21.15.135:6831","nonce":2232272678}]},"public_addr":"172.21.15.135:6827/2232272678","cluster_addr":"172.21.15.135:6829/2232272678","heartbeat_back_addr":"172.21.15.135:6833/2232272678","heartbeat_front_addr":"172.21.15.135:6831/2232272678","state":["exists","up"]},{"osd":4,"uuid":"de72ce1b-4c64-4f34-b1ba-563833e9fcf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":596,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6801","nonce":1825747977}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6803","nonce":1825747977}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6807","nonce":1825747977}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1825747977},{"type":"v1","addr":"172.21.15.144:6805","nonce":1825747977}]},"public_addr":"172.21.15.144:6801/1825747977","cluster_addr":"172.21.15.144:6803/1825747977","heartbeat_back_addr":"172.21.15.144:6807/1825747977","heartbeat_front_addr":"172.21.15.144:6805/1825747977","state":["exists","up"]},{"osd":5,"uuid":"bd0e910c-7c37-4ee6-97e8-031834dadc71","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":596,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6809","nonce":2025173223}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6811","nonce":2025173223}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6815","nonce":2025173223}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2025173223},{"type":"v1","addr":"172.21.15.144:6813","nonce":2025173223}]},"public_addr":"172.21.15.144:6809/2025173223","cluster_addr":"172.21.15.144:6811/2025173223","heartbeat_back_addr":"172.21.15.144:6815/2025173223","heartbeat_front_addr":"172.21.15.144:6813/2025173223","state":["exists","up"]},{"osd":6,"uuid":"84e8261b-0520-44c9-aeae-c25256356c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":251,"last_clean_end":501,"up_from":551,"up_thru":596,"down_at":502,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":1753202340},{"type":"v1","addr":"172.21.15.144:6817","nonce":1753202340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":1753202340},{"type":"v1","addr":"172.21.15.144:6819","nonce":1753202340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":1753202340},{"type":"v1","addr":"172.21.15.144:6823","nonce":1753202340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":1753202340},{"type":"v1","addr":"172.21.15.144:6821","nonce":1753202340}]},"public_addr":"172.21.15.144:6817/1753202340","cluster_addr":"172.21.15.144:6819/1753202340","heartbeat_back_addr":"172.21.15.144:6823/1753202340","heartbeat_front_addr":"172.21.15.144:6821/1753202340","state":["exists","up"]},{"osd":7,"uuid":"030b85ad-503a-4291-a267-97a0e0f2db6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":596,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6825","nonce":3376336059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6827","nonce":3376336059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6831","nonce":3376336059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":3376336059},{"type":"v1","addr":"172.21.15.144:6829","nonce":3376336059}]},"public_addr":"172.21.15.144:6825/3376336059","cluster_addr":"172.21.15.144:6827/3376336059","heartbeat_back_addr":"172.21.15.144:6831/3376336059","heartbeat_front_addr":"172.21.15.144:6829/3376336059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:19.000160+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:37:43.147487+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:06.246344+0000","dead_epoch":529},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:29.759765+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:38:52.057428+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:15.170970+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:39:38.236187+0000","dead_epoch":501},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T19:40:01.545755+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.d","mappings":[{"from":7,"to":2}]},{"pgid":"3.1b","mappings":[{"from":4,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":3,"to":5}]},{"pgid":"4.1d","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.135:6801/1867076080":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/1867076080":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/3668294279":"2023-09-24T19:44:16.725201+0000","172.21.15.135:6801/1642850725":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/4238089144":"2023-09-24T19:44:16.725201+0000","172.21.15.135:6801/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1005337745":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/1191504160":"2023-09-24T19:36:00.784922+0000","172.21.15.135:6800/2392681307":"2023-09-24T19:40:37.220934+0000","172.21.15.144:6834/1742442049":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/1611423028":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/775791172":"2023-09-24T19:40:37.220934+0000","172.21.15.135:0/3361160768":"2023-09-24T19:40:37.220934+0000","172.21.15.135:6800/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/173145882":"2023-09-24T19:40:37.220934+0000","172.21.15.144:0/3293023812":"2023-09-24T19:44:56.542176+0000","172.21.15.135:6801/4224665887":"2023-09-24T19:36:00.784922+0000","172.21.15.144:6835/1742442049":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/4110552877":"2023-09-24T19:36:00.784922+0000","172.21.15.144:0/3940572722":"2023-09-24T19:44:56.542176+0000","172.21.15.144:0/2226038972":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/456433686":"2023-09-24T19:45:36.991484+0000","172.21.15.135:6800/1642850725":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/3693941440":"2023-09-24T19:44:16.725201+0000","172.21.15.144:0/3756303050":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/2749491827":"2023-09-24T19:45:36.991484+0000","172.21.15.144:0/1990744051":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/645490139":"2023-09-24T19:35:36.550885+0000","172.21.15.144:0/2993818522":"2023-09-24T19:44:56.542176+0000","172.21.15.135:0/3516425720":"2023-09-24T19:44:16.725201+0000","172.21.15.135:0/1336192479":"2023-09-24T19:36:00.784922+0000","172.21.15.135:0/1910257642":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6800/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:6801/782979070":"2023-09-24T19:35:36.550885+0000","172.21.15.135:0/2123602186":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/4159692573":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/1017551752":"2023-09-24T19:45:36.991484+0000","172.21.15.135:0/1625057187":"2023-09-24T19:45:36.991484+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:24:15.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:15 smithi135 ceph-mon[221113]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:24:15.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:15 smithi135 ceph-mon[221113]: Cluster is now healthy 2023-09-23T20:24:15.222 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:15 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2850032967' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:24:15.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:15 smithi135 ceph-mon[213871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:24:15.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:15 smithi135 ceph-mon[213871]: Cluster is now healthy 2023-09-23T20:24:15.222 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:15 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2850032967' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:24:15.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:15 smithi144 ceph-mon[194833]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:24:15.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:15 smithi144 ceph-mon[194833]: Cluster is now healthy 2023-09-23T20:24:15.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:15 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2850032967' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:24:15.650 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:24:15.651 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:24:15.651 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:24:15.652 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:24:15.652 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:24:15.653 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:24:15.653 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:24:15.653 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:24:16.472 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:16 smithi135 ceph-mon[221113]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 41 KiB/s, 4 objects/s recovering 2023-09-23T20:24:16.472 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:16 smithi135 ceph-mon[213871]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 41 KiB/s, 4 objects/s recovering 2023-09-23T20:24:16.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:16 smithi144 ceph-mon[194833]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 41 KiB/s, 4 objects/s recovering 2023-09-23T20:24:17.045 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:17.096 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:17.165 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:17.246 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:17.317 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:17.349 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:17.357 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:17.864 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:18.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:18 smithi144 ceph-mon[194833]: pgmap v1670: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 KiB/s, 4 objects/s recovering 2023-09-23T20:24:18.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:18 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:18.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:18 smithi135 ceph-mon[213871]: pgmap v1670: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 KiB/s, 4 objects/s recovering 2023-09-23T20:24:18.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:18 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:18 smithi135 ceph-mon[221113]: pgmap v1670: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 KiB/s, 4 objects/s recovering 2023-09-23T20:24:18.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:18 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:20.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:20 smithi144 ceph-mon[194833]: pgmap v1671: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 31 KiB/s, 1 objects/s recovering 2023-09-23T20:24:20.644 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:20 smithi135 ceph-mon[221113]: pgmap v1671: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 31 KiB/s, 1 objects/s recovering 2023-09-23T20:24:20.644 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:20 smithi135 ceph-mon[213871]: pgmap v1671: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 31 KiB/s, 1 objects/s recovering 2023-09-23T20:24:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:22 smithi144 ceph-mon[194833]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 KiB/s, 1 objects/s recovering 2023-09-23T20:24:22.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:22 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:22 smithi135 ceph-mon[213871]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 KiB/s, 1 objects/s recovering 2023-09-23T20:24:22.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:22 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:22 smithi135 ceph-mon[221113]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 KiB/s, 1 objects/s recovering 2023-09-23T20:24:22.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:22 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:23.973 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:23 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:24:23] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2023-09-23T20:24:24.089 INFO:teuthology.orchestra.run.smithi135.stdout:231928234519 2023-09-23T20:24:24.089 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:24:24.133 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:24 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:24.133 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:24 smithi135 ceph-mon[213871]: pgmap v1673: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 KiB/s, 0 objects/s recovering 2023-09-23T20:24:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:24 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:24.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:24 smithi144 ceph-mon[194833]: pgmap v1673: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 KiB/s, 0 objects/s recovering 2023-09-23T20:24:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:24 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:24.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:24 smithi135 ceph-mon[221113]: pgmap v1673: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 KiB/s, 0 objects/s recovering 2023-09-23T20:24:25.534 INFO:teuthology.orchestra.run.smithi135.stdout:163208757793 2023-09-23T20:24:25.535 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:24:25.661 INFO:teuthology.orchestra.run.smithi135.stdout:403726926256 2023-09-23T20:24:25.662 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:24:26.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:26 smithi144 ceph-mon[194833]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:26.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:26 smithi135 ceph-mon[221113]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:26.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:26 smithi135 ceph-mon[213871]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:27.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:27.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:27 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:27.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:27 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:27.598 INFO:teuthology.orchestra.run.smithi135.stdout:2366526980118 2023-09-23T20:24:27.598 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:24:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:27.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:27 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:27.920 INFO:teuthology.orchestra.run.smithi135.stdout:468151435668 2023-09-23T20:24:27.921 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:24:28.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:28 smithi135 ceph-mon[221113]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:28.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:28 smithi135 ceph-mon[213871]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:28.748 INFO:teuthology.orchestra.run.smithi135.stdout:128849019430 2023-09-23T20:24:28.749 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:24:28.870 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:28.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:28 smithi144 ceph-mon[194833]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:29.686 INFO:teuthology.orchestra.run.smithi135.stdout:425201762719 2023-09-23T20:24:29.687 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:24:29.866 INFO:teuthology.orchestra.run.smithi135.stdout:2332167241755 2023-09-23T20:24:29.867 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:24:29.938 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:30.063 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:30.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:30 smithi135 ceph-mon[221113]: pgmap v1676: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:30.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:30 smithi135 ceph-mon[213871]: pgmap v1676: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:30.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:30 smithi144 ceph-mon[194833]: pgmap v1676: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:31.119 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:31.119 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:31.120 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:32.211 INFO:teuthology.orchestra.run.smithi135.stdout:231928234521 2023-09-23T20:24:32.669 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:32 smithi135 ceph-mon[213871]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:32.669 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:32 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/2763283827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:24:32.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:32 smithi135 ceph-mon[221113]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:32.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:32 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/2763283827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:24:32.881 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:32 smithi144 ceph-mon[194833]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:32.882 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:32 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/2763283827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:24:33.553 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234519 got 231928234521 for osd.7 2023-09-23T20:24:33.553 DEBUG:teuthology.parallel:result is None 2023-09-23T20:24:33.818 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:33 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:24:33] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2023-09-23T20:24:34.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:34 smithi135 ceph-mon[213871]: pgmap v1678: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:34.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:34 smithi135 ceph-mon[221113]: pgmap v1678: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:34.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:34 smithi144 ceph-mon[194833]: pgmap v1678: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:34.869 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:35.129 INFO:teuthology.orchestra.run.smithi135.stdout:163208757795 2023-09-23T20:24:35.405 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:35.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:35 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3567463241' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:24:35.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:35 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3567463241' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:24:35.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:35 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3567463241' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:24:35.740 INFO:teuthology.orchestra.run.smithi135.stdout:468151435670 2023-09-23T20:24:36.416 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:36 smithi135 ceph-mon[221113]: pgmap v1679: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:36.417 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:36 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/3925660228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:24:36.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:36 smithi135 ceph-mon[213871]: pgmap v1679: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:36.417 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:36 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/3925660228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:24:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:36 smithi144 ceph-mon[194833]: pgmap v1679: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:36.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:36 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/3925660228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:24:37.471 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:37 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:37.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:37 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:37.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:37 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:37.698 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757793 got 163208757795 for osd.5 2023-09-23T20:24:37.698 DEBUG:teuthology.parallel:result is None 2023-09-23T20:24:37.804 INFO:teuthology.orchestra.run.smithi135.stdout:128849019432 2023-09-23T20:24:38.033 INFO:teuthology.orchestra.run.smithi135.stdout:403726926258 2023-09-23T20:24:38.378 INFO:teuthology.orchestra.run.smithi135.stdout:2366526980120 2023-09-23T20:24:38.403 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:38 smithi135 ceph-mon[221113]: pgmap v1680: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:38.403 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1701445226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:24:38.403 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:38 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1089289724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:24:38.403 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:38 smithi135 ceph-mon[213871]: pgmap v1680: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:38.404 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1701445226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:24:38.404 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:38 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1089289724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:24:38.629 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435668 got 468151435670 for osd.3 2023-09-23T20:24:38.629 DEBUG:teuthology.parallel:result is None 2023-09-23T20:24:38.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:38 smithi144 ceph-mon[194833]: pgmap v1680: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1701445226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:24:38.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:38 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1089289724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:24:39.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:39 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/567883944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:24:39.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:39 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/567883944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:24:39.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:39 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/567883944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:24:40.350 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2366526980118 got 2366526980120 for osd.6 2023-09-23T20:24:40.350 DEBUG:teuthology.parallel:result is None 2023-09-23T20:24:40.538 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:40 smithi135 ceph-mon[221113]: pgmap v1681: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:40.539 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:40 smithi135 ceph-mon[213871]: pgmap v1681: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:40.559 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926256 got 403726926258 for osd.0 2023-09-23T20:24:40.560 DEBUG:teuthology.parallel:result is None 2023-09-23T20:24:40.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:40 smithi144 ceph-mon[194833]: pgmap v1681: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:41.541 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019430 got 128849019432 for osd.4 2023-09-23T20:24:41.541 DEBUG:teuthology.parallel:result is None 2023-09-23T20:24:41.994 INFO:teuthology.orchestra.run.smithi135.stdout:2332167241758 2023-09-23T20:24:42.306 INFO:teuthology.orchestra.run.smithi135.stdout:425201762722 2023-09-23T20:24:42.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:42 smithi144 ceph-mon[194833]: pgmap v1682: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/4097128183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:24:42.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:42 smithi144 ceph-mon[194833]: from='client.? 172.21.15.135:0/1466815314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:24:42.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:42 smithi135 ceph-mon[221113]: pgmap v1682: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/4097128183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:24:42.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:42 smithi135 ceph-mon[221113]: from='client.? 172.21.15.135:0/1466815314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:24:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:42 smithi135 ceph-mon[213871]: pgmap v1682: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/4097128183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:24:42.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:42 smithi135 ceph-mon[213871]: from='client.? 172.21.15.135:0/1466815314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:24:42.878 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2332167241755 got 2332167241758 for osd.2 2023-09-23T20:24:42.878 DEBUG:teuthology.parallel:result is None 2023-09-23T20:24:43.647 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762719 got 425201762722 for osd.1 2023-09-23T20:24:43.647 DEBUG:teuthology.parallel:result is None 2023-09-23T20:24:43.647 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:24:43.647 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:24:43.882 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 20:24:43 smithi144 conmon[173765]: logger=cleanup t=2023-09-23T20:24:43.473669113Z level=info msg="Completed cleanup jobs" duration=45.730711ms 2023-09-23T20:24:43.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:43 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:24:43] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.43.0" 2023-09-23T20:24:43.992 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:44.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:44.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:44.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:44 smithi144 ceph-mon[194833]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:44.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:24:44.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:24:44.382 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:44 smithi144 ceph-mon[194833]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:44.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:44.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:44.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[221113]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:44.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:24:44.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:24:44.411 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[221113]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:44.411 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:44.411 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:44.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[213871]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:44.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:24:44.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:24:44.412 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:44 smithi135 ceph-mon[213871]: from='mgr.25859 172.21.15.135:0/653433316' entity='mgr.y' 2023-09-23T20:24:45.805 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:24:45.807 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:24:46.545 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1684,"stamp":"2023-09-23T20:24:45.317660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11978,"num_read_kb":26422,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":8998956,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7000,"ondisk_log_size":7000,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4981180,"kb_used_data":80252,"kb_used_omap":46,"kb_used_meta":4900817,"kb_avail":744980036,"statfs":{"total":767960285184,"available":762859556864,"internally_reserved":0,"allocated":82178048,"data_stored":64601230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47576,"internal_metadata":5018437160},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003076"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":1008,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125073+0000","last_change":"2023-09-23T20:22:53.431768+0000","last_active":"2023-09-23T20:24:08.125073+0000","last_peered":"2023-09-23T20:24:08.125073+0000","last_clean":"2023-09-23T20:24:08.125073+0000","last_became_active":"2023-09-23T20:22:53.431378+0000","last_became_peered":"2023-09-23T20:22:53.431378+0000","last_unstale":"2023-09-23T20:24:08.125073+0000","last_undegraded":"2023-09-23T20:24:08.125073+0000","last_fullsized":"2023-09-23T20:24:08.125073+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:54.928103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:54.928103+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:57.671137+0000","scrub_duration":0.011135947,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":444,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134638+0000","last_change":"2023-09-23T20:23:43.209456+0000","last_active":"2023-09-23T20:24:08.134638+0000","last_peered":"2023-09-23T20:24:08.134638+0000","last_clean":"2023-09-23T20:24:08.134638+0000","last_became_active":"2023-09-23T20:23:43.208165+0000","last_became_peered":"2023-09-23T20:23:43.208165+0000","last_unstale":"2023-09-23T20:24:08.134638+0000","last_undegraded":"2023-09-23T20:24:08.134638+0000","last_fullsized":"2023-09-23T20:24:08.134638+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:59.585179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":25,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.131381+0000","last_change":"2023-09-23T20:23:40.194024+0000","last_active":"2023-09-23T20:24:08.131381+0000","last_peered":"2023-09-23T20:24:08.131381+0000","last_clean":"2023-09-23T20:24:08.131381+0000","last_became_active":"2023-09-23T20:23:40.193564+0000","last_became_peered":"2023-09-23T20:23:40.193564+0000","last_unstale":"2023-09-23T20:24:08.131381+0000","last_undegraded":"2023-09-23T20:24:08.131381+0000","last_fullsized":"2023-09-23T20:24:08.131381+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:41.394399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":633,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125291+0000","last_change":"2023-09-23T20:22:55.256917+0000","last_active":"2023-09-23T20:24:08.125291+0000","last_peered":"2023-09-23T20:24:08.125291+0000","last_clean":"2023-09-23T20:24:08.125291+0000","last_became_active":"2023-09-23T20:22:53.428227+0000","last_became_peered":"2023-09-23T20:22:53.428227+0000","last_unstale":"2023-09-23T20:24:08.125291+0000","last_undegraded":"2023-09-23T20:24:08.125291+0000","last_fullsized":"2023-09-23T20:24:08.125291+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:55.256829+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:21:21.987455+0000","last_clean_scrub_stamp":"2023-09-23T20:22:55.256829+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:14.034424+0000","scrub_duration":0.0021020539999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":119,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125006+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:24:08.125006+0000","last_peered":"2023-09-23T20:24:08.125006+0000","last_clean":"2023-09-23T20:24:08.125006+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:24:08.125006+0000","last_undegraded":"2023-09-23T20:24:08.125006+0000","last_fullsized":"2023-09-23T20:24:08.125006+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":235,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133238+0000","last_change":"2023-09-23T20:23:23.492821+0000","last_active":"2023-09-23T20:24:08.133238+0000","last_peered":"2023-09-23T20:24:08.133238+0000","last_clean":"2023-09-23T20:24:08.133238+0000","last_became_active":"2023-09-23T20:23:23.492583+0000","last_became_peered":"2023-09-23T20:23:23.492583+0000","last_unstale":"2023-09-23T20:24:08.133238+0000","last_undegraded":"2023-09-23T20:24:08.133238+0000","last_fullsized":"2023-09-23T20:24:08.133238+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:59.293610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":515,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134365+0000","last_change":"2023-09-23T20:23:44.174294+0000","last_active":"2023-09-23T20:24:08.134365+0000","last_peered":"2023-09-23T20:24:08.134365+0000","last_clean":"2023-09-23T20:24:08.134365+0000","last_became_active":"2023-09-23T20:23:43.211607+0000","last_became_peered":"2023-09-23T20:23:43.211607+0000","last_unstale":"2023-09-23T20:24:08.134365+0000","last_undegraded":"2023-09-23T20:24:08.134365+0000","last_fullsized":"2023-09-23T20:24:08.134365+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:44.174207+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:23:44.174207+0000","last_clean_scrub_stamp":"2023-09-23T20:23:44.174207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:41.963295+0000","scrub_duration":0.0017378339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":200,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123673+0000","last_change":"2023-09-23T20:23:36.167494+0000","last_active":"2023-09-23T20:24:08.123673+0000","last_peered":"2023-09-23T20:24:08.123673+0000","last_clean":"2023-09-23T20:24:08.123673+0000","last_became_active":"2023-09-23T20:23:36.167166+0000","last_became_peered":"2023-09-23T20:23:36.167166+0000","last_unstale":"2023-09-23T20:24:08.123673+0000","last_undegraded":"2023-09-23T20:24:08.123673+0000","last_fullsized":"2023-09-23T20:24:08.123673+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:32.918587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134618+0000","last_change":"2023-09-23T20:24:06.406026+0000","last_active":"2023-09-23T20:24:08.134618+0000","last_peered":"2023-09-23T20:24:08.134618+0000","last_clean":"2023-09-23T20:24:08.134618+0000","last_became_active":"2023-09-23T20:24:06.405646+0000","last_became_peered":"2023-09-23T20:24:06.405646+0000","last_unstale":"2023-09-23T20:24:08.134618+0000","last_undegraded":"2023-09-23T20:24:08.134618+0000","last_fullsized":"2023-09-23T20:24:08.134618+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:58.670587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":876,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134240+0000","last_change":"2023-09-23T20:22:53.431778+0000","last_active":"2023-09-23T20:24:08.134240+0000","last_peered":"2023-09-23T20:24:08.134240+0000","last_clean":"2023-09-23T20:24:08.134240+0000","last_became_active":"2023-09-23T20:22:53.431517+0000","last_became_peered":"2023-09-23T20:22:53.431517+0000","last_unstale":"2023-09-23T20:24:08.134240+0000","last_undegraded":"2023-09-23T20:24:08.134240+0000","last_fullsized":"2023-09-23T20:24:08.134240+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:25.555266+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:22:25.555266+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:21.690091+0000","scrub_duration":0.001735737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":224,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133052+0000","last_change":"2023-09-23T20:23:23.492260+0000","last_active":"2023-09-23T20:24:08.133052+0000","last_peered":"2023-09-23T20:24:08.133052+0000","last_clean":"2023-09-23T20:24:08.133052+0000","last_became_active":"2023-09-23T20:23:23.491987+0000","last_became_peered":"2023-09-23T20:23:23.491987+0000","last_unstale":"2023-09-23T20:24:08.133052+0000","last_undegraded":"2023-09-23T20:24:08.133052+0000","last_fullsized":"2023-09-23T20:24:08.133052+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:13.763941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":472,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123734+0000","last_change":"2023-09-23T20:24:06.403062+0000","last_active":"2023-09-23T20:24:08.123734+0000","last_peered":"2023-09-23T20:24:08.123734+0000","last_clean":"2023-09-23T20:24:08.123734+0000","last_became_active":"2023-09-23T20:24:06.402772+0000","last_became_peered":"2023-09-23T20:24:06.402772+0000","last_unstale":"2023-09-23T20:24:08.123734+0000","last_undegraded":"2023-09-23T20:24:08.123734+0000","last_fullsized":"2023-09-23T20:24:08.123734+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:56.781582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":907,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.131781+0000","last_change":"2023-09-23T20:22:24.576681+0000","last_active":"2023-09-23T20:24:09.131781+0000","last_peered":"2023-09-23T20:24:09.131781+0000","last_clean":"2023-09-23T20:24:09.131781+0000","last_became_active":"2023-09-23T20:22:22.381258+0000","last_became_peered":"2023-09-23T20:22:22.381258+0000","last_unstale":"2023-09-23T20:24:09.131781+0000","last_undegraded":"2023-09-23T20:24:09.131781+0000","last_fullsized":"2023-09-23T20:24:09.131781+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:22:24.576567+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:22:24.576567+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:58.491034+0000","scrub_duration":0.0019668609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":616,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127459+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:24:08.127459+0000","last_peered":"2023-09-23T20:24:08.127459+0000","last_clean":"2023-09-23T20:24:08.127459+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:24:08.127459+0000","last_undegraded":"2023-09-23T20:24:08.127459+0000","last_fullsized":"2023-09-23T20:24:08.127459+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":473,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124050+0000","last_change":"2023-09-23T20:24:06.412627+0000","last_active":"2023-09-23T20:24:08.124050+0000","last_peered":"2023-09-23T20:24:08.124050+0000","last_clean":"2023-09-23T20:24:08.124050+0000","last_became_active":"2023-09-23T20:24:06.412285+0000","last_became_peered":"2023-09-23T20:24:06.412285+0000","last_unstale":"2023-09-23T20:24:08.124050+0000","last_undegraded":"2023-09-23T20:24:08.124050+0000","last_fullsized":"2023-09-23T20:24:08.124050+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:56.903892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":537,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.141725+0000","last_change":"2023-09-23T20:24:09.141725+0000","last_active":"2023-09-23T20:24:09.141725+0000","last_peered":"2023-09-23T20:24:09.141725+0000","last_clean":"2023-09-23T20:24:09.141725+0000","last_became_active":"2023-09-23T20:24:09.140313+0000","last_became_peered":"2023-09-23T20:24:09.140313+0000","last_unstale":"2023-09-23T20:24:09.141725+0000","last_undegraded":"2023-09-23T20:24:09.141725+0000","last_fullsized":"2023-09-23T20:24:09.141725+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:57.231266+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:22:57.231266+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:23.246244+0000","scrub_duration":0.0013866460000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":290,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134680+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:24:08.134680+0000","last_peered":"2023-09-23T20:24:08.134680+0000","last_clean":"2023-09-23T20:24:08.134680+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:24:08.134680+0000","last_undegraded":"2023-09-23T20:24:08.134680+0000","last_fullsized":"2023-09-23T20:24:08.134680+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":143,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.126869+0000","last_change":"2023-09-23T20:24:06.407302+0000","last_active":"2023-09-23T20:24:08.126869+0000","last_peered":"2023-09-23T20:24:08.126869+0000","last_clean":"2023-09-23T20:24:08.126869+0000","last_became_active":"2023-09-23T20:24:06.406954+0000","last_became_peered":"2023-09-23T20:24:06.406954+0000","last_unstale":"2023-09-23T20:24:08.126869+0000","last_undegraded":"2023-09-23T20:24:08.126869+0000","last_fullsized":"2023-09-23T20:24:08.126869+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:23.009091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":812,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127162+0000","last_change":"2023-09-23T20:22:53.424578+0000","last_active":"2023-09-23T20:24:08.127162+0000","last_peered":"2023-09-23T20:24:08.127162+0000","last_clean":"2023-09-23T20:24:08.127162+0000","last_became_active":"2023-09-23T20:22:53.424250+0000","last_became_peered":"2023-09-23T20:22:53.424250+0000","last_unstale":"2023-09-23T20:24:08.127162+0000","last_undegraded":"2023-09-23T20:24:08.127162+0000","last_fullsized":"2023-09-23T20:24:08.127162+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:23.507023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:23.507023+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:09.885578+0000","scrub_duration":0.0014042600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":520,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.137589+0000","last_change":"2023-09-23T20:24:06.416788+0000","last_active":"2023-09-23T20:24:08.137589+0000","last_peered":"2023-09-23T20:24:08.137589+0000","last_clean":"2023-09-23T20:24:08.137589+0000","last_became_active":"2023-09-23T20:24:06.406663+0000","last_became_peered":"2023-09-23T20:24:06.406663+0000","last_unstale":"2023-09-23T20:24:08.137589+0000","last_undegraded":"2023-09-23T20:24:08.137589+0000","last_fullsized":"2023-09-23T20:24:08.137589+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:35.670573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":442,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132546+0000","last_change":"2023-09-23T20:23:43.210782+0000","last_active":"2023-09-23T20:24:08.132546+0000","last_peered":"2023-09-23T20:24:08.132546+0000","last_clean":"2023-09-23T20:24:08.132546+0000","last_became_active":"2023-09-23T20:23:43.210411+0000","last_became_peered":"2023-09-23T20:23:43.210411+0000","last_unstale":"2023-09-23T20:24:08.132546+0000","last_undegraded":"2023-09-23T20:24:08.132546+0000","last_fullsized":"2023-09-23T20:24:08.132546+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:09.197771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":465,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132234+0000","last_change":"2023-09-23T20:23:37.720659+0000","last_active":"2023-09-23T20:24:08.132234+0000","last_peered":"2023-09-23T20:24:08.132234+0000","last_clean":"2023-09-23T20:24:08.132234+0000","last_became_active":"2023-09-23T20:23:23.491730+0000","last_became_peered":"2023-09-23T20:23:23.491730+0000","last_unstale":"2023-09-23T20:24:08.132234+0000","last_undegraded":"2023-09-23T20:24:08.132234+0000","last_fullsized":"2023-09-23T20:24:08.132234+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:37.720558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:23:37.720558+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:28.730427+0000","scrub_duration":0.0018684839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":307,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134199+0000","last_change":"2023-09-23T20:22:22.384480+0000","last_active":"2023-09-23T20:24:08.134199+0000","last_peered":"2023-09-23T20:24:08.134199+0000","last_clean":"2023-09-23T20:24:08.134199+0000","last_became_active":"2023-09-23T20:22:22.384129+0000","last_became_peered":"2023-09-23T20:22:22.384129+0000","last_unstale":"2023-09-23T20:24:08.134199+0000","last_undegraded":"2023-09-23T20:24:08.134199+0000","last_fullsized":"2023-09-23T20:24:08.134199+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:15.885311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":218,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124932+0000","last_change":"2023-09-23T20:23:36.168359+0000","last_active":"2023-09-23T20:24:08.124932+0000","last_peered":"2023-09-23T20:24:08.124932+0000","last_clean":"2023-09-23T20:24:08.124932+0000","last_became_active":"2023-09-23T20:23:36.168024+0000","last_became_peered":"2023-09-23T20:23:36.168024+0000","last_unstale":"2023-09-23T20:24:08.124932+0000","last_undegraded":"2023-09-23T20:24:08.124932+0000","last_fullsized":"2023-09-23T20:24:08.124932+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:03.468455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":665,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.141354+0000","last_change":"2023-09-23T20:24:09.141354+0000","last_active":"2023-09-23T20:24:09.141354+0000","last_peered":"2023-09-23T20:24:09.141354+0000","last_clean":"2023-09-23T20:24:09.141354+0000","last_became_active":"2023-09-23T20:24:09.140356+0000","last_became_peered":"2023-09-23T20:24:09.140356+0000","last_unstale":"2023-09-23T20:24:09.141354+0000","last_undegraded":"2023-09-23T20:24:09.141354+0000","last_fullsized":"2023-09-23T20:24:09.141354+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:15.086008+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":281,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.130598+0000","last_change":"2023-09-23T20:23:23.491186+0000","last_active":"2023-09-23T20:24:08.130598+0000","last_peered":"2023-09-23T20:24:08.130598+0000","last_clean":"2023-09-23T20:24:08.130598+0000","last_became_active":"2023-09-23T20:23:23.490877+0000","last_became_peered":"2023-09-23T20:23:23.490877+0000","last_unstale":"2023-09-23T20:24:08.130598+0000","last_undegraded":"2023-09-23T20:24:08.130598+0000","last_fullsized":"2023-09-23T20:24:08.130598+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:36.133469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":595,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127755+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:24:08.127755+0000","last_peered":"2023-09-23T20:24:08.127755+0000","last_clean":"2023-09-23T20:24:08.127755+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:24:08.127755+0000","last_undegraded":"2023-09-23T20:24:08.127755+0000","last_fullsized":"2023-09-23T20:24:08.127755+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":668,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127305+0000","last_change":"2023-09-23T20:21:11.930313+0000","last_active":"2023-09-23T20:24:08.127305+0000","last_peered":"2023-09-23T20:24:08.127305+0000","last_clean":"2023-09-23T20:24:08.127305+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:24:08.127305+0000","last_undegraded":"2023-09-23T20:24:08.127305+0000","last_fullsized":"2023-09-23T20:24:08.127305+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:11.930203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.930203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:00.534581+0000","scrub_duration":0.0016982569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":311,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133942+0000","last_change":"2023-09-23T20:22:44.517176+0000","last_active":"2023-09-23T20:24:08.133942+0000","last_peered":"2023-09-23T20:24:08.133942+0000","last_clean":"2023-09-23T20:24:08.133942+0000","last_became_active":"2023-09-23T20:22:44.516833+0000","last_became_peered":"2023-09-23T20:22:44.516833+0000","last_unstale":"2023-09-23T20:24:08.133942+0000","last_undegraded":"2023-09-23T20:24:08.133942+0000","last_fullsized":"2023-09-23T20:24:08.133942+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:43.417704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":331,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133306+0000","last_change":"2023-09-23T20:23:23.492872+0000","last_active":"2023-09-23T20:24:08.133306+0000","last_peered":"2023-09-23T20:24:08.133306+0000","last_clean":"2023-09-23T20:24:08.133306+0000","last_became_active":"2023-09-23T20:23:23.492570+0000","last_became_peered":"2023-09-23T20:23:23.492570+0000","last_unstale":"2023-09-23T20:24:08.133306+0000","last_undegraded":"2023-09-23T20:24:08.133306+0000","last_fullsized":"2023-09-23T20:24:08.133306+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:43.291046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":774,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.142582+0000","last_change":"2023-09-23T20:24:09.142582+0000","last_active":"2023-09-23T20:24:09.142582+0000","last_peered":"2023-09-23T20:24:09.142582+0000","last_clean":"2023-09-23T20:24:09.142582+0000","last_became_active":"2023-09-23T20:24:09.142201+0000","last_became_peered":"2023-09-23T20:24:09.142201+0000","last_unstale":"2023-09-23T20:24:09.142582+0000","last_undegraded":"2023-09-23T20:24:09.142582+0000","last_fullsized":"2023-09-23T20:24:09.142582+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.591311+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":23,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.131498+0000","last_change":"2023-09-23T20:24:06.429779+0000","last_active":"2023-09-23T20:24:08.131498+0000","last_peered":"2023-09-23T20:24:08.131498+0000","last_clean":"2023-09-23T20:24:08.131498+0000","last_became_active":"2023-09-23T20:24:06.407037+0000","last_became_peered":"2023-09-23T20:24:06.407037+0000","last_unstale":"2023-09-23T20:24:08.131498+0000","last_undegraded":"2023-09-23T20:24:08.131498+0000","last_fullsized":"2023-09-23T20:24:08.131498+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:41.192972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124375+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:24:08.124375+0000","last_peered":"2023-09-23T20:24:08.124375+0000","last_clean":"2023-09-23T20:24:08.124375+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:24:08.124375+0000","last_undegraded":"2023-09-23T20:24:08.124375+0000","last_fullsized":"2023-09-23T20:24:08.124375+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":792,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.131254+0000","last_change":"2023-09-23T20:23:43.528783+0000","last_active":"2023-09-23T20:24:08.131254+0000","last_peered":"2023-09-23T20:24:08.131254+0000","last_clean":"2023-09-23T20:24:08.131254+0000","last_became_active":"2023-09-23T20:23:43.208308+0000","last_became_peered":"2023-09-23T20:23:43.208308+0000","last_unstale":"2023-09-23T20:24:08.131254+0000","last_undegraded":"2023-09-23T20:24:08.131254+0000","last_fullsized":"2023-09-23T20:24:08.131254+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:43.528685+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:23:43.528685+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:16.679023+0000","scrub_duration":0.00208479,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":288,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.136597+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:24:08.136597+0000","last_peered":"2023-09-23T20:24:08.136597+0000","last_clean":"2023-09-23T20:24:08.136597+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:24:08.136597+0000","last_undegraded":"2023-09-23T20:24:08.136597+0000","last_fullsized":"2023-09-23T20:24:08.136597+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":615,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125180+0000","last_change":"2023-09-23T20:24:06.413510+0000","last_active":"2023-09-23T20:24:08.125180+0000","last_peered":"2023-09-23T20:24:08.125180+0000","last_clean":"2023-09-23T20:24:08.125180+0000","last_became_active":"2023-09-23T20:24:06.413130+0000","last_became_peered":"2023-09-23T20:24:06.413130+0000","last_unstale":"2023-09-23T20:24:08.125180+0000","last_undegraded":"2023-09-23T20:24:08.125180+0000","last_fullsized":"2023-09-23T20:24:08.125180+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:40.601793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":997,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.138046+0000","last_change":"2023-09-23T20:24:09.138046+0000","last_active":"2023-09-23T20:24:09.138046+0000","last_peered":"2023-09-23T20:24:09.138046+0000","last_clean":"2023-09-23T20:24:09.138046+0000","last_became_active":"2023-09-23T20:24:09.137455+0000","last_became_peered":"2023-09-23T20:24:09.137455+0000","last_unstale":"2023-09-23T20:24:09.138046+0000","last_undegraded":"2023-09-23T20:24:09.138046+0000","last_fullsized":"2023-09-23T20:24:09.138046+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:21:11.597355+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.597355+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:39.872420+0000","scrub_duration":0.0021599240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":317,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134558+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:24:08.134558+0000","last_peered":"2023-09-23T20:24:08.134558+0000","last_clean":"2023-09-23T20:24:08.134558+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:24:08.134558+0000","last_undegraded":"2023-09-23T20:24:08.134558+0000","last_fullsized":"2023-09-23T20:24:08.134558+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":502,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135504+0000","last_change":"2023-09-23T20:23:43.207185+0000","last_active":"2023-09-23T20:24:08.135504+0000","last_peered":"2023-09-23T20:24:08.135504+0000","last_clean":"2023-09-23T20:24:08.135504+0000","last_became_active":"2023-09-23T20:23:43.206472+0000","last_became_peered":"2023-09-23T20:23:43.206472+0000","last_unstale":"2023-09-23T20:24:08.135504+0000","last_undegraded":"2023-09-23T20:24:08.135504+0000","last_fullsized":"2023-09-23T20:24:08.135504+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:23.396782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":695,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:10.182647+0000","last_change":"2023-09-23T20:24:10.182603+0000","last_active":"2023-09-23T20:24:10.182647+0000","last_peered":"2023-09-23T20:24:10.182647+0000","last_clean":"2023-09-23T20:24:10.182647+0000","last_became_active":"2023-09-23T20:24:09.141983+0000","last_became_peered":"2023-09-23T20:24:09.141983+0000","last_unstale":"2023-09-23T20:24:10.182647+0000","last_undegraded":"2023-09-23T20:24:10.182647+0000","last_fullsized":"2023-09-23T20:24:10.182647+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:24:10.182525+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:24:10.182525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.186914+0000","scrub_duration":0.0019599510000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":319,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134482+0000","last_change":"2023-09-23T20:22:44.388257+0000","last_active":"2023-09-23T20:24:08.134482+0000","last_peered":"2023-09-23T20:24:08.134482+0000","last_clean":"2023-09-23T20:24:08.134482+0000","last_became_active":"2023-09-23T20:22:44.387884+0000","last_became_peered":"2023-09-23T20:22:44.387884+0000","last_unstale":"2023-09-23T20:24:08.134482+0000","last_undegraded":"2023-09-23T20:24:08.134482+0000","last_fullsized":"2023-09-23T20:24:08.134482+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:26.931322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.137194+0000","last_change":"2023-09-23T20:24:06.408200+0000","last_active":"2023-09-23T20:24:08.137194+0000","last_peered":"2023-09-23T20:24:08.137194+0000","last_clean":"2023-09-23T20:24:08.137194+0000","last_became_active":"2023-09-23T20:24:06.407859+0000","last_became_peered":"2023-09-23T20:24:06.407859+0000","last_unstale":"2023-09-23T20:24:08.137194+0000","last_undegraded":"2023-09-23T20:24:08.137194+0000","last_fullsized":"2023-09-23T20:24:08.137194+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:18.066383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":684,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.136800+0000","last_change":"2023-09-23T20:23:46.210808+0000","last_active":"2023-09-23T20:24:08.136800+0000","last_peered":"2023-09-23T20:24:08.136800+0000","last_clean":"2023-09-23T20:24:08.136800+0000","last_became_active":"2023-09-23T20:23:43.212317+0000","last_became_peered":"2023-09-23T20:23:43.212317+0000","last_unstale":"2023-09-23T20:24:08.136800+0000","last_undegraded":"2023-09-23T20:24:08.136800+0000","last_fullsized":"2023-09-23T20:24:08.136800+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:46.210708+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:23:46.210708+0000","last_clean_scrub_stamp":"2023-09-23T20:23:46.210708+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:24.395204+0000","scrub_duration":0.001786686,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":665,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127083+0000","last_change":"2023-09-23T20:24:06.412761+0000","last_active":"2023-09-23T20:24:08.127083+0000","last_peered":"2023-09-23T20:24:08.127083+0000","last_clean":"2023-09-23T20:24:08.127083+0000","last_became_active":"2023-09-23T20:24:06.412382+0000","last_became_peered":"2023-09-23T20:24:06.412382+0000","last_unstale":"2023-09-23T20:24:08.127083+0000","last_undegraded":"2023-09-23T20:24:08.127083+0000","last_fullsized":"2023-09-23T20:24:08.127083+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:38.504383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":265,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.129865+0000","last_change":"2023-09-23T20:23:23.495138+0000","last_active":"2023-09-23T20:24:08.129865+0000","last_peered":"2023-09-23T20:24:08.129865+0000","last_clean":"2023-09-23T20:24:08.129865+0000","last_became_active":"2023-09-23T20:23:23.494769+0000","last_became_peered":"2023-09-23T20:23:23.494769+0000","last_unstale":"2023-09-23T20:24:08.129865+0000","last_undegraded":"2023-09-23T20:24:08.129865+0000","last_fullsized":"2023-09-23T20:24:08.129865+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:18.221526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1016,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133845+0000","last_change":"2023-09-23T20:22:22.386488+0000","last_active":"2023-09-23T20:24:08.133845+0000","last_peered":"2023-09-23T20:24:08.133845+0000","last_clean":"2023-09-23T20:24:08.133845+0000","last_became_active":"2023-09-23T20:22:22.386137+0000","last_became_peered":"2023-09-23T20:22:22.386137+0000","last_unstale":"2023-09-23T20:24:08.133845+0000","last_undegraded":"2023-09-23T20:24:08.133845+0000","last_fullsized":"2023-09-23T20:24:08.133845+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:47.181912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:47.181912+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:26.092523+0000","scrub_duration":0.0074047540000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":349,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123895+0000","last_change":"2023-09-23T20:23:36.165051+0000","last_active":"2023-09-23T20:24:08.123895+0000","last_peered":"2023-09-23T20:24:08.123895+0000","last_clean":"2023-09-23T20:24:08.123895+0000","last_became_active":"2023-09-23T20:23:36.164701+0000","last_became_peered":"2023-09-23T20:23:36.164701+0000","last_unstale":"2023-09-23T20:24:08.123895+0000","last_undegraded":"2023-09-23T20:24:08.123895+0000","last_fullsized":"2023-09-23T20:24:08.123895+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.493418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":482,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.137677+0000","last_change":"2023-09-23T20:24:06.407059+0000","last_active":"2023-09-23T20:24:08.137677+0000","last_peered":"2023-09-23T20:24:08.137677+0000","last_clean":"2023-09-23T20:24:08.137677+0000","last_became_active":"2023-09-23T20:24:06.406263+0000","last_became_peered":"2023-09-23T20:24:06.406263+0000","last_unstale":"2023-09-23T20:24:08.137677+0000","last_undegraded":"2023-09-23T20:24:08.137677+0000","last_fullsized":"2023-09-23T20:24:08.137677+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:29.763801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":987,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125213+0000","last_change":"2023-09-23T20:22:22.377942+0000","last_active":"2023-09-23T20:24:08.125213+0000","last_peered":"2023-09-23T20:24:08.125213+0000","last_clean":"2023-09-23T20:24:08.125213+0000","last_became_active":"2023-09-23T20:22:22.377619+0000","last_became_peered":"2023-09-23T20:22:22.377619+0000","last_unstale":"2023-09-23T20:24:08.125213+0000","last_undegraded":"2023-09-23T20:24:08.125213+0000","last_fullsized":"2023-09-23T20:24:08.125213+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:43.954204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:43.954204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:00.412768+0000","scrub_duration":0.0019293979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.137137+0000","last_change":"2023-09-23T20:24:06.410227+0000","last_active":"2023-09-23T20:24:08.137137+0000","last_peered":"2023-09-23T20:24:08.137137+0000","last_clean":"2023-09-23T20:24:08.137137+0000","last_became_active":"2023-09-23T20:24:06.409886+0000","last_became_peered":"2023-09-23T20:24:06.409886+0000","last_unstale":"2023-09-23T20:24:08.137137+0000","last_undegraded":"2023-09-23T20:24:08.137137+0000","last_fullsized":"2023-09-23T20:24:08.137137+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:52.648626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":475,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133983+0000","last_change":"2023-09-23T20:23:43.335408+0000","last_active":"2023-09-23T20:24:08.133983+0000","last_peered":"2023-09-23T20:24:08.133983+0000","last_clean":"2023-09-23T20:24:08.133983+0000","last_became_active":"2023-09-23T20:23:43.335063+0000","last_became_peered":"2023-09-23T20:23:43.335063+0000","last_unstale":"2023-09-23T20:24:08.133983+0000","last_undegraded":"2023-09-23T20:24:08.133983+0000","last_fullsized":"2023-09-23T20:24:08.133983+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:19.424719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":674,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.126696+0000","last_change":"2023-09-23T20:21:20.952512+0000","last_active":"2023-09-23T20:24:09.126696+0000","last_peered":"2023-09-23T20:24:09.126696+0000","last_clean":"2023-09-23T20:24:09.126696+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:24:09.126696+0000","last_undegraded":"2023-09-23T20:24:09.126696+0000","last_fullsized":"2023-09-23T20:24:09.126696+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:20.952352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:20.952352+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:28.266826+0000","scrub_duration":0.00171708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":436,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135285+0000","last_change":"2023-09-23T20:23:43.211174+0000","last_active":"2023-09-23T20:24:08.135285+0000","last_peered":"2023-09-23T20:24:08.135285+0000","last_clean":"2023-09-23T20:24:08.135285+0000","last_became_active":"2023-09-23T20:23:43.210874+0000","last_became_peered":"2023-09-23T20:23:43.210874+0000","last_unstale":"2023-09-23T20:24:08.135285+0000","last_undegraded":"2023-09-23T20:24:08.135285+0000","last_fullsized":"2023-09-23T20:24:08.135285+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:26.252449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":482,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.136829+0000","last_change":"2023-09-23T20:24:06.398839+0000","last_active":"2023-09-23T20:24:08.136829+0000","last_peered":"2023-09-23T20:24:08.136829+0000","last_clean":"2023-09-23T20:24:08.136829+0000","last_became_active":"2023-09-23T20:24:06.398496+0000","last_became_peered":"2023-09-23T20:24:06.398496+0000","last_unstale":"2023-09-23T20:24:08.136829+0000","last_undegraded":"2023-09-23T20:24:08.136829+0000","last_fullsized":"2023-09-23T20:24:08.136829+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:12.143805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":1019,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.140789+0000","last_change":"2023-09-23T20:24:09.140789+0000","last_active":"2023-09-23T20:24:09.140789+0000","last_peered":"2023-09-23T20:24:09.140789+0000","last_clean":"2023-09-23T20:24:09.140789+0000","last_became_active":"2023-09-23T20:24:09.140417+0000","last_became_peered":"2023-09-23T20:24:09.140417+0000","last_unstale":"2023-09-23T20:24:09.140789+0000","last_undegraded":"2023-09-23T20:24:09.140789+0000","last_fullsized":"2023-09-23T20:24:09.140789+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:12.314623+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T20:21:12.314623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:39.801719+0000","scrub_duration":0.001743334,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":251,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.125004+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:24:09.125004+0000","last_peered":"2023-09-23T20:24:09.125004+0000","last_clean":"2023-09-23T20:24:09.125004+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:24:09.125004+0000","last_undegraded":"2023-09-23T20:24:09.125004+0000","last_fullsized":"2023-09-23T20:24:09.125004+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":120,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.126024+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:24:09.126024+0000","last_peered":"2023-09-23T20:24:09.126024+0000","last_clean":"2023-09-23T20:24:09.126024+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:24:09.126024+0000","last_undegraded":"2023-09-23T20:24:09.126024+0000","last_fullsized":"2023-09-23T20:24:09.126024+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":913,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.140533+0000","last_change":"2023-09-23T20:24:09.140533+0000","last_active":"2023-09-23T20:24:09.140533+0000","last_peered":"2023-09-23T20:24:09.140533+0000","last_clean":"2023-09-23T20:24:09.140533+0000","last_became_active":"2023-09-23T20:24:09.139750+0000","last_became_peered":"2023-09-23T20:24:09.139750+0000","last_unstale":"2023-09-23T20:24:09.140533+0000","last_undegraded":"2023-09-23T20:24:09.140533+0000","last_fullsized":"2023-09-23T20:24:09.140533+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:40.219310+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:40.219310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:53.078388+0000","scrub_duration":0.011178762,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"568'268","reported_seq":1164,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.127891+0000","last_change":"2023-09-23T20:23:01.133380+0000","last_active":"2023-09-23T20:24:09.127891+0000","last_peered":"2023-09-23T20:24:09.127891+0000","last_clean":"2023-09-23T20:24:09.127891+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:24:09.127891+0000","last_undegraded":"2023-09-23T20:24:09.127891+0000","last_fullsized":"2023-09-23T20:24:09.127891+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":268,"log_dups_size":0,"ondisk_log_size":268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034039399999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1578,"num_read_kb":2522,"num_write":266,"num_write_kb":5301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":614,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127661+0000","last_change":"2023-09-23T20:22:22.379055+0000","last_active":"2023-09-23T20:24:08.127661+0000","last_peered":"2023-09-23T20:24:08.127661+0000","last_clean":"2023-09-23T20:24:08.127661+0000","last_became_active":"2023-09-23T20:22:22.378812+0000","last_became_peered":"2023-09-23T20:22:22.378812+0000","last_unstale":"2023-09-23T20:24:08.127661+0000","last_undegraded":"2023-09-23T20:24:08.127661+0000","last_fullsized":"2023-09-23T20:24:08.127661+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:26.700741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.126823+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:24:08.126823+0000","last_peered":"2023-09-23T20:24:08.126823+0000","last_clean":"2023-09-23T20:24:08.126823+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:24:08.126823+0000","last_undegraded":"2023-09-23T20:24:08.126823+0000","last_fullsized":"2023-09-23T20:24:08.126823+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":957,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125154+0000","last_change":"2023-09-23T20:22:22.379736+0000","last_active":"2023-09-23T20:24:08.125154+0000","last_peered":"2023-09-23T20:24:08.125154+0000","last_clean":"2023-09-23T20:24:08.125154+0000","last_became_active":"2023-09-23T20:22:22.379411+0000","last_became_peered":"2023-09-23T20:22:22.379411+0000","last_unstale":"2023-09-23T20:24:08.125154+0000","last_undegraded":"2023-09-23T20:24:08.125154+0000","last_fullsized":"2023-09-23T20:24:08.125154+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:21.231359+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"568'188","reported_seq":2067,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.141410+0000","last_change":"2023-09-23T20:24:09.141410+0000","last_active":"2023-09-23T20:24:09.141410+0000","last_peered":"2023-09-23T20:24:09.141410+0000","last_clean":"2023-09-23T20:24:09.141410+0000","last_became_active":"2023-09-23T20:24:09.140746+0000","last_became_peered":"2023-09-23T20:24:09.140746+0000","last_unstale":"2023-09-23T20:24:09.141410+0000","last_undegraded":"2023-09-23T20:24:09.141410+0000","last_fullsized":"2023-09-23T20:24:09.141410+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:30.688454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001683302,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2025,"num_read_kb":4391,"num_write":141,"num_write_kb":2375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":114,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":475,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132826+0000","last_change":"2023-09-23T20:24:06.408577+0000","last_active":"2023-09-23T20:24:08.132826+0000","last_peered":"2023-09-23T20:24:08.132826+0000","last_clean":"2023-09-23T20:24:08.132826+0000","last_became_active":"2023-09-23T20:24:06.408244+0000","last_became_peered":"2023-09-23T20:24:06.408244+0000","last_unstale":"2023-09-23T20:24:08.132826+0000","last_undegraded":"2023-09-23T20:24:08.132826+0000","last_fullsized":"2023-09-23T20:24:08.132826+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:56.941793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":135,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127422+0000","last_change":"2023-09-23T20:22:22.382763+0000","last_active":"2023-09-23T20:24:08.127422+0000","last_peered":"2023-09-23T20:24:08.127422+0000","last_clean":"2023-09-23T20:24:08.127422+0000","last_became_active":"2023-09-23T20:22:22.381690+0000","last_became_peered":"2023-09-23T20:22:22.381690+0000","last_unstale":"2023-09-23T20:24:08.127422+0000","last_undegraded":"2023-09-23T20:24:08.127422+0000","last_fullsized":"2023-09-23T20:24:08.127422+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:51.102791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":835,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135547+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:24:08.135547+0000","last_peered":"2023-09-23T20:24:08.135547+0000","last_clean":"2023-09-23T20:24:08.135547+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:24:08.135547+0000","last_undegraded":"2023-09-23T20:24:08.135547+0000","last_fullsized":"2023-09-23T20:24:08.135547+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"568'210","reported_seq":1592,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.131843+0000","last_change":"2023-09-23T20:23:01.134345+0000","last_active":"2023-09-23T20:24:09.131843+0000","last_peered":"2023-09-23T20:24:09.131843+0000","last_clean":"2023-09-23T20:24:09.131843+0000","last_became_active":"2023-09-23T20:22:22.387896+0000","last_became_peered":"2023-09-23T20:22:22.387896+0000","last_unstale":"2023-09-23T20:24:09.131843+0000","last_undegraded":"2023-09-23T20:24:09.131843+0000","last_fullsized":"2023-09-23T20:24:09.131843+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":210,"log_dups_size":0,"ondisk_log_size":210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:11.339762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000263816,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1470,"num_read_kb":4956,"num_write":177,"num_write_kb":4597,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.126298+0000","last_change":"2023-09-23T20:24:06.406921+0000","last_active":"2023-09-23T20:24:08.126298+0000","last_peered":"2023-09-23T20:24:08.126298+0000","last_clean":"2023-09-23T20:24:08.126298+0000","last_became_active":"2023-09-23T20:24:06.406593+0000","last_became_peered":"2023-09-23T20:24:06.406593+0000","last_unstale":"2023-09-23T20:24:08.126298+0000","last_undegraded":"2023-09-23T20:24:08.126298+0000","last_fullsized":"2023-09-23T20:24:08.126298+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:15.895040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":120,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.126103+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:24:09.126103+0000","last_peered":"2023-09-23T20:24:09.126103+0000","last_clean":"2023-09-23T20:24:09.126103+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:24:09.126103+0000","last_undegraded":"2023-09-23T20:24:09.126103+0000","last_fullsized":"2023-09-23T20:24:09.126103+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"568'393","reported_seq":2007,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125394+0000","last_change":"2023-09-23T20:23:01.130145+0000","last_active":"2023-09-23T20:24:08.125394+0000","last_peered":"2023-09-23T20:24:08.125394+0000","last_clean":"2023-09-23T20:24:08.125394+0000","last_became_active":"2023-09-23T20:22:22.379019+0000","last_became_peered":"2023-09-23T20:22:22.379019+0000","last_unstale":"2023-09-23T20:24:08.125394+0000","last_undegraded":"2023-09-23T20:24:08.125394+0000","last_fullsized":"2023-09-23T20:24:08.125394+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":393,"log_dups_size":0,"ondisk_log_size":393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:35.091435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000589447,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1993,"num_read_kb":4020,"num_write":324,"num_write_kb":1818,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":832,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.137931+0000","last_change":"2023-09-23T20:24:09.137931+0000","last_active":"2023-09-23T20:24:09.137931+0000","last_peered":"2023-09-23T20:24:09.137931+0000","last_clean":"2023-09-23T20:24:09.137931+0000","last_became_active":"2023-09-23T20:24:09.137540+0000","last_became_peered":"2023-09-23T20:24:09.137540+0000","last_unstale":"2023-09-23T20:24:09.137931+0000","last_undegraded":"2023-09-23T20:24:09.137931+0000","last_fullsized":"2023-09-23T20:24:09.137931+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:02.946807+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":227,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124187+0000","last_change":"2023-09-23T20:24:06.400847+0000","last_active":"2023-09-23T20:24:08.124187+0000","last_peered":"2023-09-23T20:24:08.124187+0000","last_clean":"2023-09-23T20:24:08.124187+0000","last_became_active":"2023-09-23T20:24:06.400550+0000","last_became_peered":"2023-09-23T20:24:06.400550+0000","last_unstale":"2023-09-23T20:24:08.124187+0000","last_undegraded":"2023-09-23T20:24:08.124187+0000","last_fullsized":"2023-09-23T20:24:08.124187+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:12.345559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133385+0000","last_change":"2023-09-23T20:24:06.408618+0000","last_active":"2023-09-23T20:24:08.133385+0000","last_peered":"2023-09-23T20:24:08.133385+0000","last_clean":"2023-09-23T20:24:08.133385+0000","last_became_active":"2023-09-23T20:24:06.408283+0000","last_became_peered":"2023-09-23T20:24:06.408283+0000","last_unstale":"2023-09-23T20:24:08.133385+0000","last_undegraded":"2023-09-23T20:24:08.133385+0000","last_fullsized":"2023-09-23T20:24:08.133385+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:08.161937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":638,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.139344+0000","last_change":"2023-09-23T20:24:09.138900+0000","last_active":"2023-09-23T20:24:09.139344+0000","last_peered":"2023-09-23T20:24:09.139344+0000","last_clean":"2023-09-23T20:24:09.139344+0000","last_became_active":"2023-09-23T20:24:09.138442+0000","last_became_peered":"2023-09-23T20:24:09.138442+0000","last_unstale":"2023-09-23T20:24:09.139344+0000","last_undegraded":"2023-09-23T20:24:09.139344+0000","last_fullsized":"2023-09-23T20:24:09.139344+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:18.875863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:18.875863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:42.121179+0000","scrub_duration":0.0015811600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"568'327","reported_seq":1719,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135879+0000","last_change":"2023-09-23T20:23:01.135093+0000","last_active":"2023-09-23T20:24:08.135879+0000","last_peered":"2023-09-23T20:24:08.135879+0000","last_clean":"2023-09-23T20:24:08.135879+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:24:08.135879+0000","last_undegraded":"2023-09-23T20:24:08.135879+0000","last_fullsized":"2023-09-23T20:24:08.135879+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":327,"log_dups_size":0,"ondisk_log_size":327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065965800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2451,"num_read_kb":3623,"num_write":260,"num_write_kb":1387,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":393,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132475+0000","last_change":"2023-09-23T20:23:23.494173+0000","last_active":"2023-09-23T20:24:08.132475+0000","last_peered":"2023-09-23T20:24:08.132475+0000","last_clean":"2023-09-23T20:24:08.132475+0000","last_became_active":"2023-09-23T20:23:23.493960+0000","last_became_peered":"2023-09-23T20:23:23.493960+0000","last_unstale":"2023-09-23T20:24:08.132475+0000","last_undegraded":"2023-09-23T20:24:08.132475+0000","last_fullsized":"2023-09-23T20:24:08.132475+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:21.604629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":280,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133078+0000","last_change":"2023-09-23T20:23:23.495694+0000","last_active":"2023-09-23T20:24:08.133078+0000","last_peered":"2023-09-23T20:24:08.133078+0000","last_clean":"2023-09-23T20:24:08.133078+0000","last_became_active":"2023-09-23T20:23:23.494719+0000","last_became_peered":"2023-09-23T20:23:23.494719+0000","last_unstale":"2023-09-23T20:24:08.133078+0000","last_undegraded":"2023-09-23T20:24:08.133078+0000","last_fullsized":"2023-09-23T20:24:08.133078+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:58.553688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":815,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.137463+0000","last_change":"2023-09-23T20:24:09.137463+0000","last_active":"2023-09-23T20:24:09.137463+0000","last_peered":"2023-09-23T20:24:09.137463+0000","last_clean":"2023-09-23T20:24:09.137463+0000","last_became_active":"2023-09-23T20:24:09.137060+0000","last_became_peered":"2023-09-23T20:24:09.137060+0000","last_unstale":"2023-09-23T20:24:09.137463+0000","last_undegraded":"2023-09-23T20:24:09.137463+0000","last_fullsized":"2023-09-23T20:24:09.137463+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:19.413043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:19.413043+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:30.009344+0000","scrub_duration":0.0014842639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"568'118","reported_seq":440,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134579+0000","last_change":"2023-09-23T20:23:01.135784+0000","last_active":"2023-09-23T20:24:08.134579+0000","last_peered":"2023-09-23T20:24:08.134579+0000","last_clean":"2023-09-23T20:24:08.134579+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:24:08.134579+0000","last_undegraded":"2023-09-23T20:24:08.134579+0000","last_fullsized":"2023-09-23T20:24:08.134579+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":118,"log_dups_size":0,"ondisk_log_size":118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030549100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":324,"num_read_kb":2262,"num_write":135,"num_write_kb":2116,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":135,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127262+0000","last_change":"2023-09-23T20:22:22.382963+0000","last_active":"2023-09-23T20:24:08.127262+0000","last_peered":"2023-09-23T20:24:08.127262+0000","last_clean":"2023-09-23T20:24:08.127262+0000","last_became_active":"2023-09-23T20:22:22.382376+0000","last_became_peered":"2023-09-23T20:22:22.382376+0000","last_unstale":"2023-09-23T20:24:08.127262+0000","last_undegraded":"2023-09-23T20:24:08.127262+0000","last_fullsized":"2023-09-23T20:24:08.127262+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:29.101975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":192,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132781+0000","last_change":"2023-09-23T20:23:23.492835+0000","last_active":"2023-09-23T20:24:08.132781+0000","last_peered":"2023-09-23T20:24:08.132781+0000","last_clean":"2023-09-23T20:24:08.132781+0000","last_became_active":"2023-09-23T20:23:23.492319+0000","last_became_peered":"2023-09-23T20:23:23.492319+0000","last_unstale":"2023-09-23T20:24:08.132781+0000","last_undegraded":"2023-09-23T20:24:08.132781+0000","last_fullsized":"2023-09-23T20:24:08.132781+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:50.763850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":922,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.138749+0000","last_change":"2023-09-23T20:24:09.138749+0000","last_active":"2023-09-23T20:24:09.138749+0000","last_peered":"2023-09-23T20:24:09.138749+0000","last_clean":"2023-09-23T20:24:09.138749+0000","last_became_active":"2023-09-23T20:24:09.138395+0000","last_became_peered":"2023-09-23T20:24:09.138395+0000","last_unstale":"2023-09-23T20:24:09.138749+0000","last_undegraded":"2023-09-23T20:24:09.138749+0000","last_fullsized":"2023-09-23T20:24:09.138749+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:45.194227+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:23:45.194227+0000","last_clean_scrub_stamp":"2023-09-23T20:23:45.194227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:10.526350+0000","scrub_duration":0.0022667260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":213,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125106+0000","last_change":"2023-09-23T20:23:36.166081+0000","last_active":"2023-09-23T20:24:08.125106+0000","last_peered":"2023-09-23T20:24:08.125106+0000","last_clean":"2023-09-23T20:24:08.125106+0000","last_became_active":"2023-09-23T20:23:36.164780+0000","last_became_peered":"2023-09-23T20:23:36.164780+0000","last_unstale":"2023-09-23T20:24:08.125106+0000","last_undegraded":"2023-09-23T20:24:08.125106+0000","last_fullsized":"2023-09-23T20:24:08.125106+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:23.300655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"568'213","reported_seq":980,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.140186+0000","last_change":"2023-09-23T20:24:09.139797+0000","last_active":"2023-09-23T20:24:09.140186+0000","last_peered":"2023-09-23T20:24:09.140186+0000","last_clean":"2023-09-23T20:24:09.140186+0000","last_became_active":"2023-09-23T20:24:09.138753+0000","last_became_peered":"2023-09-23T20:24:09.138753+0000","last_unstale":"2023-09-23T20:24:09.140186+0000","last_undegraded":"2023-09-23T20:24:09.140186+0000","last_fullsized":"2023-09-23T20:24:09.140186+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":213,"log_dups_size":0,"ondisk_log_size":213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:07.738003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053758899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":832,"num_read_kb":3069,"num_write":171,"num_write_kb":2341,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":233,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123555+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:24:08.123555+0000","last_peered":"2023-09-23T20:24:08.123555+0000","last_clean":"2023-09-23T20:24:08.123555+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:24:08.123555+0000","last_undegraded":"2023-09-23T20:24:08.123555+0000","last_fullsized":"2023-09-23T20:24:08.123555+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":926,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125019+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:24:08.125019+0000","last_peered":"2023-09-23T20:24:08.125019+0000","last_clean":"2023-09-23T20:24:08.125019+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:24:08.125019+0000","last_undegraded":"2023-09-23T20:24:08.125019+0000","last_fullsized":"2023-09-23T20:24:08.125019+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"568'233","reported_seq":470,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134787+0000","last_change":"2023-09-23T20:23:01.129481+0000","last_active":"2023-09-23T20:24:08.134787+0000","last_peered":"2023-09-23T20:24:08.134787+0000","last_clean":"2023-09-23T20:24:08.134787+0000","last_became_active":"2023-09-23T20:22:45.391193+0000","last_became_peered":"2023-09-23T20:22:45.391193+0000","last_unstale":"2023-09-23T20:24:08.134787+0000","last_undegraded":"2023-09-23T20:24:08.134787+0000","last_fullsized":"2023-09-23T20:24:08.134787+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.102405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041353899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1305,"num_read_kb":1579,"num_write":216,"num_write_kb":5455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":210,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133165+0000","last_change":"2023-09-23T20:23:23.495081+0000","last_active":"2023-09-23T20:24:08.133165+0000","last_peered":"2023-09-23T20:24:08.133165+0000","last_clean":"2023-09-23T20:24:08.133165+0000","last_became_active":"2023-09-23T20:23:23.494708+0000","last_became_peered":"2023-09-23T20:23:23.494708+0000","last_unstale":"2023-09-23T20:24:08.133165+0000","last_undegraded":"2023-09-23T20:24:08.133165+0000","last_fullsized":"2023-09-23T20:24:08.133165+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:11.928892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":405,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135365+0000","last_change":"2023-09-23T20:23:43.335243+0000","last_active":"2023-09-23T20:24:08.135365+0000","last_peered":"2023-09-23T20:24:08.135365+0000","last_clean":"2023-09-23T20:24:08.135365+0000","last_became_active":"2023-09-23T20:23:43.334827+0000","last_became_peered":"2023-09-23T20:23:43.334827+0000","last_unstale":"2023-09-23T20:24:08.135365+0000","last_undegraded":"2023-09-23T20:24:08.135365+0000","last_fullsized":"2023-09-23T20:24:08.135365+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:46.924684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":664,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123604+0000","last_change":"2023-09-23T20:23:36.709648+0000","last_active":"2023-09-23T20:24:08.123604+0000","last_peered":"2023-09-23T20:24:08.123604+0000","last_clean":"2023-09-23T20:24:08.123604+0000","last_became_active":"2023-09-23T20:23:36.169562+0000","last_became_peered":"2023-09-23T20:23:36.169562+0000","last_unstale":"2023-09-23T20:24:08.123604+0000","last_undegraded":"2023-09-23T20:24:08.123604+0000","last_fullsized":"2023-09-23T20:24:08.123604+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:36.709512+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_clean_scrub_stamp":"2023-09-23T20:23:36.709512+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:17.481909+0000","scrub_duration":0.0021278949999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":194,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132726+0000","last_change":"2023-09-23T20:23:23.492624+0000","last_active":"2023-09-23T20:24:08.132726+0000","last_peered":"2023-09-23T20:24:08.132726+0000","last_clean":"2023-09-23T20:24:08.132726+0000","last_became_active":"2023-09-23T20:23:23.491878+0000","last_became_peered":"2023-09-23T20:23:23.491878+0000","last_unstale":"2023-09-23T20:24:08.132726+0000","last_undegraded":"2023-09-23T20:24:08.132726+0000","last_fullsized":"2023-09-23T20:24:08.132726+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:56.234441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":341,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.130518+0000","last_change":"2023-09-23T20:22:22.388204+0000","last_active":"2023-09-23T20:24:08.130518+0000","last_peered":"2023-09-23T20:24:08.130518+0000","last_clean":"2023-09-23T20:24:08.130518+0000","last_became_active":"2023-09-23T20:22:22.387640+0000","last_became_peered":"2023-09-23T20:22:22.387640+0000","last_unstale":"2023-09-23T20:24:08.130518+0000","last_undegraded":"2023-09-23T20:24:08.130518+0000","last_fullsized":"2023-09-23T20:24:08.130518+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:58.015528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":721,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123789+0000","last_change":"2023-09-23T20:22:56.760430+0000","last_active":"2023-09-23T20:24:08.123789+0000","last_peered":"2023-09-23T20:24:08.123789+0000","last_clean":"2023-09-23T20:24:08.123789+0000","last_became_active":"2023-09-23T20:22:53.431327+0000","last_became_peered":"2023-09-23T20:22:53.431327+0000","last_unstale":"2023-09-23T20:24:08.123789+0000","last_undegraded":"2023-09-23T20:24:08.123789+0000","last_fullsized":"2023-09-23T20:24:08.123789+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:56.760329+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:22:56.760329+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:13.013667+0000","scrub_duration":0.001887615,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":628,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125509+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:24:08.125509+0000","last_peered":"2023-09-23T20:24:08.125509+0000","last_clean":"2023-09-23T20:24:08.125509+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:24:08.125509+0000","last_undegraded":"2023-09-23T20:24:08.125509+0000","last_fullsized":"2023-09-23T20:24:08.125509+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":638,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132922+0000","last_change":"2023-09-23T20:23:43.209497+0000","last_active":"2023-09-23T20:24:08.132922+0000","last_peered":"2023-09-23T20:24:08.132922+0000","last_clean":"2023-09-23T20:24:08.132922+0000","last_became_active":"2023-09-23T20:23:43.207902+0000","last_became_peered":"2023-09-23T20:23:43.207902+0000","last_unstale":"2023-09-23T20:24:08.132922+0000","last_undegraded":"2023-09-23T20:24:08.132922+0000","last_fullsized":"2023-09-23T20:24:08.132922+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:52.439396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":632,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127584+0000","last_change":"2023-09-23T20:24:06.418341+0000","last_active":"2023-09-23T20:24:08.127584+0000","last_peered":"2023-09-23T20:24:08.127584+0000","last_clean":"2023-09-23T20:24:08.127584+0000","last_became_active":"2023-09-23T20:24:06.413272+0000","last_became_peered":"2023-09-23T20:24:06.413272+0000","last_unstale":"2023-09-23T20:24:08.127584+0000","last_undegraded":"2023-09-23T20:24:08.127584+0000","last_fullsized":"2023-09-23T20:24:08.127584+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:51.417862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":593,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134481+0000","last_change":"2023-09-23T20:23:43.332907+0000","last_active":"2023-09-23T20:24:08.134481+0000","last_peered":"2023-09-23T20:24:08.134481+0000","last_clean":"2023-09-23T20:24:08.134481+0000","last_became_active":"2023-09-23T20:23:43.332488+0000","last_became_peered":"2023-09-23T20:23:43.332488+0000","last_unstale":"2023-09-23T20:24:08.134481+0000","last_undegraded":"2023-09-23T20:24:08.134481+0000","last_fullsized":"2023-09-23T20:24:08.134481+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:58.187317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":803,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.138970+0000","last_change":"2023-09-23T20:24:09.138970+0000","last_active":"2023-09-23T20:24:09.138970+0000","last_peered":"2023-09-23T20:24:09.138970+0000","last_clean":"2023-09-23T20:24:09.138970+0000","last_became_active":"2023-09-23T20:24:09.138546+0000","last_became_peered":"2023-09-23T20:24:09.138546+0000","last_unstale":"2023-09-23T20:24:09.138970+0000","last_undegraded":"2023-09-23T20:24:09.138970+0000","last_fullsized":"2023-09-23T20:24:09.138970+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:22:58.073284+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:22:58.073284+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:50.317070+0000","scrub_duration":0.001452982,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":219,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124974+0000","last_change":"2023-09-23T20:24:06.396306+0000","last_active":"2023-09-23T20:24:08.124974+0000","last_peered":"2023-09-23T20:24:08.124974+0000","last_clean":"2023-09-23T20:24:08.124974+0000","last_became_active":"2023-09-23T20:24:06.395984+0000","last_became_peered":"2023-09-23T20:24:06.395984+0000","last_unstale":"2023-09-23T20:24:08.124974+0000","last_undegraded":"2023-09-23T20:24:08.124974+0000","last_fullsized":"2023-09-23T20:24:08.124974+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:45.963073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":391,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123920+0000","last_change":"2023-09-23T20:23:36.169691+0000","last_active":"2023-09-23T20:24:08.123920+0000","last_peered":"2023-09-23T20:24:08.123920+0000","last_clean":"2023-09-23T20:24:08.123920+0000","last_became_active":"2023-09-23T20:23:36.169387+0000","last_became_peered":"2023-09-23T20:23:36.169387+0000","last_unstale":"2023-09-23T20:24:08.123920+0000","last_undegraded":"2023-09-23T20:24:08.123920+0000","last_fullsized":"2023-09-23T20:24:08.123920+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:54.707421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":828,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125668+0000","last_change":"2023-09-23T20:22:57.574748+0000","last_active":"2023-09-23T20:24:08.125668+0000","last_peered":"2023-09-23T20:24:08.125668+0000","last_clean":"2023-09-23T20:24:08.125668+0000","last_became_active":"2023-09-23T20:22:53.425726+0000","last_became_peered":"2023-09-23T20:22:53.425726+0000","last_unstale":"2023-09-23T20:24:08.125668+0000","last_undegraded":"2023-09-23T20:24:08.125668+0000","last_fullsized":"2023-09-23T20:24:08.125668+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:57.574606+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:22:57.574606+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:55.820932+0000","scrub_duration":0.0033644019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":225,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133048+0000","last_change":"2023-09-23T20:23:23.494966+0000","last_active":"2023-09-23T20:24:08.133048+0000","last_peered":"2023-09-23T20:24:08.133048+0000","last_clean":"2023-09-23T20:24:08.133048+0000","last_became_active":"2023-09-23T20:23:23.494649+0000","last_became_peered":"2023-09-23T20:23:23.494649+0000","last_unstale":"2023-09-23T20:24:08.133048+0000","last_undegraded":"2023-09-23T20:24:08.133048+0000","last_fullsized":"2023-09-23T20:24:08.133048+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:56.958751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":502,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132725+0000","last_change":"2023-09-23T20:24:06.403361+0000","last_active":"2023-09-23T20:24:08.132725+0000","last_peered":"2023-09-23T20:24:08.132725+0000","last_clean":"2023-09-23T20:24:08.132725+0000","last_became_active":"2023-09-23T20:24:06.403032+0000","last_became_peered":"2023-09-23T20:24:06.403032+0000","last_unstale":"2023-09-23T20:24:08.132725+0000","last_undegraded":"2023-09-23T20:24:08.132725+0000","last_fullsized":"2023-09-23T20:24:08.132725+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:37.993783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":19866,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41405,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11978,"num_read_kb":26422,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":8388722,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234524,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":609324,"kb_used_data":14564,"kb_used_omap":0,"kb_used_meta":594752,"kb_avail":93135828,"statfs":{"total":95995035648,"available":95371087872,"internally_reserved":0,"allocated":14913536,"data_stored":13286875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":609026048},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":5.79,"15min":4.4740000000000002},"min":{"1min":0.33800000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.92100000000000004,"5min":230.952,"15min":230.952},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.53200000000000003,"5min":6.1459999999999999,"15min":4.7329999999999997},"min":{"1min":0.379,"5min":0.224,"15min":0.222},"max":{"1min":0.73499999999999999,"5min":231.08699999999999,"15min":231.08699999999999},"last":0.73099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.5049999999999999,"15min":2.1429999999999998},"min":{"1min":0.371,"5min":0.253,"15min":0.21299999999999999},"max":{"1min":0.85199999999999998,"5min":61.113999999999997,"15min":91.335999999999999},"last":0.745},{"interface":"front","average":{"1min":0.52600000000000002,"5min":1.5329999999999999,"15min":2.4060000000000001},"min":{"1min":0.39800000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.79800000000000004,"5min":61.082999999999998,"15min":91.516999999999996},"last":0.69199999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51700000000000002,"15min":0.51800000000000002},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.64000000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.55700000000000005,"15min":0.54300000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.78000000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.53,"15min":3.089},"min":{"1min":0.433,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.95399999999999996,"5min":25.977,"15min":185.17099999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.716,"15min":3.3100000000000001},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.29699999999999999},"max":{"1min":0.86799999999999999,"5min":26.481999999999999,"15min":184.99100000000001},"last":0.75700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.161,"15min":0.95299999999999996},"min":{"1min":0.378,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66300000000000003,"5min":83.811999999999998,"15min":83.811999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.2290000000000001,"15min":1.0609999999999999},"min":{"1min":0.42199999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.90000000000000002,"5min":84.620000000000005,"15min":84.620000000000005},"last":0.56599999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.53400000000000003,"15min":1.0349999999999999},"min":{"1min":0.38500000000000001,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":0.77900000000000003,"5min":3.794,"15min":16.327000000000002},"last":0.622},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.48299999999999998,"15min":0.95499999999999996},"min":{"1min":0.45500000000000002,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":0.72699999999999998,"5min":2.1190000000000002,"15min":16.716000000000001},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.67600000000000005}]}]},{"osd":6,"up_from":551,"seq":2366526980122,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37468,"kb_used_data":10516,"kb_used_omap":9,"kb_used_meta":26934,"kb_avail":93707684,"statfs":{"total":95995035648,"available":95956668416,"internally_reserved":0,"allocated":10768384,"data_stored":9157229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9991,"internal_metadata":27580665},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.61899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.71999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.744,"5min":1.744,"15min":1.744},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":0.63300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.46999999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.505},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.69799999999999995}]}]},{"osd":1,"up_from":99,"seq":425201762723,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856792,"kb_used_data":11408,"kb_used_omap":7,"kb_used_meta":845368,"kb_avail":92888360,"statfs":{"total":95995035648,"available":95117680640,"internally_reserved":0,"allocated":11681792,"data_stored":8967757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":865657658},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":1.7130000000000001,"15min":2.8660000000000001},"min":{"1min":0.23599999999999999,"5min":0.191,"15min":0.185},"max":{"1min":1.5780000000000001,"5min":62.710999999999999,"15min":145.81299999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":2.0489999999999999,"15min":3.105},"min":{"1min":0.24199999999999999,"5min":0.20000000000000001,"15min":0.19800000000000001},"max":{"1min":1.0089999999999999,"5min":62.610999999999997,"15min":146.83799999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:24:25 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.308,"5min":0.249,"15min":0.249},"max":{"1min":0.84099999999999997,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41699999999999998,"15min":0.41899999999999998},"min":{"1min":0.314,"5min":0.254,"15min":0.254},"max":{"1min":0.67900000000000005,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.29699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":2.9409999999999998,"15min":2.8330000000000002},"min":{"1min":0.28399999999999997,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.69899999999999995,"5min":174.578,"15min":174.578},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":2.7719999999999998,"15min":2.1819999999999999},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.88,"5min":174.822,"15min":174.822},"last":0.31}]},{"osd":4,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.97899999999999998,"15min":1.4410000000000001},"min":{"1min":0.38100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.5089999999999999,"5min":10.387,"15min":29.791},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.0229999999999999,"15min":1.341},"min":{"1min":0.39200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.91400000000000003,"5min":10.422000000000001,"15min":27.143999999999998},"last":0.61699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.0960000000000001,"15min":1.76},"min":{"1min":0.433,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":0.94799999999999995,"5min":10.461,"15min":83.510999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":1.0680000000000001,"15min":1.8700000000000001},"min":{"1min":0.40600000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.89700000000000002,"5min":12.648,"15min":83.353999999999999},"last":0.55900000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.629},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.843,"5min":2.843,"15min":2.843},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.0149999999999999,"15min":2.1160000000000001},"min":{"1min":0.42099999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.84599999999999997,"5min":12.599,"15min":127.03100000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.038,"15min":1.9019999999999999},"min":{"1min":0.505,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.4630000000000001,"5min":10.340999999999999,"15min":125.095},"last":0.51200000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926260,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1059168,"kb_used_data":11672,"kb_used_omap":8,"kb_used_meta":1047479,"kb_avail":92685984,"statfs":{"total":95995035648,"available":94910447616,"internally_reserved":0,"allocated":11952128,"data_stored":9289956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1072619105},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":1.526,"15min":1.9590000000000001},"min":{"1min":0.245,"5min":0.20899999999999999,"15min":0.17599999999999999},"max":{"1min":0.41699999999999998,"5min":39.853999999999999,"15min":72.534000000000006},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.33000000000000002,"5min":1.851,"15min":2.4100000000000001},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":0.442,"5min":39.872999999999998,"15min":72.667000000000002},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.375,"15min":0.377},"min":{"1min":0.24099999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.51700000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40400000000000003,"15min":0.41199999999999998},"min":{"1min":0.28199999999999997,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.51300000000000001,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":3.8149999999999999,"15min":2.754},"min":{"1min":0.29499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.45700000000000002,"5min":201.655,"15min":201.655},"last":0.374},{"interface":"front","average":{"1min":0.38300000000000001,"5min":3.7109999999999999,"15min":2.8660000000000001},"min":{"1min":0.26900000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.49199999999999999,"5min":201.69399999999999,"15min":201.69399999999999},"last":0.52700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.611,"15min":1.5549999999999999},"min":{"1min":0.35199999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.755,"5min":16.593,"15min":16.593},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.613,"15min":1.754},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.27600000000000002},"max":{"1min":0.82499999999999996,"5min":21.899000000000001,"15min":105.282},"last":0.68600000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":2.456,"15min":1.8720000000000001},"min":{"1min":0.443,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2390000000000001,"5min":145.14500000000001,"15min":145.14500000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.59799999999999998,"5min":3.1040000000000001,"15min":2.2970000000000002},"min":{"1min":0.42999999999999999,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":0.80000000000000004,"5min":145.04499999999999,"15min":145.04499999999999},"last":0.61699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.98999999999999999,"15min":1.663},"min":{"1min":0.503,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.2130000000000001,"5min":6.3449999999999998,"15min":99.486000000000004},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.7829999999999999,"15min":2.2149999999999999},"min":{"1min":0.46000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.75600000000000001,"5min":16.617000000000001,"15min":99.668999999999997},"last":0.67500000000000004}]}]},{"osd":2,"up_from":543,"seq":2332167241758,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":150720,"kb_used_data":11640,"kb_used_omap":16,"kb_used_meta":139055,"kb_avail":93594432,"statfs":{"total":95995035648,"available":95840698368,"internally_reserved":0,"allocated":11919360,"data_stored":9229890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16601,"internal_metadata":142393127},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.315,"5min":0.33600000000000002,"15min":0.34000000000000002},"min":{"1min":0.188,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.47199999999999998,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.32600000000000001,"5min":0.33500000000000002,"15min":0.33700000000000002},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.47099999999999997,"5min":0.504,"15min":0.504},"last":0.36699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.378,"15min":0.38400000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.48699999999999999,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.53600000000000003,"5min":1.089,"15min":1.089},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.377,"15min":0.379},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.51700000000000002,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.42199999999999999,"15min":0.42799999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.53200000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.44500000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.54600000000000004,"15min":0.53600000000000003},"min":{"1min":0.432,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.81299999999999994}]},{"osd":5,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.40400000000000003,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.56899999999999995,"15min":0.56200000000000006},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.73099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.86799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.60799999999999998,"15min":0.60199999999999998},"min":{"1min":0.46300000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.876,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.72899999999999998,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.74099999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435672,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1029180,"kb_used_data":7988,"kb_used_omap":4,"kb_used_meta":1021179,"kb_avail":92715972,"statfs":{"total":95995035648,"available":94941155328,"internally_reserved":0,"allocated":8179712,"data_stored":5139996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1045687405},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.32300000000000001,"5min":4.8630000000000004,"15min":3.6230000000000002},"min":{"1min":0.23000000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":0.44900000000000001,"5min":219.80000000000001,"15min":219.80000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.33000000000000002,"5min":4.875,"15min":3.927},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.41699999999999998,"5min":221.36000000000001,"15min":221.36000000000001},"last":0.32400000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":1.2649999999999999,"15min":3.3239999999999998},"min":{"1min":0.223,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.50600000000000001,"5min":9.673,"15min":183.87},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.376,"5min":1.3080000000000001,"15min":3.48},"min":{"1min":0.26000000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.70999999999999996,"5min":19.513000000000002,"15min":185.84899999999999},"last":0.374}]},{"osd":2,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.39400000000000002,"15min":0.40000000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.45900000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.40999999999999998,"15min":0.41399999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.51000000000000001,"5min":0.54600000000000004,"15min":0.54600000000000004},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.5109999999999999,"15min":1.6479999999999999},"min":{"1min":0.377,"5min":0.35499999999999998,"15min":0.28999999999999998},"max":{"1min":0.85599999999999998,"5min":54.046999999999997,"15min":58.472999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.7130000000000001,"15min":2.0659999999999998},"min":{"1min":0.36799999999999999,"5min":0.34699999999999998,"15min":0.28299999999999997},"max":{"1min":0.92500000000000004,"5min":54.311,"15min":58.408000000000001},"last":0.68100000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.7390000000000001,"15min":2.8140000000000001},"min":{"1min":0.38800000000000001,"5min":0.32000000000000001,"15min":0.30299999999999999},"max":{"1min":0.86199999999999999,"5min":54.139000000000003,"15min":200.53200000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.5189999999999999,"15min":2.7480000000000002},"min":{"1min":0.44400000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.70699999999999996,"5min":56.457999999999998,"15min":200.846},"last":0.623}]},{"osd":6,"last update":"Sat Sep 23 20:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.66800000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.93400000000000005,"15min":2.1949999999999998},"min":{"1min":0.44,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77500000000000002,"5min":7.6079999999999997,"15min":139.465},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.92700000000000005,"15min":2.1230000000000002},"min":{"1min":0.432,"5min":0.36899999999999999,"15min":0.32800000000000001},"max":{"1min":0.85799999999999998,"5min":8.9019999999999992,"15min":141.56200000000001},"last":0.69099999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019434,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":583268,"kb_used_data":6108,"kb_used_omap":0,"kb_used_meta":577152,"kb_avail":93161884,"statfs":{"total":95995035648,"available":95397769216,"internally_reserved":0,"allocated":6254592,"data_stored":4635567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":591003648},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":5.3520000000000003,"15min":4.0460000000000003},"min":{"1min":0.34200000000000003,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.77100000000000002,"5min":153.108,"15min":232.89500000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":5.2830000000000004,"15min":4.0469999999999997},"min":{"1min":0.39600000000000002,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":0.74199999999999999,"5min":152.495,"15min":232.851},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.234,"15min":2.331},"min":{"1min":0.35099999999999998,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":0.623,"5min":26.420000000000002,"15min":138.47999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.109,"15min":2.331},"min":{"1min":0.35799999999999998,"5min":0.25,"15min":0.217},"max":{"1min":0.78900000000000003,"5min":26.388999999999999,"15min":138.429},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:24:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":8.3379999999999992,"15min":9.6329999999999991},"min":{"1min":0.41799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.71299999999999997,"5min":196.80500000000001,"15min":196.80500000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":8.4849999999999994,"15min":9.8010000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78400000000000003,"5min":196.77600000000001,"15min":196.77600000000001},"last":0.66100000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":5.5789999999999997,"15min":3.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.2390000000000001,"5min":156.72300000000001,"15min":156.72300000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":5.6120000000000001,"15min":4.0570000000000004},"min":{"1min":0.377,"5min":0.33200000000000002,"15min":0.32900000000000001},"max":{"1min":0.68999999999999995,"5min":155.84800000000001,"15min":155.84800000000001},"last":0.54700000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.49299999999999999,"15min":1.073},"min":{"1min":0.28699999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64600000000000002,"5min":4.3970000000000002,"15min":114.788},"last":0.626},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48899999999999999,"15min":1.103},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.77200000000000002,"5min":3.5590000000000002,"15min":113.822},"last":0.71099999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:23:44 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.75600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.504,"15min":1.1679999999999999},"min":{"1min":0.45500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":113.861},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.629,"5min":0.57699999999999996,"15min":1.2290000000000001},"min":{"1min":0.47799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.81399999999999995,"5min":4.6710000000000003,"15min":114.32299999999999},"last":0.77000000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757798,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":655260,"kb_used_data":6356,"kb_used_omap":0,"kb_used_meta":648896,"kb_avail":93089892,"statfs":{"total":95995035648,"available":95324049408,"internally_reserved":0,"allocated":6508544,"data_stored":4893960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":664469504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":4.5860000000000003,"15min":4.1120000000000001},"min":{"1min":0.375,"5min":0.253,"15min":0.20699999999999999},"max":{"1min":0.77400000000000002,"5min":152.71299999999999,"15min":166.57900000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":6.4580000000000002,"15min":4.6470000000000002},"min":{"1min":0.35299999999999998,"5min":0.22,"15min":0.21099999999999999},"max":{"1min":0.81299999999999994,"5min":256.52199999999999,"15min":256.52199999999999},"last":0.66200000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.995,"15min":2.694},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.22},"max":{"1min":1.165,"5min":8.5289999999999999,"15min":150.16999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.0109999999999999,"15min":2.641},"min":{"1min":0.34200000000000003,"5min":0.27300000000000002,"15min":0.21199999999999999},"max":{"1min":1.4330000000000001,"5min":8.3420000000000005,"15min":150.12700000000001},"last":0.58999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.38100000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.74299999999999999,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.55200000000000005,"15min":0.54100000000000004},"min":{"1min":0.38500000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":3.476,"15min":5.2450000000000001},"min":{"1min":0.32000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.69399999999999995,"5min":155.97300000000001,"15min":196.97800000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":3.4199999999999999,"15min":5.2000000000000002},"min":{"1min":0.36699999999999999,"5min":0.35199999999999998,"15min":0.29999999999999999},"max":{"1min":0.70999999999999996,"5min":155.803,"15min":196.875},"last":0.71699999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53900000000000003,"15min":0.72999999999999998},"min":{"1min":0.35899999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70499999999999996,"5min":5.7279999999999998,"15min":8},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.502,"15min":0.56100000000000005},"min":{"1min":0.42199999999999999,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":0.72499999999999998,"5min":2.6890000000000001,"15min":3.371},"last":0.749}]},{"osd":6,"last update":"Sat Sep 23 20:23:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.73499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.60599999999999998,"15min":0.94099999999999995},"min":{"1min":0.35099999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.8860000000000001,"5min":5.0469999999999997,"15min":11.725},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.54800000000000004,"15min":0.871},"min":{"1min":0.41699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.7989999999999999,"5min":3.0880000000000001,"15min":11.041},"last":0.76000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11855,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9991,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:24:46.552 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:24:46.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:46 smithi144 ceph-mon[194833]: pgmap v1684: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:46.720 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:46 smithi135 ceph-mon[213871]: pgmap v1684: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:46.721 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:46 smithi135 ceph-mon[221113]: pgmap v1684: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:46.899 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:47.470 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:47 smithi135 ceph-mon[221113]: from='client.47783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:24:47.471 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:47 smithi135 ceph-mon[213871]: from='client.47783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:24:47.632 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:47 smithi144 ceph-mon[194833]: from='client.47783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:24:48.631 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:48 smithi144 ceph-mon[194833]: pgmap v1685: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:48.695 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-09-23T20:24:48.697 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2023-09-23T20:24:48.720 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:48 smithi135 ceph-mon[221113]: pgmap v1685: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:48.721 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:48 smithi135 ceph-mon[213871]: pgmap v1685: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:24:49.462 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1685,"stamp":"2023-09-23T20:24:47.318152+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11978,"num_read_kb":26422,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":8998956,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7000,"ondisk_log_size":7000,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4981180,"kb_used_data":80252,"kb_used_omap":46,"kb_used_meta":4900817,"kb_avail":744980036,"statfs":{"total":767960285184,"available":762859556864,"internally_reserved":0,"allocated":82178048,"data_stored":64601230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47576,"internal_metadata":5018437160},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002892"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":1008,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125073+0000","last_change":"2023-09-23T20:22:53.431768+0000","last_active":"2023-09-23T20:24:08.125073+0000","last_peered":"2023-09-23T20:24:08.125073+0000","last_clean":"2023-09-23T20:24:08.125073+0000","last_became_active":"2023-09-23T20:22:53.431378+0000","last_became_peered":"2023-09-23T20:22:53.431378+0000","last_unstale":"2023-09-23T20:24:08.125073+0000","last_undegraded":"2023-09-23T20:24:08.125073+0000","last_fullsized":"2023-09-23T20:24:08.125073+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:54.928103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:54.928103+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:57.671137+0000","scrub_duration":0.011135947,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":444,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134638+0000","last_change":"2023-09-23T20:23:43.209456+0000","last_active":"2023-09-23T20:24:08.134638+0000","last_peered":"2023-09-23T20:24:08.134638+0000","last_clean":"2023-09-23T20:24:08.134638+0000","last_became_active":"2023-09-23T20:23:43.208165+0000","last_became_peered":"2023-09-23T20:23:43.208165+0000","last_unstale":"2023-09-23T20:24:08.134638+0000","last_undegraded":"2023-09-23T20:24:08.134638+0000","last_fullsized":"2023-09-23T20:24:08.134638+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:59.585179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":25,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.131381+0000","last_change":"2023-09-23T20:23:40.194024+0000","last_active":"2023-09-23T20:24:08.131381+0000","last_peered":"2023-09-23T20:24:08.131381+0000","last_clean":"2023-09-23T20:24:08.131381+0000","last_became_active":"2023-09-23T20:23:40.193564+0000","last_became_peered":"2023-09-23T20:23:40.193564+0000","last_unstale":"2023-09-23T20:24:08.131381+0000","last_undegraded":"2023-09-23T20:24:08.131381+0000","last_fullsized":"2023-09-23T20:24:08.131381+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:41.394399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":633,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125291+0000","last_change":"2023-09-23T20:22:55.256917+0000","last_active":"2023-09-23T20:24:08.125291+0000","last_peered":"2023-09-23T20:24:08.125291+0000","last_clean":"2023-09-23T20:24:08.125291+0000","last_became_active":"2023-09-23T20:22:53.428227+0000","last_became_peered":"2023-09-23T20:22:53.428227+0000","last_unstale":"2023-09-23T20:24:08.125291+0000","last_undegraded":"2023-09-23T20:24:08.125291+0000","last_fullsized":"2023-09-23T20:24:08.125291+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:55.256829+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:21:21.987455+0000","last_clean_scrub_stamp":"2023-09-23T20:22:55.256829+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:14.034424+0000","scrub_duration":0.0021020539999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":119,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125006+0000","last_change":"2023-09-23T20:17:47.264850+0000","last_active":"2023-09-23T20:24:08.125006+0000","last_peered":"2023-09-23T20:24:08.125006+0000","last_clean":"2023-09-23T20:24:08.125006+0000","last_became_active":"2023-09-23T20:17:47.264534+0000","last_became_peered":"2023-09-23T20:17:47.264534+0000","last_unstale":"2023-09-23T20:24:08.125006+0000","last_undegraded":"2023-09-23T20:24:08.125006+0000","last_fullsized":"2023-09-23T20:24:08.125006+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:37.231092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":235,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133238+0000","last_change":"2023-09-23T20:23:23.492821+0000","last_active":"2023-09-23T20:24:08.133238+0000","last_peered":"2023-09-23T20:24:08.133238+0000","last_clean":"2023-09-23T20:24:08.133238+0000","last_became_active":"2023-09-23T20:23:23.492583+0000","last_became_peered":"2023-09-23T20:23:23.492583+0000","last_unstale":"2023-09-23T20:24:08.133238+0000","last_undegraded":"2023-09-23T20:24:08.133238+0000","last_fullsized":"2023-09-23T20:24:08.133238+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:59.293610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":515,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134365+0000","last_change":"2023-09-23T20:23:44.174294+0000","last_active":"2023-09-23T20:24:08.134365+0000","last_peered":"2023-09-23T20:24:08.134365+0000","last_clean":"2023-09-23T20:24:08.134365+0000","last_became_active":"2023-09-23T20:23:43.211607+0000","last_became_peered":"2023-09-23T20:23:43.211607+0000","last_unstale":"2023-09-23T20:24:08.134365+0000","last_undegraded":"2023-09-23T20:24:08.134365+0000","last_fullsized":"2023-09-23T20:24:08.134365+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:44.174207+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:23:44.174207+0000","last_clean_scrub_stamp":"2023-09-23T20:23:44.174207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:41.963295+0000","scrub_duration":0.0017378339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":200,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123673+0000","last_change":"2023-09-23T20:23:36.167494+0000","last_active":"2023-09-23T20:24:08.123673+0000","last_peered":"2023-09-23T20:24:08.123673+0000","last_clean":"2023-09-23T20:24:08.123673+0000","last_became_active":"2023-09-23T20:23:36.167166+0000","last_became_peered":"2023-09-23T20:23:36.167166+0000","last_unstale":"2023-09-23T20:24:08.123673+0000","last_undegraded":"2023-09-23T20:24:08.123673+0000","last_fullsized":"2023-09-23T20:24:08.123673+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:32.918587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134618+0000","last_change":"2023-09-23T20:24:06.406026+0000","last_active":"2023-09-23T20:24:08.134618+0000","last_peered":"2023-09-23T20:24:08.134618+0000","last_clean":"2023-09-23T20:24:08.134618+0000","last_became_active":"2023-09-23T20:24:06.405646+0000","last_became_peered":"2023-09-23T20:24:06.405646+0000","last_unstale":"2023-09-23T20:24:08.134618+0000","last_undegraded":"2023-09-23T20:24:08.134618+0000","last_fullsized":"2023-09-23T20:24:08.134618+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:58.670587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":876,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134240+0000","last_change":"2023-09-23T20:22:53.431778+0000","last_active":"2023-09-23T20:24:08.134240+0000","last_peered":"2023-09-23T20:24:08.134240+0000","last_clean":"2023-09-23T20:24:08.134240+0000","last_became_active":"2023-09-23T20:22:53.431517+0000","last_became_peered":"2023-09-23T20:22:53.431517+0000","last_unstale":"2023-09-23T20:24:08.134240+0000","last_undegraded":"2023-09-23T20:24:08.134240+0000","last_fullsized":"2023-09-23T20:24:08.134240+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:25.555266+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:22:25.555266+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:21.690091+0000","scrub_duration":0.001735737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":224,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133052+0000","last_change":"2023-09-23T20:23:23.492260+0000","last_active":"2023-09-23T20:24:08.133052+0000","last_peered":"2023-09-23T20:24:08.133052+0000","last_clean":"2023-09-23T20:24:08.133052+0000","last_became_active":"2023-09-23T20:23:23.491987+0000","last_became_peered":"2023-09-23T20:23:23.491987+0000","last_unstale":"2023-09-23T20:24:08.133052+0000","last_undegraded":"2023-09-23T20:24:08.133052+0000","last_fullsized":"2023-09-23T20:24:08.133052+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:13.763941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":472,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123734+0000","last_change":"2023-09-23T20:24:06.403062+0000","last_active":"2023-09-23T20:24:08.123734+0000","last_peered":"2023-09-23T20:24:08.123734+0000","last_clean":"2023-09-23T20:24:08.123734+0000","last_became_active":"2023-09-23T20:24:06.402772+0000","last_became_peered":"2023-09-23T20:24:06.402772+0000","last_unstale":"2023-09-23T20:24:08.123734+0000","last_undegraded":"2023-09-23T20:24:08.123734+0000","last_fullsized":"2023-09-23T20:24:08.123734+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:56.781582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"125'162","reported_seq":907,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.131781+0000","last_change":"2023-09-23T20:22:24.576681+0000","last_active":"2023-09-23T20:24:09.131781+0000","last_peered":"2023-09-23T20:24:09.131781+0000","last_clean":"2023-09-23T20:24:09.131781+0000","last_became_active":"2023-09-23T20:22:22.381258+0000","last_became_peered":"2023-09-23T20:22:22.381258+0000","last_unstale":"2023-09-23T20:24:09.131781+0000","last_undegraded":"2023-09-23T20:24:09.131781+0000","last_fullsized":"2023-09-23T20:24:09.131781+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:22:24.576567+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:22:24.576567+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:58.491034+0000","scrub_duration":0.0019668609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":616,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127459+0000","last_change":"2023-09-23T20:17:47.267892+0000","last_active":"2023-09-23T20:24:08.127459+0000","last_peered":"2023-09-23T20:24:08.127459+0000","last_clean":"2023-09-23T20:24:08.127459+0000","last_became_active":"2023-09-23T20:17:47.267596+0000","last_became_peered":"2023-09-23T20:17:47.267596+0000","last_unstale":"2023-09-23T20:24:08.127459+0000","last_undegraded":"2023-09-23T20:24:08.127459+0000","last_fullsized":"2023-09-23T20:24:08.127459+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:55.435726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":473,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124050+0000","last_change":"2023-09-23T20:24:06.412627+0000","last_active":"2023-09-23T20:24:08.124050+0000","last_peered":"2023-09-23T20:24:08.124050+0000","last_clean":"2023-09-23T20:24:08.124050+0000","last_became_active":"2023-09-23T20:24:06.412285+0000","last_became_peered":"2023-09-23T20:24:06.412285+0000","last_unstale":"2023-09-23T20:24:08.124050+0000","last_undegraded":"2023-09-23T20:24:08.124050+0000","last_fullsized":"2023-09-23T20:24:08.124050+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:56.903892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":537,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.141725+0000","last_change":"2023-09-23T20:24:09.141725+0000","last_active":"2023-09-23T20:24:09.141725+0000","last_peered":"2023-09-23T20:24:09.141725+0000","last_clean":"2023-09-23T20:24:09.141725+0000","last_became_active":"2023-09-23T20:24:09.140313+0000","last_became_peered":"2023-09-23T20:24:09.140313+0000","last_unstale":"2023-09-23T20:24:09.141725+0000","last_undegraded":"2023-09-23T20:24:09.141725+0000","last_fullsized":"2023-09-23T20:24:09.141725+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:57.231266+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:22:57.231266+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:23.246244+0000","scrub_duration":0.0013866460000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":290,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134680+0000","last_change":"2023-09-23T20:07:37.120625+0000","last_active":"2023-09-23T20:24:08.134680+0000","last_peered":"2023-09-23T20:24:08.134680+0000","last_clean":"2023-09-23T20:24:08.134680+0000","last_became_active":"2023-09-23T20:07:37.045758+0000","last_became_peered":"2023-09-23T20:07:37.045758+0000","last_unstale":"2023-09-23T20:24:08.134680+0000","last_undegraded":"2023-09-23T20:24:08.134680+0000","last_fullsized":"2023-09-23T20:24:08.134680+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:06.911261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":143,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.126869+0000","last_change":"2023-09-23T20:24:06.407302+0000","last_active":"2023-09-23T20:24:08.126869+0000","last_peered":"2023-09-23T20:24:08.126869+0000","last_clean":"2023-09-23T20:24:08.126869+0000","last_became_active":"2023-09-23T20:24:06.406954+0000","last_became_peered":"2023-09-23T20:24:06.406954+0000","last_unstale":"2023-09-23T20:24:08.126869+0000","last_undegraded":"2023-09-23T20:24:08.126869+0000","last_fullsized":"2023-09-23T20:24:08.126869+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:23.009091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":812,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127162+0000","last_change":"2023-09-23T20:22:53.424578+0000","last_active":"2023-09-23T20:24:08.127162+0000","last_peered":"2023-09-23T20:24:08.127162+0000","last_clean":"2023-09-23T20:24:08.127162+0000","last_became_active":"2023-09-23T20:22:53.424250+0000","last_became_peered":"2023-09-23T20:22:53.424250+0000","last_unstale":"2023-09-23T20:24:08.127162+0000","last_undegraded":"2023-09-23T20:24:08.127162+0000","last_fullsized":"2023-09-23T20:24:08.127162+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:23.507023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:23.507023+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:09.885578+0000","scrub_duration":0.0014042600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":520,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.137589+0000","last_change":"2023-09-23T20:24:06.416788+0000","last_active":"2023-09-23T20:24:08.137589+0000","last_peered":"2023-09-23T20:24:08.137589+0000","last_clean":"2023-09-23T20:24:08.137589+0000","last_became_active":"2023-09-23T20:24:06.406663+0000","last_became_peered":"2023-09-23T20:24:06.406663+0000","last_unstale":"2023-09-23T20:24:08.137589+0000","last_undegraded":"2023-09-23T20:24:08.137589+0000","last_fullsized":"2023-09-23T20:24:08.137589+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:35.670573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":442,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132546+0000","last_change":"2023-09-23T20:23:43.210782+0000","last_active":"2023-09-23T20:24:08.132546+0000","last_peered":"2023-09-23T20:24:08.132546+0000","last_clean":"2023-09-23T20:24:08.132546+0000","last_became_active":"2023-09-23T20:23:43.210411+0000","last_became_peered":"2023-09-23T20:23:43.210411+0000","last_unstale":"2023-09-23T20:24:08.132546+0000","last_undegraded":"2023-09-23T20:24:08.132546+0000","last_fullsized":"2023-09-23T20:24:08.132546+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:09.197771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":465,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132234+0000","last_change":"2023-09-23T20:23:37.720659+0000","last_active":"2023-09-23T20:24:08.132234+0000","last_peered":"2023-09-23T20:24:08.132234+0000","last_clean":"2023-09-23T20:24:08.132234+0000","last_became_active":"2023-09-23T20:23:23.491730+0000","last_became_peered":"2023-09-23T20:23:23.491730+0000","last_unstale":"2023-09-23T20:24:08.132234+0000","last_undegraded":"2023-09-23T20:24:08.132234+0000","last_fullsized":"2023-09-23T20:24:08.132234+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:37.720558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:23:37.720558+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:28.730427+0000","scrub_duration":0.0018684839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":307,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134199+0000","last_change":"2023-09-23T20:22:22.384480+0000","last_active":"2023-09-23T20:24:08.134199+0000","last_peered":"2023-09-23T20:24:08.134199+0000","last_clean":"2023-09-23T20:24:08.134199+0000","last_became_active":"2023-09-23T20:22:22.384129+0000","last_became_peered":"2023-09-23T20:22:22.384129+0000","last_unstale":"2023-09-23T20:24:08.134199+0000","last_undegraded":"2023-09-23T20:24:08.134199+0000","last_fullsized":"2023-09-23T20:24:08.134199+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:15.885311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":218,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124932+0000","last_change":"2023-09-23T20:23:36.168359+0000","last_active":"2023-09-23T20:24:08.124932+0000","last_peered":"2023-09-23T20:24:08.124932+0000","last_clean":"2023-09-23T20:24:08.124932+0000","last_became_active":"2023-09-23T20:23:36.168024+0000","last_became_peered":"2023-09-23T20:23:36.168024+0000","last_unstale":"2023-09-23T20:24:08.124932+0000","last_undegraded":"2023-09-23T20:24:08.124932+0000","last_fullsized":"2023-09-23T20:24:08.124932+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:03.468455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":665,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.141354+0000","last_change":"2023-09-23T20:24:09.141354+0000","last_active":"2023-09-23T20:24:09.141354+0000","last_peered":"2023-09-23T20:24:09.141354+0000","last_clean":"2023-09-23T20:24:09.141354+0000","last_became_active":"2023-09-23T20:24:09.140356+0000","last_became_peered":"2023-09-23T20:24:09.140356+0000","last_unstale":"2023-09-23T20:24:09.141354+0000","last_undegraded":"2023-09-23T20:24:09.141354+0000","last_fullsized":"2023-09-23T20:24:09.141354+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:35.274469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:13:35.274469+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:15.086008+0000","scrub_duration":0.0028276429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":281,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.130598+0000","last_change":"2023-09-23T20:23:23.491186+0000","last_active":"2023-09-23T20:24:08.130598+0000","last_peered":"2023-09-23T20:24:08.130598+0000","last_clean":"2023-09-23T20:24:08.130598+0000","last_became_active":"2023-09-23T20:23:23.490877+0000","last_became_peered":"2023-09-23T20:23:23.490877+0000","last_unstale":"2023-09-23T20:24:08.130598+0000","last_undegraded":"2023-09-23T20:24:08.130598+0000","last_fullsized":"2023-09-23T20:24:08.130598+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:36.133469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":595,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127755+0000","last_change":"2023-09-23T20:07:36.976840+0000","last_active":"2023-09-23T20:24:08.127755+0000","last_peered":"2023-09-23T20:24:08.127755+0000","last_clean":"2023-09-23T20:24:08.127755+0000","last_became_active":"2023-09-23T20:07:36.975042+0000","last_became_peered":"2023-09-23T20:07:36.975042+0000","last_unstale":"2023-09-23T20:24:08.127755+0000","last_undegraded":"2023-09-23T20:24:08.127755+0000","last_fullsized":"2023-09-23T20:24:08.127755+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:29.330179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":668,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127305+0000","last_change":"2023-09-23T20:21:11.930313+0000","last_active":"2023-09-23T20:24:08.127305+0000","last_peered":"2023-09-23T20:24:08.127305+0000","last_clean":"2023-09-23T20:24:08.127305+0000","last_became_active":"2023-09-23T20:17:50.329591+0000","last_became_peered":"2023-09-23T20:17:50.329591+0000","last_unstale":"2023-09-23T20:24:08.127305+0000","last_undegraded":"2023-09-23T20:24:08.127305+0000","last_fullsized":"2023-09-23T20:24:08.127305+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:11.930203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.930203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:00.534581+0000","scrub_duration":0.0016982569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":311,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133942+0000","last_change":"2023-09-23T20:22:44.517176+0000","last_active":"2023-09-23T20:24:08.133942+0000","last_peered":"2023-09-23T20:24:08.133942+0000","last_clean":"2023-09-23T20:24:08.133942+0000","last_became_active":"2023-09-23T20:22:44.516833+0000","last_became_peered":"2023-09-23T20:22:44.516833+0000","last_unstale":"2023-09-23T20:24:08.133942+0000","last_undegraded":"2023-09-23T20:24:08.133942+0000","last_fullsized":"2023-09-23T20:24:08.133942+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:43.417704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":331,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133306+0000","last_change":"2023-09-23T20:23:23.492872+0000","last_active":"2023-09-23T20:24:08.133306+0000","last_peered":"2023-09-23T20:24:08.133306+0000","last_clean":"2023-09-23T20:24:08.133306+0000","last_became_active":"2023-09-23T20:23:23.492570+0000","last_became_peered":"2023-09-23T20:23:23.492570+0000","last_unstale":"2023-09-23T20:24:08.133306+0000","last_undegraded":"2023-09-23T20:24:08.133306+0000","last_fullsized":"2023-09-23T20:24:08.133306+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:43.291046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":774,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.142582+0000","last_change":"2023-09-23T20:24:09.142582+0000","last_active":"2023-09-23T20:24:09.142582+0000","last_peered":"2023-09-23T20:24:09.142582+0000","last_clean":"2023-09-23T20:24:09.142582+0000","last_became_active":"2023-09-23T20:24:09.142201+0000","last_became_peered":"2023-09-23T20:24:09.142201+0000","last_unstale":"2023-09-23T20:24:09.142582+0000","last_undegraded":"2023-09-23T20:24:09.142582+0000","last_fullsized":"2023-09-23T20:24:09.142582+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:13:31.284953+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:13:31.284953+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.591311+0000","scrub_duration":0.0033014530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":23,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.131498+0000","last_change":"2023-09-23T20:24:06.429779+0000","last_active":"2023-09-23T20:24:08.131498+0000","last_peered":"2023-09-23T20:24:08.131498+0000","last_clean":"2023-09-23T20:24:08.131498+0000","last_became_active":"2023-09-23T20:24:06.407037+0000","last_became_peered":"2023-09-23T20:24:06.407037+0000","last_unstale":"2023-09-23T20:24:08.131498+0000","last_undegraded":"2023-09-23T20:24:08.131498+0000","last_fullsized":"2023-09-23T20:24:08.131498+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:41.192972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124375+0000","last_change":"2023-09-23T20:17:47.276192+0000","last_active":"2023-09-23T20:24:08.124375+0000","last_peered":"2023-09-23T20:24:08.124375+0000","last_clean":"2023-09-23T20:24:08.124375+0000","last_became_active":"2023-09-23T20:17:47.275666+0000","last_became_peered":"2023-09-23T20:17:47.275666+0000","last_unstale":"2023-09-23T20:24:08.124375+0000","last_undegraded":"2023-09-23T20:24:08.124375+0000","last_fullsized":"2023-09-23T20:24:08.124375+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:18.790115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":792,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.131254+0000","last_change":"2023-09-23T20:23:43.528783+0000","last_active":"2023-09-23T20:24:08.131254+0000","last_peered":"2023-09-23T20:24:08.131254+0000","last_clean":"2023-09-23T20:24:08.131254+0000","last_became_active":"2023-09-23T20:23:43.208308+0000","last_became_peered":"2023-09-23T20:23:43.208308+0000","last_unstale":"2023-09-23T20:24:08.131254+0000","last_undegraded":"2023-09-23T20:24:08.131254+0000","last_fullsized":"2023-09-23T20:24:08.131254+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:43.528685+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:23:43.528685+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:16.679023+0000","scrub_duration":0.00208479,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":288,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.136597+0000","last_change":"2023-09-23T20:07:37.052408+0000","last_active":"2023-09-23T20:24:08.136597+0000","last_peered":"2023-09-23T20:24:08.136597+0000","last_clean":"2023-09-23T20:24:08.136597+0000","last_became_active":"2023-09-23T20:07:36.969689+0000","last_became_peered":"2023-09-23T20:07:36.969689+0000","last_unstale":"2023-09-23T20:24:08.136597+0000","last_undegraded":"2023-09-23T20:24:08.136597+0000","last_fullsized":"2023-09-23T20:24:08.136597+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:32.660424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":615,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125180+0000","last_change":"2023-09-23T20:24:06.413510+0000","last_active":"2023-09-23T20:24:08.125180+0000","last_peered":"2023-09-23T20:24:08.125180+0000","last_clean":"2023-09-23T20:24:08.125180+0000","last_became_active":"2023-09-23T20:24:06.413130+0000","last_became_peered":"2023-09-23T20:24:06.413130+0000","last_unstale":"2023-09-23T20:24:08.125180+0000","last_undegraded":"2023-09-23T20:24:08.125180+0000","last_fullsized":"2023-09-23T20:24:08.125180+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:40.601793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"125'162","reported_seq":997,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.138046+0000","last_change":"2023-09-23T20:24:09.138046+0000","last_active":"2023-09-23T20:24:09.138046+0000","last_peered":"2023-09-23T20:24:09.138046+0000","last_clean":"2023-09-23T20:24:09.138046+0000","last_became_active":"2023-09-23T20:24:09.137455+0000","last_became_peered":"2023-09-23T20:24:09.137455+0000","last_unstale":"2023-09-23T20:24:09.138046+0000","last_undegraded":"2023-09-23T20:24:09.138046+0000","last_fullsized":"2023-09-23T20:24:09.138046+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":4,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:21:11.597355+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:21:11.597355+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:39.872420+0000","scrub_duration":0.0021599240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":317,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134558+0000","last_change":"2023-09-23T20:17:47.286520+0000","last_active":"2023-09-23T20:24:08.134558+0000","last_peered":"2023-09-23T20:24:08.134558+0000","last_clean":"2023-09-23T20:24:08.134558+0000","last_became_active":"2023-09-23T20:17:47.269982+0000","last_became_peered":"2023-09-23T20:17:47.269982+0000","last_unstale":"2023-09-23T20:24:08.134558+0000","last_undegraded":"2023-09-23T20:24:08.134558+0000","last_fullsized":"2023-09-23T20:24:08.134558+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:34.956326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":502,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135504+0000","last_change":"2023-09-23T20:23:43.207185+0000","last_active":"2023-09-23T20:24:08.135504+0000","last_peered":"2023-09-23T20:24:08.135504+0000","last_clean":"2023-09-23T20:24:08.135504+0000","last_became_active":"2023-09-23T20:23:43.206472+0000","last_became_peered":"2023-09-23T20:23:43.206472+0000","last_unstale":"2023-09-23T20:24:08.135504+0000","last_undegraded":"2023-09-23T20:24:08.135504+0000","last_fullsized":"2023-09-23T20:24:08.135504+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:23.396782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":695,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:10.182647+0000","last_change":"2023-09-23T20:24:10.182603+0000","last_active":"2023-09-23T20:24:10.182647+0000","last_peered":"2023-09-23T20:24:10.182647+0000","last_clean":"2023-09-23T20:24:10.182647+0000","last_became_active":"2023-09-23T20:24:09.141983+0000","last_became_peered":"2023-09-23T20:24:09.141983+0000","last_unstale":"2023-09-23T20:24:10.182647+0000","last_undegraded":"2023-09-23T20:24:10.182647+0000","last_fullsized":"2023-09-23T20:24:10.182647+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:24:10.182525+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:24:10.182525+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.186914+0000","scrub_duration":0.0019599510000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":319,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134482+0000","last_change":"2023-09-23T20:22:44.388257+0000","last_active":"2023-09-23T20:24:08.134482+0000","last_peered":"2023-09-23T20:24:08.134482+0000","last_clean":"2023-09-23T20:24:08.134482+0000","last_became_active":"2023-09-23T20:22:44.387884+0000","last_became_peered":"2023-09-23T20:22:44.387884+0000","last_unstale":"2023-09-23T20:24:08.134482+0000","last_undegraded":"2023-09-23T20:24:08.134482+0000","last_fullsized":"2023-09-23T20:24:08.134482+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:26.931322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.137194+0000","last_change":"2023-09-23T20:24:06.408200+0000","last_active":"2023-09-23T20:24:08.137194+0000","last_peered":"2023-09-23T20:24:08.137194+0000","last_clean":"2023-09-23T20:24:08.137194+0000","last_became_active":"2023-09-23T20:24:06.407859+0000","last_became_peered":"2023-09-23T20:24:06.407859+0000","last_unstale":"2023-09-23T20:24:08.137194+0000","last_undegraded":"2023-09-23T20:24:08.137194+0000","last_fullsized":"2023-09-23T20:24:08.137194+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:18.066383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":684,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.136800+0000","last_change":"2023-09-23T20:23:46.210808+0000","last_active":"2023-09-23T20:24:08.136800+0000","last_peered":"2023-09-23T20:24:08.136800+0000","last_clean":"2023-09-23T20:24:08.136800+0000","last_became_active":"2023-09-23T20:23:43.212317+0000","last_became_peered":"2023-09-23T20:23:43.212317+0000","last_unstale":"2023-09-23T20:24:08.136800+0000","last_undegraded":"2023-09-23T20:24:08.136800+0000","last_fullsized":"2023-09-23T20:24:08.136800+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:46.210708+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:23:46.210708+0000","last_clean_scrub_stamp":"2023-09-23T20:23:46.210708+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:24.395204+0000","scrub_duration":0.001786686,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":665,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127083+0000","last_change":"2023-09-23T20:24:06.412761+0000","last_active":"2023-09-23T20:24:08.127083+0000","last_peered":"2023-09-23T20:24:08.127083+0000","last_clean":"2023-09-23T20:24:08.127083+0000","last_became_active":"2023-09-23T20:24:06.412382+0000","last_became_peered":"2023-09-23T20:24:06.412382+0000","last_unstale":"2023-09-23T20:24:08.127083+0000","last_undegraded":"2023-09-23T20:24:08.127083+0000","last_fullsized":"2023-09-23T20:24:08.127083+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:38.504383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":265,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.129865+0000","last_change":"2023-09-23T20:23:23.495138+0000","last_active":"2023-09-23T20:24:08.129865+0000","last_peered":"2023-09-23T20:24:08.129865+0000","last_clean":"2023-09-23T20:24:08.129865+0000","last_became_active":"2023-09-23T20:23:23.494769+0000","last_became_peered":"2023-09-23T20:23:23.494769+0000","last_unstale":"2023-09-23T20:24:08.129865+0000","last_undegraded":"2023-09-23T20:24:08.129865+0000","last_fullsized":"2023-09-23T20:24:08.129865+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:18.221526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1016,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133845+0000","last_change":"2023-09-23T20:22:22.386488+0000","last_active":"2023-09-23T20:24:08.133845+0000","last_peered":"2023-09-23T20:24:08.133845+0000","last_clean":"2023-09-23T20:24:08.133845+0000","last_became_active":"2023-09-23T20:22:22.386137+0000","last_became_peered":"2023-09-23T20:22:22.386137+0000","last_unstale":"2023-09-23T20:24:08.133845+0000","last_undegraded":"2023-09-23T20:24:08.133845+0000","last_fullsized":"2023-09-23T20:24:08.133845+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:47.181912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:47.181912+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:26.092523+0000","scrub_duration":0.0074047540000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":349,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123895+0000","last_change":"2023-09-23T20:23:36.165051+0000","last_active":"2023-09-23T20:24:08.123895+0000","last_peered":"2023-09-23T20:24:08.123895+0000","last_clean":"2023-09-23T20:24:08.123895+0000","last_became_active":"2023-09-23T20:23:36.164701+0000","last_became_peered":"2023-09-23T20:23:36.164701+0000","last_unstale":"2023-09-23T20:24:08.123895+0000","last_undegraded":"2023-09-23T20:24:08.123895+0000","last_fullsized":"2023-09-23T20:24:08.123895+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.493418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":482,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.137677+0000","last_change":"2023-09-23T20:24:06.407059+0000","last_active":"2023-09-23T20:24:08.137677+0000","last_peered":"2023-09-23T20:24:08.137677+0000","last_clean":"2023-09-23T20:24:08.137677+0000","last_became_active":"2023-09-23T20:24:06.406263+0000","last_became_peered":"2023-09-23T20:24:06.406263+0000","last_unstale":"2023-09-23T20:24:08.137677+0000","last_undegraded":"2023-09-23T20:24:08.137677+0000","last_fullsized":"2023-09-23T20:24:08.137677+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:29.763801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":987,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125213+0000","last_change":"2023-09-23T20:22:22.377942+0000","last_active":"2023-09-23T20:24:08.125213+0000","last_peered":"2023-09-23T20:24:08.125213+0000","last_clean":"2023-09-23T20:24:08.125213+0000","last_became_active":"2023-09-23T20:22:22.377619+0000","last_became_peered":"2023-09-23T20:22:22.377619+0000","last_unstale":"2023-09-23T20:24:08.125213+0000","last_undegraded":"2023-09-23T20:24:08.125213+0000","last_fullsized":"2023-09-23T20:24:08.125213+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:43.954204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:19:43.954204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:00.412768+0000","scrub_duration":0.0019293979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.137137+0000","last_change":"2023-09-23T20:24:06.410227+0000","last_active":"2023-09-23T20:24:08.137137+0000","last_peered":"2023-09-23T20:24:08.137137+0000","last_clean":"2023-09-23T20:24:08.137137+0000","last_became_active":"2023-09-23T20:24:06.409886+0000","last_became_peered":"2023-09-23T20:24:06.409886+0000","last_unstale":"2023-09-23T20:24:08.137137+0000","last_undegraded":"2023-09-23T20:24:08.137137+0000","last_fullsized":"2023-09-23T20:24:08.137137+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:52.648626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":475,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133983+0000","last_change":"2023-09-23T20:23:43.335408+0000","last_active":"2023-09-23T20:24:08.133983+0000","last_peered":"2023-09-23T20:24:08.133983+0000","last_clean":"2023-09-23T20:24:08.133983+0000","last_became_active":"2023-09-23T20:23:43.335063+0000","last_became_peered":"2023-09-23T20:23:43.335063+0000","last_unstale":"2023-09-23T20:24:08.133983+0000","last_undegraded":"2023-09-23T20:24:08.133983+0000","last_fullsized":"2023-09-23T20:24:08.133983+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:19.424719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":674,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.126696+0000","last_change":"2023-09-23T20:21:20.952512+0000","last_active":"2023-09-23T20:24:09.126696+0000","last_peered":"2023-09-23T20:24:09.126696+0000","last_clean":"2023-09-23T20:24:09.126696+0000","last_became_active":"2023-09-23T20:19:05.083002+0000","last_became_peered":"2023-09-23T20:19:05.083002+0000","last_unstale":"2023-09-23T20:24:09.126696+0000","last_undegraded":"2023-09-23T20:24:09.126696+0000","last_fullsized":"2023-09-23T20:24:09.126696+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:20.952352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:20.952352+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:28.266826+0000","scrub_duration":0.00171708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":436,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135285+0000","last_change":"2023-09-23T20:23:43.211174+0000","last_active":"2023-09-23T20:24:08.135285+0000","last_peered":"2023-09-23T20:24:08.135285+0000","last_clean":"2023-09-23T20:24:08.135285+0000","last_became_active":"2023-09-23T20:23:43.210874+0000","last_became_peered":"2023-09-23T20:23:43.210874+0000","last_unstale":"2023-09-23T20:24:08.135285+0000","last_undegraded":"2023-09-23T20:24:08.135285+0000","last_fullsized":"2023-09-23T20:24:08.135285+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:26.252449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":482,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.136829+0000","last_change":"2023-09-23T20:24:06.398839+0000","last_active":"2023-09-23T20:24:08.136829+0000","last_peered":"2023-09-23T20:24:08.136829+0000","last_clean":"2023-09-23T20:24:08.136829+0000","last_became_active":"2023-09-23T20:24:06.398496+0000","last_became_peered":"2023-09-23T20:24:06.398496+0000","last_unstale":"2023-09-23T20:24:08.136829+0000","last_undegraded":"2023-09-23T20:24:08.136829+0000","last_fullsized":"2023-09-23T20:24:08.136829+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:12.143805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":1019,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.140789+0000","last_change":"2023-09-23T20:24:09.140789+0000","last_active":"2023-09-23T20:24:09.140789+0000","last_peered":"2023-09-23T20:24:09.140789+0000","last_clean":"2023-09-23T20:24:09.140789+0000","last_became_active":"2023-09-23T20:24:09.140417+0000","last_became_peered":"2023-09-23T20:24:09.140417+0000","last_unstale":"2023-09-23T20:24:09.140789+0000","last_undegraded":"2023-09-23T20:24:09.140789+0000","last_fullsized":"2023-09-23T20:24:09.140789+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:12.314623+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:59.809323+0000","last_clean_scrub_stamp":"2023-09-23T20:21:12.314623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:39.801719+0000","scrub_duration":0.001743334,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":251,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.125004+0000","last_change":"2023-09-23T20:17:47.270596+0000","last_active":"2023-09-23T20:24:09.125004+0000","last_peered":"2023-09-23T20:24:09.125004+0000","last_clean":"2023-09-23T20:24:09.125004+0000","last_became_active":"2023-09-23T20:17:47.270279+0000","last_became_peered":"2023-09-23T20:17:47.270279+0000","last_unstale":"2023-09-23T20:24:09.125004+0000","last_undegraded":"2023-09-23T20:24:09.125004+0000","last_fullsized":"2023-09-23T20:24:09.125004+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:23.473330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":120,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.126024+0000","last_change":"2023-09-23T20:17:47.271305+0000","last_active":"2023-09-23T20:24:09.126024+0000","last_peered":"2023-09-23T20:24:09.126024+0000","last_clean":"2023-09-23T20:24:09.126024+0000","last_became_active":"2023-09-23T20:17:47.271062+0000","last_became_peered":"2023-09-23T20:17:47.271062+0000","last_unstale":"2023-09-23T20:24:09.126024+0000","last_undegraded":"2023-09-23T20:24:09.126024+0000","last_fullsized":"2023-09-23T20:24:09.126024+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.188235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":913,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.140533+0000","last_change":"2023-09-23T20:24:09.140533+0000","last_active":"2023-09-23T20:24:09.140533+0000","last_peered":"2023-09-23T20:24:09.140533+0000","last_clean":"2023-09-23T20:24:09.140533+0000","last_became_active":"2023-09-23T20:24:09.139750+0000","last_became_peered":"2023-09-23T20:24:09.139750+0000","last_unstale":"2023-09-23T20:24:09.140533+0000","last_undegraded":"2023-09-23T20:24:09.140533+0000","last_fullsized":"2023-09-23T20:24:09.140533+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:19:40.219310+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T19:52:58.592597+0000","last_clean_scrub_stamp":"2023-09-23T20:19:40.219310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:53.078388+0000","scrub_duration":0.011178762,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"568'268","reported_seq":1164,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.127891+0000","last_change":"2023-09-23T20:23:01.133380+0000","last_active":"2023-09-23T20:24:09.127891+0000","last_peered":"2023-09-23T20:24:09.127891+0000","last_clean":"2023-09-23T20:24:09.127891+0000","last_became_active":"2023-09-23T20:17:50.311146+0000","last_became_peered":"2023-09-23T20:17:50.311146+0000","last_unstale":"2023-09-23T20:24:09.127891+0000","last_undegraded":"2023-09-23T20:24:09.127891+0000","last_fullsized":"2023-09-23T20:24:09.127891+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":268,"log_dups_size":0,"ondisk_log_size":268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:35.729762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034039399999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1578,"num_read_kb":2522,"num_write":266,"num_write_kb":5301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":614,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127661+0000","last_change":"2023-09-23T20:22:22.379055+0000","last_active":"2023-09-23T20:24:08.127661+0000","last_peered":"2023-09-23T20:24:08.127661+0000","last_clean":"2023-09-23T20:24:08.127661+0000","last_became_active":"2023-09-23T20:22:22.378812+0000","last_became_peered":"2023-09-23T20:22:22.378812+0000","last_unstale":"2023-09-23T20:24:08.127661+0000","last_undegraded":"2023-09-23T20:24:08.127661+0000","last_fullsized":"2023-09-23T20:24:08.127661+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:26.700741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.126823+0000","last_change":"2023-09-23T20:17:47.266289+0000","last_active":"2023-09-23T20:24:08.126823+0000","last_peered":"2023-09-23T20:24:08.126823+0000","last_clean":"2023-09-23T20:24:08.126823+0000","last_became_active":"2023-09-23T20:17:47.265603+0000","last_became_peered":"2023-09-23T20:17:47.265603+0000","last_unstale":"2023-09-23T20:24:08.126823+0000","last_undegraded":"2023-09-23T20:24:08.126823+0000","last_fullsized":"2023-09-23T20:24:08.126823+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:26.349704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":957,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125154+0000","last_change":"2023-09-23T20:22:22.379736+0000","last_active":"2023-09-23T20:24:08.125154+0000","last_peered":"2023-09-23T20:24:08.125154+0000","last_clean":"2023-09-23T20:24:08.125154+0000","last_became_active":"2023-09-23T20:22:22.379411+0000","last_became_peered":"2023-09-23T20:22:22.379411+0000","last_unstale":"2023-09-23T20:24:08.125154+0000","last_undegraded":"2023-09-23T20:24:08.125154+0000","last_fullsized":"2023-09-23T20:24:08.125154+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.010689+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.010689+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:21.231359+0000","scrub_duration":0.0016743260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"568'188","reported_seq":2067,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.141410+0000","last_change":"2023-09-23T20:24:09.141410+0000","last_active":"2023-09-23T20:24:09.141410+0000","last_peered":"2023-09-23T20:24:09.141410+0000","last_clean":"2023-09-23T20:24:09.141410+0000","last_became_active":"2023-09-23T20:24:09.140746+0000","last_became_peered":"2023-09-23T20:24:09.140746+0000","last_unstale":"2023-09-23T20:24:09.141410+0000","last_undegraded":"2023-09-23T20:24:09.141410+0000","last_fullsized":"2023-09-23T20:24:09.141410+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:30.688454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001683302,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2025,"num_read_kb":4391,"num_write":141,"num_write_kb":2375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":114,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":475,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132826+0000","last_change":"2023-09-23T20:24:06.408577+0000","last_active":"2023-09-23T20:24:08.132826+0000","last_peered":"2023-09-23T20:24:08.132826+0000","last_clean":"2023-09-23T20:24:08.132826+0000","last_became_active":"2023-09-23T20:24:06.408244+0000","last_became_peered":"2023-09-23T20:24:06.408244+0000","last_unstale":"2023-09-23T20:24:08.132826+0000","last_undegraded":"2023-09-23T20:24:08.132826+0000","last_fullsized":"2023-09-23T20:24:08.132826+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:56.941793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":135,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127422+0000","last_change":"2023-09-23T20:22:22.382763+0000","last_active":"2023-09-23T20:24:08.127422+0000","last_peered":"2023-09-23T20:24:08.127422+0000","last_clean":"2023-09-23T20:24:08.127422+0000","last_became_active":"2023-09-23T20:22:22.381690+0000","last_became_peered":"2023-09-23T20:22:22.381690+0000","last_unstale":"2023-09-23T20:24:08.127422+0000","last_undegraded":"2023-09-23T20:24:08.127422+0000","last_fullsized":"2023-09-23T20:24:08.127422+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:51.102791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":835,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135547+0000","last_change":"2023-09-23T20:12:48.124804+0000","last_active":"2023-09-23T20:24:08.135547+0000","last_peered":"2023-09-23T20:24:08.135547+0000","last_clean":"2023-09-23T20:24:08.135547+0000","last_became_active":"2023-09-23T20:09:56.042734+0000","last_became_peered":"2023-09-23T20:09:56.042734+0000","last_unstale":"2023-09-23T20:24:08.135547+0000","last_undegraded":"2023-09-23T20:24:08.135547+0000","last_fullsized":"2023-09-23T20:24:08.135547+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:48.124686+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:03:50.811793+0000","last_clean_scrub_stamp":"2023-09-23T20:12:48.124686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:04.168183+0000","scrub_duration":0.001804302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"568'210","reported_seq":1592,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.131843+0000","last_change":"2023-09-23T20:23:01.134345+0000","last_active":"2023-09-23T20:24:09.131843+0000","last_peered":"2023-09-23T20:24:09.131843+0000","last_clean":"2023-09-23T20:24:09.131843+0000","last_became_active":"2023-09-23T20:22:22.387896+0000","last_became_peered":"2023-09-23T20:22:22.387896+0000","last_unstale":"2023-09-23T20:24:09.131843+0000","last_undegraded":"2023-09-23T20:24:09.131843+0000","last_fullsized":"2023-09-23T20:24:09.131843+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":210,"log_dups_size":0,"ondisk_log_size":210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:11.339762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000263816,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1470,"num_read_kb":4956,"num_write":177,"num_write_kb":4597,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.126298+0000","last_change":"2023-09-23T20:24:06.406921+0000","last_active":"2023-09-23T20:24:08.126298+0000","last_peered":"2023-09-23T20:24:08.126298+0000","last_clean":"2023-09-23T20:24:08.126298+0000","last_became_active":"2023-09-23T20:24:06.406593+0000","last_became_peered":"2023-09-23T20:24:06.406593+0000","last_unstale":"2023-09-23T20:24:08.126298+0000","last_undegraded":"2023-09-23T20:24:08.126298+0000","last_fullsized":"2023-09-23T20:24:08.126298+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:15.895040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":120,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.126103+0000","last_change":"2023-09-23T20:17:47.271196+0000","last_active":"2023-09-23T20:24:09.126103+0000","last_peered":"2023-09-23T20:24:09.126103+0000","last_clean":"2023-09-23T20:24:09.126103+0000","last_became_active":"2023-09-23T20:17:47.270635+0000","last_became_peered":"2023-09-23T20:17:47.270635+0000","last_unstale":"2023-09-23T20:24:09.126103+0000","last_undegraded":"2023-09-23T20:24:09.126103+0000","last_fullsized":"2023-09-23T20:24:09.126103+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:54.913626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"568'393","reported_seq":2007,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125394+0000","last_change":"2023-09-23T20:23:01.130145+0000","last_active":"2023-09-23T20:24:08.125394+0000","last_peered":"2023-09-23T20:24:08.125394+0000","last_clean":"2023-09-23T20:24:08.125394+0000","last_became_active":"2023-09-23T20:22:22.379019+0000","last_became_peered":"2023-09-23T20:22:22.379019+0000","last_unstale":"2023-09-23T20:24:08.125394+0000","last_undegraded":"2023-09-23T20:24:08.125394+0000","last_fullsized":"2023-09-23T20:24:08.125394+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":393,"log_dups_size":0,"ondisk_log_size":393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:35.091435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000589447,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1993,"num_read_kb":4020,"num_write":324,"num_write_kb":1818,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":832,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.137931+0000","last_change":"2023-09-23T20:24:09.137931+0000","last_active":"2023-09-23T20:24:09.137931+0000","last_peered":"2023-09-23T20:24:09.137931+0000","last_clean":"2023-09-23T20:24:09.137931+0000","last_became_active":"2023-09-23T20:24:09.137540+0000","last_became_peered":"2023-09-23T20:24:09.137540+0000","last_unstale":"2023-09-23T20:24:09.137931+0000","last_undegraded":"2023-09-23T20:24:09.137931+0000","last_fullsized":"2023-09-23T20:24:09.137931+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:12:54.043618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:12:54.043618+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:02.946807+0000","scrub_duration":0.0016156390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":227,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124187+0000","last_change":"2023-09-23T20:24:06.400847+0000","last_active":"2023-09-23T20:24:08.124187+0000","last_peered":"2023-09-23T20:24:08.124187+0000","last_clean":"2023-09-23T20:24:08.124187+0000","last_became_active":"2023-09-23T20:24:06.400550+0000","last_became_peered":"2023-09-23T20:24:06.400550+0000","last_unstale":"2023-09-23T20:24:08.124187+0000","last_undegraded":"2023-09-23T20:24:08.124187+0000","last_fullsized":"2023-09-23T20:24:08.124187+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:12.345559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133385+0000","last_change":"2023-09-23T20:24:06.408618+0000","last_active":"2023-09-23T20:24:08.133385+0000","last_peered":"2023-09-23T20:24:08.133385+0000","last_clean":"2023-09-23T20:24:08.133385+0000","last_became_active":"2023-09-23T20:24:06.408283+0000","last_became_peered":"2023-09-23T20:24:06.408283+0000","last_unstale":"2023-09-23T20:24:08.133385+0000","last_undegraded":"2023-09-23T20:24:08.133385+0000","last_fullsized":"2023-09-23T20:24:08.133385+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:08.161937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":638,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.139344+0000","last_change":"2023-09-23T20:24:09.138900+0000","last_active":"2023-09-23T20:24:09.139344+0000","last_peered":"2023-09-23T20:24:09.139344+0000","last_clean":"2023-09-23T20:24:09.139344+0000","last_became_active":"2023-09-23T20:24:09.138442+0000","last_became_peered":"2023-09-23T20:24:09.138442+0000","last_unstale":"2023-09-23T20:24:09.139344+0000","last_undegraded":"2023-09-23T20:24:09.139344+0000","last_fullsized":"2023-09-23T20:24:09.139344+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:18.875863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:18.875863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:42.121179+0000","scrub_duration":0.0015811600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"568'327","reported_seq":1719,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135879+0000","last_change":"2023-09-23T20:23:01.135093+0000","last_active":"2023-09-23T20:24:08.135879+0000","last_peered":"2023-09-23T20:24:08.135879+0000","last_clean":"2023-09-23T20:24:08.135879+0000","last_became_active":"2023-09-23T20:17:50.312798+0000","last_became_peered":"2023-09-23T20:17:50.312798+0000","last_unstale":"2023-09-23T20:24:08.135879+0000","last_undegraded":"2023-09-23T20:24:08.135879+0000","last_fullsized":"2023-09-23T20:24:08.135879+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":327,"log_dups_size":0,"ondisk_log_size":327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:01.750241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065965800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2451,"num_read_kb":3623,"num_write":260,"num_write_kb":1387,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":393,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132475+0000","last_change":"2023-09-23T20:23:23.494173+0000","last_active":"2023-09-23T20:24:08.132475+0000","last_peered":"2023-09-23T20:24:08.132475+0000","last_clean":"2023-09-23T20:24:08.132475+0000","last_became_active":"2023-09-23T20:23:23.493960+0000","last_became_peered":"2023-09-23T20:23:23.493960+0000","last_unstale":"2023-09-23T20:24:08.132475+0000","last_undegraded":"2023-09-23T20:24:08.132475+0000","last_fullsized":"2023-09-23T20:24:08.132475+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:21.604629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":280,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133078+0000","last_change":"2023-09-23T20:23:23.495694+0000","last_active":"2023-09-23T20:24:08.133078+0000","last_peered":"2023-09-23T20:24:08.133078+0000","last_clean":"2023-09-23T20:24:08.133078+0000","last_became_active":"2023-09-23T20:23:23.494719+0000","last_became_peered":"2023-09-23T20:23:23.494719+0000","last_unstale":"2023-09-23T20:24:08.133078+0000","last_undegraded":"2023-09-23T20:24:08.133078+0000","last_fullsized":"2023-09-23T20:24:08.133078+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:58.553688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":815,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.137463+0000","last_change":"2023-09-23T20:24:09.137463+0000","last_active":"2023-09-23T20:24:09.137463+0000","last_peered":"2023-09-23T20:24:09.137463+0000","last_clean":"2023-09-23T20:24:09.137463+0000","last_became_active":"2023-09-23T20:24:09.137060+0000","last_became_peered":"2023-09-23T20:24:09.137060+0000","last_unstale":"2023-09-23T20:24:09.137463+0000","last_undegraded":"2023-09-23T20:24:09.137463+0000","last_fullsized":"2023-09-23T20:24:09.137463+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:21:19.413043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:38:09.132356+0000","last_clean_scrub_stamp":"2023-09-23T20:21:19.413043+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:30.009344+0000","scrub_duration":0.0014842639999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"568'118","reported_seq":440,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134579+0000","last_change":"2023-09-23T20:23:01.135784+0000","last_active":"2023-09-23T20:24:08.134579+0000","last_peered":"2023-09-23T20:24:08.134579+0000","last_clean":"2023-09-23T20:24:08.134579+0000","last_became_active":"2023-09-23T20:07:37.045261+0000","last_became_peered":"2023-09-23T20:07:37.045261+0000","last_unstale":"2023-09-23T20:24:08.134579+0000","last_undegraded":"2023-09-23T20:24:08.134579+0000","last_fullsized":"2023-09-23T20:24:08.134579+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":118,"log_dups_size":0,"ondisk_log_size":118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.352584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030549100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":324,"num_read_kb":2262,"num_write":135,"num_write_kb":2116,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":135,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127262+0000","last_change":"2023-09-23T20:22:22.382963+0000","last_active":"2023-09-23T20:24:08.127262+0000","last_peered":"2023-09-23T20:24:08.127262+0000","last_clean":"2023-09-23T20:24:08.127262+0000","last_became_active":"2023-09-23T20:22:22.382376+0000","last_became_peered":"2023-09-23T20:22:22.382376+0000","last_unstale":"2023-09-23T20:24:08.127262+0000","last_undegraded":"2023-09-23T20:24:08.127262+0000","last_fullsized":"2023-09-23T20:24:08.127262+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:29.101975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":192,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132781+0000","last_change":"2023-09-23T20:23:23.492835+0000","last_active":"2023-09-23T20:24:08.132781+0000","last_peered":"2023-09-23T20:24:08.132781+0000","last_clean":"2023-09-23T20:24:08.132781+0000","last_became_active":"2023-09-23T20:23:23.492319+0000","last_became_peered":"2023-09-23T20:23:23.492319+0000","last_unstale":"2023-09-23T20:24:08.132781+0000","last_undegraded":"2023-09-23T20:24:08.132781+0000","last_fullsized":"2023-09-23T20:24:08.132781+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:50.763850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":922,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.138749+0000","last_change":"2023-09-23T20:24:09.138749+0000","last_active":"2023-09-23T20:24:09.138749+0000","last_peered":"2023-09-23T20:24:09.138749+0000","last_clean":"2023-09-23T20:24:09.138749+0000","last_became_active":"2023-09-23T20:24:09.138395+0000","last_became_peered":"2023-09-23T20:24:09.138395+0000","last_unstale":"2023-09-23T20:24:09.138749+0000","last_undegraded":"2023-09-23T20:24:09.138749+0000","last_fullsized":"2023-09-23T20:24:09.138749+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:45.194227+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:23:45.194227+0000","last_clean_scrub_stamp":"2023-09-23T20:23:45.194227+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:10.526350+0000","scrub_duration":0.0022667260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":213,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125106+0000","last_change":"2023-09-23T20:23:36.166081+0000","last_active":"2023-09-23T20:24:08.125106+0000","last_peered":"2023-09-23T20:24:08.125106+0000","last_clean":"2023-09-23T20:24:08.125106+0000","last_became_active":"2023-09-23T20:23:36.164780+0000","last_became_peered":"2023-09-23T20:23:36.164780+0000","last_unstale":"2023-09-23T20:24:08.125106+0000","last_undegraded":"2023-09-23T20:24:08.125106+0000","last_fullsized":"2023-09-23T20:24:08.125106+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:23.300655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"568'213","reported_seq":980,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.140186+0000","last_change":"2023-09-23T20:24:09.139797+0000","last_active":"2023-09-23T20:24:09.140186+0000","last_peered":"2023-09-23T20:24:09.140186+0000","last_clean":"2023-09-23T20:24:09.140186+0000","last_became_active":"2023-09-23T20:24:09.138753+0000","last_became_peered":"2023-09-23T20:24:09.138753+0000","last_unstale":"2023-09-23T20:24:09.140186+0000","last_undegraded":"2023-09-23T20:24:09.140186+0000","last_fullsized":"2023-09-23T20:24:09.140186+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":213,"log_dups_size":0,"ondisk_log_size":213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:07.738003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053758899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":832,"num_read_kb":3069,"num_write":171,"num_write_kb":2341,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":233,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123555+0000","last_change":"2023-09-23T20:11:17.703434+0000","last_active":"2023-09-23T20:24:08.123555+0000","last_peered":"2023-09-23T20:24:08.123555+0000","last_clean":"2023-09-23T20:24:08.123555+0000","last_became_active":"2023-09-23T20:11:17.703059+0000","last_became_peered":"2023-09-23T20:11:17.703059+0000","last_unstale":"2023-09-23T20:24:08.123555+0000","last_undegraded":"2023-09-23T20:24:08.123555+0000","last_fullsized":"2023-09-23T20:24:08.123555+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.805471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"125'162","reported_seq":926,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125019+0000","last_change":"2023-09-23T20:13:26.021581+0000","last_active":"2023-09-23T20:24:08.125019+0000","last_peered":"2023-09-23T20:24:08.125019+0000","last_clean":"2023-09-23T20:24:08.125019+0000","last_became_active":"2023-09-23T20:09:56.035736+0000","last_became_peered":"2023-09-23T20:09:56.035736+0000","last_unstale":"2023-09-23T20:24:08.125019+0000","last_undegraded":"2023-09-23T20:24:08.125019+0000","last_fullsized":"2023-09-23T20:24:08.125019+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:13:26.021462+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:13:26.021462+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:18.700727+0000","scrub_duration":0.0085225860000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"568'233","reported_seq":470,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134787+0000","last_change":"2023-09-23T20:23:01.129481+0000","last_active":"2023-09-23T20:24:08.134787+0000","last_peered":"2023-09-23T20:24:08.134787+0000","last_clean":"2023-09-23T20:24:08.134787+0000","last_became_active":"2023-09-23T20:22:45.391193+0000","last_became_peered":"2023-09-23T20:22:45.391193+0000","last_unstale":"2023-09-23T20:24:08.134787+0000","last_undegraded":"2023-09-23T20:24:08.134787+0000","last_fullsized":"2023-09-23T20:24:08.134787+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:40:39.228450+0000","last_clean_scrub_stamp":"2023-09-23T19:40:39.228450+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.102405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041353899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1305,"num_read_kb":1579,"num_write":216,"num_write_kb":5455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":210,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133165+0000","last_change":"2023-09-23T20:23:23.495081+0000","last_active":"2023-09-23T20:24:08.133165+0000","last_peered":"2023-09-23T20:24:08.133165+0000","last_clean":"2023-09-23T20:24:08.133165+0000","last_became_active":"2023-09-23T20:23:23.494708+0000","last_became_peered":"2023-09-23T20:23:23.494708+0000","last_unstale":"2023-09-23T20:24:08.133165+0000","last_undegraded":"2023-09-23T20:24:08.133165+0000","last_fullsized":"2023-09-23T20:24:08.133165+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:11.928892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":405,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.135365+0000","last_change":"2023-09-23T20:23:43.335243+0000","last_active":"2023-09-23T20:24:08.135365+0000","last_peered":"2023-09-23T20:24:08.135365+0000","last_clean":"2023-09-23T20:24:08.135365+0000","last_became_active":"2023-09-23T20:23:43.334827+0000","last_became_peered":"2023-09-23T20:23:43.334827+0000","last_unstale":"2023-09-23T20:24:08.135365+0000","last_undegraded":"2023-09-23T20:24:08.135365+0000","last_fullsized":"2023-09-23T20:24:08.135365+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:46.924684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":664,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123604+0000","last_change":"2023-09-23T20:23:36.709648+0000","last_active":"2023-09-23T20:24:08.123604+0000","last_peered":"2023-09-23T20:24:08.123604+0000","last_clean":"2023-09-23T20:24:08.123604+0000","last_became_active":"2023-09-23T20:23:36.169562+0000","last_became_peered":"2023-09-23T20:23:36.169562+0000","last_unstale":"2023-09-23T20:24:08.123604+0000","last_undegraded":"2023-09-23T20:24:08.123604+0000","last_fullsized":"2023-09-23T20:24:08.123604+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:23:36.709512+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:19:56.180203+0000","last_clean_scrub_stamp":"2023-09-23T20:23:36.709512+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:17.481909+0000","scrub_duration":0.0021278949999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":194,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132726+0000","last_change":"2023-09-23T20:23:23.492624+0000","last_active":"2023-09-23T20:24:08.132726+0000","last_peered":"2023-09-23T20:24:08.132726+0000","last_clean":"2023-09-23T20:24:08.132726+0000","last_became_active":"2023-09-23T20:23:23.491878+0000","last_became_peered":"2023-09-23T20:23:23.491878+0000","last_unstale":"2023-09-23T20:24:08.132726+0000","last_undegraded":"2023-09-23T20:24:08.132726+0000","last_fullsized":"2023-09-23T20:24:08.132726+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:56.234441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":341,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.130518+0000","last_change":"2023-09-23T20:22:22.388204+0000","last_active":"2023-09-23T20:24:08.130518+0000","last_peered":"2023-09-23T20:24:08.130518+0000","last_clean":"2023-09-23T20:24:08.130518+0000","last_became_active":"2023-09-23T20:22:22.387640+0000","last_became_peered":"2023-09-23T20:22:22.387640+0000","last_unstale":"2023-09-23T20:24:08.130518+0000","last_undegraded":"2023-09-23T20:24:08.130518+0000","last_fullsized":"2023-09-23T20:24:08.130518+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:58.015528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":721,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123789+0000","last_change":"2023-09-23T20:22:56.760430+0000","last_active":"2023-09-23T20:24:08.123789+0000","last_peered":"2023-09-23T20:24:08.123789+0000","last_clean":"2023-09-23T20:24:08.123789+0000","last_became_active":"2023-09-23T20:22:53.431327+0000","last_became_peered":"2023-09-23T20:22:53.431327+0000","last_unstale":"2023-09-23T20:24:08.123789+0000","last_undegraded":"2023-09-23T20:24:08.123789+0000","last_fullsized":"2023-09-23T20:24:08.123789+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:56.760329+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:42.767336+0000","last_clean_scrub_stamp":"2023-09-23T20:22:56.760329+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:13.013667+0000","scrub_duration":0.001887615,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":628,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125509+0000","last_change":"2023-09-23T20:17:47.296591+0000","last_active":"2023-09-23T20:24:08.125509+0000","last_peered":"2023-09-23T20:24:08.125509+0000","last_clean":"2023-09-23T20:24:08.125509+0000","last_became_active":"2023-09-23T20:17:47.270415+0000","last_became_peered":"2023-09-23T20:17:47.270415+0000","last_unstale":"2023-09-23T20:24:08.125509+0000","last_undegraded":"2023-09-23T20:24:08.125509+0000","last_fullsized":"2023-09-23T20:24:08.125509+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:32.681698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":638,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132922+0000","last_change":"2023-09-23T20:23:43.209497+0000","last_active":"2023-09-23T20:24:08.132922+0000","last_peered":"2023-09-23T20:24:08.132922+0000","last_clean":"2023-09-23T20:24:08.132922+0000","last_became_active":"2023-09-23T20:23:43.207902+0000","last_became_peered":"2023-09-23T20:23:43.207902+0000","last_unstale":"2023-09-23T20:24:08.132922+0000","last_undegraded":"2023-09-23T20:24:08.132922+0000","last_fullsized":"2023-09-23T20:24:08.132922+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:52.439396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":632,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.127584+0000","last_change":"2023-09-23T20:24:06.418341+0000","last_active":"2023-09-23T20:24:08.127584+0000","last_peered":"2023-09-23T20:24:08.127584+0000","last_clean":"2023-09-23T20:24:08.127584+0000","last_became_active":"2023-09-23T20:24:06.413272+0000","last_became_peered":"2023-09-23T20:24:06.413272+0000","last_unstale":"2023-09-23T20:24:08.127584+0000","last_undegraded":"2023-09-23T20:24:08.127584+0000","last_fullsized":"2023-09-23T20:24:08.127584+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:51.417862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":593,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.134481+0000","last_change":"2023-09-23T20:23:43.332907+0000","last_active":"2023-09-23T20:24:08.134481+0000","last_peered":"2023-09-23T20:24:08.134481+0000","last_clean":"2023-09-23T20:24:08.134481+0000","last_became_active":"2023-09-23T20:23:43.332488+0000","last_became_peered":"2023-09-23T20:23:43.332488+0000","last_unstale":"2023-09-23T20:24:08.134481+0000","last_undegraded":"2023-09-23T20:24:08.134481+0000","last_fullsized":"2023-09-23T20:24:08.134481+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:58.187317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"125'162","reported_seq":803,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:24:09.138970+0000","last_change":"2023-09-23T20:24:09.138970+0000","last_active":"2023-09-23T20:24:09.138970+0000","last_peered":"2023-09-23T20:24:09.138970+0000","last_clean":"2023-09-23T20:24:09.138970+0000","last_became_active":"2023-09-23T20:24:09.138546+0000","last_became_peered":"2023-09-23T20:24:09.138546+0000","last_unstale":"2023-09-23T20:24:09.138970+0000","last_undegraded":"2023-09-23T20:24:09.138970+0000","last_fullsized":"2023-09-23T20:24:09.138970+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'162","last_scrub_stamp":"2023-09-23T20:22:58.073284+0000","last_deep_scrub":"125'162","last_deep_scrub_stamp":"2023-09-23T19:53:00.704900+0000","last_clean_scrub_stamp":"2023-09-23T20:22:58.073284+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:50.317070+0000","scrub_duration":0.001452982,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":219,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.124974+0000","last_change":"2023-09-23T20:24:06.396306+0000","last_active":"2023-09-23T20:24:08.124974+0000","last_peered":"2023-09-23T20:24:08.124974+0000","last_clean":"2023-09-23T20:24:08.124974+0000","last_became_active":"2023-09-23T20:24:06.395984+0000","last_became_peered":"2023-09-23T20:24:06.395984+0000","last_unstale":"2023-09-23T20:24:08.124974+0000","last_undegraded":"2023-09-23T20:24:08.124974+0000","last_fullsized":"2023-09-23T20:24:08.124974+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:45.963073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":391,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.123920+0000","last_change":"2023-09-23T20:23:36.169691+0000","last_active":"2023-09-23T20:24:08.123920+0000","last_peered":"2023-09-23T20:24:08.123920+0000","last_clean":"2023-09-23T20:24:08.123920+0000","last_became_active":"2023-09-23T20:23:36.169387+0000","last_became_peered":"2023-09-23T20:23:36.169387+0000","last_unstale":"2023-09-23T20:24:08.123920+0000","last_undegraded":"2023-09-23T20:24:08.123920+0000","last_fullsized":"2023-09-23T20:24:08.123920+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:54.707421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":828,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.125668+0000","last_change":"2023-09-23T20:22:57.574748+0000","last_active":"2023-09-23T20:24:08.125668+0000","last_peered":"2023-09-23T20:24:08.125668+0000","last_clean":"2023-09-23T20:24:08.125668+0000","last_became_active":"2023-09-23T20:22:53.425726+0000","last_became_peered":"2023-09-23T20:22:53.425726+0000","last_unstale":"2023-09-23T20:24:08.125668+0000","last_undegraded":"2023-09-23T20:24:08.125668+0000","last_fullsized":"2023-09-23T20:24:08.125668+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":508,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:22:57.574606+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:12:52.744296+0000","last_clean_scrub_stamp":"2023-09-23T20:22:57.574606+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:55.820932+0000","scrub_duration":0.0033644019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":225,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.133048+0000","last_change":"2023-09-23T20:23:23.494966+0000","last_active":"2023-09-23T20:24:08.133048+0000","last_peered":"2023-09-23T20:24:08.133048+0000","last_clean":"2023-09-23T20:24:08.133048+0000","last_became_active":"2023-09-23T20:23:23.494649+0000","last_became_peered":"2023-09-23T20:23:23.494649+0000","last_unstale":"2023-09-23T20:24:08.133048+0000","last_undegraded":"2023-09-23T20:24:08.133048+0000","last_fullsized":"2023-09-23T20:24:08.133048+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:38.513158+0000","last_clean_scrub_stamp":"2023-09-23T19:41:38.513158+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:56.958751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":502,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:24:08.132725+0000","last_change":"2023-09-23T20:24:06.403361+0000","last_active":"2023-09-23T20:24:08.132725+0000","last_peered":"2023-09-23T20:24:08.132725+0000","last_clean":"2023-09-23T20:24:08.132725+0000","last_became_active":"2023-09-23T20:24:06.403032+0000","last_became_peered":"2023-09-23T20:24:06.403032+0000","last_unstale":"2023-09-23T20:24:08.132725+0000","last_undegraded":"2023-09-23T20:24:08.132725+0000","last_fullsized":"2023-09-23T20:24:08.132725+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:41:39.520007+0000","last_clean_scrub_stamp":"2023-09-23T19:41:39.520007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:37.993783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":19866,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41405,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4995,"ondisk_log_size":4995,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11978,"num_read_kb":26422,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":8388722,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6171,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234524,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":609324,"kb_used_data":14564,"kb_used_omap":0,"kb_used_meta":594752,"kb_avail":93135828,"statfs":{"total":95995035648,"available":95371087872,"internally_reserved":0,"allocated":14913536,"data_stored":13286875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":609026048},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":5.79,"15min":4.4740000000000002},"min":{"1min":0.33800000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.92100000000000004,"5min":230.952,"15min":230.952},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.53200000000000003,"5min":6.1459999999999999,"15min":4.7329999999999997},"min":{"1min":0.379,"5min":0.224,"15min":0.222},"max":{"1min":0.73499999999999999,"5min":231.08699999999999,"15min":231.08699999999999},"last":0.73099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.5049999999999999,"15min":2.1429999999999998},"min":{"1min":0.371,"5min":0.253,"15min":0.21299999999999999},"max":{"1min":0.85199999999999998,"5min":61.113999999999997,"15min":91.335999999999999},"last":0.745},{"interface":"front","average":{"1min":0.52600000000000002,"5min":1.5329999999999999,"15min":2.4060000000000001},"min":{"1min":0.39800000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.79800000000000004,"5min":61.082999999999998,"15min":91.516999999999996},"last":0.69199999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51700000000000002,"15min":0.51800000000000002},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.64000000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.55700000000000005,"15min":0.54300000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.78000000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.53,"15min":3.089},"min":{"1min":0.433,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.95399999999999996,"5min":25.977,"15min":185.17099999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.716,"15min":3.3100000000000001},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.29699999999999999},"max":{"1min":0.86799999999999999,"5min":26.481999999999999,"15min":184.99100000000001},"last":0.75700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.161,"15min":0.95299999999999996},"min":{"1min":0.378,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66300000000000003,"5min":83.811999999999998,"15min":83.811999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.2290000000000001,"15min":1.0609999999999999},"min":{"1min":0.42199999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.90000000000000002,"5min":84.620000000000005,"15min":84.620000000000005},"last":0.56599999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.53400000000000003,"15min":1.0349999999999999},"min":{"1min":0.38500000000000001,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":0.77900000000000003,"5min":3.794,"15min":16.327000000000002},"last":0.622},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.48299999999999998,"15min":0.95499999999999996},"min":{"1min":0.45500000000000002,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":0.72699999999999998,"5min":2.1190000000000002,"15min":16.716000000000001},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.67600000000000005}]}]},{"osd":6,"up_from":551,"seq":2366526980122,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37468,"kb_used_data":10516,"kb_used_omap":9,"kb_used_meta":26934,"kb_avail":93707684,"statfs":{"total":95995035648,"available":95956668416,"internally_reserved":0,"allocated":10768384,"data_stored":9157229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9991,"internal_metadata":27580665},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.61899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.71999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.744,"5min":1.744,"15min":1.744},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":0.63300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.46999999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.505},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.69799999999999995}]}]},{"osd":1,"up_from":99,"seq":425201762723,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856792,"kb_used_data":11408,"kb_used_omap":7,"kb_used_meta":845368,"kb_avail":92888360,"statfs":{"total":95995035648,"available":95117680640,"internally_reserved":0,"allocated":11681792,"data_stored":8967757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":865657658},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":1.7130000000000001,"15min":2.8660000000000001},"min":{"1min":0.23599999999999999,"5min":0.191,"15min":0.185},"max":{"1min":1.5780000000000001,"5min":62.710999999999999,"15min":145.81299999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":2.0489999999999999,"15min":3.105},"min":{"1min":0.24199999999999999,"5min":0.20000000000000001,"15min":0.19800000000000001},"max":{"1min":1.0089999999999999,"5min":62.610999999999997,"15min":146.83799999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:24:25 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.308,"5min":0.249,"15min":0.249},"max":{"1min":0.84099999999999997,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41699999999999998,"15min":0.41899999999999998},"min":{"1min":0.314,"5min":0.254,"15min":0.254},"max":{"1min":0.67900000000000005,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.29699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":2.9409999999999998,"15min":2.8330000000000002},"min":{"1min":0.28399999999999997,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.69899999999999995,"5min":174.578,"15min":174.578},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":2.7719999999999998,"15min":2.1819999999999999},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.88,"5min":174.822,"15min":174.822},"last":0.31}]},{"osd":4,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.97899999999999998,"15min":1.4410000000000001},"min":{"1min":0.38100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.5089999999999999,"5min":10.387,"15min":29.791},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.0229999999999999,"15min":1.341},"min":{"1min":0.39200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.91400000000000003,"5min":10.422000000000001,"15min":27.143999999999998},"last":0.61699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.0960000000000001,"15min":1.76},"min":{"1min":0.433,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":0.94799999999999995,"5min":10.461,"15min":83.510999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":1.0680000000000001,"15min":1.8700000000000001},"min":{"1min":0.40600000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.89700000000000002,"5min":12.648,"15min":83.353999999999999},"last":0.55900000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.629},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.843,"5min":2.843,"15min":2.843},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:24:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.0149999999999999,"15min":2.1160000000000001},"min":{"1min":0.42099999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.84599999999999997,"5min":12.599,"15min":127.03100000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.038,"15min":1.9019999999999999},"min":{"1min":0.505,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.4630000000000001,"5min":10.340999999999999,"15min":125.095},"last":0.51200000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926261,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1059168,"kb_used_data":11672,"kb_used_omap":8,"kb_used_meta":1047479,"kb_avail":92685984,"statfs":{"total":95995035648,"available":94910447616,"internally_reserved":0,"allocated":11952128,"data_stored":9289956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":1072619105},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":1.526,"15min":1.9590000000000001},"min":{"1min":0.245,"5min":0.20899999999999999,"15min":0.17599999999999999},"max":{"1min":0.41699999999999998,"5min":39.853999999999999,"15min":72.534000000000006},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.33000000000000002,"5min":1.851,"15min":2.4100000000000001},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":0.442,"5min":39.872999999999998,"15min":72.667000000000002},"last":0.28100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.375,"15min":0.377},"min":{"1min":0.24099999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.51700000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40400000000000003,"15min":0.41199999999999998},"min":{"1min":0.28199999999999997,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.51300000000000001,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 20:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":3.8149999999999999,"15min":2.754},"min":{"1min":0.29499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.45700000000000002,"5min":201.655,"15min":201.655},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":3.7109999999999999,"15min":2.8660000000000001},"min":{"1min":0.26900000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.49199999999999999,"5min":201.69399999999999,"15min":201.69399999999999},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.611,"15min":1.5549999999999999},"min":{"1min":0.35199999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.755,"5min":16.593,"15min":16.593},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.613,"15min":1.754},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.27600000000000002},"max":{"1min":0.82499999999999996,"5min":21.899000000000001,"15min":105.282},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":2.456,"15min":1.8720000000000001},"min":{"1min":0.443,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2390000000000001,"5min":145.14500000000001,"15min":145.14500000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":3.1040000000000001,"15min":2.2970000000000002},"min":{"1min":0.42999999999999999,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":0.80000000000000004,"5min":145.04499999999999,"15min":145.04499999999999},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.63800000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.98999999999999999,"15min":1.663},"min":{"1min":0.503,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.2130000000000001,"5min":6.3449999999999998,"15min":99.486000000000004},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.7829999999999999,"15min":2.2149999999999999},"min":{"1min":0.46000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.75600000000000001,"5min":16.617000000000001,"15min":99.668999999999997},"last":0.56000000000000005}]}]},{"osd":2,"up_from":543,"seq":2332167241759,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":150720,"kb_used_data":11640,"kb_used_omap":16,"kb_used_meta":139055,"kb_avail":93594432,"statfs":{"total":95995035648,"available":95840698368,"internally_reserved":0,"allocated":11919360,"data_stored":9229890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16601,"internal_metadata":142393127},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.315,"5min":0.33600000000000002,"15min":0.34000000000000002},"min":{"1min":0.188,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.47199999999999998,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.432},{"interface":"front","average":{"1min":0.32600000000000001,"5min":0.33500000000000002,"15min":0.33700000000000002},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.47099999999999997,"5min":0.504,"15min":0.504},"last":0.45500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.378,"15min":0.38400000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.48699999999999999,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.53600000000000003,"5min":1.089,"15min":1.089},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.377,"15min":0.379},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.51700000000000002,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.498},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.42199999999999999,"15min":0.42799999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.53200000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.54600000000000004,"15min":0.53600000000000003},"min":{"1min":0.432,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.61099999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.40400000000000003,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.56899999999999995,"15min":0.56200000000000006},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.67800000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.76000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.60799999999999998,"15min":0.60199999999999998},"min":{"1min":0.46300000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.876,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.72899999999999998,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.75}]}]},{"osd":3,"up_from":109,"seq":468151435673,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1029180,"kb_used_data":7988,"kb_used_omap":4,"kb_used_meta":1021179,"kb_avail":92715972,"statfs":{"total":95995035648,"available":94941155328,"internally_reserved":0,"allocated":8179712,"data_stored":5139996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5011,"internal_metadata":1045687405},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.32300000000000001,"5min":4.8630000000000004,"15min":3.6230000000000002},"min":{"1min":0.23000000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":0.44900000000000001,"5min":219.80000000000001,"15min":219.80000000000001},"last":0.317},{"interface":"front","average":{"1min":0.33000000000000002,"5min":4.875,"15min":3.927},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.41699999999999998,"5min":221.36000000000001,"15min":221.36000000000001},"last":0.23400000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":1.2649999999999999,"15min":3.3239999999999998},"min":{"1min":0.223,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.50600000000000001,"5min":9.673,"15min":183.87},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.376,"5min":1.3080000000000001,"15min":3.48},"min":{"1min":0.26000000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.70999999999999996,"5min":19.513000000000002,"15min":185.84899999999999},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.39400000000000002,"15min":0.40000000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.45900000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.40999999999999998,"15min":0.41399999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.51000000000000001,"5min":0.54600000000000004,"15min":0.54600000000000004},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.5109999999999999,"15min":1.6479999999999999},"min":{"1min":0.377,"5min":0.35499999999999998,"15min":0.28999999999999998},"max":{"1min":0.85599999999999998,"5min":54.046999999999997,"15min":58.472999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.7130000000000001,"15min":2.0659999999999998},"min":{"1min":0.36799999999999999,"5min":0.34699999999999998,"15min":0.28299999999999997},"max":{"1min":0.92500000000000004,"5min":54.311,"15min":58.408000000000001},"last":0.66300000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.7390000000000001,"15min":2.8140000000000001},"min":{"1min":0.38800000000000001,"5min":0.32000000000000001,"15min":0.30299999999999999},"max":{"1min":0.86199999999999999,"5min":54.139000000000003,"15min":200.53200000000001},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.5189999999999999,"15min":2.7480000000000002},"min":{"1min":0.44400000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.70699999999999996,"5min":56.457999999999998,"15min":200.846},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:24:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.41299999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73099999999999998,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.60599999999999998,"15min":0.60399999999999998},"min":{"1min":0.41599999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.81599999999999995,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.69999999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.93400000000000005,"15min":2.1949999999999998},"min":{"1min":0.44,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77500000000000002,"5min":7.6079999999999997,"15min":139.465},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.92700000000000005,"15min":2.1230000000000002},"min":{"1min":0.432,"5min":0.36899999999999999,"15min":0.32800000000000001},"max":{"1min":0.85799999999999998,"5min":8.9019999999999992,"15min":141.56200000000001},"last":0.57999999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019434,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":583268,"kb_used_data":6108,"kb_used_omap":0,"kb_used_meta":577152,"kb_avail":93161884,"statfs":{"total":95995035648,"available":95397769216,"internally_reserved":0,"allocated":6254592,"data_stored":4635567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":591003648},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":5.3520000000000003,"15min":4.0460000000000003},"min":{"1min":0.34200000000000003,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.77100000000000002,"5min":153.108,"15min":232.89500000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":5.2830000000000004,"15min":4.0469999999999997},"min":{"1min":0.39600000000000002,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":0.74199999999999999,"5min":152.495,"15min":232.851},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.234,"15min":2.331},"min":{"1min":0.35099999999999998,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":0.623,"5min":26.420000000000002,"15min":138.47999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.109,"15min":2.331},"min":{"1min":0.35799999999999998,"5min":0.25,"15min":0.217},"max":{"1min":0.78900000000000003,"5min":26.388999999999999,"15min":138.429},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:24:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":8.3379999999999992,"15min":9.6329999999999991},"min":{"1min":0.41799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.71299999999999997,"5min":196.80500000000001,"15min":196.80500000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":8.4849999999999994,"15min":9.8010000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78400000000000003,"5min":196.77600000000001,"15min":196.77600000000001},"last":0.66100000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":5.5789999999999997,"15min":3.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.2390000000000001,"5min":156.72300000000001,"15min":156.72300000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":5.6120000000000001,"15min":4.0570000000000004},"min":{"1min":0.377,"5min":0.33200000000000002,"15min":0.32900000000000001},"max":{"1min":0.68999999999999995,"5min":155.84800000000001,"15min":155.84800000000001},"last":0.54700000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.49299999999999999,"15min":1.073},"min":{"1min":0.28699999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64600000000000002,"5min":4.3970000000000002,"15min":114.788},"last":0.626},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48899999999999999,"15min":1.103},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.77200000000000002,"5min":3.5590000000000002,"15min":113.822},"last":0.71099999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:23:44 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.75600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.504,"15min":1.1679999999999999},"min":{"1min":0.45500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":113.861},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.629,"5min":0.57699999999999996,"15min":1.2290000000000001},"min":{"1min":0.47799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.81399999999999995,"5min":4.6710000000000003,"15min":114.32299999999999},"last":0.77000000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757798,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":655260,"kb_used_data":6356,"kb_used_omap":0,"kb_used_meta":648896,"kb_avail":93089892,"statfs":{"total":95995035648,"available":95324049408,"internally_reserved":0,"allocated":6508544,"data_stored":4893960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":664469504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":4.5860000000000003,"15min":4.1120000000000001},"min":{"1min":0.375,"5min":0.253,"15min":0.20699999999999999},"max":{"1min":0.77400000000000002,"5min":152.71299999999999,"15min":166.57900000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":6.4580000000000002,"15min":4.6470000000000002},"min":{"1min":0.35299999999999998,"5min":0.22,"15min":0.21099999999999999},"max":{"1min":0.81299999999999994,"5min":256.52199999999999,"15min":256.52199999999999},"last":0.66200000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.995,"15min":2.694},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.22},"max":{"1min":1.165,"5min":8.5289999999999999,"15min":150.16999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.0109999999999999,"15min":2.641},"min":{"1min":0.34200000000000003,"5min":0.27300000000000002,"15min":0.21199999999999999},"max":{"1min":1.4330000000000001,"5min":8.3420000000000005,"15min":150.12700000000001},"last":0.58999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.38100000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.74299999999999999,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.55200000000000005,"15min":0.54100000000000004},"min":{"1min":0.38500000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":3.476,"15min":5.2450000000000001},"min":{"1min":0.32000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.69399999999999995,"5min":155.97300000000001,"15min":196.97800000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":3.4199999999999999,"15min":5.2000000000000002},"min":{"1min":0.36699999999999999,"5min":0.35199999999999998,"15min":0.29999999999999999},"max":{"1min":0.70999999999999996,"5min":155.803,"15min":196.875},"last":0.71699999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53900000000000003,"15min":0.72999999999999998},"min":{"1min":0.35899999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70499999999999996,"5min":5.7279999999999998,"15min":8},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.502,"15min":0.56100000000000005},"min":{"1min":0.42199999999999999,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":0.72499999999999998,"5min":2.6890000000000001,"15min":3.371},"last":0.749}]},{"osd":6,"last update":"Sat Sep 23 20:23:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.73499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.60599999999999998,"15min":0.94099999999999995},"min":{"1min":0.35099999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.8860000000000001,"5min":5.0469999999999997,"15min":11.725},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.54800000000000004,"15min":0.871},"min":{"1min":0.41699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.7989999999999999,"5min":3.0880000000000001,"15min":11.041},"last":0.76000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4746,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1425,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8607,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7366,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11855,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3586,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9991,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:24:49.470 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:24:49.470 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T20:24:49.536 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=1a498cc713e249dfa3ac45d5db36436a Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T20:24:49.538 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T20:24:49.549 INFO:tasks.cephadm:Teardown begin 2023-09-23T20:24:49.550 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:24:49.613 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:24:49.937 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T20:24:49.937 DEBUG:teuthology.orchestra.run.smithi135:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:24:49.954 DEBUG:teuthology.orchestra.run.smithi144:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:24:49.975 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T20:24:49.975 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T20:24:49.975 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T20:24:50.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:50 smithi135 systemd[1]: Stopping Ceph mon.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:50.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:50 smithi135 conmon[213845]: 2023-09-23T20:24:50.259+0000 7f970e12c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:24:50.269 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:50 smithi135 conmon[213845]: 2023-09-23T20:24:50.259+0000 7f970e12c700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T20:24:50.632 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:50 smithi135 conmon[201366]: [23/Sep/2023:20:24:50] ENGINE Bus STOPPING 2023-09-23T20:24:50.632 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:50 smithi135 conmon[201366]: [23/Sep/2023:20:24:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:24:50.632 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:50 smithi135 conmon[201366]: [23/Sep/2023:20:24:50] ENGINE Bus STOPPED 2023-09-23T20:24:50.632 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:50 smithi135 conmon[201366]: [23/Sep/2023:20:24:50] ENGINE Bus STARTING 2023-09-23T20:24:50.632 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:50 smithi135 conmon[201366]: [23/Sep/2023:20:24:50] ENGINE Serving on http://:::9283 2023-09-23T20:24:50.632 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:50 smithi135 conmon[201366]: [23/Sep/2023:20:24:50] ENGINE Bus STARTED 2023-09-23T20:24:50.970 INFO:journalctl@ceph.mon.a.smithi135.stdout:Sep 23 20:24:50 smithi135 bash[710399]: 7fa1f78d472573b2351f8e7ad7cd571eee8e96cc94a0c3a0a69716e892041863 2023-09-23T20:24:51.312 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T20:24:51.353 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:24:51.353 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T20:24:51.354 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T20:24:51.354 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T20:24:51.665 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:51 smithi135 systemd[1]: Stopping Ceph mon.c for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:51.963 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:51 smithi135 conmon[221089]: 2023-09-23T20:24:51.662+0000 7f73b00fb700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:24:51.964 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:51 smithi135 conmon[221089]: 2023-09-23T20:24:51.662+0000 7f73b00fb700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T20:24:51.964 INFO:journalctl@ceph.mon.c.smithi135.stdout:Sep 23 20:24:51 smithi135 bash[710708]: 6b8308cd56e0454f441271967312d5e96ceeb99846fed25d0daa93ed355ef162 2023-09-23T20:24:51.964 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:51 smithi135 conmon[201366]: [23/Sep/2023:20:24:51] ENGINE Bus STOPPING 2023-09-23T20:24:51.964 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:51 smithi135 conmon[201366]: [23/Sep/2023:20:24:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:24:51.964 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:51 smithi135 conmon[201366]: [23/Sep/2023:20:24:51] ENGINE Bus STOPPED 2023-09-23T20:24:51.965 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:51 smithi135 conmon[201366]: [23/Sep/2023:20:24:51] ENGINE Bus STARTING 2023-09-23T20:24:51.965 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:51 smithi135 conmon[201366]: [23/Sep/2023:20:24:51] ENGINE Serving on http://:::9283 2023-09-23T20:24:51.965 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:51 smithi135 conmon[201366]: [23/Sep/2023:20:24:51] ENGINE Bus STARTED 2023-09-23T20:24:52.421 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T20:24:52.459 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:24:52.459 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T20:24:52.459 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T20:24:52.460 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T20:24:52.816 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:52 smithi144 systemd[1]: Stopping Ceph mon.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:52 smithi144 conmon[194809]: 2023-09-23T20:24:52.815+0000 7f8331fbc700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:24:53.132 INFO:journalctl@ceph.mon.b.smithi144.stdout:Sep 23 20:24:52 smithi144 conmon[194809]: 2023-09-23T20:24:52.815+0000 7f8331fbc700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T20:24:53.641 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T20:24:53.682 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:24:53.682 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T20:24:53.683 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T20:24:53.683 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T20:24:53.970 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:53 smithi135 conmon[201366]: ::ffff:172.21.15.144 - - [23/Sep/2023:20:24:53] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.43.0" 2023-09-23T20:24:53.971 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:53 smithi135 systemd[1]: Stopping Ceph mgr.y for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:54.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:53 smithi135 conmon[201366]: 2023-09-23T20:24:53.977+0000 7fcf8530c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:24:54.471 INFO:journalctl@ceph.mgr.y.smithi135.stdout:Sep 23 20:24:53 smithi135 conmon[201366]: 2023-09-23T20:24:53.977+0000 7fcf8530c700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-09-23T20:24:54.812 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T20:24:54.849 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:24:54.850 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T20:24:54.850 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T20:24:54.850 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T20:24:55.131 INFO:journalctl@ceph.mgr.x.smithi144.stdout:Sep 23 20:24:54 smithi144 systemd[1]: Stopping Ceph mgr.x for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:55.962 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T20:24:56.001 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:24:56.001 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T20:24:56.001 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T20:24:56.001 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T20:24:56.287 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 20:24:56 smithi135 systemd[1]: Stopping Ceph osd.0 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:56.720 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 20:24:56 smithi135 conmon[235304]: 2023-09-23T20:24:56.285+0000 7fe34ddac700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:24:56.721 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 20:24:56 smithi135 conmon[235304]: 2023-09-23T20:24:56.285+0000 7fe34ddac700 -1 osd.0 597 *** Got signal Terminated *** 2023-09-23T20:24:56.721 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 20:24:56 smithi135 conmon[235304]: 2023-09-23T20:24:56.285+0000 7fe34ddac700 -1 osd.0 597 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:25:02.970 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 20:25:02 smithi135 bash[711316]: 28ca011335d607c5ae8af5254f8b5921b8fb53099a87b43ae6c625791925e59e 2023-09-23T20:25:04.452 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T20:25:04.478 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 20:25:04 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T20:25:04.478 INFO:journalctl@ceph.osd.0.smithi135.stdout:Sep 23 20:25:04 smithi135 systemd[1]: Stopped Ceph osd.0 for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T20:25:04.492 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:25:04.492 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T20:25:04.493 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T20:25:04.493 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:25:04.796 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 20:25:04 smithi135 systemd[1]: Stopping Ceph osd.1 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:05.220 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 20:25:04 smithi135 conmon[243900]: 2023-09-23T20:25:04.794+0000 7eff03a33700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:25:05.221 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 20:25:04 smithi135 conmon[243900]: 2023-09-23T20:25:04.794+0000 7eff03a33700 -1 osd.1 597 *** Got signal Terminated *** 2023-09-23T20:25:05.221 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 20:25:04 smithi135 conmon[243900]: 2023-09-23T20:25:04.794+0000 7eff03a33700 -1 osd.1 597 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:25:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:25:09 smithi144 conmon[178718]: ts=2023-09-23T20:25:09.549Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.135:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.135:8765: connect: connection refused" 2023-09-23T20:25:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:25:09 smithi144 conmon[178718]: ts=2023-09-23T20:25:09.549Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.135:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.135:8765: connect: connection refused" 2023-09-23T20:25:09.882 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:25:09 smithi144 conmon[178718]: ts=2023-09-23T20:25:09.550Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.135:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.135:8765: connect: connection refused" 2023-09-23T20:25:09.883 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:25:09 smithi144 conmon[178718]: ts=2023-09-23T20:25:09.550Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.135:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.135:8765: connect: connection refused" 2023-09-23T20:25:11.471 INFO:journalctl@ceph.osd.1.smithi135.stdout:Sep 23 20:25:11 smithi135 bash[711894]: aa8c0ac7e9ea7c1f63854f6cc83a86ec2c95b5e3f4330a1c615187ce3c39290b 2023-09-23T20:25:12.592 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T20:25:12.633 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:25:12.633 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T20:25:12.633 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T20:25:12.633 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T20:25:12.932 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:25:12 smithi135 systemd[1]: Stopping Ceph osd.2 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:13.220 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:25:12 smithi135 conmon[673209]: 2023-09-23T20:25:12.930+0000 7ff12d34e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:25:13.221 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:25:12 smithi135 conmon[673209]: 2023-09-23T20:25:12.931+0000 7ff12d34e700 -1 osd.2 597 *** Got signal Terminated *** 2023-09-23T20:25:13.221 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:25:12 smithi135 conmon[673209]: 2023-09-23T20:25:12.931+0000 7ff12d34e700 -1 osd.2 597 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:25:18.721 INFO:journalctl@ceph.osd.2.smithi135.stdout:Sep 23 20:25:18 smithi135 bash[712476]: 4236ae85039f7fdfbabc1c46d6400d9f5852ab80f86fb86b309d59df93e8a0f0 2023-09-23T20:25:20.284 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T20:25:20.325 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:25:20.325 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T20:25:20.325 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T20:25:20.325 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T20:25:20.626 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 20:25:20 smithi135 systemd[1]: Stopping Ceph osd.3 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:20.970 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 20:25:20 smithi135 conmon[260414]: 2023-09-23T20:25:20.624+0000 7fd857604700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:25:20.971 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 20:25:20 smithi135 conmon[260414]: 2023-09-23T20:25:20.624+0000 7fd857604700 -1 osd.3 597 *** Got signal Terminated *** 2023-09-23T20:25:20.971 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 20:25:20 smithi135 conmon[260414]: 2023-09-23T20:25:20.624+0000 7fd857604700 -1 osd.3 597 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:25:23.362 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:23 smithi144 conmon[121943]: 2023-09-23T20:25:23.291+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:23.363 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:23 smithi144 conmon[400857]: 2023-09-23T20:25:23.048+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:23.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:23 smithi144 conmon[141003]: 2023-09-23T20:25:23.362+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:24.299 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:24 smithi144 conmon[400857]: 2023-09-23T20:25:24.006+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:24.300 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:24 smithi144 conmon[113051]: 2023-09-23T20:25:24.078+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:24.631 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:24 smithi144 conmon[121943]: 2023-09-23T20:25:24.299+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:24.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:24 smithi144 conmon[141003]: 2023-09-23T20:25:24.406+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:25.331 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:25 smithi144 conmon[400857]: 2023-09-23T20:25:25.046+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:25.331 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:25 smithi144 conmon[113051]: 2023-09-23T20:25:25.099+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:25.631 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:25 smithi144 conmon[121943]: 2023-09-23T20:25:25.330+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:25.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:25 smithi144 conmon[141003]: 2023-09-23T20:25:25.380+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:26.347 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:26 smithi144 conmon[400857]: 2023-09-23T20:25:26.077+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:26.347 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:26 smithi144 conmon[121943]: 2023-09-23T20:25:26.300+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:26.348 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:26 smithi144 conmon[113051]: 2023-09-23T20:25:26.081+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:26.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:26 smithi144 conmon[141003]: 2023-09-23T20:25:26.346+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:26.720 INFO:journalctl@ceph.osd.3.smithi135.stdout:Sep 23 20:25:26 smithi135 bash[713067]: 8bbface9be59e32010cc1d2a386581ccdf17d6b94e329c7562be3448353e8164 2023-09-23T20:25:27.326 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:27 smithi144 conmon[400857]: 2023-09-23T20:25:27.122+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:27.326 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:27 smithi144 conmon[113051]: 2023-09-23T20:25:27.065+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:27.631 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:27 smithi144 conmon[121943]: 2023-09-23T20:25:27.325+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:27.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:27 smithi144 conmon[141003]: 2023-09-23T20:25:27.357+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:28.266 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T20:25:28.307 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:25:28.307 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T20:25:28.307 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T20:25:28.308 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T20:25:28.322 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:28 smithi144 conmon[113051]: 2023-09-23T20:25:28.037+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:28.322 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:28 smithi144 conmon[400857]: 2023-09-23T20:25:28.073+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:28.602 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:28 smithi144 conmon[121943]: 2023-09-23T20:25:28.320+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:28.602 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:28 smithi144 conmon[141003]: 2023-09-23T20:25:28.329+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:28.603 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:28 smithi144 systemd[1]: Stopping Ceph osd.4 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:28.881 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:28 smithi144 conmon[113051]: 2023-09-23T20:25:28.600+0000 7f609454f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:25:28.882 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:28 smithi144 conmon[113051]: 2023-09-23T20:25:28.600+0000 7f609454f700 -1 osd.4 597 *** Got signal Terminated *** 2023-09-23T20:25:28.882 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:28 smithi144 conmon[113051]: 2023-09-23T20:25:28.600+0000 7f609454f700 -1 osd.4 597 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:25:29.341 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:29 smithi144 conmon[400857]: 2023-09-23T20:25:29.090+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:29.341 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:29 smithi144 conmon[113051]: 2023-09-23T20:25:29.003+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:29.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:29 smithi144 conmon[121943]: 2023-09-23T20:25:29.339+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:29.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:29 smithi144 conmon[141003]: 2023-09-23T20:25:29.340+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:30.366 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:30 smithi144 conmon[400857]: 2023-09-23T20:25:30.115+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:30.367 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:29 smithi144 conmon[113051]: 2023-09-23T20:25:29.996+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:30.631 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:30 smithi144 conmon[121943]: 2023-09-23T20:25:30.365+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:30.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:30 smithi144 conmon[141003]: 2023-09-23T20:25:30.383+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:31.346 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:31 smithi144 conmon[400857]: 2023-09-23T20:25:31.139+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:31.347 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:31 smithi144 conmon[400857]: 2023-09-23T20:25:31.139+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:31.347 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:31 smithi144 conmon[113051]: 2023-09-23T20:25:31.041+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:31.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:31 smithi144 conmon[121943]: 2023-09-23T20:25:31.399+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:31.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:31 smithi144 conmon[121943]: 2023-09-23T20:25:31.399+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:31.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:31 smithi144 conmon[141003]: 2023-09-23T20:25:31.345+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:31.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:31 smithi144 conmon[141003]: 2023-09-23T20:25:31.345+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:32.373 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:32 smithi144 conmon[400857]: 2023-09-23T20:25:32.154+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:32.373 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:32 smithi144 conmon[400857]: 2023-09-23T20:25:32.154+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:32.373 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:32 smithi144 conmon[113051]: 2023-09-23T20:25:32.073+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:32.631 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:32 smithi144 conmon[121943]: 2023-09-23T20:25:32.433+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:32.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:32 smithi144 conmon[121943]: 2023-09-23T20:25:32.433+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:32.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:32 smithi144 conmon[141003]: 2023-09-23T20:25:32.371+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:32.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:32 smithi144 conmon[141003]: 2023-09-23T20:25:32.371+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:33.328 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:33 smithi144 conmon[400857]: 2023-09-23T20:25:33.188+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:33.328 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:33 smithi144 conmon[400857]: 2023-09-23T20:25:33.188+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:33.328 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:33 smithi144 conmon[113051]: 2023-09-23T20:25:33.059+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.756895+0000 front 2023-09-23T20:25:00.756880+0000 (oldest deadline 2023-09-23T20:25:23.656727+0000) 2023-09-23T20:25:33.328 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:33 smithi144 conmon[113051]: 2023-09-23T20:25:33.059+0000 7f6092350700 -1 osd.4 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:09.259451+0000 front 2023-09-23T20:25:09.259512+0000 (oldest deadline 2023-09-23T20:25:32.159476+0000) 2023-09-23T20:25:33.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:33 smithi144 conmon[121943]: 2023-09-23T20:25:33.414+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:33.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:33 smithi144 conmon[121943]: 2023-09-23T20:25:33.414+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:33.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:33 smithi144 conmon[141003]: 2023-09-23T20:25:33.327+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:33.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:33 smithi144 conmon[141003]: 2023-09-23T20:25:33.327+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:34.477 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:34 smithi144 conmon[121943]: 2023-09-23T20:25:34.433+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:34.477 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:34 smithi144 conmon[121943]: 2023-09-23T20:25:34.433+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:34.477 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:34 smithi144 conmon[141003]: 2023-09-23T20:25:34.340+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:34.477 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:34 smithi144 conmon[141003]: 2023-09-23T20:25:34.340+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:34.478 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:34 smithi144 conmon[400857]: 2023-09-23T20:25:34.185+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:34.478 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:34 smithi144 conmon[400857]: 2023-09-23T20:25:34.185+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:34.882 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 23 20:25:34 smithi144 bash[408799]: 076c88452d16ee305f420a9035292d2360aa74937172f303fa4e988cd0303387 2023-09-23T20:25:35.498 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:35 smithi144 conmon[121943]: 2023-09-23T20:25:35.396+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:35.499 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:35 smithi144 conmon[121943]: 2023-09-23T20:25:35.396+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:35.499 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:35 smithi144 conmon[141003]: 2023-09-23T20:25:35.317+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:35.499 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:35 smithi144 conmon[141003]: 2023-09-23T20:25:35.317+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:35.500 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:35 smithi144 conmon[400857]: 2023-09-23T20:25:35.197+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:35.500 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:35 smithi144 conmon[400857]: 2023-09-23T20:25:35.197+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:36.591 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[121943]: 2023-09-23T20:25:36.429+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:36.592 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[121943]: 2023-09-23T20:25:36.429+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:36.592 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[141003]: 2023-09-23T20:25:36.301+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:36.592 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[141003]: 2023-09-23T20:25:36.301+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:36.592 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[400857]: 2023-09-23T20:25:36.229+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:36.593 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[400857]: 2023-09-23T20:25:36.229+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:36.599 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T20:25:36.682 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:25:36.682 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T20:25:36.682 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T20:25:36.682 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T20:25:36.882 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:36 smithi144 systemd[1]: Stopping Ceph osd.5 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:37.271 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:37 smithi144 conmon[400857]: 2023-09-23T20:25:37.190+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:37.272 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:37 smithi144 conmon[400857]: 2023-09-23T20:25:37.190+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:37.272 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[121943]: 2023-09-23T20:25:36.990+0000 7ff671e22700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:25:37.272 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[121943]: 2023-09-23T20:25:36.990+0000 7ff671e22700 -1 osd.5 597 *** Got signal Terminated *** 2023-09-23T20:25:37.272 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:36 smithi144 conmon[121943]: 2023-09-23T20:25:36.990+0000 7ff671e22700 -1 osd.5 597 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:25:37.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:37 smithi144 conmon[121943]: 2023-09-23T20:25:37.385+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:37.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:37 smithi144 conmon[121943]: 2023-09-23T20:25:37.385+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:37.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:37 smithi144 conmon[141003]: 2023-09-23T20:25:37.271+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:37.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:37 smithi144 conmon[141003]: 2023-09-23T20:25:37.271+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:38.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:38 smithi144 conmon[121943]: 2023-09-23T20:25:38.408+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:38.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:38 smithi144 conmon[121943]: 2023-09-23T20:25:38.408+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:38.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:38 smithi144 conmon[141003]: 2023-09-23T20:25:38.280+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:38.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:38 smithi144 conmon[141003]: 2023-09-23T20:25:38.280+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:38.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:38 smithi144 conmon[400857]: 2023-09-23T20:25:38.237+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:38.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:38 smithi144 conmon[400857]: 2023-09-23T20:25:38.237+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:39.631 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:39 smithi144 conmon[121943]: 2023-09-23T20:25:39.457+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:39.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:39 smithi144 conmon[121943]: 2023-09-23T20:25:39.457+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:39.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:39 smithi144 conmon[121943]: 2023-09-23T20:25:39.457+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:15.849358+0000 front 2023-09-23T20:25:15.849305+0000 (oldest deadline 2023-09-23T20:25:38.748920+0000) 2023-09-23T20:25:39.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:39 smithi144 conmon[141003]: 2023-09-23T20:25:39.268+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:39.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:39 smithi144 conmon[141003]: 2023-09-23T20:25:39.268+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:39.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:39 smithi144 conmon[400857]: 2023-09-23T20:25:39.269+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:39.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:39 smithi144 conmon[400857]: 2023-09-23T20:25:39.269+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:40.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:40 smithi144 conmon[121943]: 2023-09-23T20:25:40.476+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:40.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:40 smithi144 conmon[121943]: 2023-09-23T20:25:40.476+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:40.632 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:40 smithi144 conmon[121943]: 2023-09-23T20:25:40.476+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:15.849358+0000 front 2023-09-23T20:25:15.849305+0000 (oldest deadline 2023-09-23T20:25:38.748920+0000) 2023-09-23T20:25:40.633 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:40 smithi144 conmon[141003]: 2023-09-23T20:25:40.231+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:40.633 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:40 smithi144 conmon[141003]: 2023-09-23T20:25:40.231+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:40.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:40 smithi144 conmon[400857]: 2023-09-23T20:25:40.253+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:40.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:40 smithi144 conmon[400857]: 2023-09-23T20:25:40.253+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:40.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:40 smithi144 conmon[400857]: 2023-09-23T20:25:40.253+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:41.501 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[141003]: 2023-09-23T20:25:41.255+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:41.501 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[141003]: 2023-09-23T20:25:41.255+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:41.501 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[141003]: 2023-09-23T20:25:41.255+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:41.501 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[400857]: 2023-09-23T20:25:41.237+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:41.502 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[400857]: 2023-09-23T20:25:41.237+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:41.502 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[400857]: 2023-09-23T20:25:41.237+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:41.881 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[121943]: 2023-09-23T20:25:41.500+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:25:00.647093+0000 front 2023-09-23T20:25:00.647182+0000 (oldest deadline 2023-09-23T20:25:22.947023+0000) 2023-09-23T20:25:41.882 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[121943]: 2023-09-23T20:25:41.500+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:05.848046+0000 front 2023-09-23T20:25:05.847948+0000 (oldest deadline 2023-09-23T20:25:30.547934+0000) 2023-09-23T20:25:41.882 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:41 smithi144 conmon[121943]: 2023-09-23T20:25:41.500+0000 7ff66bc1b700 -1 osd.5 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:15.849358+0000 front 2023-09-23T20:25:15.849305+0000 (oldest deadline 2023-09-23T20:25:38.748920+0000) 2023-09-23T20:25:42.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:42 smithi144 conmon[141003]: 2023-09-23T20:25:42.288+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:42.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:42 smithi144 conmon[141003]: 2023-09-23T20:25:42.288+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:42.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:42 smithi144 conmon[141003]: 2023-09-23T20:25:42.288+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:42.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:42 smithi144 conmon[400857]: 2023-09-23T20:25:42.187+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:42.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:42 smithi144 conmon[400857]: 2023-09-23T20:25:42.187+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:42.632 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:42 smithi144 conmon[400857]: 2023-09-23T20:25:42.187+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:43.631 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 23 20:25:43 smithi144 bash[409390]: 30c9f1e12bc8578e59c5272542b47860bfcef6da802cc8514c189b58f8e6ecda 2023-09-23T20:25:43.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:43 smithi144 conmon[141003]: 2023-09-23T20:25:43.287+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:43.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:43 smithi144 conmon[141003]: 2023-09-23T20:25:43.287+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:43.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:43 smithi144 conmon[141003]: 2023-09-23T20:25:43.287+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:43.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:43 smithi144 conmon[400857]: 2023-09-23T20:25:43.162+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:43.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:43 smithi144 conmon[400857]: 2023-09-23T20:25:43.162+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:43.633 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:43 smithi144 conmon[400857]: 2023-09-23T20:25:43.162+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:44.382 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:44 smithi144 conmon[141003]: 2023-09-23T20:25:44.265+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:44.382 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:44 smithi144 conmon[141003]: 2023-09-23T20:25:44.265+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:44.382 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:44 smithi144 conmon[141003]: 2023-09-23T20:25:44.265+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:44.383 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:44 smithi144 conmon[400857]: 2023-09-23T20:25:44.148+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:44.383 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:44 smithi144 conmon[400857]: 2023-09-23T20:25:44.149+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:44.383 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:44 smithi144 conmon[400857]: 2023-09-23T20:25:44.149+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:44.664 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T20:25:44.705 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:25:44.705 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T20:25:44.705 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T20:25:44.706 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T20:25:45.031 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:44 smithi144 systemd[1]: Stopping Ceph osd.6 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:45.293 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[400857]: 2023-09-23T20:25:45.030+0000 7f19fc75f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:25:45.293 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[400857]: 2023-09-23T20:25:45.030+0000 7f19fc75f700 -1 osd.6 597 *** Got signal Terminated *** 2023-09-23T20:25:45.294 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[400857]: 2023-09-23T20:25:45.030+0000 7f19fc75f700 -1 osd.6 597 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:25:45.294 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[400857]: 2023-09-23T20:25:45.104+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:45.294 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[400857]: 2023-09-23T20:25:45.104+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:45.294 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[400857]: 2023-09-23T20:25:45.104+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:45.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[141003]: 2023-09-23T20:25:45.293+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:45.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[141003]: 2023-09-23T20:25:45.293+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:45.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:45 smithi144 conmon[141003]: 2023-09-23T20:25:45.293+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:46.382 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:46 smithi144 conmon[141003]: 2023-09-23T20:25:46.284+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:46.382 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:46 smithi144 conmon[141003]: 2023-09-23T20:25:46.284+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:46.383 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:46 smithi144 conmon[141003]: 2023-09-23T20:25:46.284+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:46.383 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:46 smithi144 conmon[400857]: 2023-09-23T20:25:46.073+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:46.383 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:46 smithi144 conmon[400857]: 2023-09-23T20:25:46.073+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:46.383 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:46 smithi144 conmon[400857]: 2023-09-23T20:25:46.073+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:47.313 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:47 smithi144 conmon[400857]: 2023-09-23T20:25:47.033+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:47.313 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:47 smithi144 conmon[400857]: 2023-09-23T20:25:47.033+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:47.313 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:47 smithi144 conmon[400857]: 2023-09-23T20:25:47.033+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:47.634 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:47 smithi144 conmon[141003]: 2023-09-23T20:25:47.312+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:47.634 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:47 smithi144 conmon[141003]: 2023-09-23T20:25:47.312+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:47.634 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:47 smithi144 conmon[141003]: 2023-09-23T20:25:47.312+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:48.340 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[400857]: 2023-09-23T20:25:48.022+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:48.340 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[400857]: 2023-09-23T20:25:48.022+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:48.340 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[400857]: 2023-09-23T20:25:48.022+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:48.340 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[400857]: 2023-09-23T20:25:48.022+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:22.791276+0000 front 2023-09-23T20:25:22.791288+0000 (oldest deadline 2023-09-23T20:25:47.491162+0000) 2023-09-23T20:25:48.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[141003]: 2023-09-23T20:25:48.339+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:48.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[141003]: 2023-09-23T20:25:48.339+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:48.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[141003]: 2023-09-23T20:25:48.339+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:49.311 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[400857]: 2023-09-23T20:25:48.990+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:49.311 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[400857]: 2023-09-23T20:25:48.990+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:49.311 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[400857]: 2023-09-23T20:25:48.990+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:49.311 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:48 smithi144 conmon[400857]: 2023-09-23T20:25:48.990+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:22.791276+0000 front 2023-09-23T20:25:22.791288+0000 (oldest deadline 2023-09-23T20:25:47.491162+0000) 2023-09-23T20:25:49.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:49 smithi144 conmon[141003]: 2023-09-23T20:25:49.310+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:49.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:49 smithi144 conmon[141003]: 2023-09-23T20:25:49.310+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:49.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:49 smithi144 conmon[141003]: 2023-09-23T20:25:49.310+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:50.284 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:49 smithi144 conmon[400857]: 2023-09-23T20:25:49.967+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:58.188353+0000 front 2023-09-23T20:24:58.188367+0000 (oldest deadline 2023-09-23T20:25:22.288201+0000) 2023-09-23T20:25:50.284 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:49 smithi144 conmon[400857]: 2023-09-23T20:25:49.967+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:04.589380+0000 front 2023-09-23T20:25:04.589365+0000 (oldest deadline 2023-09-23T20:25:30.489279+0000) 2023-09-23T20:25:50.285 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:49 smithi144 conmon[400857]: 2023-09-23T20:25:49.967+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:13.390337+0000 front 2023-09-23T20:25:13.390420+0000 (oldest deadline 2023-09-23T20:25:39.290203+0000) 2023-09-23T20:25:50.285 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:49 smithi144 conmon[400857]: 2023-09-23T20:25:49.967+0000 7f19f5d57700 -1 osd.6 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:22.791276+0000 front 2023-09-23T20:25:22.791288+0000 (oldest deadline 2023-09-23T20:25:47.491162+0000) 2023-09-23T20:25:50.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:50 smithi144 conmon[141003]: 2023-09-23T20:25:50.283+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:50.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:50 smithi144 conmon[141003]: 2023-09-23T20:25:50.283+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:50.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:50 smithi144 conmon[141003]: 2023-09-23T20:25:50.283+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:51.131 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 23 20:25:50 smithi144 bash[409966]: 052ceac7488dfdb2a0c0266b3062a18a827392fdaed4811ffeab55915425bd38 2023-09-23T20:25:51.552 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:51 smithi144 conmon[141003]: 2023-09-23T20:25:51.280+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:51.553 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:51 smithi144 conmon[141003]: 2023-09-23T20:25:51.280+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:51.553 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:51 smithi144 conmon[141003]: 2023-09-23T20:25:51.280+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:51.553 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:51 smithi144 conmon[141003]: 2023-09-23T20:25:51.280+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:25.234323+0000 front 2023-09-23T20:25:25.234312+0000 (oldest deadline 2023-09-23T20:25:50.534183+0000) 2023-09-23T20:25:52.396 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T20:25:52.436 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:25:52.436 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T20:25:52.437 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T20:25:52.437 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T20:25:52.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:52 smithi144 conmon[141003]: 2023-09-23T20:25:52.291+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:52.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:52 smithi144 conmon[141003]: 2023-09-23T20:25:52.291+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:52.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:52 smithi144 conmon[141003]: 2023-09-23T20:25:52.291+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:52.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:52 smithi144 conmon[141003]: 2023-09-23T20:25:52.291+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:25.234323+0000 front 2023-09-23T20:25:25.234312+0000 (oldest deadline 2023-09-23T20:25:50.534183+0000) 2023-09-23T20:25:52.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:52 smithi144 systemd[1]: Stopping Ceph osd.7 for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:53.131 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:52 smithi144 conmon[141003]: 2023-09-23T20:25:52.745+0000 7fad53635700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:25:53.132 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:52 smithi144 conmon[141003]: 2023-09-23T20:25:52.745+0000 7fad53635700 -1 osd.7 597 *** Got signal Terminated *** 2023-09-23T20:25:53.132 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:52 smithi144 conmon[141003]: 2023-09-23T20:25:52.745+0000 7fad53635700 -1 osd.7 597 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:25:53.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:53 smithi144 conmon[141003]: 2023-09-23T20:25:53.318+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:53.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:53 smithi144 conmon[141003]: 2023-09-23T20:25:53.318+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:53.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:53 smithi144 conmon[141003]: 2023-09-23T20:25:53.318+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:53.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:53 smithi144 conmon[141003]: 2023-09-23T20:25:53.318+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:25.234323+0000 front 2023-09-23T20:25:25.234312+0000 (oldest deadline 2023-09-23T20:25:50.534183+0000) 2023-09-23T20:25:54.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:54 smithi144 conmon[141003]: 2023-09-23T20:25:54.286+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:54.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:54 smithi144 conmon[141003]: 2023-09-23T20:25:54.286+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:54.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:54 smithi144 conmon[141003]: 2023-09-23T20:25:54.286+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:54.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:54 smithi144 conmon[141003]: 2023-09-23T20:25:54.286+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:25.234323+0000 front 2023-09-23T20:25:25.234312+0000 (oldest deadline 2023-09-23T20:25:50.534183+0000) 2023-09-23T20:25:55.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:55 smithi144 conmon[141003]: 2023-09-23T20:25:55.239+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:55.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:55 smithi144 conmon[141003]: 2023-09-23T20:25:55.239+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:55.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:55 smithi144 conmon[141003]: 2023-09-23T20:25:55.239+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:55.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:55 smithi144 conmon[141003]: 2023-09-23T20:25:55.239+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:25.234323+0000 front 2023-09-23T20:25:25.234312+0000 (oldest deadline 2023-09-23T20:25:50.534183+0000) 2023-09-23T20:25:56.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:56 smithi144 conmon[141003]: 2023-09-23T20:25:56.250+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:56.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:56 smithi144 conmon[141003]: 2023-09-23T20:25:56.250+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:56.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:56 smithi144 conmon[141003]: 2023-09-23T20:25:56.250+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:56.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:56 smithi144 conmon[141003]: 2023-09-23T20:25:56.250+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:25.234323+0000 front 2023-09-23T20:25:25.234312+0000 (oldest deadline 2023-09-23T20:25:50.534183+0000) 2023-09-23T20:25:56.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:56 smithi144 conmon[141003]: 2023-09-23T20:25:56.250+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.144:6804 osd.4 since back 2023-09-23T20:25:32.235016+0000 front 2023-09-23T20:25:32.235047+0000 (oldest deadline 2023-09-23T20:25:55.734954+0000) 2023-09-23T20:25:57.631 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:57 smithi144 conmon[141003]: 2023-09-23T20:25:57.281+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6806 osd.0 since back 2023-09-23T20:24:57.730814+0000 front 2023-09-23T20:24:57.730705+0000 (oldest deadline 2023-09-23T20:25:23.030635+0000) 2023-09-23T20:25:57.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:57 smithi144 conmon[141003]: 2023-09-23T20:25:57.281+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6814 osd.1 since back 2023-09-23T20:25:08.931628+0000 front 2023-09-23T20:25:08.931643+0000 (oldest deadline 2023-09-23T20:25:31.231622+0000) 2023-09-23T20:25:57.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:57 smithi144 conmon[141003]: 2023-09-23T20:25:57.281+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6822 osd.2 since back 2023-09-23T20:25:17.033089+0000 front 2023-09-23T20:25:17.033212+0000 (oldest deadline 2023-09-23T20:25:41.133098+0000) 2023-09-23T20:25:57.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:57 smithi144 conmon[141003]: 2023-09-23T20:25:57.281+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.135:6830 osd.3 since back 2023-09-23T20:25:25.234323+0000 front 2023-09-23T20:25:25.234312+0000 (oldest deadline 2023-09-23T20:25:50.534183+0000) 2023-09-23T20:25:57.632 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:57 smithi144 conmon[141003]: 2023-09-23T20:25:57.281+0000 7fad51436700 -1 osd.7 597 heartbeat_check: no reply from 172.21.15.144:6804 osd.4 since back 2023-09-23T20:25:32.235016+0000 front 2023-09-23T20:25:32.235047+0000 (oldest deadline 2023-09-23T20:25:55.734954+0000) 2023-09-23T20:25:58.882 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 23 20:25:58 smithi144 bash[410560]: 4b2a00379dc8493d0c70f0ba20e674f6759472f2b2773e8117d641852e1fed47 2023-09-23T20:26:00.354 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T20:26:00.395 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:26:00.395 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T20:26:00.396 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T20:26:00.396 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T20:26:00.631 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 systemd[1]: Stopping Ceph prometheus.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:26:01.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:26:01.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:26:01.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:26:01.132 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:26:01.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:26:01.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:26:01.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:26:01.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:26:01.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.694Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:26:01.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.696Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:26:01.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.696Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:26:01.133 INFO:journalctl@ceph.prometheus.a.smithi144.stdout:Sep 23 20:26:00 smithi144 conmon[178718]: ts=2023-09-23T20:26:00.696Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:26:01.359 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T20:26:01.395 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:26:01.395 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T20:26:01.396 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T20:26:02.721 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 20:26:02 smithi135 systemd[1]: Stopping Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:26:03.220 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 20:26:02 smithi135 conmon[199576]: ts=2023-09-23T20:26:02.768Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:26:03.630 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 20:26:03 smithi135 bash[713955]: 22e51759a65e52bf00d0348418b5f8654d9f519f952ca85c94179bab8d98f768 2023-09-23T20:26:03.630 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 20:26:03 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:26:03.630 INFO:journalctl@ceph.alertmanager.a.smithi135.stdout:Sep 23 20:26:03 smithi135 systemd[1]: Stopped Ceph alertmanager.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T20:26:03.971 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 20:26:03 smithi135 systemd[1]: Stopping Ceph node-exporter.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:26:04.690 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 20:26:04 smithi135 bash[714240]: caa9af75261e5f75e88c2c53173cb1ecb03a964ae5f232b013a43ea089586a36 2023-09-23T20:26:04.691 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 20:26:04 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:26:04.691 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 20:26:04 smithi135 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T20:26:04.691 INFO:journalctl@ceph.node-exporter.a.smithi135.stdout:Sep 23 20:26:04 smithi135 systemd[1]: Stopped Ceph node-exporter.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T20:26:12.574 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T20:26:14.127 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 20:26:13 smithi144 systemd[1]: Stopping Ceph grafana.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:26:14.751 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 20:26:14 smithi144 bash[411758]: 902bff181c8509e46c48b7724c84d8c06e55d53f5a5aaea09119e79aa3b5a3dc 2023-09-23T20:26:14.752 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 20:26:14 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:26:14.752 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 20:26:14 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T20:26:14.752 INFO:journalctl@ceph.grafana.a.smithi144.stdout:Sep 23 20:26:14 smithi144 systemd[1]: Stopped Ceph grafana.a for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T20:26:15.085 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 20:26:14 smithi144 systemd[1]: Stopping Ceph node-exporter.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7... 2023-09-23T20:26:15.632 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 20:26:15 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:26:15.632 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 20:26:15 smithi144 bash[412037]: a8b1ed1aeea77b07fec36a14302a64287c8393793a2e043b353610a175fada7f 2023-09-23T20:26:15.632 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 20:26:15 smithi144 systemd[1]: ceph-2f95c858-5a48-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T20:26:15.632 INFO:journalctl@ceph.node-exporter.b.smithi144.stdout:Sep 23 20:26:15 smithi144 systemd[1]: Stopped Ceph node-exporter.b for 2f95c858-5a48-11ee-8db4-212e2dc638e7. 2023-09-23T20:26:20.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:26:21.032 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:26:21.062 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T20:26:21.065 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812/remote/smithi135/crash 2023-09-23T20:26:21.066 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T20:26:21.130 INFO:teuthology.orchestra.run.smithi135.stderr:tar: /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T20:26:21.130 INFO:teuthology.orchestra.run.smithi135.stderr:tar: Error is not recoverable: exiting now 2023-09-23T20:26:21.132 DEBUG:teuthology.misc:Transferring archived files from smithi144:/var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812/remote/smithi144/crash 2023-09-23T20:26:21.133 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T20:26:21.188 INFO:teuthology.orchestra.run.smithi144.stderr:tar: /var/lib/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T20:26:21.189 INFO:teuthology.orchestra.run.smithi144.stderr:tar: Error is not recoverable: exiting now 2023-09-23T20:26:21.190 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T20:26:21.190 DEBUG:teuthology.orchestra.run.smithi135:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T20:26:21.226 INFO:teuthology.orchestra.run.smithi135.stderr:grep: /var/log/ceph/2f95c858-5a48-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T20:26:21.228 INFO:tasks.cephadm:Compressing logs... 2023-09-23T20:26:21.229 DEBUG:teuthology.orchestra.run.smithi135:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:26:21.271 DEBUG:teuthology.orchestra.run.smithi144:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:26:21.308 INFO:teuthology.orchestra.run.smithi135.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T20:26:21.310 INFO:teuthology.orchestra.run.smithi144.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T20:35:00.638 INFO:tasks.cephadm:Archiving logs... 2023-09-23T20:35:00.689 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812/remote/smithi135/log 2023-09-23T20:35:00.692 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T20:36:08.084 DEBUG:teuthology.misc:Transferring archived files from smithi144:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812/remote/smithi144/log 2023-09-23T20:36:08.086 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T20:36:55.277 INFO:tasks.cephadm:Removing cluster... 2023-09-23T20:36:55.278 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 --force 2023-09-23T20:36:56.463 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2f95c858-5a48-11ee-8db4-212e2dc638e7 --force 2023-09-23T20:36:57.581 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T20:36:57.582 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T20:36:57.605 DEBUG:teuthology.orchestra.run.smithi144:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T20:36:57.644 INFO:tasks.cephadm:Teardown complete 2023-09-23T20:36:57.644 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T20:36:57.656 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T20:36:57.656 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T20:36:57.659 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T20:36:57.783 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T20:36:57.793 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T20:36:57.794 DEBUG:teuthology.orchestra.run.smithi135:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:36:57.796 DEBUG:teuthology.orchestra.run.smithi144:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:36:57.812 INFO:teuthology.orchestra.run.smithi144.stderr:bash: ntpq: command not found 2023-09-23T20:36:57.812 INFO:teuthology.orchestra.run.smithi135.stderr:bash: ntpq: command not found 2023-09-23T20:36:57.836 INFO:teuthology.orchestra.run.smithi144.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T20:36:57.836 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================== 2023-09-23T20:36:57.836 INFO:teuthology.orchestra.run.smithi144.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 13 -1191us[-1172us] +/- 70ms 2023-09-23T20:36:57.837 INFO:teuthology.orchestra.run.smithi144.stdout:^* hv02.front.sepia.ceph.com 3 6 377 11 -100us[ -80us] +/- 52ms 2023-09-23T20:36:57.837 INFO:teuthology.orchestra.run.smithi144.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 13 +1358us[+1377us] +/- 72ms 2023-09-23T20:36:57.837 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:36:57.845 INFO:teuthology.orchestra.run.smithi135.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T20:36:57.845 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================== 2023-09-23T20:36:57.846 INFO:teuthology.orchestra.run.smithi135.stdout:^+ hv01.front.sepia.ceph.com 2 7 377 14 -1234us[-1325us] +/- 70ms 2023-09-23T20:36:57.846 INFO:teuthology.orchestra.run.smithi135.stdout:^* hv02.front.sepia.ceph.com 3 7 377 11 -137us[ -228us] +/- 52ms 2023-09-23T20:36:57.846 INFO:teuthology.orchestra.run.smithi135.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 11 +1300us[+1300us] +/- 72ms 2023-09-23T20:36:57.847 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:36:57.848 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T20:36:57.858 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T20:36:57.859 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T20:36:57.869 DEBUG:teuthology.orchestra.run.smithi135:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T20:36:58.030 DEBUG:teuthology.orchestra.run.smithi144:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T20:36:58.159 DEBUG:teuthology.orchestra.run.smithi135:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T20:36:58.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:36:58.192 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T20:36:58.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:36:58.223 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T20:36:58.233 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T20:36:58.247 INFO:teuthology.task.internal:Duration was 4611.845016 seconds 2023-09-23T20:36:58.247 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T20:36:58.263 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T20:36:58.263 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T20:36:58.266 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T20:36:58.332 INFO:teuthology.orchestra.run.smithi144.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T20:36:58.375 INFO:teuthology.orchestra.run.smithi135.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T20:36:58.742 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T20:36:58.743 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi135.front.sepia.ceph.com 2023-09-23T20:36:58.743 DEBUG:teuthology.orchestra.run.smithi135:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T20:36:58.764 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi144.front.sepia.ceph.com 2023-09-23T20:36:58.764 DEBUG:teuthology.orchestra.run.smithi144:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T20:36:58.797 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T20:36:58.797 DEBUG:teuthology.orchestra.run.smithi135:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:36:58.807 DEBUG:teuthology.orchestra.run.smithi144:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:36:58.961 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T20:36:58.961 DEBUG:teuthology.orchestra.run.smithi135:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T20:36:58.963 DEBUG:teuthology.orchestra.run.smithi144:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T20:36:59.305 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T20:36:59.327 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T20:36:59.328 DEBUG:teuthology.orchestra.run.smithi135:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T20:36:59.381 DEBUG:teuthology.orchestra.run.smithi144:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T20:36:59.422 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T20:36:59.507 DEBUG:teuthology.orchestra.run.smithi135:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T20:36:59.509 DEBUG:teuthology.orchestra.run.smithi144:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T20:36:59.542 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern = core 2023-09-23T20:36:59.551 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern = core 2023-09-23T20:36:59.570 DEBUG:teuthology.orchestra.run.smithi135:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T20:36:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:36:59.618 DEBUG:teuthology.orchestra.run.smithi144:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T20:36:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:36:59.636 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T20:36:59.697 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T20:36:59.709 DEBUG:teuthology.misc:Transferring archived files from smithi135:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812/remote/smithi135 2023-09-23T20:36:59.710 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T20:36:59.963 DEBUG:teuthology.misc:Transferring archived files from smithi144:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812/remote/smithi144 2023-09-23T20:36:59.964 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T20:37:00.037 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T20:37:00.037 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T20:37:00.040 DEBUG:teuthology.orchestra.run.smithi144:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T20:37:00.064 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T20:37:00.090 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T20:37:00.091 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T20:37:00.105 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T20:37:00.105 DEBUG:teuthology.orchestra.run.smithi135:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T20:37:00.108 DEBUG:teuthology.orchestra.run.smithi144:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T20:37:00.122 INFO:teuthology.orchestra.run.smithi135.stdout: 262320 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 20:37 /home/ubuntu/cephtest 2023-09-23T20:37:00.122 INFO:teuthology.orchestra.run.smithi135.stdout: 3932161 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 20:23 /home/ubuntu/cephtest/mnt.0 2023-09-23T20:37:00.122 INFO:teuthology.orchestra.run.smithi135.stdout: 3932165 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 20:23 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:37:00.125 INFO:teuthology.orchestra.run.smithi144.stdout: 262320 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 20:37 /home/ubuntu/cephtest 2023-09-23T20:37:00.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:37:00.137 INFO:teuthology.orchestra.run.smithi135.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T20:37:00.137 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi135 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T20:37:00.144 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T20:37:00.172 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T20:37:00.196 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi135 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T20:37:00.196 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T20:37:00.228 DEBUG:teuthology.nuke:shortname: smithi135 2023-09-23T20:37:00.228 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T20:37:00.260 DEBUG:teuthology.nuke:shortname: smithi144 2023-09-23T20:37:00.261 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T20:37:00.267 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi135.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 19:14:57.983037', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T20:37:00.284 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi144.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401812', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 19:14:57.984771', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T20:37:00.321 INFO:teuthology.orchestra.console.smithi135:Power off 2023-09-23T20:37:00.322 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T20:37:00.339 INFO:teuthology.orchestra.console.smithi144:Power off 2023-09-23T20:37:00.339 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T20:37:00.362 DEBUG:teuthology.orchestra.console.smithi135:power off output: Chassis Power Control: Down/Off 2023-09-23T20:37:00.363 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:37:00.376 DEBUG:teuthology.orchestra.console.smithi144:power off output: Chassis Power Control: Down/Off 2023-09-23T20:37:00.377 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:37:00.392 DEBUG:teuthology.orchestra.console.smithi135:check power output: Chassis Power is on 2023-09-23T20:37:00.409 DEBUG:teuthology.orchestra.console.smithi144:check power output: Chassis Power is on 2023-09-23T20:37:04.393 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:37:04.410 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:37:04.505 DEBUG:teuthology.orchestra.console.smithi135:check power output: Chassis Power is on 2023-09-23T20:37:04.520 DEBUG:teuthology.orchestra.console.smithi144:check power output: Chassis Power is on 2023-09-23T20:37:08.510 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:37:08.521 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T20:37:08.621 DEBUG:teuthology.orchestra.console.smithi135:check power output: Chassis Power is off 2023-09-23T20:37:08.630 DEBUG:teuthology.orchestra.console.smithi144:check power output: Chassis Power is off 2023-09-23T20:37:08.723 INFO:teuthology.orchestra.console.smithi135:Power off completed 2023-09-23T20:37:08.731 INFO:teuthology.orchestra.console.smithi144:Power off completed 2023-09-23T20:37:08.836 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 4611.845015525818 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=1a498cc713e249dfa3ac45d5db36436a status: fail success: false 2023-09-23T20:37:08.837 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T20:37:08.920 INFO:teuthology.run:FAIL